pip install selenium

#下载wheel安装文件：https://pypi.python.org/pypi/selenium/#downloads
pip install selenium-3.13.0-py2.py3-none-any.whl

from selenium import webdriver

browser=webdriver.Chrome()

#如果能弹出一个空白的chrome浏览器页面，说明配置成功

(virtualenv-36) C:\Users\Administrator\PycharmProjects\untitled\pachong>phantomjs -v
2.1.1
#在python的terminal环境下执行命令phantomjs -v，如果能打印版本信息说明安装成功

from selenium import webdriver

#browser=webdriver.Firefox()
browser=webdriver.Chrome()
#browser=webdriver.Edge()
#browser=webdriver.Safari()

print(type(browser))

#返回的是一个WebDriver对象
<class ‘selenium.webdriver.chrome.webdriver.WebDriver‘>

from selenium import webdriver
driver=webdriver.Chrome()
driver.get(‘https://www.baidu.com‘)
driver.execute_script("alert(‘are you sure‘);")

#它基本可以实现JavaScript的所有功能

from selenium import webdriver

browser=webdriver.Chrome()
browser.get(‘http://selenium-python.readthedocs.io‘)
browser.execute_script(‘window.open("https://www.baidu.com");‘)  #在标签页打开URL
browser.execute_script(‘window.open("https://www.taobao.com");‘)

browser.back()  #后退到前一个页面
browser.set_page_load_timeout(5)
browser.forward()  #前进到下一个页面
print(browser.name)
print(browser.title)
print(browser.current_url)
print(browser.current_window_handle)
print(browser.get_cookies())
print(type(browser))

#
chrome
Selenium with Python — Selenium Python Bindings 2 documentation
http://selenium-python.readthedocs.io/
CDwindow-243FD31239F20FCC0195DD522A60A0DA
[{‘domain‘: ‘.readthedocs.io‘, ‘expiry‘: 1530766561, ‘httpOnly‘: False, ‘name‘: ‘_gid‘, ‘path‘: ‘/‘, ‘secure‘: False, ‘value‘: ‘GA1.2.1126774326.1530680157‘}, {‘domain‘: ‘.readthedocs.io‘, ‘expiry‘: 1593752161, ‘httpOnly‘: False, ‘name‘: ‘_ga‘, ‘path‘: ‘/‘, ‘secure‘: False, ‘value‘: ‘GA1.2.2096958532.1530680157‘}, {‘domain‘: ‘.readthedocs.io‘, ‘expiry‘: 1530680217, ‘httpOnly‘: False, ‘name‘: ‘_gat_rtfd‘, ‘path‘: ‘/‘, ‘secure‘: False, ‘value‘: ‘1‘}]
<class ‘selenium.webdriver.chrome.webdriver.WebDriver‘>

from selenium import webdriver

driver=webdriver.Chrome()
driver.get(‘http://www.python.org‘)
driver.save_screenshot(‘screenshot.png‘)  #保持页面截图到当前路径
driver.quit()

from selenium import webdriver
driver=webdriver.Chrome()
driver.get(‘http://www.python.org‘)
#通过DOM中的window对象的scrollTo方法，将窗口位置滚动到指定位置，document.body.scrollHeight返回整个body的高度，所以页面将滚动到页面底部
driver.execute_script("window.scrollTo(0,document.body.scrollHeight);")

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2018/7/5 16:19
# @Author  : Py.qi
# @File    : test7.py
# @Software: PyCharm
from selenium import webdriver
driver=webdriver.Chrome()
driver.get(‘https://www.baidu.com‘)
print(driver.get_cookies())   #获取所有cookies
driver.add_cookie({‘name‘:‘name‘,‘domain‘:‘www.baidu.com‘,‘value‘:‘germey‘})   #添加cookie
print(driver.get_cookies())
driver.delete_all_cookies()
print(driver.get_cookies())

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2018/7/4 13:03
# @Author  : Py.qi
# @File    : test2.py
# @Software: PyCharm
from selenium import webdriver
from selenium.webdriver.common.by import By
driver=webdriver.Chrome()
driver.get(‘http://selenium-python.readthedocs.io/locating-elements.html#locating-elements‘)
data=driver.find_element(By.CLASS_NAME,‘simple‘)
#driver.find_element(By.ID,‘IDname‘) #获取ID标签定位元素
#driver.find_element(By.CSS_SELECTOR,‘cssname‘)#CSS选择器定位元素
#driver.find_element(By.LINK_TEXT,‘linktext‘) #链接文本定位元素
#driver.find_element(By.PARTIAL_LINK_TEXT,‘linktext‘) #部分链接文件定位元素
#driver.find_element(By.NAME,‘name‘) #属性名定位元素
#driver.find_element(By.TAG_NAME,‘tagname‘) #标签名定位元素

print(data.text)  #打印元素文本内容

#
find_element_by_id
find_element_by_name
find_element_by_xpath
find_element_by_link_text
find_element_by_partial_link_text
find_element_by_tag_name
find_element_by_class_name
find_element_by_css_selector

#如获取多个，可是使用：
data=driver.find_elements(By.CLASS_NAME,‘simple‘)

driver.find_element_by_id(‘idname‘)
driver.find_elements_by_id(‘idname‘)

driver.find_element_by_name(‘name‘)
driver.find_elements_by_name(‘name‘)

driver.find_element_by_xpath("//from[1]")  #查看第一个表单元素
friver.dind_element_by_xpath("//from[@id=‘loginform‘]")  #查找id为loinform的表单元素

<HTML>
 <BODY>
  <p>你确定要这么做吗？</ p>
  <a href="continue.html">继续</a>
  <a href="cancel.html">取消</a>
</ BODY>
<HTML>


driver.find_element_by_link_text(‘继续)  #通过链接文本定位到元素

driver.find_element_by_tag_name(‘h1‘)

driver.find_element_by_class_name(‘content‘)

driver.find_element_by_css_selector(‘p.content‘)

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2018/7/4 13:55
# @Author  : Py.qi
# @File    : test3.py
# @Software: PyCharm
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
opt=Options()
opt.add_argument(‘headless‘)
driver=webdriver.Chrome(chrome_options=opt)
driver.get(‘http://selenium-python.readthedocs.io/api.html#module-selenium.webdriver.remote.webelement‘)
element=driver.find_element_by_id(‘module-selenium.webdriver.remote.webelement‘)
print(element)
print(type(element))

#返回一个webelement对象
<selenium.webdriver.remote.webelement.WebElement (session="dfaee65201abdf5a931306df6e7fe421", element="0.95256057244967-1")>
<class ‘selenium.webdriver.remote.webelement.WebElement‘>

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2018/7/4 13:55
# @Author  : Py.qi
# @File    : test3.py
# @Software: PyCharm
from selenium import webdriver

driver=webdriver.Chrome()
driver.get(‘http://www.cnblogs.com/zhangxinqi/‘)
element=driver.find_element_by_id(‘q‘)  #获取输入框元素
element.send_keys(‘python3之requests‘)  #发送元素
button=driver.find_element_by_id(‘btnZzk‘)  #获取搜索按钮
button.click()  #发送搜索动作
data=driver.page_source   

print(driver.current_url)   #打印URL
print(data)
print(type(element))
driver.close()

#
http://zzk-s.cnblogs.com/s?w=blog%3Azhangxinqi%20python3%E4%B9%8Brequests
<!DOCTYPE html><html xmlns="http://www.w3.org/1999/xhtml"><head>
    <meta charset="utf-8" />
    <title> blog:zhangxinqi python3之requests - 博客园找找看</title>......

element = driver.find_element_by_name("source")
target = driver.find_element_by_name("target")
 
from selenium.webdriver import ActionChains
action_chains = ActionChains(driver)
action_chains.drag_and_drop(element, target).perform()

menu = driver.find_element_by_css_selector(".nav") #获取element对象
hidden_submenu = driver.find_element_by_css_selector(".nav #submenu1")  #获取点击对象
#创建鼠标对象
actions = ActionChains(driver)
#移动鼠标到对象
actions.move_to_element(menu)
#点击对象
actions.click(hidden_submenu)
#执行操作
actions.perform()

import time
from selenium import webdriver
from selenium.webdriver.common.alert import Alert
driver=webdriver.Chrome()
driver.get(‘https://www.baidu.com‘)
driver.execute_script("alert(‘确定‘);")  #弹出窗口
time.sleep(2)
print(driver.switch_to.alert.text) #获取alert文本
alert=Alert(driver).accept()  #自动点击确定窗口

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2018/7/5 13:05
# @Author  : Py.qi
# @File    : test5.py
# @Software: PyCharm
import requests
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.action_chains import ActionChains
driver=webdriver.Chrome()
driver.get(‘https://pypi.org/‘)

element=driver.find_element_by_id(‘search‘)  #获取输入框
element.send_keys(‘selenium‘)  #搜索selenium包
element.send_keys(Keys.ENTER)  #按回车键

element_a=driver.find_element_by_link_text(‘selenium‘) #定位selenium包链接

ActionChains(driver).move_to_element(element_a).click(element_a).perform() #按左键点击链接执行

element_down=driver.find_element_by_link_text(‘Download files‘)  #定位下载链接
ActionChains(driver).move_to_element(element_down).click(element_down).perform()  #按左键点击链接

element_selenium=driver.find_element_by_link_text(‘selenium-3.13.0.tar.gz‘)  #定位元素selenium下载包链接
data=element_selenium.get_attribute(‘href‘)   #获取链接地址
with open(‘selenium-3.13.0.tar.gz‘,‘wb‘) as f:
    source=requests.get(data).content   #请求下载链接地址获取二进制包数据
    f.write(source)  #写入数据
    f.close()
    
driver.quit()

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2018/7/5 14:41
# @Author  : Py.qi
# @File    : test6.py
# @Software: PyCharm
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
driver=webdriver.Chrome()
driver.get(‘https://www.taobao.com/‘)
wait=WebDriverWait(driver,3)  #设置监听driver等待时间3秒
input=wait.until(EC.presence_of_element_located((By.ID,‘q‘))) #设置等待条件为id为q的元素加载完成
button=wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR,‘.btn-search‘)))  #设置等待条件为class名为btn-search的元素加载完成
print(input,button)

from selenium import webdriver

driver = webdriver.Firefox()
driver.implicitly_wait(10) #隐式等待设置为10等待时间
driver.get("http://somedomain/url_that_delays_loading")
myDynamicElement = driver.find_element_by_id("myDynamicElement")

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2018/7/5 16:19
# @Author  : Py.qi
# @File    : test7.py
# @Software: PyCharm
from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException
driver=webdriver.Chrome()
driver.get(‘https://www.baidu.com‘)
try:
    element=driver.find_element_by_id(‘test‘)
    print(element)
except NoSuchElementException as e:
    print(‘元素不存在：‘,e)

#
元素不存在： Message: no such element: Unable to locate element: {"method":"id","selector":"test"}
  (Session info: chrome=67.0.3396.99)
  (Driver info: chromedriver=2.40.565498 (ea082db3280dd6843ebfb08a625e3eb905c4f5ab),platform=Windows NT 10.0.10240 x86_64)

#!/usr/bin/env python
#coding:utf-8
import re
import pymongo
from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from urllib.parse import quote
from selenium.common.exceptions import WebDriverException
from pyquery import PyQuery as pq
#链接mongodb数据库
client=pymongo.MongoClient(host=‘localhost‘,port=27017)
db=client[‘taobao‘]
#定义无头chrome
opt=webdriver.ChromeOptions()
opt.add_argument(‘--headless‘)
driver=webdriver.Chrome(chrome_options=opt)
#定义页面等待时间
wait=WebDriverWait(driver,10)
#定义搜索商品名
uname=‘iPad‘

#搜索商品
def search():
    try:
        url = ‘https://s.taobao.com/search?q=‘ + quote(uname)
        driver.get(url)
        total=wait.until(EC.presence_of_element_located((By.CSS_SELECTOR,‘#mainsrp-pager > div > div > div > div.total‘)))
    except TimeoutException:
        return search()
    return total.text

#实现翻页商品
def next_page(page):
    print(‘正在抓取第{}‘.format(page))
    try:
        if page >= 1:
            input=wait.until(EC.presence_of_element_located((By.CSS_SELECTOR,‘#mainsrp-pager > div > div > div > div.form > input‘)))
            submit=wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR,‘#mainsrp-pager > div > div > div > div.form > span.btn.J_Submit‘)))
            input.clear()
            input.send_keys(page)
            submit.click()
            wait.until(EC.text_to_be_present_in_element((By.CSS_SELECTOR,‘#mainsrp-pager > div > div > div > ul > li.item.active > span‘),str(page)))
            wait.until(EC.presence_of_element_located((By.CSS_SELECTOR,‘#mainsrp-itemlist .items .item‘)))

            get_products()
    except TimeoutException:
        next_page(page)
    except WebDriverException as e:
        print(‘index_page:‘,e)

#解析商品信息
def get_products():
    #print(‘开始解析页面...‘)
    html = driver.page_source
    doc = pq(html, parser=‘html‘)
    items = doc(‘#mainsrp-itemlist .items .item‘).items()
    for i in items:
        product = {
            ‘image‘: ‘https:‘ + i.find(‘.pic .img‘).attr(‘data-src‘),
            ‘price‘: i.find(‘.price‘).text(),
            ‘deal‘: i.find(‘.deal-cnt‘).text(),
            ‘title‘: i.find(‘.title‘).text(),
            ‘shop‘: i.find(‘.shop‘).text(),
            ‘location‘: i.find(‘.location‘).text()
        }
        #print(product)
        save_to_mongo(product)

#保存到mongodb
def save_to_mongo(result):
    try:
        if db[‘collection_taobao‘].insert(result):
            print(‘保存到mongodb成功！‘,result)
    except Exception:
        print(‘保存到mongodb失败‘,result)

#主函数调用
def main():
    try:
        total=search()
        total=int(re.compile(‘(\d+)‘).search(total).group(1))
        for i in range(1,total+1):
            next_page(i)
    finally:
        driver.quit()

#执行函数入口
if __name__ == ‘__main__‘:
    main()