python学习第五天

‘‘‘
post请求访问github
https://github.com/session

post
Referer:https://github.com/login
User-Agent:Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36
From data:
commit:Sign in
utf8:?
authenticity_token:AJdAVBwjyD7pSBlTAss/+dPNzXG4TRZCkai517a7d1tZI3HBbv57wCdWBMLmloSliZ9A7DwgGZsQ1D1B0pENIA==
login:1343805472
password:17855320635zmm
webauthn-support:unknown
‘‘‘

#访问login页获取token信息
‘‘‘
请求url:https://github.com/login
请求方式：GET
响应头：Set-Cookie
请求头：Cookie
User-Agent
‘‘‘

import requests
import re
headers={
    ‘User-Agent‘:‘Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36‘
}
response = requests.get(url=‘https://github.com/login‘, headers=headers)
#print(response.text)
login_cookies=response.cookies.get_dict()
print(login_cookies)


authenticity_token=re.findall(‘<input type="hidden" name="authenticity_token" value="(.*?)" />‘,response.text,re.S)[0]
print(authenticity_token)

#二、往sessionurl发送post请求
#
headers2={
    ‘Referer‘: ‘https://github.com/login‘,
    ‘User-Agent‘:‘Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36‘
}

# login_cookies=response.cookies.get_dict()
form_data={
    "utf8":"?",
    "authenticity_token":authenticity_token,
    "login":"1343805472@qq.com",
    "password":"17855320635zmm",
    "webauthn-support":"unsupported",
    "commit":"Sign in",
}
response2=requests.post(url=‘https://github.com/session‘,data=form_data,headers=headers2,cookies=login_cookies)
print(response2.status_code)

with open(‘github.html‘,‘w‘,encoding=‘utf-8‘) as f:
    f.write(response2.text)

import requests

response=requests.get(‘http://baidu.com‘)
print(response.status_code)#获取响应状态码
print(response.url)#获取url地址
print(response.text)#获取文本
print(response.content)#获取二进制流
print(response.headers)#获取页面请求头信息
print(response.history)#上一次跳转的信息
#1、返回cookie字典 2、返回cookie对象
print(response.cookies)#获取cookie信息
print(response.cookies.get_dict())#获取cookie信息转换成字典
print(response.cookies.items())#获取cookie信息转换成字典
print(response.cookies.items())#获取cookie信息转换成字典
print(response.encoding)#
print(response.elapsed)#访问时间

#证书验证(大部分网站都是https)
import requests
# 如果是ssl请求,首先检查证书是否合法,不合法则报错,程序终端
response = requests.get(‘https://www.xiaohuar.com‘)
print(response.status_code)

# 改进1:去掉报错,但是会报警告
import requests
response = requests.get(‘https://www.xiaohuar.com‘, verify=False)
# 不验证证书,报警告,返回200
print(response.status_code)

# 改进2:去掉报错,并且去掉警报信息
import requests
import urllib3
urllib3.disable_warnings()  # 关闭警告
response = requests.get(‘https://www.xiaohuar.com‘, verify=False)
print(response.status_code)

# 改进3:加上证书
# 很多网站都是https,但是不用证书也可以访问,大多数情况都是可以携带也可以不携带证书
# 知乎\百度等都是可带可不带
# 有硬性要求的,则必须带，比如对于定向的用户,拿到证书后才有权限访问某个特定网站
import requests
import urllib3
# urllib3.disable_warnings()  # 关闭警告
response = requests.get(
    ‘https://www.xiaohuar.com‘,
    # verify=False,
    cert=(‘/path/server.crt‘, ‘/path/key‘))
print(response.status_code)

from selenium import webdriver  # 用来驱动浏览器的

#调用得到一个动作链对象，破解滑动验证码的时候用的 可以拖动图片
from selenium.webdriver import ActionChains  # 破解滑动验证码的时候用的 可以拖动图片

#按照什么方式查找属性，By.ID,By.CSS_SELECTOR
from selenium.webdriver.common.by import By  # 按照什么方式查找，By.ID,By.CSS_SELECTOR
from selenium.webdriver.common.keys import Keys  # 键盘按键操作
from selenium.webdriver.support import expected_conditions as EC  # 和下面WebDriverWait一起用的
from selenium.webdriver.support.wait import WebDriverWait  # 等待页面加载某些元素






from selenium import webdriver
import time

chrome = webdriver.Chrome(r‘F:\软件\chromedriver_win32\chromedriver.exe‘)
#若try出现异常
try:
    #通过谷歌浏览器驱动打开谷歌浏览器
    #chrome=webdriver.Chrome(r‘F:\软件\chromedriver_win32\chromedriver.exe‘)
    # 往lyj博客发送get请求
    #chrome.get(‘https://www.cnblogs.com/lyj68/‘)


    wait=WebDriverWait(chrome,10)

    #1、访问百度
    chrome.get(‘https://www.baidu.com/‘)

    #2、查找input输入框
    input_tag=wait.until(
        #调用EC的presence_of_element_located()
        EC.presence_of_element_located((By.ID,"kw")))


    #3、搜索一拳超人
    input_tag.send_keys(‘一拳超人‘)

    #4、按键盘回车键
    input_tag.send_keys(Keys.ENTER)
    time.sleep(10)
#无论发生什么都会关闭浏览器
finally:
#关闭浏览器
    chrome.close()

try:
    # 通过谷歌浏览器驱动打开谷歌浏览器
    # chrome=webdriver.Chrome(r‘F:\软件\chromedriver_win32\chromedriver.exe‘)
    # 往lyj博客发送get请求
    # chrome.get(‘https://www.cnblogs.com/lyj68/‘)

    wait = WebDriverWait(chrome, 10)

    # 1、访问京东主页
    chrome.get(‘https://www.jd.com/‘)

    # 2、查找input输入框
    input_tag = wait.until(
        # 调用EC的presence_of_element_located()
        EC.presence_of_element_located((By.ID, "key")))

    # 3、搜索唐诗三百首
    input_tag.send_keys(‘唐诗三百首‘)

    # 4、根据class属性标签查找名称
    search_button=wait.until(EC.presence_of_element_located((By.CLASS_NAME,‘button‘)))

    #5、点击搜索按钮
    search_button.click()
    time.sleep(10)
# 无论发生什么都会关闭浏览器
finally:
    # 关闭浏览器
    chrome.close()

from selenium import webdriver  # 用来驱动浏览器的
import time

‘‘‘
隐式等待
‘‘‘
# 获取驱动对象、
driver = webdriver.Chrome()

try:
    # 显式等待: 等待某个元素加载
    # 参数1: 驱动对象  参数2: 等待时间
    # wait = WebDriverWait(chrome, 10)

    driver.get(‘https://china.nba.com/‘)

    # 隐式等待: 等待页面所有元素加载
    driver.implicitly_wait(10)
    news_tag = driver.find_element_by_class_name(‘nav-news‘)
    # 获取标签对象
    print(news_tag)
    # 获取标签的名字
    print(news_tag.tag_name)


    time.sleep(10)

finally:
    driver.close()


from selenium import webdriver  # 用来驱动浏览器的
import time

‘‘‘
===============所有方法===================
    element是查找一个标签
    elements是查找所有标签

    1、find_element_by_link_text  通过链接文本去找
    2、find_element_by_id 通过id去找
    3、find_element_by_class_name
    4、find_element_by_partial_link_text
    5、find_element_by_name
    6、find_element_by_css_selector
    7、find_element_by_tag_name
‘‘‘
# 获取驱动对象、
driver = webdriver.Chrome()

try:

    # 往百度发送请求
    driver.get(‘https://www.baidu.com/‘)
    driver.implicitly_wait(10)

    # 1、find_element_by_link_text  通过链接文本去找
    # 根据登录
    # send_tag = driver.find_element_by_link_text(‘登录‘)
    # send_tag.click()

    # 2、find_element_by_partial_link_text 通过局部文本查找a标签
    login_button = driver.find_element_by_partial_link_text(‘登‘)
    login_button.click()
    time.sleep(1)

    # 3、find_element_by_class_name 根据class属性名查找
    login_tag = driver.find_element_by_class_name(‘tang-pass-footerBarULogin‘)
    login_tag.click()
    time.sleep(1)

    # 4、find_element_by_name 根据name属性查找
    username = driver.find_element_by_name(‘userName‘)
    username.send_keys(‘15622792660‘)
    time.sleep(1)

    # 5、find_element_by_id 通过id属性名查找
    password = driver.find_element_by_id(‘TANGRAM__PSP_10__password‘)
    password.send_keys(‘*******‘)
    time.sleep(1)

    # 6、find_element_by_css_selector  根据属性选择器查找
    # 根据id查找登录按钮
    login_submit = driver.find_element_by_css_selector(‘#TANGRAM__PSP_10__submit‘)
    # driver.find_element_by_css_selector(‘.pass-button-submit‘)
    login_submit.click()

    # 7、find_element_by_tag_name  根据标签名称查找标签
    div = driver.find_element_by_tag_name(‘div‘)
    print(div.tag_name)

    time.sleep(10)

finally:
    driver.close()

‘‘‘ 爬取快代理： 
    1.访问快代理页面 
    2.通过re模块解析并提取所有代理 
    3.通过ip测试网站对爬取的代理进行测试
    4.若test_ip函数抛出异常代表代理作废，否则代理有效 
    5.利用有效的代理进行代理测试 
    <tr>
                <td data-title="IP">124.205.143.212 

</td>
                <td data-title="PORT">40585</td>
                <td data-title="匿名度">高匿名</td>
                <td data-title="类型">HTTP</td>
                <td data-title="位置">北京市北京市 鹏博士宽带</td>
                <td data-title="响应速度">2秒</td>
                <td data-title="最后验证时间">2019-06-17 16:30:54</td>
                </tr>
    re:
        <tr>.*?<td data-title="IP">(.*?)</td>.*?<td data-title="PORT">(.*?)</td>
‘‘‘
‘‘‘ 
页面链接 
第一页： 
    https://www.kuaidaili.com/free/ 

 
第二页： 
    https://www.kuaidaili.com/free/inha/2/ 

 
‘‘‘
import requests
import re
import time
headers = {
    ‘User-Agent‘: ‘Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36‘,
}
def get_index(url):
    time.sleep(1)
    response1 = requests.get(url, headers=headers)
    return response1
def parse_index(text):
    ip_list1 = re.findall(‘<tr>.*?<td data-title="IP">(.*?)</td>.*?<td data-title="PORT">(.*?)</td>‘, text, re.S)
    for ip_port in ip_list1:
        ip1 = ‘:‘.join(ip_port)
        yield ip1
def test_ip(ip2):
    print(‘测试ip: %s‘ % ip2)
    try:
        proxies = {‘https‘: ip2}
        # ip测试网站
        ip_url1 = ‘https://www.ipip.net/ 

‘
        # 使用有效与无效的代理对ip测试站点进行访问，若返回的结果为200则代表当前测试ip正常
        response2 = requests.get(ip_url1, headers=headers, proxies=proxies, timeout=1)
        if response2.status_code == 200:
            return ip
        # 若ip代理无效则抛出异常
    except Exception as e:
            print(e)
        # 使用代理爬取nba
    def spider_nba(good_ip1):
        url = ‘https://china.nba.com/ 

‘
        proxies = {‘https‘: good_ip1}
        response3 = requests.get(url, headers=headers, proxies=proxies)

        print(response3.status_code)
        print(response3.text)

    if __name__ == ‘__main__‘:
        base_url = ‘https://www.kuaidaili.com/free/inha/{}/ 

‘
        for line in range(1, 2905):
            ip_url = base_url.format(line)
            response = get_index(ip_url)
            ip_list = parse_index(response.text)
            for ip in ip_list:
                good_ip = test_ip(ip)
                if good_ip:
                    spider_nba(good_ip)

from selenium import webdriver
import time
#获取驱动对象
driver = webdriver.Chrome()
try:
 #自动登陆抽屉新热榜
 #发送get请求
    driver.get(‘https://dig.chouti.com/ ‘)
 #隐式等待
    driver.implicitly_wait(10)
     #获取 ‘登陆‘ 按钮
    send_tag = driver.find_element_by_id(‘login_btn‘)
    send_tag.click()
     #获取手机号输入框
    username = driver.find_element_by_class_name(‘login-phone‘)
    username.send_keys(‘*******‘)
    time.sleep(1)
     #获取密码输入框
    password = driver.find_element_by_class_name(‘pwd-password-input‘)
    password.send_keys(‘*******‘)
    time.sleep(1)
     #获取 ‘登陆‘ 按钮
    login = driver.find_elements_by_link_text(‘登录‘)
    login[1].click()
    time.sleep(10)
finally:
    driver.close()