requests高级用法

 1 import requests
 2 from lxml.html.clean import etree
 3 from fake_useragent import UserAgent
 4 UA=UserAgent()
 5 headers={‘User-Agent‘:UA.random,}
 6 url=‘https://www.baidu.com/s?wd=ip‘
 7 response=requests.get(url=url,headers=headers,proxies={‘https‘:‘111.231.94.44:8888‘},timeout=10)#代理IP指定
 8 response.encoding=‘utf-8‘
 9 
10 # print(response.text)#在页面中可以搜索‘本机’找到本次请求的代理服务器
11 #<span class="c-gap-right">本机IP:&nbsp;117.64.251.9</span>安徽省合肥市 电信（代理IP也使用了代理）
12 print(etree.HTML(response.text).xpath(‘//span[@class="c-gap-right"]/text()‘)[0])

 1 import random
 2 import requests
 3 from lxml.html.clean import etree
 4 from fake_useragent import UserAgent
 5 
 6 UA = UserAgent()
 7 headers = {‘User-Agent‘: UA.random, }
 8 url = ‘https://www.baidu.com/s?wd=ip‘
 9 
10 #手动配置ip代理池
11 proxy_list = [
12     {‘https‘: ‘60.13.42.29:9999‘},
13     {‘https‘: ‘111.231.94.44:8888‘},
14     {‘http‘: ‘222.182.188.17:8060‘},
15     {‘https‘: ‘111.231.91.104:8888‘},
16     {‘https‘: ‘221.178.232.130:8080‘},
17 ]
18 #随机选取ip代理
19 proxies = random.choice(proxy_list)
20 print(proxies)
21 response = requests.get(url=url, headers=headers, proxies={‘https‘: ‘111.231.94.44:8888‘}, timeout=10)  # 代理IP指定
22 print(etree.HTML(response.text).xpath(‘//span[@class="c-gap-right"]/text()‘)[0])
23 # {‘https‘: ‘111.231.94.44:8888‘}
24 # 本机IP: 117.64.251.9
25 # {‘http‘: ‘222.182.188.17:8060‘}
26 # 本机IP: 117.64.251.9

 1 import random
 2 import requests
 3 from lxml.html.clean import etree
 4 from fake_useragent import UserAgent
 5 
 6 UA = UserAgent()
 7 headers = {‘User-Agent‘: UA.random, }
 8 url = ‘https://www.baidu.com/s?wd=ip‘
 9 
10 #http://http.zhiliandaili.cn/Index-getapi.html（注册登录购买获取）
11 #API自动创建代理池
12 proxy_list = []
13 ip_url = ‘http://ip.11jsq.com/index.php/api/entry?method=proxyServer.generate_api_url&packid=1&fa=0&fetch_key=&groupid=0&qty=53&time=1&pro=&city=&port=1&format=html&ss=5&css=&dt=1&specialTxt=3&specialJson=‘
14 page_text = requests.get(ip_url,headers=headers).text
15 tree = etree.HTML(page_text)
16 ip_list = tree.xpath(‘//body//text()‘)
17 for ip in ip_list:
18     ip = {‘https‘:ip}
19     proxy_list.append(ip)
20 print(proxy_list)
21 
22 #随机选取ip代理
23 proxies = random.choice(proxy_list)
24 print(proxies)
25 response = requests.get(url=url, headers=headers, proxies={‘https‘: ‘111.231.94.44:8888‘}, timeout=10)  # 代理IP指定
26 print(etree.HTML(response.text).xpath(‘//span[@class="c-gap-right"]/text()‘)[0])

 1 ‘‘‘
 2 雪球网:https://xueqiu.com/
 3 ‘‘‘
 4 import random
 5 import requests
 6 from fake_useragent import UserAgent
 7 
 8 UA = UserAgent()
 9 headers = {‘User-Agent‘: UA.random, }
10 proxy_list = [
11     {‘https‘: ‘60.13.42.29:9999‘},
12     {‘https‘: ‘111.231.94.44:8888‘},
13     {‘http‘: ‘222.182.188.17:8060‘},
14     {‘https‘: ‘111.231.91.104:8888‘},
15     {‘https‘: ‘221.178.232.130:8080‘},
16 ]
17 proxies = random.choice(proxy_list)
18 
19 #通过抓包工具捕获的基于ajax请求的数据包中提取的url(直接报错，分析原因找到第一次访问主页面时创建了cookie)
20 # url = ‘https://stock.xueqiu.com/v5/stock/batch/quote.json?symbol=SH000001,SZ399001,SZ399006,HKHSI,HKHSCEI,HKHSCCI,.DJI,.IXIC,.INX‘
21 # json_data = requests.get(url=url,headers=headers).json()
22 # print(json_data)
23 #{‘error_description‘: ‘遇到错误，请刷新页面或者重新登录帐号后再试‘, ‘error_uri‘: ‘/v5/stock/batch/quote.json‘, ‘error_code‘: ‘400016‘}
24 
25 
26 
27 
28 #创建一个session对象
29 session = requests.Session()
30 #将cookie保存到session对象中
31 first_url = ‘https://xueqiu.com/‘
32 session.get(url=first_url,headers=headers)#为了获取cookie且将cookie存储到session中
33 print(session.cookies)
34 
35 url = ‘https://stock.xueqiu.com/v5/stock/batch/quote.json?symbol=SH000001,SZ399001,SZ399006,HKHSI,HKHSCEI,HKHSCCI,.DJI,.IXIC,.INX‘
36 response = session.get(url=url,headers=headers,proxies=proxies)#携带cookie发起的请求(根据需要可以发起各种请求get/post/put...)
37 print(response.json())