import aiohttp
import asyncio

@asyncio.coroutine
def get_page(url):
    print(‘GET:%s‘ %url)
    response=yield from aiohttp.request(‘GET‘,url)

    data=yield from response.read()

    print(url,data)
    response.close()
    return 1

tasks=[
    get_page(‘https://www.python.org/doc‘),
    get_page(‘https://www.cnblogs.com/linhaifeng‘),
    get_page(‘https://www.openstack.org‘)
]

loop=asyncio.get_event_loop()插入代码 - 博客园
results=loop.run_until_complete(asyncio.gather(*tasks))
loop.close()

print(‘=====>‘,results) #[1, 1, 1]

asyncio+aiohttp

import requests
import asyncio

@asyncio.coroutine
def get_page(func,*args):
    print(‘GET:%s‘ %args[0])
    loog=asyncio.get_event_loop()
    furture=loop.run_in_executor(None,func,*args)
    response=yield from furture

    print(response.url,len(response.text))
    return 1

tasks=[
    get_page(requests.get,‘https://www.python.org/doc‘),
    get_page(requests.get,‘https://www.cnblogs.com/linhaifeng‘),
    get_page(requests.get,‘https://www.openstack.org‘)
]

loop=asyncio.get_event_loop()
results=loop.run_until_complete(asyncio.gather(*tasks))
loop.close()

print(‘=====>‘,results) #[1, 1, 1]

asyncio+requests模块的方法

from gevent import monkey;monkey.patch_all()
import gevent
import requests

def get_page(url):
    print(‘GET:%s‘ %url)
    response=requests.get(url)
    print(url,len(response.text))
    return 1

# g1=gevent.spawn(get_page,‘https://www.python.org/doc‘)
# g2=gevent.spawn(get_page,‘https://www.cnblogs.com/linhaifeng‘)
# g3=gevent.spawn(get_page,‘https://www.openstack.org‘)
# gevent.joinall([g1,g2,g3,])
# print(g1.value,g2.value,g3.value) #拿到返回值


#协程池
from gevent.pool import Pool
pool=Pool(2)
g1=pool.spawn(get_page,‘https://www.python.org/doc‘)
g2=pool.spawn(get_page,‘https://www.cnblogs.com/linhaifeng‘)
g3=pool.spawn(get_page,‘https://www.openstack.org‘)
gevent.joinall([g1,g2,g3,])
print(g1.value,g2.value,g3.value) #拿到返回值

gevent+requests

#pip3 install grequests

import grequests

request_list=[
    grequests.get(‘https://wwww.xxxx.org/doc1‘),
    grequests.get(‘https://www.cnblogs.com/linhaifeng‘),
    grequests.get(‘https://www.openstack.org‘)
]


##### 执行并获取响应列表 #####
# response_list = grequests.map(request_list)
# print(response_list)

##### 执行并获取响应列表（处理异常） #####
def exception_handler(request, exception):
    # print(request,exception)
    print("%s Request failed" %request.url)

response_list = grequests.map(request_list, exception_handler=exception_handler)
print(response_list)

grequests

‘‘‘
#问题一：error: Microsoft Visual C++ 14.0 is required. Get it with "Microsoft Visual C++ Build Tools": http://landinghub.visualstudio.com/visual-cpp-build-tools
https://www.lfd.uci.edu/~gohlke/pythonlibs/#twisted
pip3 install C:\Users\Administrator\Downloads\Twisted-17.9.0-cp36-cp36m-win_amd64.whl
pip3 install twisted

#问题二：ModuleNotFoundError: No module named ‘win32api‘
https://sourceforge.net/projects/pywin32/files/pywin32/

#问题三：openssl
pip3 install pyopenssl
‘‘‘

#twisted基本用法
from twisted.web.client import getPage,defer
from twisted.internet import reactor

def all_done(arg):
    # print(arg)
    reactor.stop()

def callback(res):
    print(res)
    return 1

defer_list=[]
urls=[
    ‘http://www.baidu.com‘,
    ‘http://www.bing.com‘,
    ‘https://www.python.org‘,
]
for url in urls:
    obj=getPage(url.encode(‘utf=-8‘),)
    obj.addCallback(callback)
    defer_list.append(obj)

defer.DeferredList(defer_list).addBoth(all_done)

reactor.run()




#twisted的getPage的详细用法
from twisted.internet import reactor
from twisted.web.client import getPage
import urllib.parse


def one_done(arg):
    print(arg)
    reactor.stop()

post_data = urllib.parse.urlencode({‘check_data‘: ‘adf‘})
post_data = bytes(post_data, encoding=‘utf8‘)
headers = {b‘Content-Type‘: b‘application/x-www-form-urlencoded‘}
response = getPage(bytes(‘http://dig.chouti.com/login‘, encoding=‘utf8‘),
                   method=bytes(‘POST‘, encoding=‘utf8‘),
                   postdata=post_data,
                   cookies={},
                   headers=headers)
response.addBoth(one_done)

reactor.run()

twisted的用法

from tornado.httpclient import AsyncHTTPClient
from tornado.httpclient import HTTPRequest
from tornado import ioloop


def handle_response(response):
    """
    处理返回值内容（需要维护计数器，来停止IO循环），调用 ioloop.IOLoop.current().stop()
    :param response:
    :return:
    """
    if response.error:
        print("Error:", response.error)
    else:
        print(response.body)


def func():
    url_list = [
        ‘http://www.baidu.com‘,
        ‘http://www.bing.com‘,
    ]
    for url in url_list:
        print(url)
        http_client = AsyncHTTPClient()
        http_client.fetch(HTTPRequest(url), handle_response)


ioloop.IOLoop.current().add_callback(func)
ioloop.IOLoop.current().start()




#发现上例在所有任务都完毕后也不能正常结束，为了解决该问题，让我们来加上计数器
from tornado.httpclient import AsyncHTTPClient
from tornado.httpclient import HTTPRequest
from tornado import ioloop

count=0

def handle_response(response):
    """
    处理返回值内容（需要维护计数器，来停止IO循环），调用 ioloop.IOLoop.current().stop()
    :param response:
    :return:
    """
    if response.error:
        print("Error:", response.error)
    else:
        print(len(response.body))

    global count
    count-=1 #完成一次回调，计数减1
    if count == 0:
        ioloop.IOLoop.current().stop() 

def func():
    url_list = [
        ‘http://www.baidu.com‘,
        ‘http://www.bing.com‘,
    ]

    global count
    for url in url_list:
        print(url)
        http_client = AsyncHTTPClient()
        http_client.fetch(HTTPRequest(url), handle_response)
        count+=1 #计数加1

ioloop.IOLoop.current().add_callback(func)
ioloop.IOLoop.current().start()

Tornado