python学习_多进程

from multiprocessing import Pool
import os


def func(num):
    num += 1
    print(num)
    return num


if __name__ == ‘__main__‘:
    p = Pool(os.cpu_count() + 1)
    print(os.cpu_count())
    res = p.map(func, [i for i in range(20)]) . # i作为参数传入func中
    p.close() # 表示不能再向进程池中添加任务
    p.join()  # 表示等待进程池中所有任务执行完毕
    print(type(res))

from multiprocessing import Pool
import requests
import time


def func(url):
    res = requests.get(url)
    if res.status_code == 200:
        return ‘ok‘


if __name__ == ‘__main__‘:
    p = Pool(5)
    l = [‘https://www.baidu.com‘,
         ‘http://www.jd.com‘,
         ‘http://www.taobao.com‘,
         ‘http://www.mi.com‘,
         ‘http://www.cnblogs.com‘,
         ‘https://www.bilibili.com‘,
         ]
    start = time.time()
    for i in l:
        p.apply(func, args=(i,)) #即使有n个线程也是一个一个的去执行
    print(time.time() - start)

    start = time.time()
    for i in l:
        p.apply_async(func, args=(i, ))
    p.close()
    p.join()
    print(time.time() - start)

from multiprocessing import Pool
import requests
import time
import os


def func(url):
    res = requests.get(url)
    print("子进程的pid：%s, 父进程的pid：%s" % (os.getpid(), os.getppid()))
    if res.status_code == 200:
        return url


def cal_back(sta):
    url = sta
    print("回调函数的pid:%s" % os.getpid())
    with open(‘content.txt‘, ‘a+‘) as f:
        f.write(url + "\n")


if __name__ == ‘__main__‘:
    p = Pool(5)
    l = [‘https://www.baidu.com‘,
         ‘http://www.jd.com‘,
         ‘http://www.taobao.com‘,
         ‘http://www.mi.com‘,
         ‘http://www.cnblogs.com‘,
         ‘https://www.bilibili.com‘,
         ]
    print("主进程pid:%s" % os.getpid())
    for i in l:
        p.apply_async(func, args=(i,), callback=cal_back)
    p.close()
    p.join()

from multiprocessing import Process, Pool
import time


def func(num):
    num += 1
    # print(num)


if __name__ == ‘__main__‘:
    p = Pool(5)
    start = time.time()
    p.map(func, [i for i in range(1000)])
    p.close()  # 指不能再向进程池中添加任务
    p.join()  # 等待进程池中的所有任务执行完毕
    print(time.time() - start)

    p_l = []
    start = time.time()
    for i in range(1000):
        p = Process(target=func, args=(i,))
        p.start()
        p_l.append(p)
    [i.join() for i in p_l]
    print(time.time() - start)

from multiprocessing import Queue, Process
import time


def producer(q, name):
    for i in range(20):
        q.put(name)
        print("生产第%s个%s" % (i, name))
    # q.put(None)


def consumer(q, name, color):
    while 1:
        info = q.get()
        if info:
            print("%s %s拿走来%s\033[0m" % (color, name, info))
        else:
            break


if __name__ == ‘__main__‘:
    q = Queue(10)
    p = Process(target=producer, args=(q, ‘number one‘))
    p1 = Process(target=producer, args=(q, ‘number two‘))
    p2 = Process(target=producer, args=(q, ‘number three‘))
    c1 = Process(target=consumer, args=(q, ‘alex‘, ‘\033[31m‘))
    c2 = Process(target=consumer, args=(q, ‘wusir‘, ‘\033[32m‘))
    p_l = [p, p1, p2, c2, c1]
    [i.start() for i in p_l]
    p.join()
    p1.join()
    p2.join()
    q.put(None)
    q.put(None)  # 设置标志 表示没有数据了，生产者不再生产数据

from multiprocessing import JoinableQueue, Process


def producer(q, name):
    for i in range(20):
        q.put(name)
        print("生产第%s个%s" % (i, name))
    q.join() 生产者进程等待消费者进程消费完成


def consumer(q, name):
    while 1:
        q.get()
        print("%s 拿走了一个" % name)
        q.task_done()


if __name__ == ‘__main__‘:
    q = JoinableQueue(10)
    p1 = Process(target=producer, args=(q, ‘one‘))
    c1 = Process(target=consumer, args=(q, ‘alex‘))
    c1.daemon = True # 设置守护进程
    p1.start()
    c1.start()
    p1.join() # 主进程等待生产者进程

    # 主进程等待生产者进程结束
    # 程序有3个进程，主进程和生产者进程和消费者进程。  当主进程执行到35行代码时，主进程会等待生产进程结束
    # 而生产进程中（第26行）会等待消费者进程把所有数据消费完，生产者进程才结束。
    # 现在的状态就是  主进程等待生产者进程结束，生产者进程等待消费者消费完所有数据
    # 所以，把消费者设置为守护进程。  当主进程执行完，就代表生产进程已经结束，也就代表消费者进程已经把队列中数据消费完
    # 此时，主进程一旦结束，守护进程也就是消费者进程也就跟着结束。    整个程序也就能正常结束了。