网站开发php岗位职责,网络设计的目的是,采购信息网,公司网站建设策划书一#xff0e;多进程
当计算机运行程序时#xff0c;就会创建包含代码和状态的进程。这些进程会通过计算机的一个或多个CPU执行。不过#xff0c;同一时刻每个CPU只会执行一个进程#xff0c;然后不同进程间快速切换#xff0c;给我们一种错觉#xff0c;感觉好像多个程…一多进程
当计算机运行程序时就会创建包含代码和状态的进程。这些进程会通过计算机的一个或多个CPU执行。不过同一时刻每个CPU只会执行一个进程然后不同进程间快速切换给我们一种错觉感觉好像多个程序在同时进行。例如有一个大型工厂该工厂负责生产电脑工厂有很多的车间用来生产不同的电脑部件。每个车间又有很多工人互相合作共享资源来生产某个电脑部件。这里的工厂相当于一个爬虫工程每个车间相当于一个进程每个工人就相当于线程。线程是CPU调度的基本单元。
也就是进程间是独立的,这表现在内存空间,上下文环境;而线程运行在进程空间内.也就是同一进程产生的线程共享同一内存空间.
需要注意的是单核CPU系统中真正的并发是不可能的
顺序执行 多进程并发 注意除了时间的加速意外也要看看函数返回值的写法带有多进程的map,是返回一个列表
import requests
import re
import time
from multiprocessing import Pool
from multiprocessing.dummy import Pool as ThreadPool
def spyder(url):# res []res {init::hello}print(hahah:{}.format(url))time.sleep(1)# res.append(url)res.update({entr:url:url})return resdef use_process():urls [https://www.qiushibaike.com/text/page/{}/.format(str(i)) for i in range(0, 4)]start_1 time.time()#获取函数返回结果res1 []for url in urls:res_ spyder(url)res1.append(res_)end_1 time.time()print(单进程, end_1 - start_1)print(res1:, res1)# 获取函数返回结果# 进程池start_2 time.time()pool Pool(processes2)res2 pool.map(spyder, urls)pool.close()pool.join()print(res2:, res2)end_2 time.time()print(2进程, end_2 - start_2)# 获取函数返回结果# 进程池start_3 time.time()pool Pool(processes4)res3 pool.map(spyder, urls)pool.close()pool.join()print(res2:, res3)end_3 time.time()print(4进程, end_3 - start_3)
if __name__ __main__:use_process() 二.多线程
实际上由于GIL全局解释器锁的限制哪个线程想要执行代码就需要去申请锁否则只能等着所以这个锁阻碍了真正的多线程并发这是解释器cpython的锅一般不推荐用多线程而是用多进程multiprocess来绕过GIL.
2.1 thread多线程
import time
import _thread
from threading import Thread
# 使用线程锁防止线程死锁
mutex _thread.allocate_lock()
def test(d_num):d_num.append(89)print(test: %s% str(d_num))
def test1(d_num):print(test1: %s% str(d_num))
def main():d_num [100, 58]t1 Thread(targettest, args(d_num,))t2 Thread(targettest1, args(d_num,))t1.start()time.sleep(1)t2.start()time.sleep(1)if __name__ __main__:main() 2.2 多线程队列版
import time
import _thread
from threading import Thread
import queue
# 使用线程锁防止线程死锁
mutex _thread.allocate_lock()
frame_queue queue.Queue()
def test(d_num):print(test: %s % str(d_num))for i in range(d_num):frame_queue.put(i)def test1():while 1:if frame_queue.empty() ! True:# 从队列中取出图片value frame_queue.get()print(value:, value)time.sleep(1)else:break
def main():d_num 10t1 Thread(targettest, args(d_num,))t1.start()t2 Thread(targettest1)t2.start()if __name__ __main__:main() 2.3 注意传参与多进程的区别,线程池
from functools import partial
from itertools import repeat
from multiprocessing import Pool, freeze_supportdef func(a, b):return a bdef main():a_args [1, 2, 3]second_arg 1with Pool() as pool:L pool.starmap(func, [(1, 1), (2, 1), (3, 1)])print(L:, L)M pool.starmap(func, zip(a_args, repeat(second_arg)))print(M:, M)N pool.map(partial(func, bsecond_arg), a_args)print(N:, N)
main() import requests
import re
import time
from multiprocessing import Pool
from multiprocessing.dummy import Pool as ThreadPool
def spyder(url):# res []res {init::hello}print(hahah:{}.format(url))time.sleep(1)# res.append(url)res.update({entr:url:url})return resdef use_process():urls [https://www.qiushibaike.com/text/page/{}/.format(str(i)) for i in range(0, 4)]start_1 time.time()#获取函数返回结果res1 []for url in urls:res_ spyder(url)res1.append(res_)end_1 time.time()print(单进程, end_1 - start_1)print(res1:, res1)# 获取函数返回结果# 进程池start_2 time.time()pool Pool(processes2)res2 pool.map(spyder, urls)pool.close()pool.join()print(res2:, res2)end_2 time.time()print(2进程, end_2 - start_2)# 获取函数返回结果# 进程池start_3 time.time()pool Pool(processes4)res3 pool.map(spyder, urls)pool.close()pool.join()print(res2:, res3)end_3 time.time()print(4进程, end_3 - start_3)def use_threadpool():urls [[https://www.qiushibaike.com/text/page/{}/.format(str(i))] for i in range(0, 4)]print(urls:, urls)# 线程池start time.time()pool ThreadPool(processes4)res pool.starmap(spyder, urls)pool.close()pool.join()end time.time()print(res:, res)print(4线程, end - start)
if __name__ __main__:# use_process()use_threadpool() 实际应用将图片路径和名字传入用zip方式打包传参 import osimport cv2
import time
import itertools
from multiprocessing.dummy import Pool as ThreadPoolSIZE (75,75)
SAVE_DIRECTORYthumbs
def save_img(filename,save_path):save_path filename.split(/)[-1]im cv2.imread(filename)imcv2.resize(im,SIZE)cv2.imwrite(save_path,im)if __name__ __main__:path./data/testlabelprint(path)output_path./data/thumbs/if not os.path.exists(output_path):os.mkdir(output_path)print(output_path)imgs_list_path[os.path.join(path,i) for i in os.listdir(path)]print(len(imgs_list_path))start_timetime.time()pool ThreadPool(processes8)print(list(zip(imgs_list_path,[output_path]*len(imgs_list_path))))pool.starmap(save_img,zip(imgs_list_path,[output_path]*len(imgs_list_path)))pool.close()pool.join()end_timetime.time()print(use time,end_time-start_time)三.共享变量
import numpy as np
from multiprocessing import shared_memory
from multiprocessing import Process
from multiprocessing.managers import SharedMemoryManager
import time
import glob
import os
import shutilsmm SharedMemoryManager()
smm.start()
screen_img np.zeros((480, 640, 3), dtypenp.uint8)
share_screen_img smm.SharedMemory(screen_img.nbytes)def change(name):a np.array([0, 0, 0, 0, 0, 0])existing_shm shared_memory.SharedMemory(namename)b np.ndarray((6, ), dtypenp.int64, bufferexisting_shm.buf)b[:] a[:]print(changed)def show(name):while True:existing_shm shared_memory.SharedMemory(namename)show_img np.ndarray((480, 640, 3),dtypenp.uint8,bufferexisting_shm.buf)print(show_img:, show_img.sum())time.sleep(0.30)if __name__ __main__:tmp np.ndarray(screen_img.shape,dtypescreen_img.dtype,buffershare_screen_img.buf)print(init tmp:, tmp.sum())# p1 Process(targetchange, args(sm.name, ))p2 Process(targetshow, args(share_screen_img.name, ))p2.start()time.sleep(2)tmp[:] np.ones((480, 640, 3), dtypenp.uint8)# p1.join()# print(final b:, b)p2.join()
