1.信号量

image.png

  1. import asyncio
  2. import time
  4. import aiohttp
  6. from main import timethis
  8. semaphore = asyncio.Semaphore(3) # 10并发度
  9. urls = [
  10.     f'https://www.cnblogs.com/sitehome/p/{page}'
  11.     for page in range(1,10)
  12. ]
  14. async def async_craw(url):
  15.     print('start craw url',url)
  16.     async with semaphore:
  17.         async with aiohttp.ClientSession() as session:
  18.             async with session.get(url) as resp:
  19.                 result = await resp.text()
  20.                 await asyncio.sleep(5)
  21.                 print(f'end craw url:{url},time:{len(result)}')
  24. loop = asyncio.get_event_loop()
  26. tasks = [
  27.     loop.create_task(async_craw(url))
  28.     for url in urls
  29. ]
  31. @timethis
  32. def do_main():
  33.     loop.run_until_complete(asyncio.wait(tasks))
  36. if __name__ == '__main__':
  37.     do_main()
  38.  ###################
  39.  do_main 15.98940396308899