基本框架代码

基本原理是同时调用电脑的多核进行爬虫,同时下载

  1. import multiprocessing as mp
  2. import time
  3. from urllib.request import urlopen
  4. from urllib.parse import urljoin
  5. from bs4 import BeautifulSoup
  6. import re
  9. base_url='https://yulizi123.github.io/'
  12. def crawl(url):
  13.     response = urlopen(url)
  14.     time.sleep(0.1)
  15.     return response.read().decode()
  18. def parse(html):
  19.     soup=BeautifulSoup(html,features='html.parser')
  20.     urls = soup.find_all('a',{'href':re.compile('^/.+?/$')})
  21.     title=soup.find('h1').get_text().strip()
  22.     page_urls=set([urljoin(base_url,url['href']) for url in urls])
  23.     url = soup.find('meta',{'property':'og:url'})['content']
  24.     return  title,page_urls,url
  27. unseen=set([base_url,])
  28. seen=set()
  29. count, t1 = 1, time.time()
  30. while len(unseen) != 0:  # still get some url to visit
  31.     if len(seen) > 20:
  32.         break
  34.     print('\nDistributed Crawling...')
  35.     htmls = [crawl(url) for url in unseen]
  37.     print('\nDistributed Parsing...')
  38.     results = [parse(html) for html in htmls]
  40.     print('\nAnalysing...')
  41.     seen.update(unseen)  # seen the crawled
  42.     unseen.clear()  # nothing unseen
  44.     for title, page_urls, url in results:
  45.         print(count, title, url)
  46.         count += 1
  47.         unseen.update(page_urls - seen)  # get new url to crawl
  48. print('Total time: %.1f s' % (time.time() - t1,))  # 53 s

当前为一个正常的单线程的爬虫程序，多线程爬虫由于模块的种种问题，现在无法进行调试，待python的多线程学习完毕之后再来进行修改。

异步加载Asyncio

利用一个单线程的程序来控制爬虫的每一步的