一、采集公众号文章中的图片

思路：

  1. 使用requests获取数据，使用requests_html解析数据
  2. 使用DHash算法计算图片相似度去重（如二维码），图片以PIL.Image格式装载
  1. import requests
  2. from requests_html import HTMLSession
  3. from PIL import Image
  5. ROOT_FOLDER = ''
  6. # 记录所有图片的DHASH，如果量大或复杂可以用redis或数据库保存
  7. DHASH_LIST = []
  9. def downimg(url, folder=None):
  10.     '''
  11.     url: 下载链接
  12.     folder: 保存文件夹
  13.     '''
  14.     r = requests.get(url)
  15.     tmp_name = '{}.jpg'.format(int(time.time()*1000))
  16.     img_name = os.path.join(folder or ROOT_FOLDER, tmp_name)
  18.     image = Image.open(BytesIO(r.content))
  19.     dh = DHash.calculate_hash(image)
  20.     if dh in DHASH_LIST:
  21.         logger.debug('Duplicated image.')
  22.     else:
  23.         image.save(img_name)
  24.         DHASH_LIST.append(dh)
  27. def get_img_links(url):
  28.     '''
  29.     url: 公众号图文链接
  30.     '''
  31.     session = HTMLSession()
  32.     r = session.get(url)
  33.     t = r.html.find('#activity-name')
  34.     title = DEFAULT_TITLE
  35.     if len(t) > 0:
  36.         title = r.html.find('#activity-name')[0].text
  37.     content = r.html.find('#js_article', first=True)
  38.     images = content.find('img')
  39.     down_img_urls = []
  40.     for img in images:
  41.         if 'data-src' in img.attrs:
  42.             down_img_urls.append(img.attrs['data-src'])
  44.     return title, down_img_urls
  47. def download_article(url):
  48.     '''
  49.     url: 公众号图文链接
  50.     下载某个图文所有图片，同步阻塞。
  51.     '''
  52.     title, images = get_img_links(url)
  53.     logger.debug('Total {} images found for {}'.format(len(images), title))
  54.     folder = os.path.join(ROOT_FOLDER, title)
  55.     if os.path.exists(folder):
  56.         if title != DEFAULT_TITLE:
  57.             # 已下载过
  58.             logger.debug('Pass already downloaded.')
  59.             return
  60.     else:
  61.         os.mkdir(folder)
  62.     for img in images:
  63.         downimg(img, folder)
  66. def download_all():
  67.     '''
  68.     根据ROOT_FOLDER下all.txt下载所有图文里的图片
  69.     '''
  70.     txt = os.path.join(ROOT_FOLDER, 'all.txt')
  71.     with open(txt, 'r') as f:
  72.         urls = f.readlines()
  73.     for u in urls:
  74.         download_article(u)