代码

项目地址：https://github.com/xingstarx/crawl-zsxq

  1. import re
  2. import requests
  3. import json
  4. import os
  5. import pdfkit
  6. config = pdfkit.configuration(wkhtmltopdf=r"C:\Program Files\wkhtmltopdf\bin\wkhtmltopdf.exe")
  8. from bs4 import BeautifulSoup
  9. from urllib.parse import quote
  11. html_template = """
  12. <!DOCTYPE html>
  13. <html lang="en">
  14. <head>
  15.     <meta charset="UTF-8">
  16. </head>
  17. <body>
  18. <h1>{title}</h1>
  19. <p>{text}</p>
  20. </body>
  21. </html>
  22. """
  23. htmls = []
  24. num = 0
  25. def get_data(url):
  27.     global htmls, num
  29.     headers = {
  30.         'Authorization': '33A1AD5E-D03C-6EE9-1E8C-DDF321294280',
  31.         'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.79 Safari/537.36'
  32.     }
  34.     rsp = requests.get(url, headers=headers)
  35.     with open('test.json', 'w', encoding='utf-8') as f:        # 将返回数据写入 test.json 方便查看
  36.         f.write(json.dumps(rsp.json(), indent=2, ensure_ascii=False))
  38.     with open('test.json', encoding='utf-8') as f:
  39.         for topic in json.loads(f.read()).get('resp_data').get('topics'):
  40.             content = topic.get('question', topic.get('talk', topic.get('task', topic.get('solution'))))
  41.             # print(content)
  42.             text = content.get('text', '')
  43.             text = re.sub(r'<[^>]*>', '', text).strip()
  44.             text = text.replace('\n', '<br>')
  45.             title = str(num) + text[:9]
  46.             num += 1
  48.             if content.get('images'):
  49.                 soup = BeautifulSoup(html_template, 'html.parser')
  50.                 for img in content.get('images'):
  51.                     url = img.get('large').get('url')
  52.                     img_tag = soup.new_tag('img', src=url)
  53.                     soup.body.append(img_tag)
  54.                     html_img = str(soup)
  55.                     html = html_img.format(title=title, text=text)
  56.             else:
  57.                 html = html_template.format(title=title, text=text)
  59.             if topic.get('question'):
  60.                 answer = topic.get('answer').get('text', "")
  61.                 soup = BeautifulSoup(html, 'html.parser')
  62.                 answer_tag = soup.new_tag('p')
  63.                 answer_tag.string = answer
  64.                 soup.body.append(answer_tag)
  65.                 html_answer = str(soup)
  66.                 html = html_answer.format(title=title, text=text)
  68.             htmls.append(html)
  70.     next_page = rsp.json().get('resp_data').get('topics')
  71.     if next_page:
  72.         create_time = next_page[-1].get('create_time')
  73.         if create_time[20:23] == "000":
  74.             end_time = create_time[:20]+"999"+create_time[23:]
  75.         else :
  76.             res = int(create_time[20:23])-1
  77.             end_time = create_time[:20]+str(res).zfill(3)+create_time[23:] # zfill 函数补足结果前面的零，始终为3位数
  78.         end_time = quote(end_time)
  79.         if len(end_time) == 33:
  80.             end_time = end_time[:24] + '0' + end_time[24:]
  81.         next_url = start_url + '&end_time=' + end_time
  82.         print(next_url)
  83.         get_data(next_url)
  85.     return htmls
  87. def make_pdf(htmls):
  88.     html_files = []
  89.     for index, html in enumerate(htmls):
  90.         file = str(index) + ".html"
  91.         html_files.append(file)
  92.         with open(file, "w", encoding="utf-8") as f:
  93.             f.write(html)
  95.     options = {
  96.         "user-style-sheet": "test.css",
  97.         "page-size": "Letter",
  98.         "margin-top": "0.75in",
  99.         "margin-right": "0.75in",
  100.         "margin-bottom": "0.75in",
  101.         "margin-left": "0.75in",
  102.         "encoding": "UTF-8",
  103.         "custom-header": [("Accept-Encoding", "gzip")],
  104.         "cookie": [
  105.             ("cookie-name1", "cookie-value1"), ("cookie-name2", "cookie-value2")
  106.         ],
  107.         "outline-depth": 10,
  108.     }
  109.     try:
  110.         pdfkit.from_file(html_files, "电子书.pdf", options=options)
  111.     except Exception as e:
  112.         #pass
  113.         print(e)
  115.     for file in html_files:
  116.         os.remove(file)
  118.     print("已制作电子书在当前目录！")
  120. if __name__ == '__main__':
  121.     start_url = 'https://api.zsxq.com/v1.10/groups/5825251554/topics?scope=digests&count=20'
  122.     make_pdf(get_data(start_url))

效果

image.png

遇到的坑

1.No wkhtmltopdf executable found: “b’’”
python中需要安装wkhtmltopdf模块
image.png
然后windows系统中还要安装wkhtmltopdf软件
下载地址：https://wkhtmltopdf.org/downloads.html
image.png
2.找关键信息
image.png

以管理员方式运行pycharm