爬取笑话网站

代码

  1. from lxml import etree
  2. import requests
  3. import re
  4. import csv
  7. class XiaoHua:
  8.     # 初始化函数
  9.     def __init__(self):
  10.         self.headers = {
  11.             'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36'
  12.         }
  13.         self.url = 'http://www.17989.com/xiaohua/1.htm'
  14.         self.li = []
  15.         self.header = ['title', 'section']
  17.     # 发送请求函数
  18.     def read_url(self, url):
  19.         res = requests.get(url, self.headers)
  20.         html = res.content.decode('utf-8')
  21.         # print(html)
  22.         return html
  24.     # 数据解析函数
  25.     def parse_html(self, html):
  26.         tree = etree.HTML(html)
  27.         li_tags = tree.xpath('//div[@class="module articlelist"]//li')
  28.         # print(li_tags)
  29.         for li_tag in li_tags:
  30.             pat = '\r\n'
  31.             item = {}
  32.             title = li_tag.xpath('./div[@class="hd"]/text()')[0].strip()
  33.             item['title'] = title
  34.             section = li_tag.xpath('./pre/text()')[0].strip()
  35.             item['section'] = re.sub(pat, '', section)
  36.             # print(item)
  37.             self.li.append(item)
  39.     # 数据保存函数
  40.     def write_data(self):
  41.         with open('笑话.csv', 'w', encoding='utf-8', newline='') as f:
  42.             w = csv.DictWriter(f, self.header)
  43.             w.writeheader()
  44.             w.writerows(self.li)
  45.             print("保存成功")
  47.     # 主函数
  48.     def main(self):
  49.         # 发送请求
  50.         html = self.read_url(self.url)
  51.         # 使用xpath数据解析
  52.         self.parse_html(html)
  53.         # print(self.li)
  55.         # 翻页操作
  56.         # # 1.爬取所有
  57.         # while True:
  58.         #     tree2 = etree.HTML(html)
  59.         #     next_url = 'http://www.17989.com/' + tree2.xpath('//a[text()="下一页"]/@href')[0]
  60.         #     if next_url:
  61.         #         # 使用新的url发送请求
  62.         #         html = self.read_url(next_url)
  63.         #         # 数据解析
  64.         #         self.parse_html(html)
  65.         #         print(next_url)
  66.         #     else:
  67.         #         break
  69.         # 2.选择性爬取   由于爬取全部太多了
  70.         num = int(input("请输入你要爬取的页数："))
  71.         for i in range(num - 1):
  72.             tree2 = etree.HTML(html)
  73.             next_url = 'http://www.17989.com/' + tree2.xpath('//a[text()="下一页"]/@href')[0]
  74.             html = self.read_url(next_url)
  75.             self.parse_html(html)
  76.             print(next_url)
  78.         # 保存
  79.         self.write_data()
  82. # 启动
  83. if __name__ == '__main__':
  84.     xh = XiaoHua()
  85.     xh.main()

展示

image.png

image.png