爬取豆瓣

代码

  1. import requests
  2. from bs4 import BeautifulSoup
  3. import csv
  5. '''
  6. 处理翻页操作，分析url
  7. https://movie.douban.com/top250?start=0
  8. https://movie.douban.com/top250?start=25
  9. https://movie.douban.com/top250?start=50
  10. '''
  13. class DouBan:
  14.     def __init__(self):
  15.         self.headers = {
  16.             'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36'
  17.         }
  18.         self.url = 'https://movie.douban.com/top250?start={}'
  19.         self.li = []
  20.         self.header = ['name', 'score', 'review']
  22.     def read_url(self, url):
  23.         res = requests.get(url, headers=self.headers)
  24.         html = res.content.decode('utf-8')
  25.         # print(html)
  26.         # print('-'*100)
  27.         return html
  29.     def parse_html(self, html):
  30.         soup = BeautifulSoup(html, 'lxml')
  31.         ol_tag = soup.find('ol', class_="grid_view")
  32.         # print(ol_tag)
  33.         li_tags = ol_tag.find_all('li')
  34.         for li_tag in li_tags:
  35.             item = {}
  36.             div_hd_tag = li_tag.find('div', class_="hd")
  37.             div_bd_tag = li_tag.find('div', class_="bd")
  38.             name = div_hd_tag.find('span', class_="title").string
  39.             item['name'] = name
  40.             score = div_bd_tag.find('span', class_="rating_num").string
  41.             item['score'] = score
  42.             # 处理没有影评的操作
  43.             try:
  44.                 review = div_bd_tag.find('span', class_="inq").string
  45.                 item['review'] = review
  46.             except:
  47.                 item['review'] = '无'
  48.             self.li.append(item)
  50.     def write_data(self):
  51.         with open("豆瓣top250.csv", 'w', encoding='utf-8', newline='') as f:
  52.             w = csv.DictWriter(f, self.header)
  53.             w.writeheader()
  54.             w.writerows(self.li)
  56.     def main(self):
  57.         num = int(input("请输入你要爬取的页数：（一共有十页）"))
  58.         for i in range(num):
  59.             url = self.url.format(i*25)
  60.             html = self.read_url(url)
  61.             self.parse_html(html)
  62.         # print(self.li)
  63.         self.write_data()
  67. if __name__ == '__main__':
  68.     db = DouBan()
  69.     db.main()

结果展示

image.png
image.png