1. import requests
    2. from fake_useragent import UserAgent
    3. from bs4 import BeautifulSoup as bs
    4. import csv
    5. from bs4 import BeautifulSoup
    7. """
    8. 翻页：
    9. 华北:http://www.weather.com.cn/textFC/hb.shtml
    10. 东北：http://www.weather.com.cn/textFC/db.shtml
    11. 华东: http://www.weather.com.cn/textFC/hd.shtml
    12. """
    13. class WeatherSpider:
    14.     def __init__(self):
    16.         self.headers = {
    17.             'User-Agent':UserAgent().random
    18.         }
    19.         self.data_list = []
    20.     # 发送请求
    21.     def get_source(self,url):
    22.         try:
    23.             response = requests.get(url, headers=self.headers)
    24.             if response.status_code == 200:
    25.                 return response.content.decode('utf-8')
    26.         except Exception as error:
    27.             print(error)
    28.     # 解析数据
    29.     def parse_html(self,html):
    30.         soup = bs(html,'lxml')
    31.         # 找到class='conMidtab'的div标签,只需要找一个
    32.         conMidtab = soup.find(class_='conMidtab')
    33.         # 2 找到table标签
    34.         tables = conMidtab.find_all('table')
    35.         for table in tables:
    36.             #3、找到table标签下面的所有tr
    37.             trs = table.find_all('tr')[2:]
    38.             for index,tr in enumerate(trs):
    39.                 item = {}
    40.                 # 找到tr标签下面的所有的td，其中第一个td是城市名，倒数第二个td是最低温度
    41.                 tds = tr.find_all('td')
    42.                 if index == 0:
    43.                     city_td = list(tds[1].stripped_strings)[0]
    44.                 else:
    45.                     city_td = list(tds[0].stripped_strings)[0]
    46.                 temps_td = list(tds[-2].stripped_strings)[0]
    47.                 item['city'] = city_td
    48.                 item['temp'] = temps_td
    49.                 self.data_list.append(item)
    52.     def save_data(self):
    53.         pass
    54. weather = WeatherSpider()
    55. dicts = ['hb','db','hd','hz','hn','xb','xn','gat']
    56. for dict in dicts:
    57.     url = f'http://www.weather.com.cn/textFC/{dict}.shtml'
    58.     html = weather.get_source(url)
    59.     weather.parse_html(html)
    60. print(weather.data_list)