1.  import requests
    2. from bs4 import BeautifulSoup
    3. import json
    4. class QiuSpiker(object):
    5.     def init(self):
    6.     self.baseURL = 'https://www.qiushibaike.com/8hr/page/{}/'
    7.     self.URL = 'https://www.qiushibaike.com'
    8.     self.list = []
    9.     self.page = '1'
    10.     def get_URL(self):
    11.         url_List = []
    12.         for i in range(1, (int(self.page) + 1)):
    13.             url_List.append(self.baseURL.format(str(i)))
    14.         return url_List
    16.     def get_Request(self, url):
    17.         data = requests.get(url).content.decode()
    18.         return data
    20.     def xpath(self, data):
    21.         soup = BeautifulSoup(data, 'lxml')
    22.         html_xpath = soup.select('.item')
    23.         for xpath in html_xpath:
    24.             xpathDic = {}
    25.             # 1、调整连接
    26.             xpathDic['detailURL'] = self.URL + xpath.select_one(
    27.                 '.recmd-left').get('href')
    28.             # 2、标题
    29.             xpathDic['title'] = xpath.select_one('.recmd-content').get_text()
    30.             # 3、好笑数
    31.             xpathDic['goodsNum'] = xpath.select_one(
    32.                 '.recmd-num').get_text().replace('\n', '').replace('\r', '')
    33.             # 4、作者
    34.             xpathDic['nick'] = xpath.select_one('.recmd-name').get_text()
    35.             # 5、头像
    36.             xpathDic['headURL'] = 'https:' + xpath.select_one(
    37.             '.recmd-user img').get('src')
    38.             # 6、图片
    39.             xpathDic['URL'] = 'https:' + xpath.select_one('.recmd-left img').get(
    40.             'src')
    41.             self.list.append(xpathDic)
    43.     def get_Page(self):
    44.         url = self.baseURL.format(self.page)
    45.         data = self.get_Request(url)
    46.         soup = BeautifulSoup(data, 'lxml')
    47.         self.page = soup.select('.page-numbers')[-1].get_text().replace('\n',
    48.             '')
    50.     def save_data(self, data):
    51.         json.dump(data, open('qiushibaike.json', 'w'))
    53.     def start(self):
    54.         #获取总页码数
    55.         self.get_Page()
    56.         url_list = self.get_URL()
    57.         for url in url_list:
    58.               print(url)
    59.             data = self.get_Request(url)
    60.             self.xpath(data)
    61.             self.save_data(self.list)
    63. if name == 'main':
    64.     QiuSpiker().start()