1. import requests
    2. from bs4 import BeautifulSoup
    3. import json
    4. class QiuVideoSpider(object):
    5.     def init(self):
    6.     self.baseURL = 'https://www.qiushibaike.com/video/page/{}/'
    7.     self.URL = 'https://www.qiushibaike.com'
    8.     self.list = []
    9.     self.page = '1'
    10.     def get_URL(self):
    11.         url_List = []
    12.         for i in range(1, (int(self.page) + 1)):
    13.             url_List.append(self.baseURL.format(str(i)))
    14.         return url_List
    16.     def get_Request(self, url):
    17.         data = requests.get(url).content.decode()
    18.         return data
    20.     def xpath(self, data):
    21.         soup = BeautifulSoup(data, 'lxml')
    22.         html_xpath = soup.select('.article')
    23.         for xpath in html_xpath:
    24.             xpathDic = {}
    25.             # 1、头像
    26.             xpathDic['headerURL'] = 'https:' + xpath.select_one(
    27.                 '.author img').get('src')
    28.             # 2、昵称
    29.             xpathDic['nick'] = xpath.select_one(
    30.                 '.author h2').get_text().replace('\n', '')
    31.             # 3、标题
    32.             xpathDic['title'] = xpath.select_one('.content').get_text().replace(
    33.                 '\n', '')
    34.             # 4、性别
    35.             xpathDic['age'] = xpath.select_one('.articleGender').get_text()
    36.             # 5、视频链接
    37.             xpathDic['video'] = 'https:' + xpath.select_one('source').get('src')
    38.             self.list.append(xpathDic)
    40.     def get_Page(self):
    41.         url = self.baseURL.format(self.page)
    42.         data = self.get_Request(url)
    43.         soup = BeautifulSoup(data, 'lxml')
    44.         self.page = soup.select('.page-numbers')[-1].get_text().replace('\n',
    45.                                                                         '')
    47.     def save_data(self, data):
    48.         json.dump(data, open('qiushibaikeVideo.json', 'w'))
    50.     def start(self):
    51.         self.get_Page()
    52.         url_list = self.get_URL()
    53.         for url in url_list:
    54.             print(url)
    55.             data = self.get_Request(url)
    56.             self.xpath(data)
    57.             self.save_data(self.list)
    59. if name == 'main':
    60.     QiuVideoSpider().start()