1. import requests
    2. import threading
    3. from queue import Queue
    4. import csv
    6. class Producer(threading.Thread):
    7.     headers = {
    8.         'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.82 Safari/537.36'
    9.     }
    10.     def __init__(self,url_queue,data_queue):
    11.         super(Producer, self).__init__()
    12.         self.url_queue = url_queue
    13.         self.data_queue = data_queue
    15.     def run(self):
    16.         while True:
    17.             if url_queue.empty():
    18.                 break
    19.             url = self.url_queue.get()
    20.             self.parse_page(url)
    22.     def parse_page(self, url):
    23.         res = requests.get(url, headers=self.headers)
    24.         result = res.json()
    25.         dat = result['Data']['Posts']
    26.         data = {}
    27.         for item in dat:
    28.             data['职位'] = item['RecruitPostName']
    29.             data['地址'] = item['LocationName']
    30.             data['网址'] = item['PostURL']
    31.             self.data_queue.put(data)
    34. class Consumer(threading.Thread):
    36.     f = open('腾讯招聘.csv', 'a', encoding='utf-8-sig', newline='')
    37.     writer = csv.DictWriter(f, fieldnames=['职位', '地址', '网址'])
    38.     writer.writeheader()
    40.     def __init__(self,data_queue):
    41.         super(Consumer, self).__init__()
    42.         self.data_queue = data_queue
    44.     def run(self):
    45.         while True:
    46.             if data_queue.empty():
    47.                 break
    48.             all_data = self.data_queue.get()
    49.             self.writer.writerow(all_data)
    52. if __name__ == '__main__':
    53.     url_queue = Queue()
    54.     data_queue = Queue()
    55.     for i in range(1,11):
    56.         url = f'https://careers.tencent.com/tencentcareer/api/post/Query?categoryId=&parentCategoryId=40001&&pageIndex={i}&pageSize=10&language=zh-cn&area=cn'
    57.         url_queue.put(url)
    59.     p_list = []
    60.     for i in range(3):
    61.         t = Producer(url_queue, data_queue)
    62.         t.start()
    63.         p_list.append(t)
    65.     for p in p_list:
    66.         p.join()
    68.     c_list = []
    69.     for i in range(3):
    70.         t = Consumer(data_queue)
    71.         t.start()
    72.         c_list.append(t)
    74.     for c in c_list:
    75.         c.join()