代码

    1. import requests
    2. import csv
    3. import time
    4. import threading
    5. from queue import Queue
    7. class Producer(threading.Thread):
    8.     headers = {
    9.         'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36 Edg/100.0.1185.50'
    10.     }
    11.     def __init__(self,Page_queue, data_queue):
    12.         super(Producer, self).__init__()
    13.         self.Page_queue = Page_queue
    14.         self.data_queue = data_queue
    17.     def run(self):
    18.         while True:
    19.             if self.Page_queue.empty():
    20.                 break
    21.             url = self.Page_queue.get()
    22.             self.parse_data(url)
    25.     def parse_data(self,url):
    26.         response = requests.get(url, headers=self.headers)
    27.         data = response.json()['Data']['Posts']  # data是个列表，依次遍历
    28.         for post in data:
    29.             post_dic = {}
    30.             post_dic['Post'] = post['RecruitPostName']
    31.             post_dic['Location'] = post['LocationName']
    32.             post_dic['PostURL'] = post['PostURL']
    33.             self.data_queue.put(post_dic)
    37. class Consumer(threading.Thread):
    38.     header = ['Post', 'Location', 'PostURL']
    39.     f = open('腾讯招聘职位.csv', 'w',encoding='utf-8',newline='')
    40.     writer = csv.DictWriter(f, header)
    41.     writer.writeheader()
    44.     def __init__(self, data_queue):
    45.         super(Consumer, self).__init__()
    46.         self.data_queue = data_queue
    48.     def run(self):
    49.         while True:
    50.             if self.data_queue.empty():
    51.                 break
    52.             job = self.data_queue.get()
    53.             self.writer.writerow(job)
    57. if __name__ == '__main__':
    58.     Page_q = Queue()
    59.     data_q = Queue()
    60.     for i in range(1, 11):
    61.         page_url = f'https://careers.tencent.com/tencentcareer/api/post/Query?timestamp=1651906031606&countryId=&cityId=&bgIds=&productId=&categoryId=&parentCategoryId=&attrId=&keyword=&pageIndex={i}&pageSize=10&language=zh-cn&area=cn'
    62.         Page_q.put(page_url)
    64.     producer_lis = []
    65.     for i in range(3):
    66.         t1 = Producer(Page_q, data_q)
    67.         t1.start()
    68.         producer_lis.append(t1)
    71.     for j in producer_lis:
    72.         j.join()  # 保护线程，子线程结束后主线程在
    74.     for c in range(3):
    75.         t2 = Consumer(data_q)
    76.         t2.start()