8、BOSS直聘爬取

from selenium import webdriver
import time
import csv
class boss_work:
    lst = []
    def __init__(self):
        self.driver = webdriver.Chrome()
        self.driver.get('https://www.zhipin.com/c101250100/?ka=sel-city-101250100')
    def parse_html(self):
        self.driver.execute_script(
            'window.scrollTo(0, document.body.scrollHeight)'
        )
        time.sleep(5)
        for i in range(1, 6):
            list_ul = self.driver.find_elements_by_class_name('job-primary')
             # print(list_ul)
            for ul in list_ul:
                span = {}
                span['职位名称'] = ul.find_element_by_xpath('.//span[@class="job-name"]/a').text
                span['薪资'] = ul.find_element_by_xpath('.//span[@class="red"]').text
                span['公司名称'] = ul.find_element_by_xpath('.//h3[@class="name"]/a').text
                span['公司地址'] = ul.find_element_by_xpath('.//span[@class="job-area"]').text
                boss_work.lst.append(span)
            div_tag = self.driver.find_element_by_class_name('next')
            self.driver.execute_script('arguments[0].click();', div_tag)
            time.sleep(4)
    def save_data(self):
        with open('BOSS直聘.csv', 'w', encoding='utf-8-sig', newline='') as f:
            write = csv.DictWriter(f, fieldnames=('职位名称', '薪资', '公司名称', '公司地址'))
            write.writeheader()
            write.writerows(boss_work.lst)
    def main(self):
        self.parse_html()
        self.save_data()
work = boss_work()
work.main()