尚未解决：150页左右收到反爬机制暂停

Crawl_data.py

  1. import requests
  2. import pandas as pd
  3. from pprint import pprint
  4. from lxml import etree
  5. import time
  6. import warnings
  8. warnings.filterwarnings("ignore")
  10. for i in range(1,2001):
  11.     print("正在爬取第" + str(i) + "页的数据")
  12.     url_pre = "https://search.51job.com/list/000000,000000,0000,00,9,99,%25E6%2595%25B0%25E6%258D%25AE,2,"
  13.     url_end = ".html?"
  14.     url = url_pre + str(i) + url_end
  15.     headers = {
  16.         'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36'
  17.     }
  18.     web = requests.get(url, headers=headers)
  19.     web.encoding = "gbk"
  20.     dom = etree.HTML(web.text)
  21.     # 1、岗位名称
  22.     job_name = dom.xpath('//div[@class="dw_table"]/div[@class="el"]//p/span/a[@target="_blank"]/@title')
  23.     # 2、公司名称
  24.     company_name = dom.xpath('//div[@class="dw_table"]/div[@class="el"]/span[@class="t2"]/a[@target="_blank"]/@title')
  25.     # 3、工作地点
  26.     address = dom.xpath('//div[@class="dw_table"]/div[@class="el"]/span[@class="t3"]/text()')
  27.     # 4、工资
  28.     salary_mid = dom.xpath('//div[@class="dw_table"]/div[@class="el"]/span[@class="t4"]')
  29.     salary = [i.text for i in salary_mid]
  30.     # 5、发布日期
  31.     release_time = dom.xpath('//div[@class="dw_table"]/div[@class="el"]/span[@class="t5"]/text()')
  32.     # 6、获取二级网址url
  33.     deep_url = dom.xpath('//div[@class="dw_table"]/div[@class="el"]//p/span/a[@target="_blank"]/@href')
  34.     RandomAll = []
  35.     JobDescribe = []
  36.     CompanyType = []
  37.     CompanySize = []
  38.     Industry = []
  39.     for i in range(len(deep_url)):
  40.         web_test = requests.get(deep_url[i], headers=headers)
  41.         web_test.encoding = "gbk"
  42.         dom_test = etree.HTML(web_test.text)
  43.         # 7、爬取经验、学历信息，先合在一个字段里面，以后再做数据清洗。命名为random_all
  44.         random_all = dom_test.xpath('//div[@class="tHeader tHjob"]//div[@class="cn"]/p[@class="msg ltype"]/text()')
  45.         # 8、岗位描述性息
  46.         job_describe = dom_test.xpath('//div[@class="tBorderTop_box"]//div[@class="bmsg job_msg inbox"]/p/text()')
  47.         # 9、公司类型
  48.         company_type = dom_test.xpath('//div[@class="tCompany_sidebar"]//div[@class="com_tag"]/p[1]/@title')
  49.         # 10、公司规模(人数)
  50.         company_size = dom_test.xpath('//div[@class="tCompany_sidebar"]//div[@class="com_tag"]/p[2]/@title')
  51.         # 11、所属行业(公司)
  52.         industry = dom_test.xpath('//div[@class="tCompany_sidebar"]//div[@class="com_tag"]/p[3]/@title')
  54.         #print(industry,company_size,company_type,job_describe,random_all)
  55.         # 将上述信息保存到各自的列表中
  56.         RandomAll.append(random_all)
  57.         JobDescribe.append(job_describe)
  58.         CompanyType.append(company_type)
  59.         CompanySize.append(company_size)
  60.         Industry.append(industry)
  62.         # 为了反爬，设置睡眠时间
  63.         time.sleep(1)
  64.     # 由于我们需要爬取很多页，为了防止最后一次性保存所有数据出现的错误，因此，我们每获取一夜的数据，就进行一次数据存取。
  65.     df = pd.DataFrame()
  66.     df["岗位名称"] = job_name
  67.     df["公司名称"] = company_name
  68.     df["工作地点"] = address
  69.     df["工资"] = salary
  70.     df["发布日期"] = release_time
  71.     df["经验、学历"] = RandomAll
  72.     df["公司类型"] = CompanyType
  73.     df["公司规模"] = CompanySize
  74.     df["所属行业"] = Industry
  75.     df["岗位描述"] = JobDescribe
  76.     # 这里在写出过程中，有可能会写入失败，为了解决这个问题，我们使用异常处理。
  77.     try:
  78.         df.to_csv("job_info.csv", mode="a+", header=None, index=None, encoding="gbk")
  79.     except:
  80.         print("当页数据写入失败")
  81.     time.sleep(1)
  82. print("数据爬取完毕，是不是很开心！！！")

Analysis_data.py

  1. import re
  2. import numpy as np
  3. import pandas as pd
  4. import jieba
  5. import openpyxl
  6. import warnings
  7. warnings.filterwarnings("ignore")
  10. df = pd.read_csv(r"job_info.csv",engine="python",header=None,encoding='gbk')
  11. # 为数据框指定行索引
  12. df.index = range(len(df))
  13. # 为数据框指定列索引
  14. df.columns = ["岗位名","公司名","工作地点","工资","发布日期","经验与学历","公司类型","公司规模","行业","工作描述"]
  16. # 去重之前的记录数
  17. print("去重之前的记录数",df.shape)
  18. # 记录去重
  19. df.drop_duplicates(subset=["公司名","岗位名"],inplace=True)
  20. # 去重之后的记录数
  21. print("去重之后的记录数",df.shape)
  23. df["岗位名"].value_counts()
  24. df["岗位名"] = df["岗位名"].apply(lambda x:x.lower())
  26. #job_info.shape
  27. target_job = ['算法', '开发', '分析', '工程师', '数据', '运营', '运维']
  28. index = [df["岗位名"].str.count(i) for i in target_job]# 筛选和上面七个职位有关的岗位，本来700多，筛选之后3000多
  29. index = np.array(index).sum(axis=0) > 0
  30. job_info = df[index]
  31. job_info.shape
  33. job_list = ['数据分析', "数据统计","数据专员",'数据挖掘', '算法',
  34.             '大数据','开发工程师', '运营', '软件工程', '前端开发',
  35.             '深度学习', 'ai', '数据库', '数据库', '数据产品',
  36.             '客服', 'java', '.net', 'andrio', '人工智能', 'c++',
  37.             '数据管理',"测试","运维"]
  38. job_list = np.array(job_list)
  39. def rename(x=None,job_list=job_list):
  40.     index = [i in x for i in job_list]
  41.     if sum(index) > 0:
  42.         return job_list[index][0]
  43.     else:
  44.         return x
  45. job_info["岗位名"] = job_info["岗位名"].apply(rename)
  46. job_info["岗位名"].value_counts()   # 查看岗位名这一列下面有哪些不同的值，并计算每个值有多少个重复值
  47. # 数据统计、数据专员、数据分析统一归为数据分析
  48. job_info["岗位名"] = job_info["岗位名"].apply(lambda x:re.sub("数据专员","数据分析",x))
  49. job_info["岗位名"] = job_info["岗位名"].apply(lambda x:re.sub("数据统计","数据分析",x))
  51. job_info["工资"].str[-1].value_counts()
  52. job_info["工资"].str[-3].value_counts()
  54. index1 = job_info["工资"].str[-1].isin(["年","月"])
  55. index2 = job_info["工资"].str[-3].isin(["万","千"])
  56. job_info = job_info[index1 & index2]
  58. def get_money_max_min(x):
  59.     try:
  60.         if x[-3] == "万":
  61.             z = [float(i)*10000 for i in re.findall("[0-9]+\.?[0-9]*",x)]
  62.         elif x[-3] == "千":
  63.             z = [float(i) * 1000 for i in re.findall("[0-9]+\.?[0-9]*", x)]
  64.         if x[-1] == "年":
  65.             z = [i/12 for i in z]
  66.         return z
  67.     except:
  68.         return x
  70. salary = job_info["工资"].apply(get_money_max_min)
  71. job_info["最低工资"] = salary.str[0]
  72. job_info["最高工资"] = salary.str[1]
  73. job_info["工资水平"] = job_info[["最低工资","最高工资"]].mean(axis=1)
  75. #job_info["工作地点"].value_counts()
  76. address_list = ['北京', '上海', '广州', '深圳', '杭州', '苏州', '长沙',
  77.                 '武汉', '天津', '成都', '西安', '东莞', '合肥', '佛山',
  78.                 '宁波', '南京', '重庆', '长春', '郑州', '常州', '福州',
  79.                 '沈阳', '济南', '宁波', '厦门', '贵州', '珠海', '青岛',
  80.                 '中山', '大连','昆山',"惠州","哈尔滨","昆明","南昌","无锡"]
  81. address_list = np.array(address_list)
  83. def rename(x=None,address_list=address_list):
  84.     index = [i in x for i in address_list]
  85.     if sum(index) > 0:
  86.         return address_list[index][0]
  87.     else:
  88.         return x
  89. job_info["工作地点"] = job_info["工作地点"].apply(rename)
  91. job_info.loc[job_info["公司类型"].apply(lambda x:len(x)<6),"公司类型"] = np.nan
  92. job_info["公司类型"] = job_info["公司类型"].str[2:-2]
  94. # job_info["行业"].value_counts()
  95. job_info["行业"] = job_info["行业"].apply(lambda x:re.sub(",","/",x))
  96. job_info.loc[job_info["行业"].apply(lambda x:len(x)<6),"行业"] = np.nan
  97. job_info["行业"] = job_info["行业"].str[2:-2].str.split("/").str[0]
  100. #学历字段
  101. job_info["学历"] = job_info["经验与学历"].apply(lambda x:re.findall("本科|大专|应届生|在校生|硕士",x))
  102. #re.findall返回的是在"经验与学历"这一列里边与"本科|大专|应届生|在校生|硕士"相匹配的全部字符串，返回形式为数组
  103. # 最好加r表示raw string，不加会把\n识别成换行，而是\和n
  104. # 把找到的学历插入到新建的“学历”中，这是每一行的数据是数组形式
  105. def func(x):
  106.     if len(x) == 0:
  107.         return np.nan
  108.     elif len(x) == 1 or len(x) == 2:
  109.         return x[0]
  110.     else:
  111.         return x[2]
  112. job_info["学历"] = job_info["学历"].apply(func)
  113. #这部分是将数组中的字符串提取并重新插入
  115. #实习经验字段
  116. job_info["经验"] = job_info["经验与学历"].apply(lambda x:re.findall("1年|2年|3-4年|5-7年|8-9年|10年以上",x))
  117. def func2(x):
  118.     if len(x) == 0:
  119.         return np.nan
  120.     else:
  121.         return x[0]
  122. job_info["经验"] = job_info["经验"].apply(func2)
  125. with open(r"stopword.txt","r",encoding='UTF-8') as f:
  126.     stopword = f.read()
  127. stopword = stopword.split()
  128. stopword = stopword + ["任职","职位"," "]
  130. job_info["工作描述"] = job_info["工作描述"].str[2:-2].apply(lambda x:x.lower()).apply(lambda x:"".join(x))\
  131.     .apply(jieba.lcut).apply(lambda x:[i for i in x if i not in stopword])
  132. job_info.loc[job_info["工作描述"].apply(lambda x:len(x) < 6),"工作描述"] = np.nan
  135. #job_info["公司规模"].value_counts()
  136. def func(x):
  137.     if x == "['少于50人']":
  138.         return "<50"
  139.     elif x == "['50-150人']":
  140.         return "50-150"
  141.     elif x == "['150-500人']":
  142.         return '150-500'
  143.     elif x == "['500-1000人']":
  144.         return '500-1000'
  145.     elif x == "['1000-5000人']":
  146.         return '1000-5000'
  147.     elif x == "['5000-10000人']":
  148.         return '5000-10000'
  149.     elif x == "['10000人以上']":
  150.         return ">10000"
  151.     else:
  152.         return np.nan
  153. job_info["公司规模"] = job_info["公司规模"].apply(func)
  155. feature = ["公司名","岗位名","工作地点","工资水平","发布日期","学历","经验","公司类型","公司规模","行业"]
  156. final_df = job_info[feature]
  157. final_df.to_excel(r"词云图.xlsx",encoding="gbk",index=None)
  158. job_info.to_excel(r"new_job_info1.xlsx",encoding="gbk",index=None)
  163. df = pd.read_excel(r"new_job_info1.xlsx",encoding="gbk")
  164. df
  166. def get_word_cloud(data=None, job_name=None):
  167.     words = []
  168.     describe = data['工作描述'][data['岗位名'] == job_name].str[1:-1]
  169.     describe.dropna(inplace=True)
  170.     [words.extend(i.split(',')) for i in describe]
  171.     words = pd.Series(words)
  172.     word_fre = words.value_counts()
  173.     return word_fre
  175. zz = ['数据分析', '算法', '大数据','开发工程师', '运营', '软件工程','运维', '数据库','java',"测试"]
  176. for i in zz:
  177.     word_fre = get_word_cloud(data=df, job_name='{}'.format(i))
  178.     word_fre = word_fre[1:].reset_index()[:100]
  179.     word_fre["岗位名"] = pd.Series("{}".format(i),index=range(len(word_fre)))
  180.     word_fre.to_csv(r"bb.csv", mode='a',index=False, header=None,encoding="gbk")