Day01

1.初识mongoDo数据库

将小说的每一行都保存在mongodb中

  1. import pymongo
  2. #将小说保存到mongodb数据库中
  3. client = pymongo.MongoClient('localhost',27017)
  4. walden = client['walden']
  5. sheet_tab = walden['sheet_tab']
  7. path = '/Users/Hou/Desktop/walden.txt'
  8. with open(path2,'r') as f:
  9.     lines = f.readlines()
  10.     for index,line in enumerate(lines):
  11.         data = {
  12.             'index':index,
  13.             'line' :line,
  14.             'words':len(line.split())
  15.         }
  16.         sheet_tab.insert_one(data)
  18. # $lt/$lte/$gt/$gte/$ne，依次等价于</<=/>/>=/!=。（l表示less g表示greater e表示equal n表示not  ）
  19. for item in sheet_tab.find({'words':{'$lt':5}}):
  20.     print(item)

image.png

Day02

1.爬取58同城的多个页面的数据（不一定成功的）

分析流程

  1. 观察页面特征（不同的页面是否具有一致性）

不同页面不同规则的问题，58同城的分页问题

  1. 设计工作流程（确保工作能搞笑运行）

设定有两个爬虫spider1(先爬取所有的列表页，保存到url_list表中)和spider2（从表url_list中获取数据爬取详情页，将数据保存到item_info表中）
Week02 - 图2

1.1 爬取58的所有分类页

  1. from bs4 import BeautifulSoup
  2. import requests
  4. headers = {
  5.     'user-agent': ''
  6. }
  7. start_url = 'https://gz.58.com/sale.shtml'
  8. url_host = 'https://gz.58.com'
  9. def get_channel_url(url):
  10.     wb_data = requests.get(start_url, headers = headers)
  11.     #print(wb_data.text)
  12.     soup = BeautifulSoup(wb_data.text, 'lxml')
  13.     links = soup.select('ul.ym-submnu > li > b > a')
  14.     print(links)
  15.     for link in links:
  16.         page_url = url_host + link.get('href')
  17.         print(page_url)
  19. #ymenu-side > ul > li:nth-child(1) > ul > li:nth-child(1) > b > a
  20. # ul.ym-submnu > li > b > a
  21. get_channel_url(start_url)

image.png

1.2 根据url解析页面（获取商家链接）

  1. from bs4 import BeautifulSoup
  2. import requests
  3. import time
  4. import pymongo
  6. client = pymongo.MongoClient('localhost', 27017)
  7. tmp_58city = client['tmp_test']
  8. url_list = tmp_58city['url_list']
  9. item_info = tmp_58city['item_info']
  11. # spider 1
  13. def get_links_from(channel, pages, who_sells=0):
  14.     # https://bj.58.com/bijiben/0/pn2/
  15.     # 0表示个人，pn2表示第二页
  16.     list_view = '{}{}/pn{}'.format(channel, str(who_sells), str(pages))
  17.     #print(list_view)
  18.     wb_data = requests.get(list_view, headers=headers)
  19.     time.sleep(2)
  20.     soup = BeautifulSoup(wb_data.text, 'lxml')
  21.     # print(wb_data.text)
  22.     if soup.find('td', 't'):
  23.         for link in soup.select('td.t a.t'):
  24.             item_link = link.get('href').split('?')[0]
  25.             url_list.insert_one({'url': item_link})
  26.             print(item_link)
  27.     else:
  28.         pass
  30. def get_item_info(url):
  31.     wb_data = requests.get(url, headers =  headers)
  32.     soup = BeautifulSoup(wb_data_text, 'lxml')
  33.     title = soup.title.text
  34.     price = soup.select('span.price.c_f50')[0].text
  35.     date = soup.select('.time')[0].text
  36.     area = list(soup.select('.c_25d a')[0].stripped_strings) if soup.find_all('span', 'c_25d') else None
  37.     item_info.insert_one({'title': title, 'price': price, 'date': date, 'area': area})
  41. get_links_from('https://gz.58.com/bijiben/',2)

image.png

1.3 根据商家链接爬取详细信息

  1. path = './test/test.html'
  2. html_page = open(path, 'r', encoding='utf-8')
  3. soup = BeautifulSoup(html_page, 'lxml')
  4. # title = soup.select('div.detail-title > h1').text
  5. title = soup.title.text
  6. price = soup.select('span.infocard__container__item__main__text--price')[0].text.strip()
  7. date = soup.select('div.detail-title > div.detail-title__info > div:nth-child(1)')[0].text
  8. # area = list(soup.select('div.infocard__container__item__main a')[0].stripped_strings) if soup.find_all('span', 'c_25d') else None
  9. area = list(soup.select('div.infocard__container__item__main a'))[0].text
  10. # item_info.insert_one({'title': title, 'price': price, 'date': date, 'area': area})
  11. ans = {'title': title, 'price': price, 'date': date, 'area': area}
  12. print(ans)

image.png

1.4 多进程爬虫得数据获取

创建main.py文件调用上述的几个函数
1.导入所需要的库
2.用函数填入页码
3.创建进程池

  1. from multiprocessing import Pool
  2. from ahcnnel_extract import channel_list
  3. from page_parsing import get_links_form
  5. def get_all_links_from(channel):
  6.     for num in range(1, 101):
  7.         get_links_from(channel, num)
  9. if __name__ == '__main__':
  10.     pool = Pool()
  11.     pool.map(get_all_links_from, channel_list.split())