1. from selenium import webdriver
    2. from selenium.common.exceptions import TimeoutException
    3. from selenium.webdriver.common.by import By
    4. from selenium.webdriver.support import expected_conditions as EC
    5. from selenium.webdriver.support.wait import WebDriverWait
    6. from pyquery import PyQuery as pq
    7. import time
    8. import os
    9. import urllib
    11. browser = webdriver.Chrome("E:\soft\chromedriver\chromedriver_win32\chromedriver.exe")
    12. wait = WebDriverWait(browser, 10)
    13. url = 'http://www.mafengwo.cn/jd/10206/gonglve.html'
    14. browser.get(url)
    15. aomeng = {}
    18. def index_page():
    19.     try:
    20.         # 获取总页数
    21.         page_total = browser.find_elements_by_css_selector('span.count')
    22.         total = page_total[0].text
    23.         total_page = total[1:3]
    24.         for i in range(1, int(total_page) - 4):
    25.             print('正在爬取第', str(i), '页')
    26.             # 实现下一页
    27.             submit = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, '.m-pagination > a.pi.pg-next')))
    28.             submit.click()
    30.             html = browser.page_source
    31.             parse_page(html)
    32.             # print(html)
    33.     except TimeoutException:
    34.         print('超时')
    36.     finally:
    37.         browser.close()
    40. def parse_page(html):
    41.     doc = pq(html, parser="html")
    42.     items = doc('.bd .scenic-list.clearfix .img').parent().items()
    43.     for item in items:
    44.         # 标题
    45.         aomeng['title'] = item.find('h3').text()
    46.         # 子页链接
    47.         aomeng['href'] = item.attr('href')
    49.         # 图片链接
    50.         aomeng['img'] = item.find('.img').find('img').attr('src')
    51.         # 延迟3秒
    52.         time.sleep(3)
    53.         # 存储图片
    54.         save_img(aomeng)
    57. def save_img(param):
    58.     # 判断是否能存在路径
    59.     if not os.path.exists(param['title']):
    60.         os.makedirs(param['title'])
    61.     # 文件写入
    62.     with open('{0}/{0}.{1}'.format(param['title'], 'txt'), 'a', encoding='utf-8')as f:
    63.         f.write(param['href'])
    64.         print('写入txt文件成功')
    65.     try:
    66.         file_path = '{0}/{0}.{1}'.format(param['title'], 'jpg')
    67.         if not os.path.exists(file_path):
    68.             urllib.request.urlretrieve(param['img'], file_path)
    69.             print('img 存入success!')
    70.         else:
    71.             print('Download Complete')
    72.     except:
    73.         print('img 存入fail!')
    74.         pass
    77. index_page()