cainiao天地数据爬取

获取token和其他参数

首先分析获取该网站的隐藏链接, 必须得点击下图界面中查询按钮后才会通过cainaio的后端返回一个url,包含验证参数token, 以及其他的请求参数等
image.png

伪代码示例

对于点击网页的控制, 利用webdrive工具pyppeteer库中的api进行js控制获取

  2. import asyncio
  4. import urllib3
  5. from pyppeteer import launch, launcher
  7. from app.constant import WIDTH, HEIGHT, HEADLESS
  8. from utils.chromium import intercept_request, get_cookies, set_user_agent
  11. # launcher.AUTOMATION_ARGS.remove('--enable-automation')  # linux
  12. launcher.DEFAULT_ARGS.remove('--enable-automation')  # Windows
  14. POINT_DOMAIN_URL = 'https://fly.cainiao.com/login.htm?fromUrl=https://fly.cainiao.com/branch/branchOverallForBranch.htm'
  16. async def get_cookies_token(username, password, stat_date):
  17.     browser = await launch(headless=HEADLESS, slowMo=15, autoClose=False,
  18.                            args=['--disable-infobars', '--disable-extensions', '--hide-scrollbars', '--mute-audio',
  19.                                  '--no-sandbox', '--disable-gpu', '--disable-setuid-sandbox',
  20.                                  '--disable-translate', '--safebrowsing-disable-auto-update',
  21.                                  '--disable-bundled-ppapi-flash', '--window-size={},{}'.format(WIDTH, HEIGHT),
  22.                                  ], dumpio=True)
  23.     page = await browser.newPage()
  24.     user_agent = set_user_agent()  # set chromium info
  25.     await page.setUserAgent(user_agent)
  26.     await page.setViewport({'width': WIDTH, 'height': HEIGHT})
  27.     await page.goto(POINT_DOMAIN_URL)
  29.     await page.evaluate('''() =>{ Object.defineProperties(navigator,{ webdriver:{ get: () => undefined } }) }''')
  30.     await page.evaluate('''() =>{ window.navigator.chrome = { runtime: {},  }; }''')
  31.     await page.evaluate('''() =>{ Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] }); }''')
  32.     await page.evaluate('''() =>{ Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3, 4, 5,6], }); }''')
  34.     ####
  35.     # login fly.cainiao.com
  36.     await asyncio.sleep(1)
  37.     frames = page.frames
  38.     user = await frames[2].querySelector('#fm-login-id')
  39.     await user.type(username, {'delay': 100})
  40.     await asyncio.sleep(2)
  41.     passwd = await frames[2].querySelector('#fm-login-password')
  42.     await passwd.type(password, {'delay': 100})
  43.     await asyncio.sleep(2)
  44.     log_in = await frames[2].querySelector('#login-form > div.fm-btn > button')
  45.     await log_in.click({'delay': 50})
  46.     await asyncio.sleep(3)
  48.     # await page.reload({'delay': 50})
  49.     # await asyncio.sleep(2)
  51.     # page_content = await page.content()
  52.     # with open('login.html', 'w', encoding='utf8') as wf:
  53.     #     wf.write(page_content)
  55.     ####
  56.     # close web popup if exist
  57.     mask = await page.evaluate('''document.getElementsByClassName('edpl-icon icon-edpl-close')''')
  59.     while 1:
  60.         try:
  61.             await page.click(
  62.                 'body > div.edpl-wrap > div > div.edpl-mask > div > span',
  63.                 delay=200)
  64.         except Exception as warning:
  65.             logger.info(f'popup windows warning: {warning}')
  66.             if warning:
  67.                 break
  68.     await asyncio.sleep(2)
  69.     await page.reload({'delay': 80})
  71.     # ####
  72.     # # move mouse cursor to left sidebar and click current select(express or point url)
  73.     # await page.mouse.move(25, 150, steps=5)
  74.     # try:
  75.     #     await page.click(
  76.     #         'body > div.edpl-wrap > div > div.edpl-nav > div:nth-child(2) > ul:nth-child(1) > ul:nth-child(3) > li > a',
  77.     #         delay=200)
  78.     #     await asyncio.sleep(2)
  79.     #     await page.reload({'delay': 50})
  80.     # except Exception as err:
  81.     #     logger.warning(f'warning: {err}')
  82.     #     pass
  84.     ####
  85.     # save cookies and token to redis
  86.     cookies = await get_cookies(page)
  87.     await asyncio.sleep(2)
  88.     await page.setRequestInterception(True)
  89.     page.on('request', lambda req: asyncio.ensure_future(
  90.         intercept_request(req, cookies, stat_date)))
  92.     try:
  93.         await page.click('#J_SearchBtn', {'delay': 30})
  94.     except Exception as e:
  95.         logger.error(f'{e}')
  96.         await page.close()
  97.         await browser.close()
  98.         return None
  100.     await page.close()
  101.     await browser.close()
  103.     return True
  106. if __name__ == '__main__':
  107.     urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
  109.     from datetime import datetime, timedelta
  110.     from app.constant import ACCOUNT
  112.     stat_date = str(datetime.today().date() - timedelta(days=1))
  113.     username = ACCOUNT[1]['username']
  114.     password = ACCOUNT[1]['password']
  116.     login = asyncio.get_event_loop().run_until_complete(get_cookies_token(username, password, stat_date))
  117.     if not login:
  118.         print('failed')
  119.     else:
  120.         print('success')