获取京东的评论和搜索信息

    1. # endocing:utf-8
    2. # from bs4 import BeautifulSoup
    3. import re, requests, json
    5. # https://club.jd.com/comment/productPageComments.action?callback=fetchJSON_comment98&productId=100007386276&score=0&sortType=5&page=0&pageSize=10&isShadowSku=0&fold=1
    6. from  lxml import etree
    8. keyword=input("请输入查询商品：")
    9. # print(type(keyword))
    10. g=[]
    11. for i in range(1,2):
    12.     url="https://search.jd.com/Search?keyword={}&enc=utf-8&page={}".format(keyword,i*2-1)
    13.     header={
    14.         # ":authority":"search.jd.com",
    15.         # ":method":"GET",
    16.         # ":scheme":" https",
    17.         "upgrade - insecure - requests": "1",
    18.         "user-agent":"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.92 Safari/537.36"
    19.     }
    20.     html=requests.get(url=url,headers=header)
    21.     html.encoding="utf-8"
    22.     # print(html.text)
    23.     html=html.text
    24.     newhtml=etree.HTML(html)
    25.     # print(newhtml)
    26.     # log_id=re.findall("log_id:'(.*?)'",html,re.S)[0]
    27.     # cid=re.findall("LogParm.*?cid:(.*?),",html,re.S)[0]
    28.     # print(cid)
    29.     sku_id=newhtml.xpath('//*[@id="J_goodsList"]/ul/li/@data-sku')
    30.     pro_name=newhtml.xpath('//*[@id="J_goodsList"]/ul/li/@data-sku')
    32. # p_list=",".join('%s' % id for id in sku_id)
    33.     print(sku_id)
    34.     g = sku_id
    36. s = requests.session()
    37. url = 'https://club.jd.com/comment/productPageComments.action'
    38. data = {
    39.     'callback': 'fetchJSON_comment98',
    40.     'productId': '100007386276',
    41.     'score': 0,
    42.     'sortType': 5,
    43.     'pageSize': 10,
    44.     'isShadowSku': 0,
    45.     'page': 0
    46. }
    48. for i in g:
    49.     data['productId'] = i
    50.     t = s.get(url, params=data).text
    51.     try:
    52.         t = re.search(r'(?<=fetchJSON_comment98\().*(?=\);)', t).group(0)
    53.     except Exception as e:
    54.         print("出错了")
    55.         continue
    57.     j = json.loads(t)
    58.     commentSummary = j['hotCommentTagStatistics']
    59.     for comment in commentSummary:
    60.         c_name = comment['name']
    61.         c_count = comment['count']
    62.         # c_client = comment['userClientShow']
    63.         print('{}[{}]'.format(c_name, c_count,), end=',')
    64.     print()

    e48e614593004a0bf315a7e953e055b.png

    https://www.zhihu.com/question/28981353?sort=created
    https://blog.csdn.net/weixin_42357472/article/details/83793386