1. class JdItem(scrapy.Item):
    2.     bookName = scrapy.Field()
    3.     sellPrice = scrapy.Field()
    4.     authors = scrapy.Field()
    5.     coverUrl = scrapy.Field()
    6.     bookId = scrapy.Field()
    7.     definePrice = scrapy.Field()
    8.     publisher = scrapy.Field()
    9.     discount = scrapy.Field()
    1. import scrapy
    2. import json
    3. from jd.items import JdItem
    4. class JdShopSpider(scrapy.Spider):
    5.     name = 'jd_shop'
    6.     # allowed_domains = ['jd.com']
    7.     # start_urls = ['http://jd.com/']
    8.     def start_requests(self):
    9.         url = 'https://gw-e.jd.com/client.action?callback=func&body=%7B%22moduleType%22%3A1%2C%22page%22%3A4%2C%22pageSize%22%3A20%2C%22scopeType%22%3A1%7D&functionId=bookRank&client=e.jd.com&_=1650724495119'
    10.         yield scrapy.Request(url=url,callback=self.parse)
    12.     def parse(self, response):
    13.         data = response.text.lstrip('func(')
    14.         data = data.rstrip(')')
    15.         all_books = json.loads(data)['data']['books']
    16.         for book in all_books:
    17.             item = JdItem()
    18.             item['bookName'] = book['bookName']
    19.             item['sellPrice'] = book['sellPrice']
    20.             item['authors'] = book['authors']
    21.             item['coverUrl'] = book['coverUrl']
    22.             item['bookId'] = book['bookId']
    23.             item['definePrice'] = book['definePrice']
    24.             item['publisher'] = book['publisher']
    25.             item['discount'] = book['discount']
    26.             yield item
    1. from scrapy.crawler import CrawlerProcess
    2. from scrapy.utils.project import get_project_settings
    3. if __name__ == '__main__':
    4.     process = CrawlerProcess(get_project_settings())
    5.     process.crawl('jd_shop')
    6.     process.start()
    1. from itemadapter import ItemAdapter
    2. import pymysql
    3. import json
    4. class JdPipeline(object):
    5.     # 初始化数据库参数
    6.     def __init__(self,host,database,user,password,port):
    7.         self.host = host
    8.         self.database = database
    9.         self.user = user
    10.         self.password = password
    11.         self.port = port
    13.     @classmethod
    14.     def from_crawler(cls,crawler):
    15.         return cls(
    16.             host = crawler.settings.get('SQL_HOST'),
    17.             user = crawler.settings.get('SQL_USER'),
    18.             password = crawler.settings.get('SQL_PASSWORD'),
    19.             database = crawler.settings.get('SQL_DATABASE'),
    20.             port = crawler.settings.get('SQL_PORT'),
    21.         )
    23.     # 打开爬虫时调用
    24.     def open_spider(self,spider):
    25.         self.db = pymysql.connect(host=self.host, user=self.user, password=self.password, database=self.database, port=self.port,autocommit=True)
    26.         self.cursor = self.db.cursor() # 创建游标
    28.     # 关闭爬虫时调用
    29.     def close_spider(self,spider):
    30.         self.db.close()
    31.     # 重写process_item()方法，首先先将item对象转换为字典类型的数据，然后将数据通过zip()函数转换为每条数据为
    32.     # ['book_name','press','author']类型的数据，然后提交并返回item对象
    33.     def process_item(self, item, spider):
    34.         data = dict(item)
    35.         authors = json.dumps(data['authors'])
    36.         query = """insert into ranking (bookName,sellPrice,authors,coverUrl,bookId,definePrice,publisher,discount) values (%s,%s,%s,%s,%s,%s,%s,%s)"""
    37.         values = (str(data["bookName"]), str(data["sellPrice"]), str(authors),str(data["coverUrl"]),str(data["bookId"]),str(data["definePrice"]),str(data["publisher"]),str(data["discount"]))
    38.         self.cursor.execute(query, values)
    39.         self.db.commit()
    40.         return item
    1. SQL_HOST = 'localhost'
    2. SQL_USER = 'root'
    3. SQL_PASSWORD = 'root'
    4. SQL_DATABASE = 'mysoft'
    5. SQL_PORT = 3306
    7. ITEM_PIPELINES = {
    8.    'jd.pipelines.JdPipeline': 300,
    9. }

    表结构：
    image.png