1. import requests
    2. import re
    3. from bs4 import BeautifulSoup
    4. from lxml import etree
    5. import time
    6. from pip._vendor.retrying import retry
    9. def getBook(book):
    10.     murl = 'https://www.xbiquge.la/modules/article/waps.php'
    11.     headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:77.0) Gecko/20100101 Firefox/77.0"}
    12.     data={'searchkey':book}
    13.     response=requests.post(url=murl,data=data,headers=headers)
    14.     response.encoding = "utf-8"
    15.     code=response.text
    16.     soup=BeautifulSoup(code,'lxml')
    17.     tab=soup.select('.even')
    18.     all=re.findall(r'<td class="even">(.*?)</td>',str(tab))
    19.     if len(all)==0:
    20.         return None
    21.     name=re.findall(r'target="_blank">(.*?)</a>',str(all))
    22.     author=[]
    23.     url=re.findall(r'href="(.*?)"',str(all))
    24.     for i,n in enumerate(all):
    25.         if i%2!=0:
    26.             author.append(n)
    27.     for i in range(len(name)):
    28.         if i == 0:
    29.             print('序号\t书名\t作者\t网址')
    30.         print('['+str(i)+']\t'+name[i]+'\t'+author[i]+'\t'+url[i])
    31.     burl=input("请输入你想获得txt的书的序号：")
    32.     return url[int(burl)]
    35. def getChap(url):
    36.     headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:77.0) Gecko/20100101 Firefox/77.0"}
    37.     response = requests.get(url=url,headers=headers)
    38.     response.encoding = "utf-8"
    39.     code = response.text
    40.     tree=etree.HTML(code)
    41.     nurl = tree.xpath('//div[@id="list"]/dl/dd/a/@href')
    42.     name=str(tree.xpath('//div[@id="info"]/h1/text()')).split("'")[1]
    43.     print(name+"共发现"+str(len(nurl))+"章，即将开始爬取")
    44.     for i in range(len(nurl)):
    45.         turl='http://www.xbiquge.la'+nurl[i]
    46.         getContent(turl,name)
    47.         print("爬取完成，剩余" + str(len(nurl)-i-1)+'章')
    48. @retry
    49. def getContent(url,name):
    50.     headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:77.0) Gecko/20100101 Firefox/77.0"}
    51.     response = requests.get(url=url, headers=headers)
    52.     response.encoding='utf-8'
    53.     time.sleep(1)
    54.     code = response.text
    55.     tree = etree.HTML(code)
    56.     chap = tree.xpath('//div[@class="bookname"]/h1/text()')[0]
    57.     content = tree.xpath('//div[@id="content"]/text()')
    58.     with open(name + '.txt', 'a+', encoding='utf-8') as file:
    59.         print("开始爬取:" + chap)
    60.         file.write(chap + '\n\n')
    61.         for i in content:
    62.             text = str(i)
    63.             text.strip()
    64.             file.write(text)
    65.         file.write("\n\n")
    66.         file.close()
    68. book=input("请输入你要获取的书名或作者(请您少字也别输错字)：")
    69. url=getBook(book)
    70. if url==None:
    71.     print("未找到相关书籍")
    72. else:
    73.     getChap(url)
    74.     print("爬取完毕")