ArxivExtractor.py

    1. #!/usr/bin/env python
    2. # -*- encoding: utf-8 -*-
    3. '''
    4. @File    :   ArxivExtractor.py    
    5. @Contact :   htkstudy@163.com
    7. @Modify Time      @Author    @Version    @Desciption
    8. ------------      -------    --------    -----------
    9. 2021/5/10 21:40   Armor(htk)     1.0         None
    10. '''
    12. import re
    13. import time
    14. import numpy as np
    15. import random
    16. from selenium import webdriver
    17. from selenium.webdriver.chrome.options import Options
    18. import smtplib
    19. from email.mime.text import MIMEText
    20. from email.mime.multipart import MIMEMultipart
    21. from email.mime.image import MIMEImage
    24. def seed_email(file_name,receivers_list):
    25.     #设置服务器所需信息
    26.     #163邮箱服务器地址
    27.     mail_host = 'smtp.163.com'  
    28.     #163用户名
    29.     mail_user = '*****@163.com'  
    30.     #密码(部分邮箱为授权码) 
    31.     mail_pass = '**********'   
    32.     #邮件发送方邮箱地址
    33.     sender = '*****@163.com'  
    34.     #邮件接受方邮箱地址，注意需要[]包裹，这意味着你可以写多个邮件地址群发
    35.     receivers = receivers_list  #如 receivers = ['*****@qq.com']
    37.     #设置email信息
    38.     #邮件内容设置
    39.     message = MIMEMultipart()
    40.     message['From'] = sender
    41.     message['To'] = receivers[0]
    42.     #邮件主题       
    43.     message['Subject'] = 'Arxiv每日推送' 
    44.     #推荐使用html格式的正文内容，这样比较灵活，可以附加图片地址，调整格式等
    45.     with open(file_name,'r',encoding="utf-8") as f:
    46.         content = f.read()
    47.     #设置html格式参数
    48.     part1 = MIMEText(content,'html','utf-8')
    49.     #将内容附加到邮件主体中
    50.     message.attach(part1)
    52.     #登录并发送邮件
    53.     try:
    54.         smtpObj = smtplib.SMTP() 
    55.         #连接到服务器
    56.         smtpObj.connect(mail_host,25)
    57.         #登录到服务器
    58.         smtpObj.login(mail_user,mail_pass) 
    59.         #发送
    60.         smtpObj.sendmail(
    61.             sender,receivers,message.as_string()) 
    62.         #退出
    63.         smtpObj.quit() 
    64.         print('success')
    65.     except smtplib.SMTPException as e:
    66.         print('error',e) #打印错误
    68. class GeneralArxivExtractor(object):
    70.     def extract(self,html):
    71.         title = re.findall(r'<title>(.*?) - arXiv每日学术速递</title>',html)[0]
    72.         print(title)
    73.         subhtml = re.findall(r'<article data-v-f3c566ae="" data-v-04c933e6="" class="global">(.*?)</article>', html)[0]
    74.         subjects = re.findall(r'<center>(.*?)</center>', subhtml)[1:]
    75.         return 0
    77.     def simple(self,html,spical):
    78.         title = re.findall(r'<title>(.*?) - arXiv每日学术速递</title>', html)[0]
    79.         print(title)
    80.         flags = [1 if key in title else 0 for key in spical]
    81.         if np.array(flags).sum() > 0:
    82.             subhtml = re.findall(r'<article data-v-f3c566ae="" data-v-04c933e6="" class="global">(.*?)</article>', html)[0]
    83.             front = '<!DOCTYPE html><html lang="en"><head><meta charset="UTF-8"></head><body>'
    84.             tail = '</body></html>'
    85.             html = front + subhtml + front
    86.             with open("html/{}.html".format(title),"w",encoding='utf-8') as f:
    87.                 f.write(html)
    88.     pass
    90. class SelenuimLoading(object):
    91.     def __init__(self):
    92.         # option配置
    93.         self.chrome_options = Options()
    94.         self.chrome_options.add_argument('--disable-gpu')  # 谷歌文档提到需要加上这个属性来规避bug
    95.         self.chrome_options.add_argument('--hide-scrollbars')  # 隐藏滚动条, 应对一些特殊页面
    96.         self.chrome_options.add_argument('blink-settings=imagesEnabled=false')  # 不加载图片, 提升速度
    97.         self.driver = webdriver.Chrome(executable_path='D:\HuProject\chromedriver_win32\chromedriver.exe',options=self.chrome_options)
    99.     def Spider_Html(self,url):
    100.         self.driver.implicitly_wait(60)
    101.         self.driver.get(url)
    102.         time.sleep(random.randint(5,8))
    103.         html = self.driver.page_source
    104.         return html
    106.     def close(self):
    107.         self.driver.close()
    109.     pass
    111. if __name__ == '__main__':
    112.     org = 12056
    113.     sp = ['人工智能','机器学习','计算机视觉','自然语言处理']
    114.     # 生成urls
    115.     url_format = "https://www.arxivdaily.com/thread/{}"
    116.     urls = [url_format.format(i) for i in np.arange(970,11152)[::-1]]
    117.     # 加载 浏览器
    118.     web = SelenuimLoading()
    119.     # 加载提取器
    120.     extractor = GeneralArxivExtractor()
    121.     # 抓取urls中的html页面
    122.     for idx,url in enumerate(urls):
    123.         try:
    124.             # 提取html
    125.             html = web.Spider_Html(url)
    126.             timeput = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
    127.             print("{} {}/{} ".format(timeput,idx+1,len(urls)),sep=" ")
    128.             # 抓取网页
    129.             extractor.simple(html.replace("\n",""),sp)
    130.         except:
    131.             print(urls[idx],"出错")
    132.     web.close()

    SeedArxiv.py

    1. import re
    2. import time
    3. import numpy as np
    4. import random
    5. import os
    6. import datetime
    7. from ArxivExtractor import seed_email
    8. from selenium import webdriver
    9. from selenium.webdriver.chrome.options import Options
    10. from selenium.webdriver import ChromeOptions
    12. class GeneralArxivExtractor(object):
    14.     def extract(self,html):
    15.         title = re.findall(r'<title>(.*?) - arXiv每日学术速递</title>',html)[0]
    16.         print(title)
    17.         subhtml = re.findall(r'<article data-v-f3c566ae="" data-v-04c933e6="" class="global">(.*?)</article>', html)[0]
    18.         subjects = re.findall(r'<center>(.*?)</center>', subhtml)[1:]
    19.         return 0
    21.     def simple(self,html,spical):
    22.         title = re.findall(r'<title>(.*?) - arXiv每日学术速递</title>', html)[0]
    23.         print(title)
    24.         flags = [1 if key in title else 0 for key in spical ]
    25.         if np.array(flags).sum() > 0:
    26.             subhtml = re.findall(r'<article data-v-f3c566ae="" data-v-04c933e6="" class="global">(.*?)</article>', html)[0]
    27.             front = '<!DOCTYPE html><html lang="en"><head><meta charset="UTF-8"></head><body>'
    28.             tail = '</body></html>'
    29.             html = front + subhtml + front
    30.             with open("html/{}.html".format(title),"w",encoding='utf-8') as f:
    31.                 f.write(html)
    32.     pass
    34. class SelenuimLoading(object):
    35.     def __init__(self):
    36.         # option配置
    37.         self.options = Options()
    38.         self.chrome_options = ChromeOptions()
    39.         self.chrome_options.add_argument('headless') # 设置option
    40.         self.options.add_argument('--disable-gpu')  # 谷歌文档提到需要加上这个属性来规避bug
    41.         self.options.add_argument('--hide-scrollbars')  # 隐藏滚动条, 应对一些特殊页面
    42.         self.options.add_argument('blink-settings=imagesEnabled=false')  # 不加载图片, 提升速度
    43.         self.driver = webdriver.Chrome(executable_path='D:\HuProject\chromedriver_win32\chromedriver.exe',options=self.options,chrome_options=self.chrome_options)
    45.     def Spider_Html(self,url):
    46.         self.driver.implicitly_wait(60)
    47.         self.driver.get(url)
    48.         first = self.driver.find_element_by_xpath(r'//*[@id="__layout"]/div/div[2]/div/main/div[2]/div[1]/div[1]/div/div/div/div[2]/a/div/span')
    49.         first.click()
    50.         self.driver.switch_to.window(self.driver.window_handles[-1])
    51.         time.sleep(random.randint(5,8))
    52.         html = self.driver.page_source
    53.         return html
    55.     def close(self):
    56.         self.driver.quit()
    57.     pass
    59. def scray(choose):
    60.     urls = {'计算机视觉':"https://www.arxivdaily.com/category/19?search_ids=19",
    61.             '自然语言处理':"https://www.arxivdaily.com/category/20?search_ids=20",
    62.             '人工智能':'https://www.arxivdaily.com/category/21?search_ids=21',
    63.             '机器学习':'https://www.arxivdaily.com/category/22?search_ids=22'
    64.            }
    65.     url = urls[choose]
    66.     # 加载 浏览器
    67.     web = SelenuimLoading()
    68.     # 加载提取器
    69.     extractor = GeneralArxivExtractor()
    70.     # 抓取urls中的html页面
    71.     try:
    72.         # 提取html
    73.         html = web.Spider_Html(url)
    74.         timeput = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
    75.         print("{}".format(timeput),sep=" ")
    76.         # 抓取网页
    77.         extractor.simple(html.replace("\n",""),urls.keys())
    78.     except Exception as e:
    79.         # 访问异常的错误编号和详细信息
    80.         print(url,"出错")
    81.         print(e.args)
    82.     web.close()
    84. if __name__ == '__main__':
    85.     choose = ["自然语言处理","计算机视觉","人工智能","机器学习"]
    86.     receiver_lists = ['1939454633@qq.com','htkstudy@163.com','htkstudy@163.com']
    87.     flag = True
    88.     while True:
    89.         scray("自然语言处理")
    90.         current_time = datetime.datetime.now()
    91.         current_time_string = str(current_time.year)+"_"+str(current_time.month)+"_"+str(current_time.day)
    92.         for file_name in os.listdir("html"):
    93.             if current_time_string in file_name and "自然语言处理" in file_name and ".html" in file_name:
    94.                 seed_email("html/"+file_name,receiver_lists)
    95.                 flag = False
    96.                 break
    97.         if flag:
    98.             print("未更新")
    99.         else: 
    100.             break
    101.         time.sleep(20)