Java SpringBoot WebMagic
WebMagic是一个开源的java爬虫框架。WebMagic框架的具体如何使用请参考官方文档：http://webmagic.io/docs/。
这里对spring boot+WebMagic+MyBatis做了整合，使用WebMagic爬取数据，然后通过MyBatis持久化爬取的数据到mysql数据库。
基于 SpringBoot WebMagic 实现一个的爬虫框架 - 图1

1、添加maven依赖

  1. <?xml version="1.0" encoding="UTF-8"?>
  2. <project xmlns="http://maven.apache.org/POM/4.0.0"
  3.          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  4.          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  5.     <modelVersion>4.0.0</modelVersion>
  7.     <groupId>hyzx</groupId>
  8.     <artifactId>qbasic-crawler</artifactId>
  9.     <version>1.0.0</version>
  11.     <parent>
  12.         <groupId>org.springframework.boot</groupId>
  13.         <artifactId>spring-boot-starter-parent</artifactId>
  14.         <version>1.5.21.RELEASE</version>
  15.         <relativePath/> <!-- lookup parent from repository -->
  16.     </parent>
  18.     <properties>
  19.         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
  20.         <maven.test.skip>true</maven.test.skip>
  21.         <java.version>1.8</java.version>
  22.         <maven.compiler.plugin.version>3.8.1</maven.compiler.plugin.version>
  23.         <maven.resources.plugin.version>3.1.0</maven.resources.plugin.version>
  25.         <mysql.connector.version>5.1.47</mysql.connector.version>
  26.         <druid.spring.boot.starter.version>1.1.17</druid.spring.boot.starter.version>
  27.         <mybatis.spring.boot.starter.version>1.3.4</mybatis.spring.boot.starter.version>
  28.         <fastjson.version>1.2.58</fastjson.version>
  29.         <commons.lang3.version>3.9</commons.lang3.version>
  30.         <joda.time.version>2.10.2</joda.time.version>
  31.         <webmagic.core.version>0.7.3</webmagic.core.version>
  32.     </properties>
  34.     <dependencies>
  35.         <dependency>
  36.             <groupId>org.springframework.boot</groupId>
  37.             <artifactId>spring-boot-devtools</artifactId>
  38.             <scope>runtime</scope>
  39.             <optional>true</optional>
  40.         </dependency>
  42.         <dependency>
  43.             <groupId>org.springframework.boot</groupId>
  44.             <artifactId>spring-boot-starter-test</artifactId>
  45.             <scope>test</scope>
  46.         </dependency>
  48.         <dependency>
  49.             <groupId>org.springframework.boot</groupId>
  50.             <artifactId>spring-boot-configuration-processor</artifactId>
  51.             <optional>true</optional>
  52.         </dependency>
  54.         <dependency>
  55.             <groupId>mysql</groupId>
  56.             <artifactId>mysql-connector-java</artifactId>
  57.             <version>${mysql.connector.version}</version>
  58.         </dependency>
  60.         <dependency>
  61.             <groupId>com.alibaba</groupId>
  62.             <artifactId>druid-spring-boot-starter</artifactId>
  63.             <version>${druid.spring.boot.starter.version}</version>
  64.         </dependency>
  66.         <dependency>
  67.             <groupId>org.mybatis.spring.boot</groupId>
  68.             <artifactId>mybatis-spring-boot-starter</artifactId>
  69.             <version>${mybatis.spring.boot.starter.version}</version>
  70.         </dependency>
  72.         <dependency>
  73.             <groupId>com.alibaba</groupId>
  74.             <artifactId>fastjson</artifactId>
  75.             <version>${fastjson.version}</version>
  76.         </dependency>
  78.         <dependency>
  79.             <groupId>org.apache.commons</groupId>
  80.             <artifactId>commons-lang3</artifactId>
  81.             <version>${commons.lang3.version}</version>
  82.         </dependency>
  84.         <dependency>
  85.             <groupId>joda-time</groupId>
  86.             <artifactId>joda-time</artifactId>
  87.             <version>${joda.time.version}</version>
  88.         </dependency>
  90.         <dependency>
  91.             <groupId>us.codecraft</groupId>
  92.             <artifactId>webmagic-core</artifactId>
  93.             <version>${webmagic.core.version}</version>
  94.             <exclusions>
  95.                 <exclusion>
  96.                     <groupId>org.slf4j</groupId>
  97.                     <artifactId>slf4j-log4j12</artifactId>
  98.                 </exclusion>
  99.             </exclusions>
  100.         </dependency>
  101.     </dependencies>
  103.     <build>
  104.         <plugins>
  105.             <plugin>
  106.                 <groupId>org.apache.maven.plugins</groupId>
  107.                 <artifactId>maven-compiler-plugin</artifactId>
  108.                 <version>${maven.compiler.plugin.version}</version>
  109.                 <configuration>
  110.                     <source>${java.version}</source>
  111.                     <target>${java.version}</target>
  112.                     <encoding>${project.build.sourceEncoding}</encoding>
  113.                 </configuration>
  114.             </plugin>
  116.             <plugin>
  117.                 <groupId>org.apache.maven.plugins</groupId>
  118.                 <artifactId>maven-resources-plugin</artifactId>
  119.                 <version>${maven.resources.plugin.version}</version>
  120.                 <configuration>
  121.                     <encoding>${project.build.sourceEncoding}</encoding>
  122.                 </configuration>
  123.             </plugin>
  125.             <plugin>
  126.                 <groupId>org.springframework.boot</groupId>
  127.                 <artifactId>spring-boot-maven-plugin</artifactId>
  128.                 <configuration>
  129.                     <fork>true</fork>
  130.                     <addResources>true</addResources>
  131.                 </configuration>
  132.                 <executions>
  133.                     <execution>
  134.                         <goals>
  135.                             <goal>repackage</goal>
  136.                         </goals>
  137.                     </execution>
  138.                 </executions>
  139.             </plugin>
  140.         </plugins>
  141.     </build>
  143.     <repositories>
  144.         <repository>
  145.             <id>public</id>
  146.             <name>aliyun nexus</name>
  147.             <url>http://maven.aliyun.com/nexus/content/groups/public/</url>
  148.             <releases>
  149.                 <enabled>true</enabled>
  150.             </releases>
  151.         </repository>
  152.     </repositories>
  154.     <pluginRepositories>
  155.         <pluginRepository>
  156.             <id>public</id>
  157.             <name>aliyun nexus</name>
  158.             <url>http://maven.aliyun.com/nexus/content/groups/public/</url>
  159.             <releases>
  160.                 <enabled>true</enabled>
  161.             </releases>
  162.             <snapshots>
  163.                 <enabled>false</enabled>
  164.             </snapshots>
  165.         </pluginRepository>
  166.     </pluginRepositories>
  167. </project>

2、项目配置文件 application.properties

配置mysql数据源，druid数据库连接池以及MyBatis的mapper文件的位置。

  1. # mysql数据源配置
  2. spring.datasource.name=mysql
  3. spring.datasource.type=com.alibaba.druid.pool.DruidDataSource
  4. spring.datasource.driver-class-name=com.mysql.jdbc.Driver
  5. spring.datasource.url=jdbc:mysql://192.168.0.63:3306/gjhzjl?useUnicode=true&characterEncoding=utf8&useSSL=false&allowMultiQueries=true
  6. spring.datasource.username=root
  7. spring.datasource.password=root
  9. # druid数据库连接池配置
  10. spring.datasource.druid.initial-size=5
  11. spring.datasource.druid.min-idle=5
  12. spring.datasource.druid.max-active=10
  13. spring.datasource.druid.max-wait=60000
  14. spring.datasource.druid.validation-query=SELECT 1 FROM DUAL
  15. spring.datasource.druid.test-on-borrow=false
  16. spring.datasource.druid.test-on-return=false
  17. spring.datasource.druid.test-while-idle=true
  18. spring.datasource.druid.time-between-eviction-runs-millis=60000
  19. spring.datasource.druid.min-evictable-idle-time-millis=300000
  20. spring.datasource.druid.max-evictable-idle-time-millis=600000
  22. # mybatis配置
  23. mybatis.mapperLocations=classpath:mapper/**/*.xml

3、数据库表结构

  1. CREATE TABLE `cms_content` (
  2.   `contentId` varchar(40) NOT NULL COMMENT '内容ID',
  3.   `title` varchar(150) NOT NULL COMMENT '标题',
  4.   `content` longtext COMMENT '文章内容',
  5.   `releaseDate` datetime NOT NULL COMMENT '发布日期',
  6.   PRIMARY KEY (`contentId`)
  7. ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='CMS内容表';

4、实体类

  1. import java.util.Date;
  3. public class CmsContentPO {
  4.     private String contentId;
  6.     private String title;
  8.     private String content;
  10.     private Date releaseDate;
  12.     public String getContentId() {
  13.         return contentId;
  14.     }
  16.     public void setContentId(String contentId) {
  17.         this.contentId = contentId;
  18.     }
  20.     public String getTitle() {
  21.         return title;
  22.     }
  24.     public void setTitle(String title) {
  25.         this.title = title;
  26.     }
  28.     public String getContent() {
  29.         return content;
  30.     }
  32.     public void setContent(String content) {
  33.         this.content = content;
  34.     }
  36.     public Date getReleaseDate() {
  37.         return releaseDate;
  38.     }
  40.     public void setReleaseDate(Date releaseDate) {
  41.         this.releaseDate = releaseDate;
  42.     }
  43. }

5、mapper接口

  1. public interface CrawlerMapper {
  2.     int addCmsContent(CmsContentPO record);
  3. }

6、CrawlerMapper.xml文件

  1. <?xml version="1.0" encoding="UTF-8"?>
  2. <!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
  3. <mapper namespace="com.hyzx.qbasic.dao.CrawlerMapper">
  5.     <insert id="addCmsContent" parameterType="com.hyzx.qbasic.model.CmsContentPO">
  6.         insert into cms_content (contentId,
  7.                                  title,
  8.                                  releaseDate,
  9.                                  content)
  10.         values (#{contentId,jdbcType=VARCHAR},
  11.                 #{title,jdbcType=VARCHAR},
  12.                 #{releaseDate,jdbcType=TIMESTAMP},
  13.                 #{content,jdbcType=LONGVARCHAR})
  14.     </insert>
  15. </mapper>

7、知乎页面内容处理类ZhihuPageProcessor

主要用于解析爬取到的知乎html页面。

  1. @Component
  2. public class ZhihuPageProcessor implements PageProcessor {
  4.     private Site site = Site.me().setRetryTimes(3).setSleepTime(1000);
  6.     @Override
  7.     public void process(Page page) {
  8.         page.addTargetRequests(page.getHtml().links().regex("https://www\\.zhihu\\.com/question/\\d+/answer/\\d+.*").all());
  9.         page.putField("title", page.getHtml().xpath("//h1[@class='QuestionHeader-title']/text()").toString());
  10.         page.putField("answer", page.getHtml().xpath("//div[@class='QuestionAnswer-content']/tidyText()").toString());
  11.         if (page.getResultItems().get("title") == null) {
  12.             // 如果是列表页，跳过此页，pipeline不进行后续处理
  13.             page.setSkip(true);
  14.         }
  15.     }
  17.     @Override
  18.     public Site getSite() {
  19.         return site;
  20.     }
  21. }

8、知乎数据处理类ZhihuPipeline

主要用于将知乎html页面解析出的数据存储到mysql数据库。

  1. @Component
  2. public class ZhihuPipeline implements Pipeline {
  3.     private static final Logger LOGGER = LoggerFactory.getLogger(ZhihuPipeline.class);
  5.     @Autowired
  6.     private CrawlerMapper crawlerMapper;
  8.     public void process(ResultItems resultItems, Task task) {
  9.         String title = resultItems.get("title");
  10.         String answer = resultItems.get("answer");
  12.         CmsContentPO contentPO = new CmsContentPO();
  13.         contentPO.setContentId(UUID.randomUUID().toString());
  14.         contentPO.setTitle(title);
  15.         contentPO.setReleaseDate(new Date());
  16.         contentPO.setContent(answer);
  18.         try {
  19.             boolean success = crawlerMapper.addCmsContent(contentPO) > 0;
  20.             LOGGER.info("保存知乎文章成功：{}", title);
  21.         } catch (Exception ex) {
  22.             LOGGER.error("保存知乎文章失败", ex);
  23.         }
  24.     }
  25. }

9、知乎爬虫任务类ZhihuTask

每十分钟启动一次爬虫。

  1. @Component
  2. public class ZhihuTask {
  3.     private static final Logger LOGGER = LoggerFactory.getLogger(ZhihuPipeline.class);
  5.     @Autowired
  6.     private ZhihuPipeline zhihuPipeline;
  8.     @Autowired
  9.     private ZhihuPageProcessor zhihuPageProcessor;
  11.     private ScheduledExecutorService timer = Executors.newSingleThreadScheduledExecutor();
  13.     public void crawl() {
  14.         // 定时任务，每10分钟爬取一次
  15.         timer.scheduleWithFixedDelay(() -> {
  16.             Thread.currentThread().setName("zhihuCrawlerThread");
  18.             try {
  19.                 Spider.create(zhihuPageProcessor)
  20.                         // 从https://www.zhihu.com/explore开始抓
  21.                         .addUrl("https://www.zhihu.com/explore")
  22.                         // 抓取到的数据存数据库
  23.                         .addPipeline(zhihuPipeline)
  24.                         // 开启2个线程抓取
  25.                         .thread(2)
  26.                         // 异步启动爬虫
  27.                         .start();
  28.             } catch (Exception ex) {
  29.                 LOGGER.error("定时抓取知乎数据线程执行异常", ex);
  30.             }
  31.         }, 0, 10, TimeUnit.MINUTES);
  32.     }
  33. }

10、Spring boot程序启动类

  1. @SpringBootApplication
  2. @MapperScan(basePackages = "com.hyzx.qbasic.dao")
  3. public class Application implements CommandLineRunner {
  5.     @Autowired
  6.     private ZhihuTask zhihuTask;
  8.     public static void main(String[] args) throws IOException {
  9.         SpringApplication.run(Application.class, args);
  10.     }
  12.     @Override
  13.     public void run(String... strings) throws Exception {
  14.         // 爬取知乎数据
  15.         zhihuTask.crawl();
  16.     }
  17. }