网络爬虫就是用程序帮助我们访问网络上的资源，我们一直以来都是使用HTTP协议访问互联网的网页，网络爬虫需要编写程序，在这里使用同样的HTTP协议访问网页。
这里我们使用Java 的 HTTP协议客户端HttpClient这个技术，来实现抓取网页数据。


2.1 Get请求

  1. package cn.itbuild.crawler.test;
  3. import org.apache.http.client.methods.CloseableHttpResponse;
  4. import org.apache.http.client.methods.HttpGet;
  5. import org.apache.http.impl.client.CloseableHttpClient;
  6. import org.apache.http.impl.client.HttpClients;
  7. import org.apache.http.util.EntityUtils;
  9. import java.io.IOException;
  11. /**
  12.  * @Date 2020/12/21 19:16
  13.  * @Version 10.21
  14.  * @Author DuanChaojie
  15.  */
  16. public class HttpGetTest {
  17.     public static void main(String[] args) {
  18.         // 1.创建HttpClient对象
  19.         CloseableHttpClient httpClient = HttpClients.createDefault();
  21.         // 2.创建HttpGet对象, 设置url访问地址
  22.         HttpGet httpGet = new HttpGet("http://www.itcast.cn");
  24.         // 3.使用 HttpClient 发起请求, 获取 response
  25.         CloseableHttpResponse response = null;
  27.         try {
  28.             response = httpClient.execute(httpGet);
  29.             // 4.解析响应
  30.             if (response.getStatusLine().getStatusCode() == 200){
  31.                 String content = EntityUtils.toString(response.getEntity(), "utf-8");
  32.                 System.out.println(content.length());
  33.                 System.out.println(content);
  34.             }
  35.         } catch (IOException e) {
  36.             e.printStackTrace();
  37.         }finally {
  38.             // 5.关闭资源
  39.             if (response != null) {
  40.                 try {
  41.                     response.close();
  42.                 } catch (IOException e) {
  43.                     e.printStackTrace();
  44.                 }
  45.             }
  46.             if (httpClient != null) {
  47.                 try {
  48.                     httpClient.close();
  49.                 } catch (IOException e) {
  50.                     e.printStackTrace();
  51.                 }
  52.             }
  53.         }
  54.     }
  55. }


2.2 带参数的Get请求

在传智中搜索学习视频，地址为http://yun.itheima.com/search?keys=Java

  1. package cn.itbuild.crawler.test;
  3. import org.apache.http.client.methods.CloseableHttpResponse;
  4. import org.apache.http.client.methods.HttpGet;
  5. import org.apache.http.client.utils.URIBuilder;
  6. import org.apache.http.impl.client.CloseableHttpClient;
  7. import org.apache.http.impl.client.HttpClients;
  8. import org.apache.http.util.EntityUtils;
  11. /**
  12.  * @Date 2020/12/21 19:16
  13.  * @Version 10.21
  14.  * @Author DuanChaojie
  15.  */
  16. public class HttpGetParamTest {
  17.     public static void main(String[] args) throws Exception {
  18.         // 1.创建HttpClient对象
  19.         CloseableHttpClient httpClient = HttpClients.createDefault();
  21.         // 设置请求地址是: http://yun.itheima.com/search?keys=Java
  22.         // 创建uriBuilder
  23.         URIBuilder uriBuilder = new URIBuilder("http://yun.itheima.com/search");
  24.         // 多个参数，使用连式编程
  25.         uriBuilder.setParameter("keys","Java");
  27.         // 2.创建HttpGet对象, 设置url访问地址☆
  28.         HttpGet httpGet = new HttpGet(uriBuilder.build());
  30.         System.out.println("发送请求的信息：");
  32.         // 3.使用 HttpClient 发起请求, 获取 response
  33.         CloseableHttpResponse response = httpClient.execute(httpGet);
  34.         // 4.解析响应
  35.         if (response.getStatusLine().getStatusCode() == 200){
  36.             String content = EntityUtils.toString(response.getEntity(), "utf-8");
  37.             System.out.println(content.length());
  38.             System.out.println(content);
  39.         }
  41.     }
  42. }


2.3 Post请求

使用POST 请求访问传智官网，请求url地址：http://www.itcast.cn/

  1. package cn.itbuild.crawler.test;
  3. import org.apache.http.client.methods.CloseableHttpResponse;
  4. import org.apache.http.client.methods.HttpPost;
  5. import org.apache.http.impl.client.CloseableHttpClient;
  6. import org.apache.http.impl.client.HttpClients;
  7. import org.apache.http.util.EntityUtils;
  9. import java.io.IOException;
  11. /**
  12.  * @Date 2020/12/21 19:16
  13.  * @Version 10.21
  14.  * @Author DuanChaojie
  15.  */
  16. public class HttpPostTest {
  17.     public static void main(String[] args) {
  18.         // 1.创建HttpClient对象
  19.         CloseableHttpClient httpClient = HttpClients.createDefault();
  21.         // 2.创建HttpPost对象, 设置url访问地址
  22.         HttpPost httpPost = new HttpPost("http://www.itcast.cn");
  25.         // 3.使用 HttpClient 发起请求, 获取 response
  26.         CloseableHttpResponse response = null;
  27.         try {
  28.             response = httpClient.execute(httpPost);
  29.             // 4.解析响应
  30.             if (response.getStatusLine().getStatusCode() == 200){
  31.                 String content = EntityUtils.toString(response.getEntity(), "utf-8");
  32.                 System.out.println(content.length());
  33.                 System.out.println(content);
  34.             }
  35.         } catch (IOException e) {
  36.             e.printStackTrace();
  37.         }finally {
  38.             // 5.关闭资源
  39.             if (response != null) {
  40.                 try {
  41.                     response.close();
  42.                 } catch (IOException e) {
  43.                     e.printStackTrace();
  44.                 }
  45.             }
  46.             if (httpClient != null) {
  47.                 try {
  48.                     httpClient.close();
  49.                 } catch (IOException e) {
  50.                     e.printStackTrace();
  51.                 }
  52.             }
  53.         }
  54.     }
  55. }


2.4 带参数的Post请求

在传智中搜索学习视频，使用POST请求，url地址为：http://yun.itheima.com/search
url地址没有参数，参数 keys=java放到表单中进行提交。

  1. package cn.itbuild.crawler.test;
  3. import org.apache.http.NameValuePair;
  4. import org.apache.http.client.entity.UrlEncodedFormEntity;
  5. import org.apache.http.client.methods.CloseableHttpResponse;
  6. import org.apache.http.client.methods.HttpPost;
  7. import org.apache.http.impl.client.CloseableHttpClient;
  8. import org.apache.http.impl.client.HttpClients;
  9. import org.apache.http.message.BasicNameValuePair;
  10. import org.apache.http.util.EntityUtils;
  12. import java.util.ArrayList;
  13. import java.util.List;
  15. /**
  16.  * @Date 2020/12/21 19:16
  17.  * @Version 10.21
  18.  * @Author DuanChaojie
  19.  */
  20. public class HttpPostParamTest {
  21.     public static void main(String[] args) throws Exception {
  22.         // 1.创建HttpClient对象
  23.         CloseableHttpClient httpClient = HttpClients.createDefault();
  25.         // 2.创建HttpPost对象, 设置url访问地址
  26.         HttpPost httpPost = new HttpPost("http://yun.itheima.com/search");
  28.         /** 声明List集合, 封装表单中的参数
  29.          * public interface NameValuePair {
  30.          *     String getName();
  31.          *
  32.          *     String getValue();
  33.          * }
  34.          */
  35.         List<NameValuePair> params = new ArrayList<NameValuePair>();
  37.         // 注意BasicNameValuePair是NameValuePair唯一的实现类
  38.         params.add(new BasicNameValuePair("keys","Java"));
  40.         // 创建表单的Entity对象, 第一个参数就是封装好的表单数据, 第二个参数就是编码
  41.         UrlEncodedFormEntity formEntity = new UrlEncodedFormEntity(params,"utf-8");
  43.         // 设置表单的Entity对象到 Post 请求中
  44.         httpPost.setEntity(formEntity);
  46.         // 3.使用 HttpClient 发起请求, 获取 response
  47.         CloseableHttpResponse response = httpClient.execute(httpPost);
  49.         // 4.解析响应
  50.         if (response.getStatusLine().getStatusCode() == 200){
  51.             String content = EntityUtils.toString(response.getEntity(), "utf-8");
  52.             System.out.println(content.length());
  53.         }
  54.     }
  55. }


2.5 连接池

如果每次请求都要创建HttpClient，会有频繁创建和销毁的问题，可以使用连接池来解决这个问题。
测试以下代码，并断点查看每次获取的HttpClient都是不一样的。

  1. package cn.itbuild.crawler.test;
  3. import org.apache.http.client.methods.CloseableHttpResponse;
  4. import org.apache.http.client.methods.HttpGet;
  5. import org.apache.http.impl.client.CloseableHttpClient;
  6. import org.apache.http.impl.client.HttpClients;
  7. import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
  8. import org.apache.http.util.EntityUtils;
  10. import java.io.IOException;
  12. /**
  13.  * @Date 2020/12/21 20:08
  14.  * @Version 10.21
  15.  * @Author DuanChaojie
  16.  */
  17. public class HttpClientPoolTest {
  18.     public static void main(String[] args) {
  19.         // 创建连接池管理器
  20.         PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
  21.         // 设置连接数
  22.         cm.setMaxTotal(100);
  23.         // 设置每个主机的最大连接数
  24.         cm.setDefaultMaxPerRoute(10);
  26.         // 使用连接池管理器发起请求
  27.         doGet(cm);
  28.         doGet(cm);
  29.     }
  31.     /**
  32.      * @param cm
  33.      */
  34.     private static void doGet(PoolingHttpClientConnectionManager cm) {
  35.         // 不是每次创建新的HttpClient, 而是从连接池中获取 HttpClient 对象
  36.         CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(cm).build();
  38.         HttpGet httpGet = new HttpGet("http://www.itcast.cn");
  40.         CloseableHttpResponse response = null;
  41.         try {
  42.             // 使用 HttpClient 发起请求, 获取 response
  43.             response = httpClient.execute(httpGet);
  44.             // 解析响应
  45.             if (response.getStatusLine().getStatusCode() == 200) {
  46.                 String content = EntityUtils.toString(response.getEntity(), "utf8");
  47.                 System.out.println(content.length());
  48.             }
  49.         } catch(IOException e) {
  50.             e.printStackTrace();
  51.         } finally {
  52.             // 关闭 response
  53.             if (response != null) {
  54.                 try {
  55.                     response.close();
  56.                 } catch (IOException e) {
  57.                     e.printStackTrace();
  58.                 }
  59.                 // 不能关闭 HttpClient, 由连接池管理 HttpClient
  60.                 // httpClient. close();
  61.             }
  62.         }
  63.     }
  64. }


2.6 请求参数

有时候因为网络，或者目标服务器的原因，请求需要更长的时间才能完成，我们需要自定义相关时间。

package cn.itbuild.crawler.test;

import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

/**
 * @Date 2020/12/21 20:12
 * @Version 10.21
 * @Author DuanChaojie
 */
public class HttpConfigTest {
    public static void main(String[] args) throws Exception{
        // 1. 创建HttpClient对象
        CloseableHttpClient httpClient = HttpClients.createDefault();

        // 2. 创建HttpGet对象, 设置url访问地址
        HttpGet httpGet = new HttpGet("http://yun.itheima.com/search?keys=Java");

        // 3. 配置请求信息
        RequestConfig config = RequestConfig.custom().setConnectTimeout(1000) // 创建连接的最长时间, 单位是毫秒
                .setConnectionRequestTimeout(500) // 设置获取连接的最长时间, 单位是毫秒
                .setSocketTimeout(10 * 1000) // 设置数据传输的最长时间, 单位是毫秒
                .build();

        // 4. 给请求设置请求信息
        httpGet.setConfig(config);

        // 5. 使用 HttpClient 发起请求, 获取 response
        CloseableHttpResponse response = httpClient.execute(httpGet);

        // 6. 解析响应
        if (response.getStatusLine().getStatusCode() == 200) {
            String content = EntityUtils.toString(response.getEntity(), "utf8");
            System.out.println(content.length());
        }
    }
}