3.3. 配置apiserver L4代理

3.3.1. nginx配置

L4 代理涉及的服务器：hdss7-11，hdss7-12

  1. [root@hdss7-11 ~]# yum install -y nginx
  2. [root@hdss7-11 ~]# vim /etc/nginx/nginx.conf  
  3. # stream应该添加在末尾，不要写在http
  4. # 此配置只进行转发配置
  5. stream {
  6.     log_format proxy '$time_local|$remote_addr|$upstream_addr|$protocol|$status|'
  7.                      '$session_time|$upstream_connect_time|$bytes_sent|$bytes_received|'
  8.                      '$upstream_bytes_sent|$upstream_bytes_received' ;
  10.     upstream kube-apiserver {
  11.         server 10.4.7.21:6443     max_fails=3 fail_timeout=30s;
  12.         server 10.4.7.22:6443     max_fails=3 fail_timeout=30s;
  13.     }
  14.     server {
  15.         listen 7443;
  16.         proxy_connect_timeout 2s;
  17.         proxy_timeout 900s;
  18.         proxy_pass kube-apiserver;
  19.         access_log /var/log/nginx/proxy.log proxy;
  20.     }
  21. }
  22. [root@hdss7-11 ~]# systemctl start nginx; systemctl enable nginx
  23. [root@hdss7-11 ~]# curl 127.0.0.1:7443  # 测试几次
  24. Client sent an HTTP request to an HTTPS server.
  25. [root@hdss7-11 ~]# cat /var/log/nginx/proxy.log 
  26. 27/Jul/2020:15:20:56 +0800|127.0.0.1|10.4.7.21:6443|TCP|200|0.001|0.000|76|78|78|76
  27. 27/Jul/2020:15:21:03 +0800|127.0.0.1|10.4.7.22:6443|TCP|200|0.002|0.001|76|78|78|76
  28. 27/Jul/2020:15:21:03 +0800|127.0.0.1|10.4.7.21:6443|TCP|200|0.001|0.000|76|78|78|76

3.3.2. keepalived配置

aipserver L4 代理涉及的服务器：hdss7-11，hdss7-12

  • 安装keepalive

    [root@hdss7-11 ~]# yum install -y keepalived
[root@hdss7-11 ~]# vim /etc/keepalived/check_port.sh # 配置检查脚本
#!/bin/bash
if [ $# -eq 1 ] && [[ $1 =~ ^[0-9]+ ]];then
  [ $(netstat -lntp|grep ":$1 " |wc -l) -eq 0 ] && echo "[ERROR] nginx may be not running!" && exit 1 || exit 0
else
  echo "[ERROR] need one port!"
  exit 1
fi
[root@hdss7-11 ~]# chmod +x /etc/keepalived/check_port.sh

  • 配置主节点：vim /etc/keepalived/keepalived.conf

主节点中，必须加上 **nopreempt
因为一旦因为网络抖动导致VIP漂移，不能让它自动飘回来，必须要分析原因后手动迁移VIP到主节点！如主节点确认正常后，重启备节点的keepalive，让VIP飘到主节点.
keepalived 的日志输出配置此处省略，生产中需要进行处理。
主从节点配置项的网卡名称（eth、ens等）必须和系统里网卡名称一致，否则启动失败！！！**

! Configuration File for keepalived
global_defs {
   router_id 10.4.7.11
}
vrrp_script chk_nginx {
    script "/etc/keepalived/check_port.sh 7443"
    interval 2
    weight -20
}
vrrp_instance VI_1 {
    state MASTER
    interface ens32
    virtual_router_id 251
    priority 100
    advert_int 1
    mcast_src_ip 10.4.7.11
    nopreempt

    authentication {
        auth_type PASS
        auth_pass 11111111
    }
    track_script {
         chk_nginx
    }
    virtual_ipaddress {
        10.4.7.10
    }
}

  • 配置备节点：vim /etc/keepalived/keepalived.conf

    ! Configuration File for keepalived
global_defs {
  router_id 10.4.7.12
}
vrrp_script chk_nginx {
  script "/etc/keepalived/check_port.sh 7443"
  interval 2
  weight -20
}
vrrp_instance VI_1 {
  state BACKUP
  interface ens32
  virtual_router_id 251
  mcast_src_ip 10.4.7.12
  priority 90
  advert_int 1
  authentication {
      auth_type PASS
      auth_pass 11111111
  }
  track_script {
      chk_nginx
  }
  virtual_ipaddress {
      10.4.7.10
  }
}

  • 启动keepalived

    [root@hdss7-11 ~]# systemctl start keepalived ; systemctl enable keepalived
[root@hdss7-11 ~]# ip addr show ens32
2: ens32: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
  link/ether 00:0c:29:6d:b8:82 brd ff:ff:ff:ff:ff:ff
  inet 10.4.7.11/24 brd 10.4.7.255 scope global noprefixroute ens32
     valid_lft forever preferred_lft forever
  inet 10.4.7.10/32 scope global ens32
     valid_lft forever preferred_lft forever
......