04. 运算节点部署 - 04.2. kube-proxy部署 - 《kubernetes(K8S)容器编排工具》

4.2. kube-proxy部署

4.2. kube-proxy部署

4.2.1. 签发证书

证书签发在 hdss7-200 操作

[root@hdss7-200 ~]# cd /opt/certs/
[root@hdss7-200 certs]# vim kube-proxy-csr.json  # CN 其实是k8s中的角色
{
    "CN": "system:kube-proxy",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "ST": "beijing",
            "L": "beijing",
            "O": "od",
            "OU": "ops"
        }
    ]
}
[root@hdss7-200 certs]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client kube-proxy-csr.json |cfssl-json -bare kube-proxy-client
2020/01/07 21:45:53 [INFO] generate received request
2020/01/07 21:45:53 [INFO] received CSR
2020/01/07 21:45:53 [INFO] generating key: rsa-2048
2020/01/07 21:45:53 [INFO] encoded CSR
2020/01/07 21:45:53 [INFO] signed certificate with serial number 620191685968917036075463174423999296907693104226
2020/01/07 21:45:53 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
[root@hdss7-200 certs]# ls kube-proxy-c* -l  # 因为kube-proxy使用的用户是kube-proxy，不能使用client证书，必须要重新签发自己的证书
-rw-r--r-- 1 root root 1005 Jan  7 21:45 kube-proxy-client.csr
-rw------- 1 root root 1675 Jan  7 21:45 kube-proxy-client-key.pem
-rw-r--r-- 1 root root 1375 Jan  7 21:45 kube-proxy-client.pem
-rw-r--r-- 1 root root  267 Jan  7 21:45 kube-proxy-csr.json

[root@hdss7-200 certs]# scp kube-proxy-client-key.pem kube-proxy-client.pem hdss7-21:/opt/kubernetes/server/bin/certs/
100% 1375   870.6KB/s   00:00    
[root@hdss7-200 certs]# scp kube-proxy-client-key.pem kube-proxy-client.pem hdss7-22:/opt/kubernetes/server/bin/certs/

4.2.2. 创建kube-proxy配置

在所有node节点创建，涉及服务器：hdss7-21 ，hdss7-22

[root@hdss7-21 ~]# kubectl config set-cluster myk8s \
--certificate-authority=/opt/kubernetes/server/bin/certs/ca.pem \
--embed-certs=true \
--server=https://10.4.7.10:7443 \
--kubeconfig=/opt/kubernetes/conf/kube-proxy.kubeconfig

[root@hdss7-21 ~]# kubectl config set-credentials kube-proxy \
--client-certificate=/opt/kubernetes/server/bin/certs/kube-proxy-client.pem \
--client-key=/opt/kubernetes/server/bin/certs/kube-proxy-client-key.pem \
--embed-certs=true \
--kubeconfig=/opt/kubernetes/conf/kube-proxy.kubeconfig

[root@hdss7-21 ~]# kubectl config set-context myk8s-context \
--cluster=myk8s \
--user=kube-proxy \
--kubeconfig=/opt/kubernetes/conf/kube-proxy.kubeconfig

[root@hdss7-21 ~]# kubectl config use-context myk8s-context --kubeconfig=/opt/kubernetes/conf/kube-proxy.kubeconfig

4.2.3. 加载ipvs模块

kube-proxy 共有3种流量调度模式，分别是 namespace，iptables，ipvs，其中ipvs性能最好。

[root@hdss7-21 ~]# for i in $(ls /usr/lib/modules/$(uname -r)/kernel/net/netfilter/ipvs|grep -o "^[^.]*");do echo $i; /sbin/modinfo -F filename $i >/dev/null 2>&1 && /sbin/modprobe $i;done
[root@hdss7-21 ~]# lsmod | grep ip_vs  # 查看ipvs模块

4.2.4. 创建启动脚本

[root@hdss7-21 ~]# vim /opt/kubernetes/server/bin/kube-proxy-startup.sh
#!/bin/sh

WORK_DIR=$(dirname $(readlink -f $0))
[ $? -eq 0 ] && cd $WORK_DIR || exit

/opt/kubernetes/server/bin/kube-proxy \
  --cluster-cidr 172.7.0.0/16 \
  --hostname-override hdss7-21.host.com \
  --proxy-mode=ipvs \
  --ipvs-scheduler=nq \
  --kubeconfig ../../conf/kube-proxy.kubeconfig
[root@hdss7-21 ~]# chmod u+x /opt/kubernetes/server/bin/kube-proxy-startup.sh
[root@hdss7-21 ~]# mkdir -p /data/logs/kubernetes/kube-proxy
[root@hdss7-21 ~]# vim /etc/supervisord.d/kube-proxy.ini
[program:kube-proxy-7-21]
command=/opt/kubernetes/server/bin/kube-proxy-startup.sh                
numprocs=1                                                      
directory=/opt/kubernetes/server/bin                            
autostart=true                                                  
autorestart=true                                                
startsecs=30                                                    
startretries=3                                                  
exitcodes=0,2                                                   
stopsignal=QUIT                                                 
stopwaitsecs=10                                                 
user=root                                                       
redirect_stderr=true                                            
stdout_logfile=/data/logs/kubernetes/kube-proxy/proxy.stdout.log
stdout_logfile_maxbytes=64MB                                    
stdout_logfile_backups=5                                       
stdout_capture_maxbytes=1MB                                     
stdout_events_enabled=false

[root@hdss7-21 ~]# supervisorctl update
[root@hdss7-21 ~]# supervisorctl status
etcd-server-7-21                 RUNNING   pid 23637, uptime 2 days, 0:27:18
kube-apiserver-7-21              RUNNING   pid 32591, uptime 1 day, 2:06:47
kube-controller-manager-7-21     RUNNING   pid 33357, uptime 1 day, 0:11:02
kube-kubelet-7-21                RUNNING   pid 37232, uptime 9:32:01
kube-proxy-7-21                  RUNNING   pid 47088, uptime 0:06:19
kube-scheduler-7-21              RUNNING   pid 33450, uptime 1 day, 0:01:43

4.2.5. 验证集群


[root@hdss7-21 ~]# yum install -y ipvsadm
[root@hdss7-21 ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.0.1:443 nq
  -> 10.4.7.21:6443               Masq    1      0          0         
  -> 10.4.7.22:6443               Masq    1      0          0

[root@hdss7-21 ~]# curl -I 172.7.21.2
HTTP/1.1 200 OK
Server: nginx/1.17.6
Date: Tue, 07 Jan 2020 14:28:46 GMT
Content-Type: text/html
Content-Length: 612
Last-Modified: Tue, 19 Nov 2019 12:50:08 GMT
Connection: keep-alive
ETag: "5dd3e500-264"
Accept-Ranges: bytes

[root@hdss7-21 ~]# curl -I 172.7.22.2  # 缺少网络插件，无法跨节点通信