k8s之etcd集群 - 《Kubernetes云容器技术专题》

1. hdss7-200创建etcd证书
2.hdss7-12安装etcd
3 hdss7-21机器安装etcd
4.hdss7-22机器安装etcd

集群规划
主机名角色 ip
HDss7-12.host.com ectc lead 192.168.12.12
HDss7-21.host.com ectc follow 192.168.12.21
HDss7-22.host.com ectc follow 192.168.12.22

1. hdss7-200创建etcd证书

[root@hdss7-200 ~]# cat /opt/certs/ca-config.json
{
    "signing": {
        "default": {
            "expiry": "175200h"
        },
        "profiles": {
            "server": {
                "expiry": "175200h",
                "usages": [
                    "signing",
                    "key encipherment",
                    "server auth"
                ]
            },
            "client": {
                "expiry": "175200h",
                "usages": [
                    "signing",
                    "key encipherment",
                    "client auth"
                ]
            },
            "peer": {                
                "expiry": "175200h",
                "usages": [
                    "signing",
                    "key encipherment",
                    "server auth",
                    "client auth"
                ]
            }
        }
    }
}
[root@hdss7-200 ~]# cat /opt/certs/etcd-peer-csr.json
{
    "CN": "k8s-etcd",
    "hosts": [
        "192.168.12.11",
        "192.168.12.12",
        "192.168.12.21",
        "192.168.12.22"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "ST": "beijing",
            "L": "beijing",
            "O": "od",
            "OU": "ops"
        }
    ]
}
[root@hdss7-200 certs]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=peer etcd-peer-csr.json |cfssl-json -bare etcd-peer
[root@hdss7-200 certs]# ll
total 36
-rw-r--r-- 1 root root  840 Jun  6 13:03 ca-config.json
-rw-r--r-- 1 root root  989 Jun  6 11:19 ca.csr
-rw-r--r-- 1 root root  334 Jun  6 11:18 ca-csr.json
-rw------- 1 root root 1679 Jun  6 11:19 ca-key.pem
-rw-r--r-- 1 root root 1334 Jun  6 11:19 ca.pem
-rw-r--r-- 1 root root 1062 Jun  6 13:04 etcd-peer.csr
-rw-r--r-- 1 root root  379 Jun  6 13:04 etcd-peer-csr.json
-rw------- 1 root root 1679 Jun  6 13:04 etcd-peer-key.pem
-rw-r--r-- 1 root root 1424 Jun  6 13:04 etcd-peer.pem

2.hdss7-12安装etcd

[root@hdss7-12 ~]# useradd -s /sbin/nologin -M etcd
[root@hdss7-12 ~]# mkdir -p /opt/src
[root@hdss7-12 ~]# cd /opt/src
[root@hdss7-12 src]# ll
total 9620
-rw-r--r-- 1 root root 9850227 Apr  1 18:44 etcd-v3.1.20-linux-amd64.tar.gz
[root@hdss7-12 src]# tar xf etcd-v3.1.20-linux-amd64.tar.gz -C /opt/
[root@hdss7-12 src]# cd ..
[root@hdss7-12 opt]# mv etcd-v3.1.20-linux-amd64/ etcd-v3.1.20
[root@hdss7-12 opt]# ls
etcd-v3.1.20  src
[root@hdss7-12 opt]# ll
total 0
drwxr-xr-x 3 478493 89939 123 Oct 11  2018 etcd-v3.1.20
drwxr-xr-x 2 root   root   45 Jun  6 13:08 src
[root@hdss7-12 opt]# ln -s /opt/etcd-v3.1.20/ /opt/etcd
创建目录，拷贝证书、私钥
[root@hdss7-12 opt]# mkdir -p /opt/etcd/certs /data/etcd /data/logs/etcd-server
[root@hdss7-12 opt]# cd /opt/etcd/certs/
将运维主机上生成的ca.pem  etc-peer-key.pem etc-peer.pem 拷贝到/opt/etcd/certs目录中，私钥文件权限为600
[root@hdss7-12 certs]#  scp hdss7-200:/opt/certs/ca.pem .
[root@hdss7-12 certs]#  scp hdss7-200:/opt/certs/etcd-peer-key.pem .
[root@hdss7-12 certs]#  scp hdss7-200:/opt/certs/etcd-peer.pem .
[root@hdss7-12 certs]# chown -R etcd.etcd /opt/etcd/certs /data/etcd /data/logs/etcd-server
[root@hdss7-12 certs]# ll
total 12
-rw-r--r-- 1 etcd etcd 1334 Jun  6 13:10 ca.pem
-rw------- 1 etcd etcd 1679 Jun  6 13:10 etcd-peer-key.pem
-rw-r--r-- 1 etcd etcd 1424 Jun  6 13:10 etcd-peer.pem
[root@hdss7-12 certs]# 
创建etcd服务启动脚本IP地址改成本机IP -- 此文件本目录中有，上传修改即可，不要粘贴复制,格式容易出错
[root@hdss7-12 certs]# cat /opt/etcd/etcd-server-startup.sh
#!/bin/sh
./etcd --name etcd-server-7-12 \
       --data-dir /data/etcd/etcd-server \
       --listen-peer-urls https://192.168.12.12:2380 \
       --listen-client-urls https://192.168.12.12:2379,http://127.0.0.1:2379 \
       --quota-backend-bytes 8000000000 \
       --initial-advertise-peer-urls https://192.168.12.12:2380 \
       --advertise-client-urls https://192.168.12.12:2379,http://127.0.0.1:2379 \
       --initial-cluster  etcd-server-7-12=https://192.168.12.12:2380,etcd-server-7-21=https://192.168.12.21:2380,etcd-server-7-22=https://192.168.12.22:2380 \
       --ca-file ./certs/ca.pem \
       --cert-file ./certs/etcd-peer.pem \
       --key-file ./certs/etcd-peer-key.pem \
       --client-cert-auth  \
       --trusted-ca-file ./certs/ca.pem \
       --peer-ca-file ./certs/ca.pem \
       --peer-cert-file ./certs/etcd-peer.pem \
       --peer-key-file ./certs/etcd-peer-key.pem \
       --peer-client-cert-auth \
       --peer-trusted-ca-file ./certs/ca.pem \
       --log-output stdout

[root@hdss7-12 certs]#  chmod +x /opt/etcd/etcd-server-startup.sh
[root@hdss7-12 certs]# chown -R etcd.etcd /opt/etcd-v3.1.20/ /data/etcd /data/logs/etcd-server
[root@hdss7-12 certs]#  yum install supervisor -y
[root@hdss7-12 certs]# systemctl start supervisord
更改supervisord的配置文件：[program:etcd-server-7-12]名字需要根据实际更改
[root@hdss7-12 certs]# cat /etc/supervisord.d/etcd-server.ini 
[program:etcd-server-7-12]
command=/opt/etcd/etcd-server-startup.sh                        ; the program (relative uses PATH, can take args)    
numprocs=1                                                      ; number of processes copies to start (def 1)
directory=/opt/etcd                                             ; directory to cwd to before exec (def no cwd)
autostart=true                                                  ; start at supervisord start (default: true)
autorestart=true                                                ; retstart at unexpected quit (default: true)
startsecs=30                                                    ; number of secs prog must stay running (def. 1)
startretries=3                                                  ; max # of serial start failures (default 3)
exitcodes=0,2                                                   ; 'expected' exit codes for process (default 0,2)
stopsignal=QUIT                                                 ; signal used to kill process (default TERM)
stopwaitsecs=10                                                 ; max num secs to wait b4 SIGKILL (default 10)
user=etcd                                                       ; setuid to this UNIX account to run the program
redirect_stderr=true                                            ; redirect proc stderr to stdout (default false)
stdout_logfile=/data/logs/etcd-server/etcd.stdout.log           ; stdout log path, NONE for none; default AUTO
stdout_logfile_maxbytes=64MB                                    ; max # logfile bytes b4 rotation (default 50MB)
stdout_logfile_backups=4                                        ; # of stdout logfile backups (default 10)
stdout_capture_maxbytes=1MB                                     ; number of bytes in 'capturemode' (default 0)
stdout_events_enabled=false                                     ; emit events on stdout writes (default false)
[root@hdss7-12 certs]# supervisorctl  update
etcd-server-7-12: added process group
[root@hdss7-12 certs]# supervisorctl  status
etcd-server-7-12                 RUNNING   pid 2164, uptime 0:00:51

3 hdss7-21机器安装etcd

[root@hdss7-21 ~]#  useradd -s /sbin/nologin -M etcd
[root@hdss7-21 ~]# mkdir /opt/src
[root@hdss7-21 ~]# cd /opt/src/
[root@hdss7-21 src]# tar xf etcd-v3.1.20-linux-amd64.tar.gz -C /opt/
[root@hdss7-21 opt]# mv etcd-v3.1.20-linux-amd64/ etcd-v3.1.20
[root@hdss7-21 opt]# ln -s /opt/etcd-v3.1.20/ /opt/etcd
[root@hdss7-21 opt]# mkdir -p /opt/etcd/certs /data/etcd /data/logs/etcd-server
[root@hdss7-21 opt]# cd /opt/etcd/certs
[root@hdss7-21 certs]# scp hdss7-200:/opt/certs/ca.pem .
[root@hdss7-21 certs]# scp hdss7-200:/opt/certs/etcd-peer-key.pem .
[root@hdss7-21 certs]# scp hdss7-200:/opt/certs/etcd-peer.pem .
[root@hdss7-21 certs]#  chown -R etcd.etcd /opt/etcd/certs /data/etcd /data/logs/etcd-server
[root@hdss7-21 certs]# cat /opt/etcd/etcd-server-startup.sh
#!/bin/sh
./etcd --name etcd-server-7-21 \
       --data-dir /data/etcd/etcd-server \
       --listen-peer-urls https://192.168.12.21:2380 \
       --listen-client-urls https://192.168.12.21:2379,http://127.0.0.1:2379 \
       --quota-backend-bytes 8000000000 \
       --initial-advertise-peer-urls https://192.168.12.21:2380 \
       --advertise-client-urls https://192.168.12.21:2379,http://127.0.0.1:2379 \
       --initial-cluster  etcd-server-7-12=https://192.168.12.12:2380,etcd-server-7-21=https://192.168.12.21:2380,etcd-server-7-22=https://192.168.12.22:2380 \
       --ca-file ./certs/ca.pem \
       --cert-file ./certs/etcd-peer.pem \
       --key-file ./certs/etcd-peer-key.pem \
       --client-cert-auth  \
       --trusted-ca-file ./certs/ca.pem \
       --peer-ca-file ./certs/ca.pem \
       --peer-cert-file ./certs/etcd-peer.pem \
       --peer-key-file ./certs/etcd-peer-key.pem \
       --peer-client-cert-auth \
       --peer-trusted-ca-file ./certs/ca.pem \
       --log-output stdout
[root@hdss7-21 certs]#  chmod +x /opt/etcd/etcd-server-startup.sh
\[root@hdss7-21 certs]# 
[root@hdss7-21 certs]# 
[root@hdss7-21 certs]#  chown -R etcd.etcd /opt/etcd-v3.1.20/ /data/etcd /data/logs/etcd-server
[root@hdss7-21 certs]#  yum install supervisor -y
[root@hdss7-21 certs]# cat /etc/supervisord.d/etcd-server.ini
[program:etcd-server-7-21]
command=/opt/etcd/etcd-server-startup.sh                        ; the program (relative uses PATH, can take args)    
numprocs=1                                                      ; number of processes copies to start (def 1)
directory=/opt/etcd                                             ; directory to cwd to before exec (def no cwd)
autostart=true                                                  ; start at supervisord start (default: true)
autorestart=true                                                ; retstart at unexpected quit (default: true)
startsecs=30                                                    ; number of secs prog must stay running (def. 1)
startretries=3                                                  ; max # of serial start failures (default 3)
exitcodes=0,2                                                   ; 'expected' exit codes for process (default 0,2)
stopsignal=QUIT                                                 ; signal used to kill process (default TERM)
stopwaitsecs=10                                                 ; max num secs to wait b4 SIGKILL (default 10)
user=etcd                                                       ; setuid to this UNIX account to run the program
redirect_stderr=true                                            ; redirect proc stderr to stdout (default false)
stdout_logfile=/data/logs/etcd-server/etcd.stdout.log           ; stdout log path, NONE for none; default AUTO
stdout_logfile_maxbytes=64MB                                    ; max # logfile bytes b4 rotation (default 50MB)
stdout_logfile_backups=4                                        ; # of stdout logfile backups (default 10)
stdout_capture_maxbytes=1MB                                     ; number of bytes in 'capturemode' (default 0)
stdout_events_enabled=false                                     ; emit events on stdout writes (default false)
[root@hdss7-21 certs]# supervisorctl update
etcd-server-7-21: added process group
[root@hdss7-21 certs]# supervisorctl status
etcd-server-7-21                 RUNNING   pid 2753, uptime 0:01:17

4.hdss7-22机器安装etcd

[root@hdss7-22 ~]# useradd -s /sbin/nologin -M etcd
[root@hdss7-22 ~]#  mkdir /opt/src
[root@hdss7-22 ~]# cd /opt/src
[root@hdss7-22 src]# rz

[root@hdss7-22 src]# tar xf etcd-v3.1.20-linux-amd64.tar.gz -C /opt/
[root@hdss7-22 src]# cd ..
[root@hdss7-22 opt]# mv etcd-v3.1.20-linux-amd64/ etcd-v3.1.20
[root@hdss7-22 opt]# ln -s /opt/etcd-v3.1.20/ /opt/etcd
[root@hdss7-22 opt]# mkdir -p /opt/etcd/certs /data/etcd /data/logs/etcd-server
[root@hdss7-22 opt]# cd /opt/etcd/certs/
[root@hdss7-22 certs]# scp hdss7-200:/opt/certs/ca.pem .
[root@hdss7-22 certs]# scp hdss7-200:/opt/certs/etcd-peer-key.pem .
[root@hdss7-22 certs]# scp hdss7-200:/opt/certs/etcd-peer.pem .
[root@hdss7-22 certs]#  chown -R etcd.etcd /opt/etcd/certs /data/etcd /data/logs/etcd-server
[root@hdss7-22 certs]# cat /opt/etcd/etcd-server-startup.sh
#!/bin/sh
./etcd --name etcd-server-7-22 \
       --data-dir /data/etcd/etcd-server \
       --listen-peer-urls https://192.168.12.22:2380 \
       --listen-client-urls https://192.168.12.22:2379,http://127.0.0.1:2379 \
       --quota-backend-bytes 8000000000 \
       --initial-advertise-peer-urls https://192.168.12.22:2380 \
       --advertise-client-urls https://192.168.12.22:2379,http://127.0.0.1:2379 \
       --initial-cluster  etcd-server-7-12=https://192.168.12.12:2380,etcd-server-7-21=https://192.168.12.21:2380,etcd-server-7-22=https://192.168.12.22:2380 \
       --ca-file ./certs/ca.pem \
       --cert-file ./certs/etcd-peer.pem \
       --key-file ./certs/etcd-peer-key.pem \
       --client-cert-auth  \
       --trusted-ca-file ./certs/ca.pem \
       --peer-ca-file ./certs/ca.pem \
       --peer-cert-file ./certs/etcd-peer.pem \
       --peer-key-file ./certs/etcd-peer-key.pem \
       --peer-client-cert-auth \
       --peer-trusted-ca-file ./certs/ca.pem \
       --log-output stdout
[root@hdss7-22 certs]# chmod +x /opt/etcd/etcd-server-startup.sh
[root@hdss7-22 certs]# yum install supervisor -y
[root@hdss7-22 certs]# yum install supervisor -y
[root@hdss7-22 certs]#  systemctl start supervisord
 systemctl enable supervisord
[root@hdss7-22 certs]#  systemctl enable supervisord
[root@hdss7-22 certs]# cat /etc/supervisord.d/etcd-server.ini
[program:etcd-server-7-22]
command=/opt/etcd/etcd-server-startup.sh                        ; the program (relative uses PATH, can take args)    
numprocs=1                                                      ; number of processes copies to start (def 1)
directory=/opt/etcd                                             ; directory to cwd to before exec (def no cwd)
autostart=true                                                  ; start at supervisord start (default: true)
autorestart=true                                                ; retstart at unexpected quit (default: true)
startsecs=30                                                    ; number of secs prog must stay running (def. 1)
startretries=3                                                  ; max # of serial start failures (default 3)
exitcodes=0,2                                                   ; 'expected' exit codes for process (default 0,2)
stopsignal=QUIT                                                 ; signal used to kill process (default TERM)
stopwaitsecs=10                                                 ; max num secs to wait b4 SIGKILL (default 10)
user=etcd                                                       ; setuid to this UNIX account to run the program
redirect_stderr=true                                            ; redirect proc stderr to stdout (default false)
stdout_logfile=/data/logs/etcd-server/etcd.stdout.log           ; stdout log path, NONE for none; default AUTO
stdout_logfile_maxbytes=64MB                                    ; max # logfile bytes b4 rotation (default 50MB)
stdout_logfile_backups=4                                        ; # of stdout logfile backups (default 10)
stdout_capture_maxbytes=1MB                                     ; number of bytes in 'capturemode' (default 0)
stdout_events_enabled=false                                     ; emit events on stdout writes (default false)
[root@hdss7-22 certs]# supervisorctl update
[root@hdss7-22 certs]# supervisorctl status
etcd-server-7-22                 RUNNING   pid 4264, uptime 0:00:39