1. 服务器架构图

2. 基础环境准备

2.1. 基础安装

所有节点配置

# 关闭selinux
sed -i 's/SELINUX=.*/SELINUX=disabled/g' /etc/selinux/config
# 关闭firewalld防火墙
systemctl stop firewalld ;systemctl disable firewalld
# 关闭NetworkManager服务
systemctl stop NetworkManager ;systemctl disable NetworkManager
# 删除基础yum源 新增阿里云yum源
rm -f /etc/yum.repos.d/*
curl -o /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
yum -y install epel-release
yum clean all ; yum makecache
# 安装基础
yum -y install vim wget net-tools telnet tree nmap sysstat lrzsz dos2unix bind-utils
# kubectl命令行自动补全(bash-completion) (可选)
yum install bash-completion -y

2.2. 安装bind9 (DNS)

在节点hdss7-11.host.com安装bind9

2.2.1. 安装bind9

[root@hdss7-11 ~]# yum install -y bind
[root@hdss7-11 ~]# rpm -qa bind
bind-9.11.4-16.P2.el7_8.6.x86_64

2.2.2. 配置主配置文件

# 新增或修改以下配置信息
[root@hdss7-11 ~]# vim /etc/named.conf
options {
        listen-on port 53 { 10.4.7.11; };
        # listen-on-v6 port 53 { ::1; };
        allow-query     { any; };
        forwarders      { 10.4.7.254; };
        recursion yes;
        dnssec-enable no;
        dnssec-validation no;
};
# 检查配置是否错误        
[root@hdss7-11 ~]# named-checkconf

2.2.3. 配置区域文件

增加两个zone配置, odl.com 为业务域 host,com.zone 为主机域

cat > /etc/named.rfc1912.zones <<EOF
zone "host.com" IN {
    type master;
    file "host.com.zone";
    allow-update { 10.4.7.11; };
};
zone "odl.com" IN{
    type master;
    file "odl.com.zone";
    allow-update { 10.4.7.11; };
};
EOF

2.2.4. 配置主机域文件

配置文件中分号后面都是注释, 其中 serial的2020091701, 当有新的A记录添加,需要增加一位值,即2020091702

cat > /var/named/host.com.zone <<EOF
$ORIGIN host.com.
$TTL 600        ; 10 minutes
@    IN   SOA   dns.host.com.   dnsadmin.host.com. (
                                                2020091701  ; serial
                                10800       ; refresh (3 hours)
                                900         ; retry (15 minutes) 
                                604800      ; expire (1 week)                  
                                86400       ; minimum (1 day)
                )
                     NS   dns.host.com.
$TTL 60 ; 1 minute
dns            A           10.4.7.11
HDSS7-11       A           10.4.7.11
HDSS7-12       A           10.4.7.12
HDSS7-21       A           10.4.7.21
HDSS7-22       A           10.4.7.22
HDSS7-200      A           10.4.7.200
EOF

2.2.5. 配置业务域文件

cat >/var/named/odl.com.zone<<EOF
$ORIGIN odl.com.
$TTL 600  ; 10 minutes
@       IN SOA  dns.odl.com. dnsadmin.odl.com. (
        2020091701 ; serial
        10800      ; refresh (3 hours)
        900        ; retry (15 minutes)
        604800     ; expire (1 week)
        86400      ; minimum (1 day)
        )
        NS   dns.odl.com.
$TTL 60 ; 1 minute
dns                A    10.4.7.11
EOF
# 检查配置是否正常
[root@hdss7-11 ~]# named-checkconf

2.2.6. 启动bind并检查是否解析成功

[root@hdss7-11 ~]# systemctl start named ; systemctl enable named
# 检查是否解析成功
[root@hdss7-11 !]# dig -t A hdss7-200.host.com @10.4.7.11 +short
10.4.7.200
[root@hdss7-11 !]# dig -t A hdss7-11.host.com @10.4.7.11 +short
10.4.7.11

2.2.7. 修改各个主机DNS指向DNS

所有节点的操作
search 的设置可以将访问的后缀省略 即 访问hdss-7-200.host.com 可直接访问hdss-7-200

[root@hdss7-12 ~]# cat /etc/resolv.conf
# Generated by NetworkManager
search host.com
nameserver 10.4.7.11
# 测试是否成功测试
[root@hdss7-12 ~]# ping hdss7-200
PING HDSS7-200.host.com (10.4.7.200) 56(84) bytes of data.
64 bytes from 10.4.7.200 (10.4.7.200): icmp_seq=1 ttl=64 time=0.578 ms
64 bytes from 10.4.7.200 (10.4.7.200): icmp_seq=2 ttl=64 time=0.327 ms
64 bytes from 10.4.7.200 (10.4.7.200): icmp_seq=3 ttl=64 time=0.345 ms

2.3 准备签发证书环境

在运维主机hdss7-200.host.com中部署

2.3.1 安装cfssl

[root@hdss7-200 ~]# wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 -O /usr/local/bin/cfssl
[root@hdss7-200 ~]# wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 -O /usr/local/bin/cfssl-json
[root@hdss7-200 ~]# wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64 -O /usr/local/bin/cfssl-certinfo
[root@hdss7-200 ~]# chmod u+x /usr/local/bin/cfssl*
[root@hdss7-200 ~]# ll /usr/local/bin/
总用量 18808
-rwxr--r--. 1 root root 10376657 3月  30 2016 cfssl
-rwxr--r--. 1 root root  6595195 3月  30 2016 cfssl-certinfo
-rwxr--r--. 1 root root  2277873 3月  30 2016 cfssl-json

2.3.2. 创建CA证书请求文件(csr)的JSON配置文件

[root@hdss7-200 ~]# mkdir /opt/certs ; cd /opt/certs
[root@hdss7-200 certs]# vim ca-csr.json
{
    "CN": "OldboyEdu",
    "hosts": [
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "ST": "beijing",
            "L": "beijing",
            "O": "odl",
            "OU": "ops"
        }
    ],
    "ca": {
        "expiry": "175200h"
    }
}

CN 浏览器使用该字段验证网站是否合法,一般写域名

C: Country 国家

ST State 州,省

L Locality 地区 城市

O Oraganizetion Name 组织名称 公司名称

OU Oraganizetion Unit Name 组织单位名称 公司部门

expiry 过期时间

2.3.3. 生成CA证书和私钥

[root@hdss7-200 certs]# cfssl gencert -initca ca-csr.json | cfssl-json -bare ca
2020/09/17 20:51:52 [INFO] generating a new CA key and certificate from CSR
2020/09/17 20:51:52 [INFO] generate received request
2020/09/17 20:51:52 [INFO] received CSR
2020/09/17 20:51:52 [INFO] generating key: rsa-2048
2020/09/17 20:51:52 [INFO] encoded CSR
2020/09/17 20:51:52 [INFO] signed certificate with serial number 295886028821387664475820071483087242159712639122
[root@hdss7-200 certs]# ll
总用量 16
-rw-r--r--. 1 root root  993 9月  17 20:51 ca.csr
-rw-r--r--. 1 root root  328 9月  17 20:51 ca-csr.json
-rw-------. 1 root root 1679 9月  17 20:51 ca-key.pem  #根证书私钥
-rw-r--r--. 1 root root 1346 9月  17 20:51 ca.pem   # 根证书

2.3.4. 查看证书时间

当接触一套新的K8s集群时候,使用cfssl-certinfo查看证书有效期还有多久,以免过期

[root@hdss7-200 certs]# cfssl-certinfo -cert ca.pem
  "serial_number": "174577544452067437880255882036172381106489948922",
  "not_before": "2020-09-18T01:23:00Z",
  "not_after": "2040-09-13T01:23:00Z",

2.4. 安装docker环境

hdss-200.host.com hdss-21.host.com hdss-22.host.com 注意一些配置上的不同

# 通过脚本下载yum源,安装docker
[root@hdss7-200 ~]# curl -fsSL https://get.docker.com | bash -s docker --mirror Aliyun
[root@hdss7-21 ~]# mkdir /etc/docker/
# 不安全的registry中增加了harbor地址
# 各个机器上bip网段不一致，bip中间两段与宿主机最后两段相同，目的是方便定位问题 
# 不同主机需要修改bip  
# 比如 hdss-200.host.com 为 172.7.200.1/24 方便问题出现可以排查问题
[root@hdss7-21 ~]# vim /etc/docker/daemon.json
{
  "graph": "/data/docker",
  "storage-driver": "overlay2",
  "insecure-registries": ["registry.access.redhat.com","quay.io","harbor.odl.com"],
  "registry-mirrors": ["https://4nxg9p7s.mirror.aliyuncs.com"],
  "bip": "172.7.21.1/24",
  "exec-opts": ["native.cgroupdriver=systemd"],
  "live-restore": true
}
[root@hdss7-21 ~]# mkdir -p /data/docker
[root@hdss7-21 ~]# systemctl start docker ; systemctl enable docker

2.5. 安装私有仓库harbor

hdss-200.host.com

2.5.1 准备harbor环境

[root@hdss7-200 ~]# tar -xf harbor-offline-installer-v2.0.2-rc1.tgz -C /opt/
[root@hdss7-200 ~]# cd /opt
# 做软链接方便以后进行升级
[root@hdss7-200 opt]# mv harbor harbor-v2.0.2
[root@hdss7-200 opt]# ln -s harbor-v2.0.2 harbor
[root@hdss7-200 opt]# cd harbor
[root@hdss7-200 harbor]# cp harbor.yml.tmpl harbor.yml
# 注释https相关配置,否则安装时候报错
# external_url必须,因为后面使用nginx反向代理harbor,需要配置该参数,否则在访问时候报错
[root@hdss7-200 harbor]# vim harbor.yml
hostname: harbor.odl.com
http:
    port: 180
external_url: http://harbor.odl.com:80
harbor_admin_password: Harbor12345
data_volume: /data/harbor
log:
    location:  /data/harbor/logs
# 创建harbor数据目录和日志目录
[root@hdss7-200 harbor]# mkdir -p /data/harbor /data/harbor/logs

2.5.2. 安装docker-compose并安装harbor

在安装harbor的时候可—with-chartmuseum参数, 该参数会添加Helm-Charts仓库,详情可查看Helm章节 ./install.sh —with-chartmuseum

# daocloud 安装源
[root@hdss7-200 harbor]# curl -L https://get.daocloud.io/docker/compose/releases/download/1.25.0/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose
# 查看是否正常安装并赋予执行权限
[root@hdss7-200 harbor]# ls -l /usr/local/bin/docker-compose 
-rw-r--r-- 1 root root 17031320 8月   2 16:25 /usr/local/bin/docker-compose
[root@real-server-2 harbor]# chmod +x  /usr/local/bin/docker-compose
[root@hdss7-200 harbor]# docker-compose version
docker-compose version 1.25.0, build 0a186604
docker-py version: 4.1.0
CPython version: 3.7.4
OpenSSL version: OpenSSL 1.1.0l  10 Sep 2019
[root@hdss7-200 harbor]# ./install.sh
[Step 0]: checking if docker is installed ...
Note: docker version: 19.03.12
[Step 1]: checking docker-compose is installed ...
Note: docker-compose version: 1.25.0
[Step 2]: loading Harbor images ...
Loaded image: goharbor/prepare:v2.0.2
Loaded image: goharbor/harbor-jobservice:v2.0.2
Loaded image: goharbor/harbor-registryctl:v2.0.2
Loaded image: goharbor/registry-photon:v2.0.2
Loaded image: goharbor/harbor-core:v2.0.2
Loaded image: goharbor/notary-signer-photon:v2.0.2
Loaded image: goharbor/clair-photon:v2.0.2
Loaded image: goharbor/trivy-adapter-photon:v2.0.2
Loaded image: goharbor/harbor-log:v2.0.2
Loaded image: goharbor/nginx-photon:v2.0.2
Loaded image: goharbor/clair-adapter-photon:v2.0.2
Loaded image: goharbor/chartmuseum-photon:v2.0.2
Loaded image: goharbor/harbor-portal:v2.0.2
Loaded image: goharbor/harbor-db:v2.0.2
Loaded image: goharbor/redis-photon:v2.0.2
Loaded image: goharbor/notary-server-photon:v2.0.2
[Step 3]: preparing environment ...
[Step 4]: preparing harbor configs ...
prepare base dir is set to /opt/harbor-v2.0.2
WARNING:root:WARNING: HTTP protocol is insecure. Harbor will deprecate http protocol in the future. Please make sure to upgrade to https
Generated configuration file: /config/log/logrotate.conf
Generated configuration file: /config/log/rsyslog_docker.conf
Generated configuration file: /config/nginx/nginx.conf
Generated configuration file: /config/core/env
Generated configuration file: /config/core/app.conf
Generated configuration file: /config/registry/config.yml
Generated configuration file: /config/registryctl/env
Generated configuration file: /config/registryctl/config.yml
Generated configuration file: /config/db/env
Generated configuration file: /config/jobservice/env
Generated configuration file: /config/jobservice/config.yml
Generated and saved secret to file: /data/secret/keys/secretkey
Successfully called func: create_root_cert
Generated configuration file: /compose_location/docker-compose.yml
Clean up the input dir
[Step 5]: starting Harbor ...
Creating network "harbor-v202_harbor" with the default driver
Creating harbor-log ... done
Creating registry      ... done
Creating harbor-db     ... done
Creating redis         ... done
Creating harbor-portal ... done
Creating registryctl   ... done
Creating harbor-core   ... done
Creating harbor-jobservice ... done
Creating nginx             ... done
✔ ----Harbor has been installed and started successfully.----
[root@hdss7-200 harbor]# docker-compose ps
      Name                     Command                  State                 Ports          
---------------------------------------------------------------------------------------------
harbor-core         /harbor/entrypoint.sh            Up (healthy)                            
harbor-db           /docker-entrypoint.sh            Up (healthy)   5432/tcp                 
harbor-jobservice   /harbor/entrypoint.sh            Up (healthy)                            
harbor-log          /bin/sh -c /usr/local/bin/ ...   Up (healthy)   127.0.0.1:1514->10514/tcp
harbor-portal       nginx -g daemon off;             Up (healthy)   8080/tcp                 
nginx               nginx -g daemon off;             Up (healthy)   0.0.0.0:180->8080/tcp    
redis               redis-server /etc/redis.conf     Up (healthy)   6379/tcp                 
registry            /home/harbor/entrypoint.sh       Up (healthy)   5000/tcp                 
registryctl         /home/harbor/start.sh            Up (healthy)  
# 设置开机启动
[root@hdss7-200 ~]# vim /etc/rc.d/rc.local
# start harbor
cd /opt/harbor/
/usr/bin/docker-compose stop
/usr/bin/docker-compose start

2.5.3. 安装nginx

[root@hdss7-200 ~]# yum install nginx -y
# 或者使用源码安装.当前源码安装
[root@hdss7-200 ~]# vim /opt/nginx/conf/conf.d/harbor.odl.com.conf
server {
    listen       80;
    server_name  harbor.odl.com;
    # 避免出现上传失败的情况
    client_max_body_size 1000m;
    location / {
        proxy_pass http://127.0.0.1:180;
    }
}
[root@hdss7-200 ~]# /opt/nginx/sbin/nginx -t
[root@hdss7-200 ~]# /opt/nginx/sbin/nginx

不添加client_max_body_size参数, 上传镜像时候会报错 error parsing HTTP 413 response body: invalid character ‘<’ looking for beginning of value: “\r\n\r\n\r\n

413 Request Entity Too Large

\r\n

---

nginx/1.16.1\r\n\r\n\r\n”

2.5.4. 配置hdss7-11的DNS配置

serial 序列号需要滚动一位,即 2020091701 —> 2020091702 新增harbor.odl.com 的A记录

[root@hdss7-11 ~]# vim /var/named/odl.com.zone
$ORIGIN odl.com.
$TTL 600  ; 10 minutes
@       IN SOA  dns.odl.com. dnsadmin.odl.com. (
        2020091702 ; serial
        10800      ; refresh (3 hours)
        900        ; retry (15 minutes)
        604800     ; expire (1 week)
        86400      ; minimum (1 day)
        )
        NS   dns.odl.com.
$TTL 60 ; 1 minute
dns                A    10.4.7.11
harbor             A    10.4.7.200
[root@hdss7-11 ~]# systemctl restart named
[root@hdss7-11 ~]# dig -t A harbor.odl.com +short
10.4.7.200

2.5.5. 下载nginx镜像上传至harbor

前提需要: 需要在harbor目录上创建public创建项目, 否则上传会失败 harbor初始登录账号 admin Harbor12345

[root@hdss7-200 ~]# docker pull nginx:1.7.9
[root@hdss7-200 ~]# docker tag nginx:1.7.9 harbor.odl.com/public/nginx:v1.7.9
[root@hdss7-200 ~]#  docker login harbor.odl.com
Username: admin
Password: Harbor12345
[root@hdss7-200 ~]# docker push harbor.odl.com/public/nginx:v1.7.9

2.6. 配置yaml文件目录树

在hdss7-200的nginx中配置yaml数目录, 后期通过http方式去使用yaml清单文件

2.6.1. 配置nginx虚拟主机

[root@hdss7-200 ~]# vim /etc/nginx/conf.d/k8s-yaml.odl.com.comf 
server {
    listen       80;
    server_name  k8s-yaml.odl.com;
    location / {
        autoindex on;
        default_type text/plain;
        root /data/k8s-yaml;
    }
}
[root@hdss7-200 ~]# mkdir /data/k8s-yaml

2.6.2. 配置DNS解析

[root@hdss7-11 ~]# vim /var/named/odl.com.zone 
$ORIGIN odl.com.
$TTL 600  ; 10 minutes
@       IN SOA  dns.odl.com. dnsadmin.odl.com. (
        2020091705 ; serial
        10800      ; refresh (3 hours)
        900        ; retry (15 minutes)
        604800     ; expire (1 week)
        86400      ; minimum (1 day)
        )
        NS   dns.odl.com.
$TTL 60 ; 1 minute
dns                A    10.4.7.11
harbor             A    10.4.7.200
k8s-yaml           A    10.4.7.200
[root@hdss7-11 ~]# systemctl restart named

3. master节点组件部署

• server 表示服务端连接客户端时携带的证书，用于客户端验证服务端身份
• client 表示客户端连接服务端时携带的证书，用于服务端验证客户端身份
• peer 表示相互之间连接时使用的证书，如etcd节点之间验证

  3.1 创建所需证书

  3.1.1. 创建基于根证书的config配置文件

在hdss7-200 操作

[root@hdss7-200 ~]# vim /opt/certs/ca-config.json
{
    "signing": {
        "default": {
            "expiry": "175200h"
        },
        "profiles": {
            "server": {
                "expiry": "175200h",
                "usages": [
                    "signing",
                    "key encipherment",
                    "server auth"
                ]
            },
            "client": {
                "expiry": "175200h",
                "usages": [
                    "signing",
                    "key encipherment",
                    "client auth"
                ]
            },
            "peer": {
                "expiry": "175200h",
                "usages": [
                    "signing",
                    "key encipherment",
                    "server auth",
                    "client auth"
                ]
            }
        }
    }
}

3.1.2. 创建etcd证书配置：/opt/certs/etcd-peer-csr.json

[root@hdss7-200 ~]# vim /opt/certs/etcd-peer-csr.json
{
    "CN": "k8s-etcd",
    "hosts": [
        "10.4.7.11",
        "10.4.7.12",
        "10.4.7.21",
        "10.4.7.22"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "ST": "beijing",
            "L": "beijing",
            "O": "odl",
            "OU": "ops"
        }
    ]
}

3.1.3. 签发证书

[root@hdss7-200 ~]# cd /opt/certs/
[root@hdss7-200 certs]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=peer etcd-peer-csr.json |cfssl-json -bare etcd-peer
2020/09/18 21:35:54 [INFO] generate received request
2020/09/18 21:35:54 [INFO] received CSR
2020/09/18 21:35:54 [INFO] generating key: rsa-2048
2020/09/18 21:35:54 [INFO] encoded CSR
2020/09/18 21:35:54 [INFO] signed certificate with serial number 337386994541440179594156263415059535813502948433
2020/09/18 21:35:54 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
[root@hdss7-200 certs]# ll
总用量 36
-rw-r--r--. 1 root root  836 9月  18 17:59 ca-config.json
-rw-r--r--. 1 root root  997 9月  18 09:28 ca.csr
-rw-r--r--. 1 root root  329 9月  18 09:28 ca-csr.json
-rw-------. 1 root root 1679 9月  18 09:28 ca-key.pem
-rw-r--r--. 1 root root 1346 9月  18 09:28 ca.pem
-rw-r--r--. 1 root root 1066 9月  18 21:35 etcd-peer.csr
-rw-r--r--. 1 root root  364 9月  18 21:34 etcd-peer-csr.json
-rw-------. 1 root root 1675 9月  18 21:35 etcd-peer-key.pem
-rw-r--r--. 1 root root 1432 9月  18 21:35 etcd-peer.pem

3.2. 安装etcd组件

涉及：hdss7-12，hdss7-21，hdss7-22 , etcd启动

下载地址: https://github.com/etcd-io/etcd/releases/tag/v3.1.20

当前ectd版本: etcd-v3.1.20-linux-amd64.tar.gz

3.2.1. 准备etcd组件

# 创建etcd用户
[root@hdss7-12 ~]# useradd -M -s /sbin/nologin etcd
# 解压etcd
[root@hdss7-12 ~]# cd /opt; tar -xf etcd-v3.1.20-linux-amd64.tar.gz
[root@hdss7-12 opt]# mv etcd-v3.1.20-linux-amd64 etcd-v3.1.20
[root@hdss7-12 opt]# ln -s etcd-v3.1.20 etcd
[root@hdss7-12 opt]# mkdir -p /opt/etcd/certs /data/etcd /data/logs/etcd-server
[root@hdss7-12 opt]# scp hdss7-200:/opt/certs/{ca.pem,etcd-peer.pem,etcd-peer-key.pem} /opt/etcd/certs/
[root@hdss7-12 opt]# ll /opt/etcd/certs/
总用量 12
-rw-r--r--. 1 root root 1346 9月  18 21:49 ca.pem
-rw-------. 1 root root 1675 9月  18 21:49 etcd-peer-key.pem
-rw-r--r--. 1 root root 1432 9月  18 21:49 etcd-peer.pem

3.2.2. 设置etcd启动脚本

需要修改的参数：

name listen-peer-urls listen-client-urls initial-advertise-peer-urls advertise-client-urls

[root@hdss7-12 ~ ]# vim /opt/etcd/etcd-server-startup.sh
#!/bin/sh
# listen-peer-urls etcd节点之间通信端口
# listen-client-urls 客户端与etcd通信端口
# quota-backend-bytes 配额大小
WORK_DIR=$(dirname $(readlink -f $0))
[ $? -eq 0 ] && cd $WORK_DIR || exit
/opt/etcd/etcd --name etcd-server-7-12 \
    --data-dir /data/etcd/etcd-server \
    --listen-peer-urls https://10.4.7.12:2380 \
    --listen-client-urls https://10.4.7.12:2379,http://127.0.0.1:2379 \
    --quota-backend-bytes 8000000000 \
    --initial-advertise-peer-urls https://10.4.7.12:2380 \
    --advertise-client-urls https://10.4.7.12:2379,http://127.0.0.1:2379 \
    --initial-cluster  etcd-server-7-12=https://10.4.7.12:2380,etcd-server-7-21=https://10.4.7.21:2380,etcd-server-7-22=https://10.4.7.22:2380 \
    --ca-file ./certs/ca.pem \
    --cert-file ./certs/etcd-peer.pem \
    --key-file ./certs/etcd-peer-key.pem \
    --client-cert-auth  \
    --trusted-ca-file ./certs/ca.pem \
    --peer-ca-file ./certs/ca.pem \
    --peer-cert-file ./certs/etcd-peer.pem \
    --peer-key-file ./certs/etcd-peer-key.pem \
    --peer-client-cert-auth \
    --peer-trusted-ca-file ./certs/ca.pem \
    --log-output stdout

3.2.3. 启动etcd

采用后台管理工具 supervisor
当没指定下列两个参数,在停止服务的时候并不会杀死子进程,只会杀死父进程
stopasgroup=true ; 停止父进程和子进程
killasgroup=true ; 杀死父进程和子进程

# 修改目录权限
[root@hdss7-12 ~]# chmod u+x /opt/etcd/etcd-server-startup.sh
[root@hdss7-12 ~]# chown -R etcd:etcd /opt/etcd /opt/etcd-v3.1.20 /data/etcd/ /data/logs/etcd-server/
[root@hdss7-12 ~]# yum install -y supervisor
[root@hdss7-12 ~]# systemctl start supervisord ; systemctl enable supervisord
[root@hdss7-12 ~]# vim /etc/supervisord.d/etcd-server.ini
[program:etcd-server-7-12]
command=/opt/etcd/etcd-server-startup.sh         ; the program (relative uses PATH, can take args)
numprocs=1                                            ; number of processes copies to start (def 1)
directory=/opt/etcd                              ; directory to cwd to before exec (def no cwd)
autostart=true                                        ; start at supervisord start (default: true)
autorestart=true                                      ; retstart at unexpected quit (default: true)
startsecs=30                                          ; number of secs prog must stay running (def. 1)
startretries=3                                        ; max # of serial start failures (default 3)
exitcodes=0,2                                         ; 'expected' exit codes for process (default 0,2)
stopsignal=QUIT                                       ; signal used to kill process (default TERM)
stopwaitsecs=10                                       ; max num secs to wait b4 SIGKILL (default 10)
user=etcd                                             ; setuid to this UNIX account to run the program
redirect_stderr=true                                  ; redirect proc stderr to stdout (default false)
stdout_logfile=/data/logs/etcd-server/etcd.stdout.log ; stdout log path, NONE for none; default AUTO
stdout_logfile_maxbytes=64MB                          ; max # logfile bytes b4 rotation (default 50MB)
stdout_logfile_backups=5                              ; # of stdout logfile backups (default 10)
stdout_capture_maxbytes=1MB                           ; number of bytes in 'capturemode' (default 0)
stdout_events_enabled=false                           ; emit events on stdout writes (default false)
stopasgroup=true                                      
killasgroup=true
# 加入到supervisorctl后会自动启动
[root@hdss7-12 ~]# supervisorctl update
etcd-server-7-12: added process group
[root@hdss7-12 ~]# supervisorctl  start etcd-server-7-12
[root@hdss7-12 ~]# supervisorctl status
etcd-server-7-12                 RUNNING   pid 30305, uptime 0:00:41

3.2.4. 检查etcd集群情况

当只部署了一个etcd节点的时候会报错,因无法连接其他节点

# 查看节点成员
[root@hdss7-12 ~]#  /opt/etcd/etcdctl member list
client: etcd cluster is unavailable or misconfigured; error #0: client: endpoint http://127.0.0.1:2379 exceeded header timeout
; error #1: dial tcp 127.0.0.1:4001: getsockopt: connection refused
# 查看集群健康
[root@hdss7-12 ~]# /opt/etcd/etcdctl cluster-health
cluster may be unhealthy: failed to list members
Error:  client: etcd cluster is unavailable or misconfigured; error #0: dial tcp 127.0.0.1:4001: getsockopt: connection refused
; error #1: client: endpoint http://127.0.0.1:2379 exceeded header timeout
error #0: dial tcp 127.0.0.1:4001: getsockopt: connection refused
error #1: client: endpoint http://127.0.0.1:2379 exceeded header timeout

当正常部署两台以上节点时,(此处已将hdss7-12 hdss7-21 hdss-22 的etcd部署完毕)

# 查看节点成员
[root@hdss7-12 ~]#  /opt/etcd/etcdctl member list
988139385f78284: name=etcd-server-7-22 peerURLs=https://10.4.7.22:2380 clientURLs=http://127.0.0.1:2379,https://10.4.7.22:2379 isLeader=false
5a0ef2a004fc4349: name=etcd-server-7-21 peerURLs=https://10.4.7.21:2380 clientURLs=http://127.0.0.1:2379,https://10.4.7.21:2379 isLeader=false
f4a0cb0a765574a8: name=etcd-server-7-12 peerURLs=https://10.4.7.12:2380 clientURLs=http://127.0.0.1:2379,https://10.4.7.12:2379 isLeader=true
# 查看集群健康
[root@hdss7-12 ~]# /opt/etcd/etcdctl cluster-health
member 988139385f78284 is healthy: got healthy result from http://127.0.0.1:2379
member 5a0ef2a004fc4349 is healthy: got healthy result from http://127.0.0.1:2379
member f4a0cb0a765574a8 is healthy: got healthy result from http://127.0.0.1:2379
cluster is healthy

关停一个节点,但无法显示正确的健康状态,可使用以下命令

- 当关停10.4.7.12后, 常用命令还是显示healthy
# ./etcdctl cluster-health
member 988139385f78284 is healthy: got healthy result from http://127.0.0.1:2379
member 5a0ef2a004fc4349 is healthy: got healthy result from http://127.0.0.1:2379
member f4a0cb0a765574a8 is healthy: got healthy result from http://127.0.0.1:2379
cluster is healthy
- 解决办法
# ETCDCTL_API=3  /opt/etcd/etcdctl --endpoints=https://10.4.7.12:2379,https://10.4.7.21:2379,https://10.4.7.22:2379 --cacert=/opt/etcd/certs/ca.pem --cert=/opt/etcd/certs/etcd-peer.pem --key=/opt/etcd/certs/etcd-peer-key.pem       endpoint health --write-out=table
2020-11-16 09:45:07.452263 I | warning: ignoring ServerName for user-provided CA for backwards compatibility is deprecated
2020-11-16 09:45:07.452773 I | warning: ignoring ServerName for user-provided CA for backwards compatibility is deprecated
2020-11-16 09:45:07.453196 I | warning: ignoring ServerName for user-provided CA for backwards compatibility is deprecated
https://10.4.7.21:2379 is healthy: successfully committed proposal: took = 2.849422ms
https://10.4.7.22:2379 is healthy: successfully committed proposal: took = 3.636646ms
https://10.4.7.12:2379 is unhealthy: failed to connect: grpc: timed out when dialing
- 警告信息warning将在 etcd 3.2版本取消, 但在apiserver日志中显示

3.3. 安装apiserver组件

涉及到主机 hdss-21 hdss-22 root启动

kubernetes当前版本为1.15.3
科学上网下载: https://dl.k8s.io/v1.15.3/kubernetes-server-linux-amd64.tar.gz
当前压缩包包含了kubernetes集群所需的所有必要组件

3.3.1. 准备apiserver环境

• 创建kubernetes目录,方便升级

  [root@hdss7-21 opt]# tar -xf kubernetes-server-linux-amd64.tar.gz -C /opt/
  [root@hdss7-21 opt]# mv kubernetes/ kubernetes-1.15.3
  [root@hdss7-21 opt]# ln -s kubernetes-1.15.3 kubernetes

• 解压后kubernetes目录下有kubernetes的源码包和docker的压缩包,可删除 ```bash [root@hdss7-21 opt]# rm -f /opt/kubernetes/kubernetes-src.tar.gz [root@hdss7-21 opt]# rm -f /opt/kubernetes/server/bin/.tar /opt/kubernetes/server/bin/_tag

[root@hdss7-21 opt]# ll /opt/kubernetes/server/bin/ 总用量 884764 -rwxr-xr-x. 1 root root 43534816 8月 19 2019 apiextensions-apiserver -rwxr-xr-x. 1 root root 100569120 8月 19 2019 cloud-controller-manager -rwxr-xr-x. 1 root root 200681104 8月 19 2019 hyperkube -rwxr-xr-x. 1 root root 40186304 8月 19 2019 kubeadm -rwxr-xr-x. 1 root root 164522400 8月 19 2019 kube-apiserver -rwxr-xr-x. 1 root root 116421664 8月 19 2019 kube-controller-manager -rwxr-xr-x. 1 root root 42985504 8月 19 2019 kubectl -rwxr-xr-x. 1 root root 119645232 8月 19 2019 kubelet -rwxr-xr-x. 1 root root 36987488 8月 19 2019 kube-proxy -rwxr-xr-x. 1 root root 38786144 8月 19 2019 kube-scheduler -rwxr-xr-x. 1 root root 1648224 8月 19 2019 mounter

<a name="96JUl"></a>
### 3.3.2. 签发证书
> 签发证书涉及的服务器: hdss7-200
<a name="7dlwu"></a>
#### 3.3.2.1. 签发client证书(apiserver和etcd通信证书)
```bash
[root@hdss7-200 ~]# cd /opt/certs/
[root@hdss7-200 certs]# vim /opt/certs/client-csr.json
{
    "CN": "k8s-node",
    "hosts": [
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "ST": "beijing",
            "L": "beijing",
            "O": "od",
            "OU": "ops"
        }
    ]
}
[root@hdss7-200 certs]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client client-csr.json |cfssl-json -bare client
2020/01/06 13:42:47 [INFO] generate received request
2020/01/06 13:42:47 [INFO] received CSR
2020/01/06 13:42:47 [INFO] generating key: rsa-2048
2020/01/06 13:42:47 [INFO] encoded CSR
2020/01/06 13:42:47 [INFO] signed certificate with serial number 268276380983442021656020268926931973684313260543
2020/01/06 13:42:47 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
[root@hdss7-200 certs]# ls client* -l
-rw-r--r-- 1 root root  993 Jan  6 13:42 client.csr
-rw-r--r-- 1 root root  280 Jan  6 13:42 client-csr.json
-rw------- 1 root root 1679 Jan  6 13:42 client-key.pem
-rw-r--r-- 1 root root 1363 Jan  6 13:42 client.pem

3.3.2.2. 签发server证书(apiserver和其他k8s组件通信使用)

hosts中将所有可能作为apiserver的ip添加进去，VIP 10.4.7.10 也要加入

[root@hdss7-200 certs]# vim /opt/certs/apiserver-csr.json
{
    "CN": "k8s-apiserver",
    "hosts": [
        "127.0.0.1",
        "192.168.0.1",
        "kubernetes.default",
        "kubernetes.default.svc",
        "kubernetes.default.svc.cluster",
        "kubernetes.default.svc.cluster.local",
        "10.4.7.10",
        "10.4.7.21",
        "10.4.7.22",
        "10.4.7.23"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "ST": "beijing",
            "L": "beijing",
            "O": "od",
            "OU": "ops"
        }
    ]
}
[root@hdss7-200 certs]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server apiserver-csr.json |cfssl-json -bare apiserver
2020/01/06 13:46:56 [INFO] generate received request
2020/01/06 13:46:56 [INFO] received CSR
2020/01/06 13:46:56 [INFO] generating key: rsa-2048
2020/01/06 13:46:56 [INFO] encoded CSR
2020/01/06 13:46:56 [INFO] signed certificate with serial number 573076691386375893093727554861295529219004473872
2020/01/06 13:46:56 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
[root@hdss7-200 certs]# ls apiserver* -l
-rw-r--r-- 1 root root 1249 Jan  6 13:46 apiserver.csr
-rw-r--r-- 1 root root  566 Jan  6 13:45 apiserver-csr.json
-rw------- 1 root root 1675 Jan  6 13:46 apiserver-key.pem
-rw-r--r-- 1 root root 1598 Jan  6 13:46 apiserver.pem

3.3.2.3. 证书下发至对应主机目录

前提已免密钥登录, 涉及到hdss7-21 hdss7-22 hdss7-200

[root@hdss7-21 ~]# mkdir /opt/kubernetes/server/bin/certs
[root@hdss7-21 ~]# scp hdss7-200:/opt/certs/{apiserver-key.pem,apiserver.pem,ca-key.pem,ca.pem,client-key.pem,client.pem} /opt/kubernetes/server/bin/certs/
[root@hdss7-22 ~]# mkdir /opt/kubernetes/server/bin/certs
[root@hdss7-22 ~]# scp hdss7-200:/opt/certs/{apiserver-key.pem,apiserver.pem,ca-key.pem,ca.pem,client-key.pem,client.pem} /opt/kubernetes/server/bin/certs/

3.3.3. 配置apiserver日志审计

aipserver 涉及的服务器：hdss7-21，hdss7-22

[root@hdss7-21 bin]# mkdir /opt/kubernetes/conf
[root@hdss7-21 bin]# vi /opt/kubernetes/conf/audit.yaml # 打开文件后，设置 :set paste，避免自动缩进
apiVersion: audit.k8s.io/v1beta1 # This is required.
kind: Policy
# Don't generate audit events for all requests in RequestReceived stage.
omitStages:
  - "RequestReceived"
rules:
  # Log pod changes at RequestResponse level
  - level: RequestResponse
    resources:
    - group: ""
      # Resource "pods" doesn't match requests to any subresource of pods,
      # which is consistent with the RBAC policy.
      resources: ["pods"]
  # Log "pods/log", "pods/status" at Metadata level
  - level: Metadata
    resources:
    - group: ""
      resources: ["pods/log", "pods/status"]
  # Don't log requests to a configmap called "controller-leader"
  - level: None
    resources:
    - group: ""
      resources: ["configmaps"]
      resourceNames: ["controller-leader"]
  # Don't log watch requests by the "system:kube-proxy" on endpoints or services
  - level: None
    users: ["system:kube-proxy"]
    verbs: ["watch"]
    resources:
    - group: "" # core API group
      resources: ["endpoints", "services"]
  # Don't log authenticated requests to certain non-resource URL paths.
  - level: None
    userGroups: ["system:authenticated"]
    nonResourceURLs:
    - "/api*" # Wildcard matching.
    - "/version"
  # Log the request body of configmap changes in kube-system.
  - level: Request
    resources:
    - group: "" # core API group
      resources: ["configmaps"]
    # This rule only applies to resources in the "kube-system" namespace.
    # The empty string "" can be used to select non-namespaced resources.
    namespaces: ["kube-system"]
  # Log configmap and secret changes in all other namespaces at the Metadata level.
  - level: Metadata
    resources:
    - group: "" # core API group
      resources: ["secrets", "configmaps"]
  # Log all other resources in core and extensions at the Request level.
  - level: Request
    resources:
    - group: "" # core API group
    - group: "extensions" # Version of group should NOT be included.
  # A catch-all rule to log all other requests at the Metadata level.
  - level: Metadata
    # Long-running requests like watches that fall under this rule will not
    # generate an audit event in RequestReceived.
    omitStages:
      - "RequestReceived"