本次实验主机说明

一、下载apiServer安装包

我是手工到github上下载安装包,然后传到 5.138 和 5.139服务器上
https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.18.md#downloads-for-v1188

1.1 解压 (5.138和5.139服务器上操作)

1.2 签发证书 (在5.141服务器上操作)

1.2.1 签发client证书(apiserver和etcd通信证书)

  1. mkdir /opt/kube-cert
  2. vim /opt/kube-cert/client-csr.json
  4. {
  5.    "CN":"k8s-node",
  6.     "hosts": [
  7.     ],
  8.     "key": {
  9.         "algo": "rsa",
  10.         "size": 2048
  11.     },
  12.     "names": [
  13.         {
  14.             "C": "CN",
  15.             "ST": "GuangDong",
  16.             "L": "ShenZhen",
  17.             "O": "batar",
  18.             "OU": "zhonggu"
  19.         }
  20.     ]
  21. }
  23. kube-cert]# cfssl gencert -ca=../ca.pem -ca-key=../ca-key.pem -config=../ca-config.json -profile=client client-csr.json |cfssl-json -bare client

1.2.2 签发server证书（apiserver和其它k8s组件通信使用）

  1. # hosts中将所有可能作为apiserver的ip添加进去，VIP 192.168.5.137 也要加入
  2. [root@hdss7-200 certs]# vim /opt/certs/apiserver-csr.json
  3. {
  4.     "CN": "k8s-apiserver",
  5.     "hosts": [
  6.         "127.0.0.1",
  7.         "192.168.0.1",
  8.         "kubernetes.default",
  9.         "kubernetes.default.svc",
  10.         "kubernetes.default.svc.cluster",
  11.         "kubernetes.default.svc.cluster.local",
  12.         "192.168.5.138",
  13.         "192.168.5.139",
  14.         "192.168.5.137",
  15.         "192.168.5.140"
  16.     ],
  17.     "key": {
  18.         "algo": "rsa",
  19.         "size": 2048
  20.     },
  21.     "names": [
  22.         {
  23.             "C": "CN",
  24.             "ST": "GuangDong",
  25.             "L": "ShenZhen",
  26.             "O": "Batar",
  27.             "OU": "zhonggu"
  28.         }
  29.     ]
  30. }
  31. [root@k8s-5-141 certs]# cfssl gencert -ca=../ca.pem -ca-key=../ca-key.pem -config=../ca-config.json -profile=server apiserver-csr.json |cfssl-json -bare apiserver
  32. 2021/03/19 16:21:01 [INFO] generate received request
  33. 2021/03/19 16:21:01 [INFO] received CSR
  34. 2021/03/19 16:21:01 [INFO] generating key: rsa-2048
  35. 2021/03/19 16:21:01 [INFO] encoded CSR
  36. 2021/03/19 16:21:01 [INFO] signed certificate with serial number 372720519750825053414527052445324187571968604187
  37. 2021/03/19 16:21:01 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
  38. websites. For more information see the Baseline Requirements for the Issuance and Management
  39. of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
  40. specifically, section 10.2.3 ("Information Requirements").
  41. [root@k8s-5-141 certs]# ls apiserver* -l
  42. -rw-r--r-- 1 root root 1249 Jan  6 13:46 apiserver.csr
  43. -rw-r--r-- 1 root root  566 Jan  6 13:45 apiserver-csr.json
  44. -rw------- 1 root root 1675 Jan  6 13:46 apiserver-key.pem
  45. -rw-r--r-- 1 root root 1598 Jan  6 13:46 apiserver.pem

1.3 证书下发(分别到 5.138 和 5.139 服务器上执行)

因为我没有做短域名,所以要加host.com

  1. certs]# for i in 138 139;do echo k8s-5-$i;ssh k8s-5-$i.host.com "mkdir /opt/kubernetes/server/bin/certs";scp apiserver-key.pem apiserver.pem ../ca-key.pem ../ca.pem client-key.pem client.pem k8s-5-$i.host.com:/opt/kubernetes/server/bin/certs/;done

1.4 配置apiserver日志审计

aipserver 涉及的服务器：k8s-5-137, k8s-5-138

  1.  bin]# mkdir /opt/kubernetes/conf
  2.  bin]# vim /opt/kubernetes/conf/audit.yaml # 打开文件后，设置 :set paste，避免自动缩进
  3. apiVersion: audit.k8s.io/v1beta1 # This is required.
  4. kind: Policy
  5. # Don't generate audit events for all requests in RequestReceived stage.
  6. omitStages:
  7.   - "RequestReceived"
  8. rules:
  9.   # Log pod changes at RequestResponse level
  10.   - level: RequestResponse
  11.     resources:
  12.     - group: ""
  13.       # Resource "pods" doesn't match requests to any subresource of pods,
  14.       # which is consistent with the RBAC policy.
  15.       resources: ["pods"]
  16.   # Log "pods/log", "pods/status" at Metadata level
  17.   - level: Metadata
  18.     resources:
  19.     - group: ""
  20.       resources: ["pods/log", "pods/status"]
  22.   # Don't log requests to a configmap called "controller-leader"
  23.   - level: None
  24.     resources:
  25.     - group: ""
  26.       resources: ["configmaps"]
  27.       resourceNames: ["controller-leader"]
  29.   # Don't log watch requests by the "system:kube-proxy" on endpoints or services
  30.   - level: None
  31.     users: ["system:kube-proxy"]
  32.     verbs: ["watch"]
  33.     resources:
  34.     - group: "" # core API group
  35.       resources: ["endpoints", "services"]
  37.   # Don't log authenticated requests to certain non-resource URL paths.
  38.   - level: None
  39.     userGroups: ["system:authenticated"]
  40.     nonResourceURLs:
  41.     - "/api*" # Wildcard matching.
  42.     - "/version"
  44.   # Log the request body of configmap changes in kube-system.
  45.   - level: Request
  46.     resources:
  47.     - group: "" # core API group
  48.       resources: ["configmaps"]
  49.     # This rule only applies to resources in the "kube-system" namespace.
  50.     # The empty string "" can be used to select non-namespaced resources.
  51.     namespaces: ["kube-system"]
  53.   # Log configmap and secret changes in all other namespaces at the Metadata level.
  54.   - level: Metadata
  55.     resources:
  56.     - group: "" # core API group
  57.       resources: ["secrets", "configmaps"]
  59.   # Log all other resources in core and extensions at the Request level.
  60.   - level: Request
  61.     resources:
  62.     - group: "" # core API group
  63.     - group: "extensions" # Version of group should NOT be included.
  65.   # A catch-all rule to log all other requests at the Metadata level.
  66.   - level: Metadata
  67.     # Long-running requests like watches that fall under this rule will not
  68.     # generate an audit event in RequestReceived.
  69.     omitStages:
  70.       - "RequestReceived"

1.5 配置启动脚本

  1. #!/bin/bash
  4. WORK_DIR=$(dirname $(readlink -f $0))
  6. [ $? -eq 0 ] && cd $WORK_DIR || exit
  9. /opt/kubernetes/server/bin/kube-apiserver \
  10.     --apiserver-count 2 \
  11.     --audit-log-path /data/logs/kubernetes/kube-apiserver/audit-log \
  12.     --audit-policy-file ./conf/audit.yaml \
  13.     --authorization-mode RBAC \
  14.     --client-ca-file ./certs/ca.pem \
  15.     --requestheader-client-ca-file ./certs/ca.pem \
  16.     --enable-admission-plugins NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota \
  17.     --etcd-cafile ./certs/ca.pem \
  18.     --etcd-certfile ./certs/client.pem \
  19.     --etcd-keyfile ./certs/client-key.pem \
  20.     --etcd-servers https://192.168.5.137:2379,https://192.168.5.138:2379,https://192.168.5.139:2379 \
  21.     --service-account-key-file ./certs/ca-key.pem \
  22.     --service-cluster-ip-range 192.168.0.0/16 \
  23.     --service-node-port-range 3000-29999 \
  24.     --target-ram-mb=1024 \
  25.     --kubelet-client-certificate ./certs/client.pem \
  26.     --kubelet-client-key ./certs/client-key.pem \
  27.     --log-dir  /data/logs/kubernetes/kube-apiserver \
  28.     --tls-cert-file ./certs/apiserver.pem \
  29.     --tls-private-key-file ./certs/apiserver-key.pem \
  30.     --v 2

在生产环境中,最好配置自动检测运行状态, 可使用 supervisor 来实现自动监控和程序拉起

也可以配置成系统服务

  1. #] vim /etc/systemd/system/kube-apiserver.service
  3. # 输入如下内容
  4. [Unit]
  5. Description=kube apiserver service
  6. Documentation=https://github.com/kubernetes
  7. Conflicts=kube-apiserver.service
  9. [Service]
  10. Type=notify
  11. Restart=always
  12. RestartSec=5s
  13. LimitNOFILE=40000
  14. TimeoutStartSec=0
  16. ExecStart=/opt/kubernetes/server/bin/kube-apiserver \
  17.     --apiserver-count 2 \
  18.     --audit-log-path /data/logs/kubernetes/kube-apiserver/audit-log \
  19.     --audit-policy-file /opt/kubernetes/server/bin/conf/audit.yaml \
  20.     --authorization-mode RBAC \
  21.     --client-ca-file /opt/kubernetes/server/bin/certs/ca.pem \
  22.     --requestheader-client-ca-file /opt/kubernetes/server/bin/certs/ca.pem \
  23.     --enable-admission-plugins NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota \
  24.     --etcd-cafile /opt/kubernetes/server/bin/certs/ca.pem \
  25.     --etcd-certfile /opt/kubernetes/server/bin/certs/client.pem \
  26.     --etcd-keyfile /opt/kubernetes/server/bin/certs/client-key.pem \
  27.     --etcd-servers https://192.168.5.137:2379,https://192.168.5.138:2379,https://192.168.5.139:2379 \
  28.     --service-account-key-file /opt/kubernetes/server/bin/certs/ca-key.pem \
  29.     --service-cluster-ip-range 192.168.0.0/16 \
  30.     --service-node-port-range 3000-29999 \
  31.     --target-ram-mb=1024 \
  32.     --kubelet-client-certificate /opt/kubernetes/server/bin/certs/client.pem \
  33.     --kubelet-client-key /opt/kubernetes/server/bin/certs/client-key.pem \
  34.     --log-dir  /data/logs/kubernetes/kube-apiserver \
  35.     --tls-cert-file /opt/kubernetes/server/bin/certs/apiserver.pem \
  36.     --tls-private-key-file /opt/kubernetes/server/bin/certs/apiserver-key.pem \
  37.     --v 2 \
  38.     --log-file /data/logs/kubernetes/kube-apiserver/apiserver.log
  40. [Install]
  41. WantedBy=multi-user.target
  43. #然后执行命令添加系统服务并设置开机自动启动
  45. #] systemctl daemon-reload
  46. #] systemctl enable kube-apiserver
  48. # 启动kube-apiserver
  49. #] systemctl start kube-apiserver
  51. # 查看是否已经启动成功
  52. #] netstat -unltp |grep kube-apiserver
  53. tcp        0      0 127.0.0.1:8080          0.0.0.0:*               LISTEN      6546/kube-apiserver 
  54. tcp6       0      0 :::6443                 :::*                    LISTEN      6546/kube-apiserver

配置系统服务时,踩了一个坑, 通过命令systemctl start kube-apiserver时,提示启动失败,但是通过systemctl status kube-apiserver 并没有看到具体的报错, 后来从网上看到说可以通过 筛选 /var/log/messages 日志信息找到报错,我使用的命令如下: 由于在我系统里 kube-apiserver 是一个服务,所以必须加上单引号 cat /var/log/messages |grep ‘kube-apiserver’ |grep error

1.6 配置apiserver L4代理

nginx配置

这里只做了一台机器 所以没有做keepalived