请参考github该项目https://github.com/easzlab/kubeasz

1 基本信息

1.1 kubernetes组件通信示意图

image.png

1.2 部署结构图

image.png

1.3 主机IP

主机 IP 作用
VIP 10.0.0.248 集群VIP
master-01 10.0.0.11 master节点
master-02 10.0.0.12 master节点
master-03 10.0.0.13 master节点
etcd-01 10.0.0.14 etcd节点
etcd-02 10.0.0.15 etcd节点
etcd-03 10.0.0.16 etcd节点
haproxy-01 10.0.0.17 负载均衡
haproxy-01 10.0.0.18 负载均衡
node-01 10.0.0.19 node节点
node-02 10.0.0.20 node节点

2 部署

2.1 安装keepalived

在两个负载节点上分别安装Keepalived

  1. root@haproxy-01/02:~# apt install -y keepalived

配置节点01

  1. root@haproxy-01:~# cat /etc/keepalived/keepalived.conf 
  2. ! Configuration File for keepalived
  4. global_defs {
  5.    notification_email {
  6.      acassen
  7.    }
  8.    notification_email_from Alexandre.Cassen@firewall.loc
  9.    smtp_server 192.168.200.1
  10.    smtp_connect_timeout 30
  11.    router_id LVS_DEVEL
  12. }
  14. vrrp_instance VI_1 {
  15.     state MASTER
  16.     interface ens32
  17.     garp_master_delay 10
  18.     smtp_alert
  19.     virtual_router_id 60
  20.     priority 100
  21.     advert_int 1
  22.     authentication {
  23.         auth_type PASS
  24.         auth_pass 1111
  25.     }
  26.     virtual_ipaddress {
  27.         10.0.0.248 label ens32:1
  28.         10.0.0.249 label eth32:2
  29.         10.0.0.250 label eth32:3
  30.         10.0.0.251 label ens32:4
  31.         10.0.0.252 label ens32:5
  32.     }
  33. }

配置节点02

  1. root@haproxy-01:~# cat /etc/keepalived/keepalived.conf 
  2. ! Configuration File for keepalived
  4. global_defs {
  5.    notification_email {
  6.      acassen
  7.    }
  8.    notification_email_from Alexandre.Cassen@firewall.loc
  9.    smtp_server 192.168.200.1
  10.    smtp_connect_timeout 30
  11.    router_id LVS_DEVEL
  12. }
  14. vrrp_instance VI_1 {
  15.     state MASTER
  16.     interface ens32
  17.     garp_master_delay 10
  18.     smtp_alert
  19.     virtual_router_id 60
  20.     priority 99
  21.     advert_int 1
  22.     authentication {
  23.         auth_type PASS
  24.         auth_pass 1111
  25.     }
  26.     virtual_ipaddress {
  27.         10.0.0.248 label ens32:1
  28.         10.0.0.249 label eth32:2
  29.         10.0.0.250 label eth32:3
  30.         10.0.0.251 label ens32:4
  31.         10.0.0.252 label ens32:5
  32.     }
  33. }

启动并开机自启

  1. root@haproxy-01/02:~# systemctl enable --now keepalived.service

2.2 安装Haproxy

在两个负载节点上安装haproxy

  1. root@haproxy-01/02:~# apt install -y haproxy

配置节点01

  1. root@haproxy-01:~# cat /etc/haproxy/haproxy.cfg 
  2. global
  3.         log /dev/log    local0
  4.         log /dev/log    local1 notice
  5.         chroot /var/lib/haproxy
  6.         stats socket /run/haproxy/admin.sock mode 660 level admin expose-fd listeners
  7.         stats timeout 30s
  8.         user haproxy
  9.         group haproxy
  10.         daemon
  12.         # Default SSL material locations
  13.         ca-base /etc/ssl/certs
  14.         crt-base /etc/ssl/private
  16.         # See: https://ssl-config.mozilla.org/#server=haproxy&server-version=2.0.3&config=intermediate
  17.         ssl-default-bind-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384
  18.         ssl-default-bind-ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256
  19.         ssl-default-bind-options ssl-min-ver TLSv1.2 no-tls-tickets
  21. defaults
  22.         log     global
  23.         mode    http
  24.         option  httplog
  25.         option  dontlognull
  26.         timeout connect 5000
  27.         timeout client  50000
  28.         timeout server  50000
  29.         errorfile 400 /etc/haproxy/errors/400.http
  30.         errorfile 403 /etc/haproxy/errors/403.http
  31.         errorfile 408 /etc/haproxy/errors/408.http
  32.         errorfile 500 /etc/haproxy/errors/500.http
  33.         errorfile 502 /etc/haproxy/errors/502.http
  34.         errorfile 503 /etc/haproxy/errors/503.http
  35.         errorfile 504 /etc/haproxy/errors/504.http
  36. listen k8s-6443
  37.         bind 10.0.0.248:6443
  38.         mode tcp
  39.         server 10.0.0.11 10.0.0.11:6443 check inter 2s fall 3 rise 3
  40.         server 10.0.0.12 10.0.0.12:6443 check inter 2s fall 3 rise 3
  41.         server 10.0.0.13 10.0.0.13:6443 check inter 2s fall 3 rise 3

配置节点02

  1. root@haproxy-01:~# cat /etc/haproxy/haproxy.cfg 
  2. global
  3.         log /dev/log    local0
  4.         log /dev/log    local1 notice
  5.         chroot /var/lib/haproxy
  6.         stats socket /run/haproxy/admin.sock mode 660 level admin expose-fd listeners
  7.         stats timeout 30s
  8.         user haproxy
  9.         group haproxy
  10.         daemon
  12.         # Default SSL material locations
  13.         ca-base /etc/ssl/certs
  14.         crt-base /etc/ssl/private
  16.         # See: https://ssl-config.mozilla.org/#server=haproxy&server-version=2.0.3&config=intermediate
  17.         ssl-default-bind-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384
  18.         ssl-default-bind-ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256
  19.         ssl-default-bind-options ssl-min-ver TLSv1.2 no-tls-tickets
  21. defaults
  22.         log     global
  23.         mode    http
  24.         option  httplog
  25.         option  dontlognull
  26.         timeout connect 5000
  27.         timeout client  50000
  28.         timeout server  50000
  29.         errorfile 400 /etc/haproxy/errors/400.http
  30.         errorfile 403 /etc/haproxy/errors/403.http
  31.         errorfile 408 /etc/haproxy/errors/408.http
  32.         errorfile 500 /etc/haproxy/errors/500.http
  33.         errorfile 502 /etc/haproxy/errors/502.http
  34.         errorfile 503 /etc/haproxy/errors/503.http
  35.         errorfile 504 /etc/haproxy/errors/504.http
  36. listen k8s-6443
  37.         bind 10.0.0.248:6443
  38.         mode tcp
  39.         server 10.0.0.11 10.0.0.11:6443 check inter 2s fall 3 rise 3
  40.         server 10.0.0.12 10.0.0.12:6443 check inter 2s fall 3 rise 3
  41.         server 10.0.0.13 10.0.0.13:6443 check inter 2s fall 3 rise 3

启动并开机自启

  1. root@haproxy-01/02:~# systemctl enable --now haproxy

注意
在两个节点都是正常的情况下，由于一个节点没有VIP地址，所以会存在haproxy无法监听的情况，需要开启一个内核参数——net.ipv4.ip_nonlocal_bind = 1

  1. root@haproxy-02:~# vim /etc/sysctl.conf 
  2. net.ipv4.ip_nonlocal_bind = 1
  3. root@haproxy-02:~# sysctl -p

开启此内核参数后重启haproxy即可正常监听了

2.3 安装k8s

参考github项目https://github.com/easzlab/kubeasz
此安装步骤以master01为分发节点，安装ansible，安装步骤都在这个节点上面执行
安装ansible

  1. root@master-01:~#  apt install -y ansible

配置免密

  1. root@master-01:~# ssh-keygen 
  2. root@master-01:~# ssh-copy-id 10.0.0.11
  3. root@master-01:~# ssh-copy-id 10.0.0.12
  4. root@master-01:~# ssh-copy-id 10.0.0.13
  5. root@master-01:~# ssh-copy-id 10.0.0.14
  6. root@master-01:~# ssh-copy-id 10.0.0.15
  7. root@master-01:~# ssh-copy-id 10.0.0.16
  8. root@master-01:~# ssh-copy-id 10.0.0.17
  9. root@master-01:~# ssh-copy-id 10.0.0.18
  10. root@master-01:~# ssh-copy-id 10.0.0.19

下载项目源码、二进制及离线镜像

  1. # 下载工具脚本ezdown，举例使用kubeasz版本3.1.1
  2. root@master-01:~# export release=3.1.1
  3. root@master-01:~# wget https://github.com/easzlab/kubeasz/releases/download/${release}/ezdown
  4. root@master-01:~# chmod +x ./ezdown
  5. # 使用工具脚本下载
  6. root@master-01:~# ./ezdown -D

所有包以及相关文件都在/etc/kubeasz目录下面

  1. root@master-01:~# cd /etc/kubeasz/
  2. root@master-01:/etc/kubeasz# ll
  3. total 100
  4. drwxrwxr-x  12 root root   225 Jan  6 02:29 ./
  5. drwxr-xr-x 105 root root  8192 Jan  6 06:52 ../
  6. -rw-rw-r--   1 root root 20304 Sep 25 03:41 ansible.cfg
  7. drwxr-xr-x   3 root root  4096 Jan  6 02:25 bin/
  8. drwxr-xr-x   3 root root    28 Jan  6 02:29 clusters/
  9. drwxrwxr-x   8 root root    92 Sep 25 07:30 docs/
  10. drwxr-xr-x   2 root root  4096 Jan  6 02:27 down/
  11. drwxrwxr-x   2 root root    70 Sep 25 07:30 example/
  12. -rwxrwxr-x   1 root root 24629 Sep 25 03:41 ezctl*
  13. -rwxrwxr-x   1 root root 15031 Sep 25 03:41 ezdown*
  14. -rw-rw-r--   1 root root   301 Sep 25 03:41 .gitignore
  15. drwxrwxr-x  10 root root   145 Sep 25 07:30 manifests/
  16. drwxrwxr-x   2 root root   322 Sep 25 07:30 pics/
  17. drwxrwxr-x   2 root root  4096 Sep 25 07:30 playbooks/
  18. -rw-rw-r--   1 root root  6222 Sep 25 03:41 README.md
  19. drwxrwxr-x  22 root root   323 Sep 25 07:30 roles/
  20. drwxrwxr-x   2 root root    48 Sep 25 07:30 tools/

创建集群配置实例
创建一个新的集群名称为k8s-cluster-01

  1. root@master-01:/etc/kubeasz# ./ezctl new k8s-cluster-01

然后根据提示配置’/etc/kubeasz/clusters/k8s-cluster-01/hosts’ 和 ‘/etc/kubeasz/clusters/k8s-cluster-01/config.yml’：根据前面节点规划修改hosts 文件和其他集群层面的主要配置选项；其他集群组件等配置项可以在config.yml 文件中修改。
修改hosts文件

  1. root@master-01:/etc/kubeasz/clusters/k8s-cluster-01# cat hosts 
  2. # 'etcd' cluster should have odd member(s) (1,3,5,...)
  3. [etcd]
  4. 10.0.0.14
  5. 10.0.0.15
  6. 10.0.0.16
  8. # master node(s)
  9. [kube_master]
  10. 10.0.0.11
  11. 10.0.0.12
  12. 10.0.0.13
  14. # work node(s)
  15. [kube_node]
  16. 10.0.0.19
  17. 10.0.0.20
  19. # [optional] harbor server, a private docker registry
  20. # 'NEW_INSTALL': 'true' to install a harbor server; 'false' to integrate with existed one
  21. [harbor]
  22. #10.0.0.8 NEW_INSTALL=false
  24. # [optional] loadbalance for accessing k8s from outside
  25. [ex_lb]
  26. 10.0.0.17 LB_ROLE=master EX_APISERVER_VIP=10.0.0.248 EX_APISERVER_PORT=6443
  27. 10.0.0.18 LB_ROLE=backup EX_APISERVER_VIP=10.0.0.248 EX_APISERVER_PORT=6443
  29. # [optional] ntp server for the cluster
  30. [chrony]
  31. #10.0.0.1
  33. [all:vars]
  34. # --------- Main Variables ---------------
  35. # Secure port for apiservers
  36. SECURE_PORT="6443"
  38. # Cluster container-runtime supported: docker, containerd
  39. CONTAINER_RUNTIME="docker"
  41. # Network plugins supported: calico, flannel, kube-router, cilium, kube-ovn
  42. CLUSTER_NETWORK="calico"
  44. # Service proxy mode of kube-proxy: 'iptables' or 'ipvs'
  45. PROXY_MODE="ipvs"
  47. # K8S Service CIDR, not overlap with node(host) networking
  48. SERVICE_CIDR="10.96.0.0/16"
  50. # Cluster CIDR (Pod CIDR), not overlap with node(host) networking
  51. CLUSTER_CIDR="10.224.0.0/16"
  53. # NodePort Range
  54. NODE_PORT_RANGE="30000-40000"
  56. # Cluster DNS Domain
  57. CLUSTER_DNS_DOMAIN="cluster.local"
  59. # -------- Additional Variables (don't change the default value right now) ---
  60. # Binaries Directory
  61. bin_dir="/usr/bin"
  63. # Deploy Directory (kubeasz workspace)
  64. base_dir="/etc/kubeasz"
  66. # Directory for a specific cluster
  67. cluster_dir="{{ base_dir }}/clusters/k8s-cluster-01"
  69. # CA and other components cert/key Directory
  70. ca_dir="/etc/kubernetes/ssl"

修改config.yml
哪些组件需要安装或者不安装，配置ture或者no即可

  1. root@master-01:/etc/kubeasz/clusters/k8s-cluster-01# cat config.yml 
  2. ############################
  3. # prepare
  4. ############################
  5. # 可选离线安装系统软件包 (offline|online)
  6. INSTALL_SOURCE: "online"
  8. # 可选进行系统安全加固 github.com/dev-sec/ansible-collection-hardening
  9. OS_HARDEN: false
  11. # 设置时间源服务器【重要：集群内机器时间必须同步】
  12. ntp_servers:
  13.   - "ntp1.aliyun.com"
  14.   - "time1.cloud.tencent.com"
  15.   - "0.cn.pool.ntp.org"
  17. # 设置允许内部时间同步的网络段，比如"10.0.0.0/8"，默认全部允许
  18. local_network: "0.0.0.0/0"
  21. ############################
  22. # role:deploy
  23. ############################
  24. # default: ca will expire in 100 years
  25. # default: certs issued by the ca will expire in 50 years
  26. CA_EXPIRY: "876000h"
  27. CERT_EXPIRY: "438000h"
  29. # kubeconfig 配置参数
  30. CLUSTER_NAME: "k8s-cluster-01"
  31. CONTEXT_NAME: "context-{{ CLUSTER_NAME }}"
  34. ############################
  35. # role:etcd
  36. ############################
  37. # 设置不同的wal目录，可以避免磁盘io竞争，提高性能
  38. ETCD_DATA_DIR: "/var/lib/etcd"
  39. ETCD_WAL_DIR: ""
  42. ############################
  43. # role:runtime [containerd,docker]
  44. ############################
  45. # ------------------------------------------- containerd
  46. # [.]启用容器仓库镜像
  47. ENABLE_MIRROR_REGISTRY: true
  49. # [containerd]基础容器镜像
  50. SANDBOX_IMAGE: "easzlab/pause-amd64:3.5"
  52. # [containerd]容器持久化存储目录
  53. CONTAINERD_STORAGE_DIR: "/var/lib/containerd"
  55. # ------------------------------------------- docker
  56. # [docker]容器存储目录
  57. DOCKER_STORAGE_DIR: "/var/lib/docker"
  59. # [docker]开启Restful API
  60. ENABLE_REMOTE_API: false
  62. # [docker]信任的HTTP仓库
  63. INSECURE_REG: '["127.0.0.1/8"]'
  66. ############################
  67. # role:kube-master
  68. ############################
  69. # k8s 集群 master 节点证书配置，可以添加多个ip和域名（比如增加公网ip和域名）
  70. MASTER_CERT_HOSTS:
  71.   - "10.0.0.250"
  72.   - "ikubernetes.xsc.org"
  73.   #- "www.test.com"
  75. # node 节点上 pod 网段掩码长度（决定每个节点最多能分配的pod ip地址）
  76. # 如果flannel 使用 --kube-subnet-mgr 参数，那么它将读取该设置为每个节点分配pod网段
  77. # https://github.com/coreos/flannel/issues/847
  78. NODE_CIDR_LEN: 24
  81. ############################
  82. # role:kube-node
  83. ############################
  84. # Kubelet 根目录
  85. KUBELET_ROOT_DIR: "/var/lib/kubelet"
  87. # node节点最大pod 数
  88. MAX_PODS: 200
  90. # 配置为kube组件（kubelet,kube-proxy,dockerd等）预留的资源量
  91. # 数值设置详见templates/kubelet-config.yaml.j2
  92. KUBE_RESERVED_ENABLED: "no"
  94. # k8s 官方不建议草率开启 system-reserved, 除非你基于长期监控，了解系统的资源占用状况；
  95. # 并且随着系统运行时间，需要适当增加资源预留，数值设置详见templates/kubelet-config.yaml.j2
  96. # 系统预留设置基于 4c/8g 虚机，最小化安装系统服务，如果使用高性能物理机可以适当增加预留
  97. # 另外，集群安装时候apiserver等资源占用会短时较大，建议至少预留1g内存
  98. SYS_RESERVED_ENABLED: "no"
  100. # haproxy balance mode
  101. BALANCE_ALG: "roundrobin"
  104. ############################
  105. # role:network [flannel,calico,cilium,kube-ovn,kube-router]
  106. ############################
  107. # ------------------------------------------- flannel
  108. # [flannel]设置flannel 后端"host-gw","vxlan"等
  109. FLANNEL_BACKEND: "vxlan"
  110. DIRECT_ROUTING: false
  112. # [flannel] flanneld_image: "quay.io/coreos/flannel:v0.10.0-amd64"
  113. flannelVer: "v0.13.0-amd64"
  114. flanneld_image: "easzlab/flannel:{{ flannelVer }}"
  116. # [flannel]离线镜像tar包
  117. flannel_offline: "flannel_{{ flannelVer }}.tar"
  119. # ------------------------------------------- calico
  120. # [calico]设置 CALICO_IPV4POOL_IPIP=“off”,可以提高网络性能，条件限制详见 docs/setup/calico.md
  121. CALICO_IPV4POOL_IPIP: "Always"
  123. # [calico]设置 calico-node使用的host IP，bgp邻居通过该地址建立，可手工指定也可以自动发现
  124. IP_AUTODETECTION_METHOD: "can-reach={{ groups['kube_master'][0] }}"
  126. # [calico]设置calico 网络 backend: brid, vxlan, none
  127. CALICO_NETWORKING_BACKEND: "brid"
  129. # [calico]更新支持calico 版本: [v3.3.x] [v3.4.x] [v3.8.x] [v3.15.x]
  130. calico_ver: "v3.19.2"
  132. # [calico]calico 主版本
  133. calico_ver_main: "{{ calico_ver.split('.')[0] }}.{{ calico_ver.split('.')[1] }}"
  135. # [calico]离线镜像tar包
  136. calico_offline: "calico_{{ calico_ver }}.tar"
  138. # ------------------------------------------- cilium
  139. # [cilium]CILIUM_ETCD_OPERATOR 创建的 etcd 集群节点数 1,3,5,7...
  140. ETCD_CLUSTER_SIZE: 1
  142. # [cilium]镜像版本
  143. cilium_ver: "v1.4.1"
  145. # [cilium]离线镜像tar包
  146. cilium_offline: "cilium_{{ cilium_ver }}.tar"
  148. # ------------------------------------------- kube-ovn
  149. # [kube-ovn]选择 OVN DB and OVN Control Plane 节点，默认为第一个master节点
  150. OVN_DB_NODE: "{{ groups['kube_master'][0] }}"
  152. # [kube-ovn]离线镜像tar包
  153. kube_ovn_ver: "v1.5.3"
  154. kube_ovn_offline: "kube_ovn_{{ kube_ovn_ver }}.tar"
  156. # ------------------------------------------- kube-router
  157. # [kube-router]公有云上存在限制，一般需要始终开启 ipinip；自有环境可以设置为 "subnet"
  158. OVERLAY_TYPE: "full"
  160. # [kube-router]NetworkPolicy 支持开关
  161. FIREWALL_ENABLE: "true"
  163. # [kube-router]kube-router 镜像版本
  164. kube_router_ver: "v0.3.1"
  165. busybox_ver: "1.28.4"
  167. # [kube-router]kube-router 离线镜像tar包
  168. kuberouter_offline: "kube-router_{{ kube_router_ver }}.tar"
  169. busybox_offline: "busybox_{{ busybox_ver }}.tar"
  172. ############################
  173. # role:cluster-addon
  174. ############################
  175. # coredns 自动安装
  176. dns_install: "no"
  177. corednsVer: "1.8.4"
  178. ENABLE_LOCAL_DNS_CACHE: no
  179. dnsNodeCacheVer: "1.17.0"
  180. # 设置 local dns cache 地址
  181. LOCAL_DNS_CACHE: "169.254.20.10"
  183. # metric server 自动安装
  184. metricsserver_install: "no"
  185. metricsVer: "v0.5.0"
  187. # dashboard 自动安装
  188. dashboard_install: "no"
  189. dashboardVer: "v2.3.1"
  190. dashboardMetricsScraperVer: "v1.0.6"
  192. # ingress 自动安装
  193. ingress_install: "no"
  194. ingress_backend: "traefik"
  195. traefik_chart_ver: "9.12.3"
  197. # prometheus 自动安装
  198. prom_install: "no"
  199. prom_namespace: "monitor"
  200. prom_chart_ver: "12.10.6"
  202. # nfs-provisioner 自动安装
  203. nfs_provisioner_install: "no"
  204. nfs_provisioner_namespace: "kube-system"
  205. nfs_provisioner_ver: "v4.0.1"
  206. nfs_storage_class: "managed-nfs-storage"
  207. nfs_server: "192.168.1.10"
  208. nfs_path: "/data/nfs"
  210. ############################
  211. # role:harbor
  212. ############################
  213. # harbor version，完整版本号
  214. HARBOR_VER: "v2.1.3"
  215. HARBOR_DOMAIN: "harbor.yourdomain.com"
  216. HARBOR_TLS_PORT: 8443
  218. # if set 'false', you need to put certs named harbor.pem and harbor-key.pem in directory 'down'
  219. HARBOR_SELF_SIGNED_CERT: true
  221. # install extra component
  222. HARBOR_WITH_NOTARY: false
  223. HARBOR_WITH_TRIVY: false
  224. HARBOR_WITH_CLAIR: false
  225. HARBOR_WITH_CHARTMUSEUM: true

安装
等待以下执行完毕，及安装完成，也可以分开执行，具体步骤可以—help查看

  1. root@master-01:/etc/kubeasz# ./ezctl setup k8s-cluster-01 all