背景

服务器配置

节点 内网IP 公网IP 配置
ren 10.0.4.17 1.15.230.38 4C8G
yan 10.0.4.15 101.34.64.205 4C8G
bai 192.168.0.4 106.12.145.172 2C8G

软件版本

软件 版本
centos 7.6
docker 20.10.7
kubelet 1.20.9
kubeadm 1.20.9
kubectl 1.20.9

镜像版本

镜像 版本
k8s.gcr.io/kube-apiserver 1.20.9
k8s.gcr.io/kube-controller-manager 1.20.9
k8s.gcr.io/kube-scheduler 1.20.9
k8s.gcr.io/kube-proxy 1.20.9
k8s.gcr.io/pause 3.2
k8s.gcr.io/etcd 3.4.13-0
k8s.gcr.io/coredns 1.7.0

创建初始文件夹

  1. #/Users/keyboardone/同步空间/software
  2. mkdir -p /opt/software
  3. cd /opt/software/k8s/
  4. chmod 755 /opt/software/k8s/*.sh

配置ssh免密

ren

ren-ssh.sh

  1. cd /opt/software/k8s/
  2. vi ren-ssh.sh
  1. #修改主机名
  2. sudo hostnamectl set-hostname ren
  3. sudo hostnamectl set-hostname "ren" --pretty
  4. sudo hostnamectl set-hostname ren --static
  5. sudo hostnamectl set-hostname ren --transient
  6. #标识其他主机名
  7. cat > /etc/hosts <<EOF
  8. 127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
  9. ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
  10. 10.0.4.17 ren
  11. 101.34.64.205 yan
  12. 106.12.145.172 bai
  13. EOF
  14. #清空密钥
  15. cd ~/.ssh/
  16. rm -rf *
  17. #用户目录下生成公钥、私钥文件
  18. ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
  19. cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
  20. chmod 600 ~/.ssh/authorized_keys
  21. #分发公钥到其他主机
  22. for ip in yan bai; # 请将此处主机名替换为自己要部署的机器的 hostname
  23. do
  24. ssh-copy-id $ip # 该操作执行过程中需要手动输入用户的密码
  25. done

yan

yan-ssh.sh

  1. cd /opt/software/k8s/
  2. vi yan-ssh.sh
  1. #修改主机名
  2. sudo hostnamectl set-hostname yan
  3. sudo hostnamectl set-hostname "yan" --pretty
  4. sudo hostnamectl set-hostname yan --static
  5. sudo hostnamectl set-hostname yan --transient
  6. #标识其他主机名
  7. cat > /etc/hosts <<EOF
  8. 127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
  9. ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
  10. 1.15.230.38 ren
  11. 10.0.4.15 yan
  12. 106.12.145.172 bai
  13. EOF
  14. #清空密钥
  15. cd ~/.ssh/
  16. rm -rf *
  17. #用户目录下生成公钥、私钥文件
  18. ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
  19. cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
  20. chmod 600 ~/.ssh/authorized_keys
  21. #分发公钥到其他主机
  22. for ip in ren bai; # 请将此处主机名替换为自己要部署的机器的 hostname
  23. do
  24. ssh-copy-id $ip # 该操作执行过程中需要手动输入用户的密码
  25. done

bai

bai-ssh.sh

  1. cd /opt/software/k8s/
  2. vi bai-ssh.sh
  1. #修改主机名
  2. sudo hostnamectl set-hostname bai
  3. sudo hostnamectl set-hostname "bai" --pretty
  4. sudo hostnamectl set-hostname bai --static
  5. sudo hostnamectl set-hostname bai --transient
  6. #标识其他主机名
  7. cat > /etc/hosts <<EOF
  8. 127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
  9. ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
  10. 1.15.230.38 ren
  11. 101.34.64.205 yan
  12. 192.168.0.4 bai
  13. EOF
  14. #清空密钥
  15. cd ~/.ssh/
  16. rm -rf *
  17. #用户目录下生成公钥、私钥文件
  18. ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
  19. cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
  20. chmod 600 ~/.ssh/authorized_keys
  21. #分发公钥到其他主机
  22. for ip in ren yan; # 请将此处主机名替换为自己要部署的机器的 hostname
  23. do
  24. ssh-copy-id $ip # 该操作执行过程中需要手动输入用户的密码
  25. done

文件准备

上传相关文件到 /opt/software/

  1. scp -r /opt/software/ yan:/opt/
  2. scp -r /opt/software/ bai:/opt/

准备集群基础环境

ren

ren-os.sh nohup /opt/software/k8s/ren-os.sh > ren-os.log 2>&1 &

  1. cd /opt/software/k8s/
  2. vi ren-os.sh
  1. #设置重启自动加载模块
  2. modprobe br_netfilter
  3. sysctl -p /etc/sysctl.conf
  4. #查看
  5. lsmod | grep br_netfilter
  6. #永久设置
  7. #新建 rc.sysinit
  8. cat > /etc/rc.sysinit <<EOF
  9. #!/bin/bash
  10. for file in /etc/sysconfig/modules/*.modules ; do
  11. [ -x $file ] && $file
  12. done
  13. EOF
  14. #新建 br_netfilter.modules
  15. cat > /etc/sysconfig/modules/br_netfilter.modules <<EOF
  16. modprobe br_netfilter
  17. EOF
  18. #授权br_netfilter.modules文件执行权限
  19. chmod 755 /etc/sysconfig/modules/br_netfilter.modules
  20. #查看
  21. lsmod |grep br_netfilter
  22. #新建k8s网桥配置文件
  23. cat > /root/k8s.conf <<EOF
  24. #开启网桥模式
  25. net.bridge.bridge-nf-call-ip6tables = 1
  26. net.bridge.bridge-nf-call-iptables = 1
  27. #开启转发
  28. net.ipv4.ip_forward = 1
  29. ##关闭ipv6
  30. net.ipv6.conf.all.disable_ipv6=1
  31. EOF
  32. #拷贝k8s网桥配置文件到系统目录下
  33. cp /root/k8s.conf /etc/sysctl.d/k8s.conf
  34. sysctl -p /etc/sysctl.d/k8s.conf
  35. #设置时区
  36. # 设置系统时区为 中国/上海
  37. timedatectl set-timezone Asia/Shanghai
  38. # 将当前的UTC时间写入硬件时钟
  39. timedatectl set-local-rtc 0
  40. # 重启依赖于系统时间的服务
  41. systemctl restart rsyslog
  42. systemctl restart crond
  43. #关闭邮件服务
  44. systemctl stop postfix && systemctl disable postfix
  45. #设置rsyslogd、systemd、journald
  46. mkdir /var/log/journal # 持久化保存日志的目录
  47. mkdir /etc/systemd/journald.conf.d
  48. #新建 journald 配置文件
  49. cat > /etc/systemd/journald.conf.d/99-prophet.conf <<EOF
  50. [Journal]
  51. # 持久化
  52. Storage=persistent
  53. # 压缩历史日志
  54. Compress=yes
  55. SysnIntervalSec=5m
  56. RateLimitInterval=30s
  57. RateLimitBurst=1000
  58. # 最大占用空间 10G
  59. SystemMaxUse=10G
  60. # 单日志文件最大 200M
  61. SystemMaxFileSize=200M
  62. # 日志保存时间 2 周
  63. MaxRetentionSec=2week
  64. # 不将日志转发到 syslog
  65. ForwardToSyslog=no
  66. EOF
  67. #重启 journald 使生效
  68. systemctl restart systemd-journald
  69. #ipvs前置条件准备
  70. modprobe br_netfilter
  71. #新建 ipvs.modules 配置文件
  72. cat > /etc/sysconfig/modules/ipvs.modules <<EOF
  73. #!/bin/bash
  74. modprobe -- ip_vs
  75. modprobe -- ip_vs_rr
  76. modprobe -- ip_vs_wrr
  77. modprobe -- ip_vs_sh
  78. modprobe -- nf_conntrack_ipv4
  79. EOF
  80. #授权 ipvs.modules 文件执行权限
  81. chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules
  82. #查看已载入系统的模块
  83. lsmod | grep -e ip_vs -e nf_conntrack_ipv4
  84. #关闭swap分区
  85. free -lh
  86. #删除 swap 区所有内容
  87. swapoff -a
  88. free -lh
  89. #开启ipv4
  90. cat /proc/sys/net/ipv4/ip_forward
  91. echo "1" > /proc/sys/net/ipv4/ip_forward
  92. #安装docker
  93. cd /opt/software/docker
  94. tar xzvf docker-20.10.7.tgz
  95. chmod +x docker/*
  96. mv docker/* /usr/local/bin/
  97. #创建docker配置文件
  98. echo '[Unit]
  99. Description=Docker Application Container Engine
  100. Documentation=http://docs.docker.io
  101. After=network.target
  102. [Service]
  103. Environment="PATH=/usr/local/bin:/bin:/sbin:/usr/bin:/usr/sbin"
  104. ExecStart=/usr/local/bin/dockerd -H unix:///var/run/docker.sock -H tcp://0.0.0.0:2375
  105. ExecReload=/bin/kill -s HUP $MAINPID
  106. Restart=always
  107. RestartSec=5
  108. TimeoutSec=0
  109. LimitNOFILE=infinity
  110. LimitNPROC=infinity
  111. LimitCORE=infinity
  112. Delegate=yes
  113. KillMode=process
  114. [Install]
  115. WantedBy=multi-user.target
  116. ' >> /etc/systemd/system/docker.service
  117. #重新加载docker配置文件
  118. cd /usr/local/bin
  119. #重新加载配置文件
  120. systemctl daemon-reload
  121. #设置开机启动
  122. systemctl enable docker.service
  123. #启动
  124. systemctl start docker.service
  125. #重启
  126. systemctl daemon-reload
  127. systemctl restart docker
  128. #等待
  129. sleep 30s
  130. #添加docker源
  131. mkdir -p /etc/docker/
  132. touch /etc/docker/daemon.json
  133. cat > /etc/docker/daemon.json <<EOF
  134. {
  135. "registry-mirrors":["https://docker.mirrors.ustc.edu.cn/"],
  136. "exec-opts": ["native.cgroupdriver=systemd"]
  137. }
  138. EOF
  139. #重启docker
  140. systemctl daemon-reload
  141. systemctl restart docker
  142. #等待
  143. sleep 1m
  144. #查看验证docker
  145. docker info
  146. #等待
  147. sleep 1m
  148. #安装Kubeadm、Kubelet、Kubectl
  149. #添加yum源
  150. cat <<EOF | sudo tee /etc/yum.repos.d/kubernetes.repo
  151. [kubernetes]
  152. name=Kubernetes
  153. baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
  154. enabled=1
  155. gpgcheck=0
  156. repo_gpgcheck=0
  157. gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
  158. http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
  159. exclude=kubelet kubeadm kubectl
  160. EOF
  161. #关闭SELinux
  162. getenforce
  163. sestatus
  164. setenforce 0
  165. getenforce
  166. sestatus
  167. #yum安装kubelet、kubeadm、kubectl
  168. #sudo yum install -y kubelet-1.20.9 kubeadm-1.20.9 kubectl-1.20.9 --disableexcludes=kubernetes
  169. sudo yum install -y kubelet-1.23.3 kubeadm-1.23.3 kubectl-1.23.3 --disableexcludes=kubernetes
  170. #kubelet设置为开机自启
  171. sudo systemctl enable --now kubelet
  172. #检查 kubelet 服务
  173. systemctl status kubelet
  174. #建立虚拟网卡
  175. cat > /etc/sysconfig/network-scripts/ifcfg-eth0:1 <<EOF
  176. BOOTPROTO=static
  177. DEVICE=eth0:1
  178. IPADDR=1.15.230.38
  179. PREFIX=32
  180. TYPE=Ethernet
  181. USERCTL=no
  182. ONBOOT=yes
  183. EOF
  184. #重启网卡,使生效
  185. systemctl restart network
  186. #等待
  187. sleep 30s
  188. ip addr
  189. #重启网卡后,重新开启ipv4
  190. cat /proc/sys/net/ipv4/ip_forward
  191. echo "1" > /proc/sys/net/ipv4/ip_forward
  192. cat /proc/sys/net/ipv4/ip_forward
  193. #修改kubelet启动参数
  194. mkdir -p /usr/lib/systemd/system/kubelet.service.d/
  195. cp /opt/software/k8s/ren-10-kubeadm.conf /usr/lib/systemd/system/kubelet.service.d/
  196. rm -rf /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf
  197. mv /usr/lib/systemd/system/kubelet.service.d/ren-10-kubeadm.conf /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf
  198. #准备kubeadm初始化环境
  199. #编写 kubeadm-config.yaml 文件,准备初始化主节点
  200. cat > /root/kubeadm-config.yaml <<EOF
  201. apiVersion: kubeadm.k8s.io/v1beta2
  202. kind: ClusterConfiguration
  203. kubernetesVersion: v1.23.3
  204. apiServer:
  205. certSANs: #填写所有kube-apiserver节点的hostname、IP、VIP
  206. - ren #替换为hostname
  207. - 1.15.230.38 #替换为公网
  208. - 10.0.4.17 #替换为私网
  209. - 10.96.0.1 #不要替换,此IP是API的集群地址,部分服务会用到
  210. controlPlaneEndpoint: 1.15.230.38:6443 #替换为公网IP
  211. networking:
  212. podSubnet: 10.244.0.0/16
  213. serviceSubnet: 10.96.0.0/12
  214. ---
  215. apiVersion: kubeproxy-config.k8s.io/v1alpha1
  216. kind: KubeProxyConfiguration
  217. featureGates:
  218. SupportIPVSProxyMode: true
  219. mode: ipvs
  220. EOF
  221. #查看要下载的镜像
  222. kubeadm config images list
  223. #等待
  224. sleep 30s
  225. #编写镜像拉取脚本,准备分发各节点执行
  226. cat >/root/pull_k8s_images.sh << "EOF"
  227. # 内容为
  228. set -o errexit
  229. set -o nounset
  230. set -o pipefail
  231. ##这里定义需要下载的版本
  232. #KUBE_VERSION=v1.20.9
  233. #KUBE_PAUSE_VERSION=3.2
  234. #ETCD_VERSION=3.4.13-0
  235. #DNS_VERSION=1.7.0
  236. KUBE_VERSION=v1.23.3
  237. KUBE_PAUSE_VERSION=3.6
  238. ETCD_VERSION=3.5.1-0
  239. DNS_VERSION=1.8.6
  240. ##这是原来被墙的仓库
  241. GCR_URL=k8s.gcr.io
  242. ##这里就是写你要使用的仓库,也可以使用gotok8s
  243. DOCKERHUB_URL=registry.cn-hangzhou.aliyuncs.com/google_containers
  244. ##这里是镜像列表
  245. images=(
  246. kube-proxy:${KUBE_VERSION}
  247. kube-scheduler:${KUBE_VERSION}
  248. kube-controller-manager:${KUBE_VERSION}
  249. kube-apiserver:${KUBE_VERSION}
  250. pause:${KUBE_PAUSE_VERSION}
  251. etcd:${ETCD_VERSION}
  252. coredns:${DNS_VERSION}
  253. )
  254. ## 这里是拉取和改名的循环语句, 先下载, 再tag重命名生成需要的镜像, 再删除下载的镜像
  255. for imageName in ${images[@]} ; do
  256. docker pull $DOCKERHUB_URL/$imageName
  257. docker tag $DOCKERHUB_URL/$imageName $GCR_URL/$imageName
  258. docker rmi $DOCKERHUB_URL/$imageName
  259. done
  260. EOF
  261. #授权 镜像拉取脚本 执行权限
  262. chmod +x /root/pull_k8s_images.sh
  263. #执行 镜像拉取 脚本
  264. bash /root/pull_k8s_images.sh
  265. #验证镜像拉取
  266. docker images

主节点执行 kubeadm ,进行初始化
kubeadm init —config=kubeadm-config.yaml

yan

yan-os.sh nohup /opt/software/k8s/yan-os.sh > yan-os.log 2>&1 &

  1. cd /opt/software/k8s/
  2. vi yan-os.sh
  1. #设置重启自动加载模块
  2. modprobe br_netfilter
  3. sysctl -p /etc/sysctl.conf
  4. #查看
  5. lsmod | grep br_netfilter
  6. #永久设置
  7. #新建 rc.sysinit
  8. cat > /etc/rc.sysinit <<EOF
  9. #!/bin/bash
  10. for file in /etc/sysconfig/modules/*.modules ; do
  11. [ -x $file ] && $file
  12. done
  13. EOF
  14. #新建 br_netfilter.modules
  15. cat > /etc/sysconfig/modules/br_netfilter.modules <<EOF
  16. modprobe br_netfilter
  17. EOF
  18. #授权br_netfilter.modules文件执行权限
  19. chmod 755 /etc/sysconfig/modules/br_netfilter.modules
  20. #查看
  21. lsmod |grep br_netfilter
  22. #新建k8s网桥配置文件
  23. cat > /root/k8s.conf <<EOF
  24. #开启网桥模式
  25. net.bridge.bridge-nf-call-ip6tables = 1
  26. net.bridge.bridge-nf-call-iptables = 1
  27. #开启转发
  28. net.ipv4.ip_forward = 1
  29. ##关闭ipv6
  30. net.ipv6.conf.all.disable_ipv6=1
  31. EOF
  32. #拷贝k8s网桥配置文件到系统目录下
  33. cp /root/k8s.conf /etc/sysctl.d/k8s.conf
  34. sysctl -p /etc/sysctl.d/k8s.conf
  35. #设置时区
  36. # 设置系统时区为 中国/上海
  37. timedatectl set-timezone Asia/Shanghai
  38. # 将当前的UTC时间写入硬件时钟
  39. timedatectl set-local-rtc 0
  40. # 重启依赖于系统时间的服务
  41. systemctl restart rsyslog
  42. systemctl restart crond
  43. #关闭邮件服务
  44. systemctl stop postfix && systemctl disable postfix
  45. #设置rsyslogd、systemd、journald
  46. mkdir /var/log/journal # 持久化保存日志的目录
  47. mkdir /etc/systemd/journald.conf.d
  48. #新建 journald 配置文件
  49. cat > /etc/systemd/journald.conf.d/99-prophet.conf <<EOF
  50. [Journal]
  51. # 持久化
  52. Storage=persistent
  53. # 压缩历史日志
  54. Compress=yes
  55. SysnIntervalSec=5m
  56. RateLimitInterval=30s
  57. RateLimitBurst=1000
  58. # 最大占用空间 10G
  59. SystemMaxUse=10G
  60. # 单日志文件最大 200M
  61. SystemMaxFileSize=200M
  62. # 日志保存时间 2 周
  63. MaxRetentionSec=2week
  64. # 不将日志转发到 syslog
  65. ForwardToSyslog=no
  66. EOF
  67. #重启 journald 使生效
  68. systemctl restart systemd-journald
  69. #ipvs前置条件准备
  70. modprobe br_netfilter
  71. #新建 ipvs.modules 配置文件
  72. cat > /etc/sysconfig/modules/ipvs.modules <<EOF
  73. #!/bin/bash
  74. modprobe -- ip_vs
  75. modprobe -- ip_vs_rr
  76. modprobe -- ip_vs_wrr
  77. modprobe -- ip_vs_sh
  78. modprobe -- nf_conntrack_ipv4
  79. EOF
  80. #授权 ipvs.modules 文件执行权限
  81. chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules
  82. #查看已载入系统的模块
  83. lsmod | grep -e ip_vs -e nf_conntrack_ipv4
  84. #关闭swap分区
  85. free -lh
  86. #删除 swap 区所有内容
  87. swapoff -a
  88. free -lh
  89. #开启ipv4
  90. cat /proc/sys/net/ipv4/ip_forward
  91. echo "1" > /proc/sys/net/ipv4/ip_forward
  92. #安装docker
  93. cd /opt/software/docker
  94. tar xzvf docker-20.10.7.tgz
  95. chmod +x docker/*
  96. mv docker/* /usr/local/bin/
  97. #创建docker配置文件
  98. echo '[Unit]
  99. Description=Docker Application Container Engine
  100. Documentation=http://docs.docker.io
  101. After=network.target
  102. [Service]
  103. Environment="PATH=/usr/local/bin:/bin:/sbin:/usr/bin:/usr/sbin"
  104. ExecStart=/usr/local/bin/dockerd -H unix:///var/run/docker.sock -H tcp://0.0.0.0:2375
  105. ExecReload=/bin/kill -s HUP $MAINPID
  106. Restart=always
  107. RestartSec=5
  108. TimeoutSec=0
  109. LimitNOFILE=infinity
  110. LimitNPROC=infinity
  111. LimitCORE=infinity
  112. Delegate=yes
  113. KillMode=process
  114. [Install]
  115. WantedBy=multi-user.target
  116. ' >> /etc/systemd/system/docker.service
  117. #重新加载docker配置文件
  118. cd /usr/local/bin
  119. #重新加载配置文件
  120. systemctl daemon-reload
  121. #设置开机启动
  122. systemctl enable docker.service
  123. #启动
  124. systemctl start docker.service
  125. #重启
  126. systemctl daemon-reload
  127. systemctl restart docker
  128. #等待
  129. sleep 30s
  130. #添加docker源
  131. mkdir -p /etc/docker/
  132. touch /etc/docker/daemon.json
  133. cat > /etc/docker/daemon.json <<EOF
  134. {
  135. "registry-mirrors":["https://docker.mirrors.ustc.edu.cn/"],
  136. "exec-opts": ["native.cgroupdriver=systemd"]
  137. }
  138. EOF
  139. #重启docker
  140. systemctl daemon-reload
  141. systemctl restart docker
  142. #等待
  143. sleep 1m
  144. #查看验证docker
  145. docker info
  146. #等待
  147. sleep 1m
  148. #安装Kubeadm、Kubelet、Kubectl
  149. #添加yum源
  150. cat <<EOF | sudo tee /etc/yum.repos.d/kubernetes.repo
  151. [kubernetes]
  152. name=Kubernetes
  153. baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
  154. enabled=1
  155. gpgcheck=0
  156. repo_gpgcheck=0
  157. gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
  158. http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
  159. exclude=kubelet kubeadm kubectl
  160. EOF
  161. #关闭SELinux
  162. getenforce
  163. sestatus
  164. setenforce 0
  165. getenforce
  166. sestatus
  167. #yum安装kubelet、kubeadm、kubectl
  168. #sudo yum install -y kubelet-1.20.9 kubeadm-1.20.9 kubectl-1.20.9 --disableexcludes=kubernetes
  169. sudo yum install -y kubelet-1.23.3 kubeadm-1.23.3 kubectl-1.23.3 --disableexcludes=kubernetes
  170. #kubelet设置为开机自启
  171. sudo systemctl enable --now kubelet
  172. #检查 kubelet 服务
  173. systemctl status kubelet
  174. #建立虚拟网卡
  175. cat > /etc/sysconfig/network-scripts/ifcfg-eth0:1 <<EOF
  176. BOOTPROTO=static
  177. DEVICE=eth0:1
  178. IPADDR=101.34.64.205
  179. PREFIX=32
  180. TYPE=Ethernet
  181. USERCTL=no
  182. ONBOOT=yes
  183. EOF
  184. #重启网卡,使生效
  185. systemctl restart network
  186. #等待
  187. sleep 30s
  188. ip addr
  189. #重启网卡后,重新开启ipv4
  190. cat /proc/sys/net/ipv4/ip_forward
  191. echo "1" > /proc/sys/net/ipv4/ip_forward
  192. cat /proc/sys/net/ipv4/ip_forward
  193. #修改kubelet启动参数
  194. mkdir -p /usr/lib/systemd/system/kubelet.service.d/
  195. cp /opt/software/k8s/yan-10-kubeadm.conf /usr/lib/systemd/system/kubelet.service.d/
  196. rm -rf /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf
  197. mv /usr/lib/systemd/system/kubelet.service.d/yan-10-kubeadm.conf /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf
  198. #准备kubeadm初始化环境
  199. #编写 kubeadm-config.yaml 文件,准备初始化主节点
  200. cat > /root/kubeadm-config.yaml <<EOF
  201. apiVersion: kubeadm.k8s.io/v1beta2
  202. kind: ClusterConfiguration
  203. kubernetesVersion: v1.23.3
  204. apiServer:
  205. certSANs: #填写所有kube-apiserver节点的hostname、IP、VIP
  206. - ren #替换为hostname
  207. - 1.15.230.38 #替换为公网
  208. - 10.0.4.17 #替换为私网
  209. - 10.96.0.1 #不要替换,此IP是API的集群地址,部分服务会用到
  210. controlPlaneEndpoint: 1.15.230.38:6443 #替换为公网IP
  211. networking:
  212. podSubnet: 10.244.0.0/16
  213. serviceSubnet: 10.96.0.0/12
  214. --- 将默认调度方式改为ipvs
  215. apiVersion: kubeproxy-config.k8s.io/v1alpha1
  216. kind: KubeProxyConfiguration
  217. featureGates:
  218. SupportIPVSProxyMode: true
  219. mode: ipvs
  220. EOF
  221. #查看要下载的镜像
  222. kubeadm config images list
  223. #等待
  224. sleep 30s
  225. #编写镜像拉取脚本,准备分发各节点执行
  226. cat >/root/pull_k8s_images.sh << "EOF"
  227. # 内容为
  228. set -o errexit
  229. set -o nounset
  230. set -o pipefail
  231. ##这里定义需要下载的版本
  232. #KUBE_VERSION=v1.20.9
  233. #KUBE_PAUSE_VERSION=3.2
  234. #ETCD_VERSION=3.4.13-0
  235. #DNS_VERSION=1.7.0
  236. KUBE_VERSION=v1.23.13
  237. KUBE_PAUSE_VERSION=3.6
  238. ETCD_VERSION=3.5.1-0
  239. DNS_VERSION=1.8.6
  240. ##这是原来被墙的仓库
  241. GCR_URL=k8s.gcr.io
  242. ##这里就是写你要使用的仓库,也可以使用gotok8s
  243. DOCKERHUB_URL=registry.cn-hangzhou.aliyuncs.com/google_containers
  244. ##这里是镜像列表
  245. images=(
  246. kube-proxy:${KUBE_VERSION}
  247. kube-scheduler:${KUBE_VERSION}
  248. kube-controller-manager:${KUBE_VERSION}
  249. kube-apiserver:${KUBE_VERSION}
  250. pause:${KUBE_PAUSE_VERSION}
  251. etcd:${ETCD_VERSION}
  252. coredns:${DNS_VERSION}
  253. )
  254. ## 这里是拉取和改名的循环语句, 先下载, 再tag重命名生成需要的镜像, 再删除下载的镜像
  255. for imageName in ${images[@]} ; do
  256. docker pull $DOCKERHUB_URL/$imageName
  257. docker tag $DOCKERHUB_URL/$imageName $GCR_URL/$imageName
  258. docker rmi $DOCKERHUB_URL/$imageName
  259. done
  260. EOF
  261. #授权 镜像拉取脚本 执行权限
  262. chmod +x /root/pull_k8s_images.sh
  263. #执行 镜像拉取 脚本
  264. bash /root/pull_k8s_images.sh
  265. #验证镜像拉取
  266. docker images

bai

bai-os.sh nohup /opt/software/k8s/bai-os.sh > bai-os.log 2>&1 &

  1. cd /opt/software/k8s/
  2. vi bai-os.sh
  1. #设置重启自动加载模块
  2. modprobe br_netfilter
  3. sysctl -p /etc/sysctl.conf
  4. #查看
  5. lsmod | grep br_netfilter
  6. #永久设置
  7. #新建 rc.sysinit
  8. cat > /etc/rc.sysinit <<EOF
  9. #!/bin/bash
  10. for file in /etc/sysconfig/modules/*.modules ; do
  11. [ -x $file ] && $file
  12. done
  13. EOF
  14. #新建 br_netfilter.modules
  15. cat > /etc/sysconfig/modules/br_netfilter.modules <<EOF
  16. modprobe br_netfilter
  17. EOF
  18. #授权br_netfilter.modules文件执行权限
  19. chmod 755 /etc/sysconfig/modules/br_netfilter.modules
  20. #查看
  21. lsmod |grep br_netfilter
  22. #新建k8s网桥配置文件
  23. cat > /root/k8s.conf <<EOF
  24. #开启网桥模式
  25. net.bridge.bridge-nf-call-ip6tables = 1
  26. net.bridge.bridge-nf-call-iptables = 1
  27. #开启转发
  28. net.ipv4.ip_forward = 1
  29. ##关闭ipv6
  30. net.ipv6.conf.all.disable_ipv6=1
  31. EOF
  32. #拷贝k8s网桥配置文件到系统目录下
  33. cp /root/k8s.conf /etc/sysctl.d/k8s.conf
  34. sysctl -p /etc/sysctl.d/k8s.conf
  35. #设置时区
  36. # 设置系统时区为 中国/上海
  37. timedatectl set-timezone Asia/Shanghai
  38. # 将当前的UTC时间写入硬件时钟
  39. timedatectl set-local-rtc 0
  40. # 重启依赖于系统时间的服务
  41. systemctl restart rsyslog
  42. systemctl restart crond
  43. #关闭邮件服务
  44. systemctl stop postfix && systemctl disable postfix
  45. #设置rsyslogd、systemd、journald
  46. mkdir /var/log/journal # 持久化保存日志的目录
  47. mkdir /etc/systemd/journald.conf.d
  48. #新建 journald 配置文件
  49. cat > /etc/systemd/journald.conf.d/99-prophet.conf <<EOF
  50. [Journal]
  51. # 持久化
  52. Storage=persistent
  53. # 压缩历史日志
  54. Compress=yes
  55. SysnIntervalSec=5m
  56. RateLimitInterval=30s
  57. RateLimitBurst=1000
  58. # 最大占用空间 10G
  59. SystemMaxUse=10G
  60. # 单日志文件最大 200M
  61. SystemMaxFileSize=200M
  62. # 日志保存时间 2 周
  63. MaxRetentionSec=2week
  64. # 不将日志转发到 syslog
  65. ForwardToSyslog=no
  66. EOF
  67. #重启 journald 使生效
  68. systemctl restart systemd-journald
  69. #ipvs前置条件准备
  70. modprobe br_netfilter
  71. #新建 ipvs.modules 配置文件
  72. cat > /etc/sysconfig/modules/ipvs.modules <<EOF
  73. #!/bin/bash
  74. modprobe -- ip_vs
  75. modprobe -- ip_vs_rr
  76. modprobe -- ip_vs_wrr
  77. modprobe -- ip_vs_sh
  78. modprobe -- nf_conntrack_ipv4
  79. EOF
  80. #授权 ipvs.modules 文件执行权限
  81. chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules
  82. #查看已载入系统的模块
  83. lsmod | grep -e ip_vs -e nf_conntrack_ipv4
  84. #关闭swap分区
  85. free -lh
  86. #删除 swap 区所有内容
  87. swapoff -a
  88. free -lh
  89. #开启ipv4
  90. cat /proc/sys/net/ipv4/ip_forward
  91. echo "1" > /proc/sys/net/ipv4/ip_forward
  92. #安装docker
  93. cd /opt/software/docker
  94. tar xzvf docker-20.10.7.tgz
  95. chmod +x docker/*
  96. mv docker/* /usr/local/bin/
  97. #创建docker配置文件
  98. echo '[Unit]
  99. Description=Docker Application Container Engine
  100. Documentation=http://docs.docker.io
  101. After=network.target
  102. [Service]
  103. Environment="PATH=/usr/local/bin:/bin:/sbin:/usr/bin:/usr/sbin"
  104. ExecStart=/usr/local/bin/dockerd -H unix:///var/run/docker.sock -H tcp://0.0.0.0:2375
  105. ExecReload=/bin/kill -s HUP $MAINPID
  106. Restart=always
  107. RestartSec=5
  108. TimeoutSec=0
  109. LimitNOFILE=infinity
  110. LimitNPROC=infinity
  111. LimitCORE=infinity
  112. Delegate=yes
  113. KillMode=process
  114. [Install]
  115. WantedBy=multi-user.target
  116. ' >> /etc/systemd/system/docker.service
  117. #重新加载docker配置文件
  118. cd /usr/local/bin
  119. #重新加载配置文件
  120. systemctl daemon-reload
  121. #设置开机启动
  122. systemctl enable docker.service
  123. #启动
  124. systemctl start docker.service
  125. #重启
  126. systemctl daemon-reload
  127. systemctl restart docker
  128. #等待
  129. sleep 30s
  130. #添加docker源
  131. mkdir -p /etc/docker/
  132. touch /etc/docker/daemon.json
  133. cat > /etc/docker/daemon.json <<EOF
  134. {
  135. "registry-mirrors":["https://docker.mirrors.ustc.edu.cn/"],
  136. "exec-opts": ["native.cgroupdriver=systemd"]
  137. }
  138. EOF
  139. #重启docker
  140. systemctl daemon-reload
  141. systemctl restart docker
  142. #等待
  143. sleep 1m
  144. #查看验证docker
  145. docker info
  146. #等待
  147. sleep 1m
  148. #安装Kubeadm、Kubelet、Kubectl
  149. #添加yum源
  150. cat <<EOF | sudo tee /etc/yum.repos.d/kubernetes.repo
  151. [kubernetes]
  152. name=Kubernetes
  153. baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
  154. enabled=1
  155. gpgcheck=0
  156. repo_gpgcheck=0
  157. gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
  158. http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
  159. exclude=kubelet kubeadm kubectl
  160. EOF
  161. #关闭SELinux
  162. getenforce
  163. sestatus
  164. setenforce 0
  165. getenforce
  166. sestatus
  167. #yum安装kubelet、kubeadm、kubectl
  168. #sudo yum install -y kubelet-1.20.9 kubeadm-1.20.9 kubectl-1.20.9 --disableexcludes=kubernetes
  169. sudo yum install -y kubelet-1.23.3 kubeadm-1.23.3 kubectl-1.23.3 --disableexcludes=kubernetes
  170. #kubelet设置为开机自启
  171. sudo systemctl enable --now kubelet
  172. #检查 kubelet 服务
  173. systemctl status kubelet
  174. #建立虚拟网卡
  175. cat > /etc/sysconfig/network-scripts/ifcfg-eth0:1 <<EOF
  176. BOOTPROTO=static
  177. DEVICE=eth0:1
  178. IPADDR=106.12.145.172
  179. PREFIX=32
  180. TYPE=Ethernet
  181. USERCTL=no
  182. ONBOOT=yes
  183. EOF
  184. #重启网卡,使生效
  185. systemctl restart network
  186. #等待
  187. sleep 30s
  188. ip addr
  189. #重启网卡后,重新开启ipv4
  190. cat /proc/sys/net/ipv4/ip_forward
  191. echo "1" > /proc/sys/net/ipv4/ip_forward
  192. cat /proc/sys/net/ipv4/ip_forward
  193. #修改kubelet启动参数
  194. mkdir -p /usr/lib/systemd/system/kubelet.service.d/
  195. cp /opt/software/k8s/bai-10-kubeadm.conf /usr/lib/systemd/system/kubelet.service.d/
  196. rm -rf /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf
  197. mv /usr/lib/systemd/system/kubelet.service.d/bai-10-kubeadm.conf /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf
  198. #准备kubeadm初始化环境
  199. #编写 kubeadm-config.yaml 文件,准备初始化主节点
  200. cat > /root/kubeadm-config.yaml <<EOF
  201. apiVersion: kubeadm.k8s.io/v1beta2
  202. kind: ClusterConfiguration
  203. kubernetesVersion: v1.23.3
  204. apiServer:
  205. certSANs: #填写所有kube-apiserver节点的hostname、IP、VIP
  206. - ren #替换为hostname
  207. - 1.15.230.38 #替换为公网
  208. - 10.0.4.17 #替换为私网
  209. - 10.96.0.1 #不要替换,此IP是API的集群地址,部分服务会用到
  210. controlPlaneEndpoint: 1.15.230.38:6443 #替换为公网IP
  211. networking:
  212. podSubnet: 10.244.0.0/16
  213. serviceSubnet: 10.96.0.0/12
  214. --- 将默认调度方式改为ipvs
  215. apiVersion: kubeproxy-config.k8s.io/v1alpha1
  216. kind: KubeProxyConfiguration
  217. featureGates:
  218. SupportIPVSProxyMode: true
  219. mode: ipvs
  220. EOF
  221. #查看要下载的镜像
  222. kubeadm config images list
  223. #等待
  224. sleep 30s
  225. #编写镜像拉取脚本,准备分发各节点执行
  226. cat >/root/pull_k8s_images.sh << "EOF"
  227. # 内容为
  228. set -o errexit
  229. set -o nounset
  230. set -o pipefail
  231. ##这里定义需要下载的版本
  232. #KUBE_VERSION=v1.20.9
  233. #KUBE_PAUSE_VERSION=3.2
  234. #ETCD_VERSION=3.4.13-0
  235. #DNS_VERSION=1.7.0
  236. KUBE_VERSION=v1.23.13
  237. KUBE_PAUSE_VERSION=3.6
  238. ETCD_VERSION=3.5.1-0
  239. DNS_VERSION=1.8.6
  240. ##这是原来被墙的仓库
  241. GCR_URL=k8s.gcr.io
  242. ##这里就是写你要使用的仓库,也可以使用gotok8s
  243. DOCKERHUB_URL=registry.cn-hangzhou.aliyuncs.com/google_containers
  244. ##这里是镜像列表
  245. images=(
  246. kube-proxy:${KUBE_VERSION}
  247. kube-scheduler:${KUBE_VERSION}
  248. kube-controller-manager:${KUBE_VERSION}
  249. kube-apiserver:${KUBE_VERSION}
  250. pause:${KUBE_PAUSE_VERSION}
  251. etcd:${ETCD_VERSION}
  252. coredns:${DNS_VERSION}
  253. )
  254. ## 这里是拉取和改名的循环语句, 先下载, 再tag重命名生成需要的镜像, 再删除下载的镜像
  255. for imageName in ${images[@]} ; do
  256. docker pull $DOCKERHUB_URL/$imageName
  257. docker tag $DOCKERHUB_URL/$imageName $GCR_URL/$imageName
  258. docker rmi $DOCKERHUB_URL/$imageName
  259. done
  260. EOF
  261. #授权 镜像拉取脚本 执行权限
  262. chmod +x /root/pull_k8s_images.sh
  263. #执行 镜像拉取 脚本
  264. bash /root/pull_k8s_images.sh
  265. #验证镜像拉取
  266. docker images

节点初始化

ren

ren-init.sh nohup /opt/software/k8s/ren-init.sh > ren-init.log 2>&1 &

  1. cd /opt/software/k8s/
  2. vi ren-init.sh
  1. systemctl status kubelet
  2. #Warning: kubelet.service changed on disk. Run 'systemctl daemon-reload' to reload units.
  3. systemctl daemon-reload
  4. #等待
  5. sleep 30s
  6. #初始化
  7. kubeadm init --config=/root/kubeadm-config.yaml
  8. #等待
  9. sleep 3m
  10. mkdir -p $HOME/.kube
  11. sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  12. sudo chown $(id -u):$(id -g) $HOME/.kube/config
  13. export KUBECONFIG=/etc/kubernetes/admin.conf
  14. #修改kube-apiserver参数
  15. cp /opt/software/k8s/kube-apiserver.yaml /etc/kubernetes/manifests/

yan

yan-join.sh nohup /opt/software/k8s/yan-join.sh > yan-join.log 2>&1 &

  1. systemctl status kubelet
  2. #Warning: kubelet.service changed on disk. Run 'systemctl daemon-reload' to reload units.
  3. systemctl daemon-reload
  4. #worker节点加入集群
  5. kubeadm join 1.15.230.38:6443 --token uifpif.fyr2s4f5gtemqgnn \
  6. --discovery-token-ca-cert-hash sha256:80accd0bf78574cd8e0df8b3d276e2a8c1453277b510eb02507f8e5a0675676e

bai

bai-join.sh nohup /opt/software/k8s/bai-join.sh > bai-join.log 2>&1 &

  1. systemctl status kubelet
  2. #Warning: kubelet.service changed on disk. Run 'systemctl daemon-reload' to reload units.
  3. systemctl daemon-reload
  4. #worker节点加入集群
  5. kubeadm join 1.15.230.38:6443 --token uifpif.fyr2s4f5gtemqgnn \
  6. --discovery-token-ca-cert-hash sha256:80accd0bf78574cd8e0df8b3d276e2a8c1453277b510eb02507f8e5a0675676e

查看初始化进度

  1. #监听应用启动情况
  2. kubectl get pod -A -w
  3. #或者
  4. watch -n 1 kubectl get pod -A
  5. #检查各节点连接状态
  6. kubectl get pods -o wide --all-namespaces
  7. #或者
  8. watch -n 1 kubectl get pods -o wide --all-namespaces

安装网络插件 flannel

ren

ren-flannel.sh nohup /opt/software/k8s/ren-flannel.sh > ren-flannel.log 2>&1 &

  1. cd /opt/software/k8s/
  2. vi ren-flannel.sh
  1. kubectl get pod -A
  2. #等待
  3. sleep 3m
  4. kubectl get pod -A
  5. kubectl apply -f /opt/software/k8s/kube-flannel.yml
  6. #等待
  7. sleep 11m
  8. kubectl get pod -A

检查网络是否连通

  1. # 检查pod是否都是ready状态
  2. kubectl get pods -o wide --all-namespaces
  3. ...
  4. # 手动创建一个pod
  5. kubectl create deployment nginx --image=nginx
  6. # 查看pod的ip
  7. kubectl get pods -o wide
  8. # 主节点或其它节点,ping一下此ip,看看是否能ping通