背景

服务器配置

节点 内网IP 公网IP 配置
ren 10.0.4.17 1.15.230.38 4C8G
yan 10.0.4.15 101.34.64.205 4C8G
bai 192.168.0.4 106.12.145.172 2C8G

软件版本

软件 版本
centos 7.6
docker 20.10.7
kubelet 1.20.9
kubeadm 1.20.9
kubectl 1.20.9

镜像版本

镜像 版本
k8s.gcr.io/kube-apiserver 1.20.9
k8s.gcr.io/kube-controller-manager 1.20.9
k8s.gcr.io/kube-scheduler 1.20.9
k8s.gcr.io/kube-proxy 1.20.9
k8s.gcr.io/pause 3.2
k8s.gcr.io/etcd 3.4.13-0
k8s.gcr.io/coredns 1.7.0

创建初始文件夹

  1. #/Users/keyboardone/同步空间/software
  2. mkdir -p /opt/software
  3. cd /opt/software/k8s/
  4. chmod 755 /opt/software/k8s/*.sh

配置ssh免密

ren

ren-ssh.sh

  1. cd /opt/software/k8s/
  2. vi ren-ssh.sh
  1. #修改主机名
  2. sudo hostnamectl set-hostname ren
  3. sudo hostnamectl set-hostname "ren" --pretty
  4. sudo hostnamectl set-hostname ren --static
  5. sudo hostnamectl set-hostname ren --transient
  6. #标识其他主机名
  7. cat > /etc/hosts <<EOF
  8. 127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
  9. ::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
  10. 10.0.4.17 ren
  11. 101.34.64.205 yan
  12. 106.12.145.172 bai
  13. EOF
  14. #清空密钥
  15. cd ~/.ssh/
  16. rm -rf *
  17. #用户目录下生成公钥、私钥文件
  18. ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
  19. cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
  20. chmod 600 ~/.ssh/authorized_keys
  21. #分发公钥到其他主机
  22. for ip in yan bai;     # 请将此处主机名替换为自己要部署的机器的 hostname
  23. do
  24. ssh-copy-id  $ip   # 该操作执行过程中需要手动输入用户的密码
  25. done

yan

yan-ssh.sh

  1. cd /opt/software/k8s/
  2. vi yan-ssh.sh
  1. #修改主机名
  2. sudo hostnamectl set-hostname yan
  3. sudo hostnamectl set-hostname "yan" --pretty
  4. sudo hostnamectl set-hostname yan --static
  5. sudo hostnamectl set-hostname yan --transient
  6. #标识其他主机名
  7. cat > /etc/hosts <<EOF
  8. 127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
  9. ::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
  10. 1.15.230.38 ren
  11. 10.0.4.15 yan
  12. 106.12.145.172 bai
  13. EOF
  14. #清空密钥
  15. cd ~/.ssh/
  16. rm -rf *
  17. #用户目录下生成公钥、私钥文件
  18. ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
  19. cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
  20. chmod 600 ~/.ssh/authorized_keys
  21. #分发公钥到其他主机
  22. for ip in ren bai;     # 请将此处主机名替换为自己要部署的机器的 hostname
  23. do
  24. ssh-copy-id  $ip   # 该操作执行过程中需要手动输入用户的密码
  25. done

bai

bai-ssh.sh

  1. cd /opt/software/k8s/
  2. vi bai-ssh.sh
  1. #修改主机名
  2. sudo hostnamectl set-hostname bai
  3. sudo hostnamectl set-hostname "bai" --pretty
  4. sudo hostnamectl set-hostname bai --static
  5. sudo hostnamectl set-hostname bai --transient
  6. #标识其他主机名
  7. cat > /etc/hosts <<EOF
  8. 127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
  9. ::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
  10. 1.15.230.38 ren
  11. 101.34.64.205 yan
  12. 192.168.0.4 bai
  13. EOF
  14. #清空密钥
  15. cd ~/.ssh/
  16. rm -rf *
  17. #用户目录下生成公钥、私钥文件
  18. ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
  19. cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
  20. chmod 600 ~/.ssh/authorized_keys
  21. #分发公钥到其他主机
  22. for ip in ren yan;     # 请将此处主机名替换为自己要部署的机器的 hostname
  23. do
  24. ssh-copy-id  $ip   # 该操作执行过程中需要手动输入用户的密码
  25. done

文件准备

上传相关文件到 /opt/software/

  1. scp -r /opt/software/ yan:/opt/
  3. scp -r /opt/software/ bai:/opt/

准备集群基础环境

ren

ren-os.sh nohup /opt/software/k8s/ren-os.sh > ren-os.log 2>&1 &

  1. cd /opt/software/k8s/
  2. vi ren-os.sh
  1. #设置重启自动加载模块
  2. modprobe br_netfilter
  3. sysctl -p /etc/sysctl.conf
  5. #查看
  6. lsmod | grep br_netfilter
  8. #永久设置
  9. #新建 rc.sysinit
  10. cat > /etc/rc.sysinit <<EOF
  11. #!/bin/bash
  12. for file in /etc/sysconfig/modules/*.modules ; do
  13. [ -x $file ] && $file
  14. done
  15. EOF
  17. #新建 br_netfilter.modules
  18. cat > /etc/sysconfig/modules/br_netfilter.modules <<EOF
  19. modprobe br_netfilter
  20. EOF
  22. #授权br_netfilter.modules文件执行权限
  23. chmod 755 /etc/sysconfig/modules/br_netfilter.modules
  25. #查看
  26. lsmod |grep br_netfilter
  28. #新建k8s网桥配置文件
  29. cat > /root/k8s.conf <<EOF
  30. #开启网桥模式
  31. net.bridge.bridge-nf-call-ip6tables = 1
  32. net.bridge.bridge-nf-call-iptables = 1
  33. #开启转发
  34. net.ipv4.ip_forward = 1
  35. ##关闭ipv6
  36. net.ipv6.conf.all.disable_ipv6=1
  37. EOF
  39. #拷贝k8s网桥配置文件到系统目录下
  40. cp /root/k8s.conf /etc/sysctl.d/k8s.conf
  41. sysctl -p /etc/sysctl.d/k8s.conf
  43. #设置时区
  44. # 设置系统时区为 中国/上海
  45. timedatectl set-timezone Asia/Shanghai
  46. # 将当前的UTC时间写入硬件时钟
  47. timedatectl set-local-rtc 0
  48. # 重启依赖于系统时间的服务
  49. systemctl restart rsyslog
  50. systemctl restart crond
  52. #关闭邮件服务
  53. systemctl stop postfix && systemctl disable postfix
  55. #设置rsyslogd、systemd、journald
  56. mkdir /var/log/journal # 持久化保存日志的目录
  57. mkdir /etc/systemd/journald.conf.d
  59. #新建 journald 配置文件
  60. cat > /etc/systemd/journald.conf.d/99-prophet.conf <<EOF
  61. [Journal]
  62. # 持久化
  63. Storage=persistent
  64. # 压缩历史日志
  65. Compress=yes
  66. SysnIntervalSec=5m
  67. RateLimitInterval=30s
  68. RateLimitBurst=1000
  69. # 最大占用空间 10G
  70. SystemMaxUse=10G
  71. # 单日志文件最大 200M
  72. SystemMaxFileSize=200M
  73. # 日志保存时间 2 周
  74. MaxRetentionSec=2week
  75. # 不将日志转发到 syslog
  76. ForwardToSyslog=no
  77. EOF
  79. #重启 journald 使生效
  80. systemctl restart systemd-journald
  82. #ipvs前置条件准备
  83. modprobe br_netfilter
  84. #新建 ipvs.modules 配置文件
  85. cat > /etc/sysconfig/modules/ipvs.modules <<EOF
  86. #!/bin/bash
  87. modprobe -- ip_vs
  88. modprobe -- ip_vs_rr
  89. modprobe -- ip_vs_wrr
  90. modprobe -- ip_vs_sh
  91. modprobe -- nf_conntrack_ipv4
  92. EOF
  94. #授权 ipvs.modules 文件执行权限
  95. chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules
  97. #查看已载入系统的模块
  98. lsmod | grep -e ip_vs -e nf_conntrack_ipv4
  100. #关闭swap分区
  101. free -lh
  102. #删除 swap 区所有内容
  103. swapoff -a
  104. free -lh
  106. #开启ipv4
  107. cat /proc/sys/net/ipv4/ip_forward
  108. echo "1" > /proc/sys/net/ipv4/ip_forward
  110. #安装docker
  111. cd /opt/software/docker
  112. tar xzvf docker-20.10.7.tgz
  113. chmod +x docker/*
  114. mv docker/* /usr/local/bin/
  116. #创建docker配置文件
  117. echo '[Unit]
  118. Description=Docker Application Container Engine
  119. Documentation=http://docs.docker.io
  120. After=network.target
  121. [Service]
  122. Environment="PATH=/usr/local/bin:/bin:/sbin:/usr/bin:/usr/sbin"
  123. ExecStart=/usr/local/bin/dockerd -H unix:///var/run/docker.sock -H tcp://0.0.0.0:2375
  124. ExecReload=/bin/kill -s HUP $MAINPID
  125. Restart=always
  126. RestartSec=5
  127. TimeoutSec=0
  128. LimitNOFILE=infinity
  129. LimitNPROC=infinity
  130. LimitCORE=infinity
  131. Delegate=yes
  132. KillMode=process
  133. [Install]
  134. WantedBy=multi-user.target
  135. ' >> /etc/systemd/system/docker.service
  137. #重新加载docker配置文件
  138. cd /usr/local/bin
  139. #重新加载配置文件
  140. systemctl daemon-reload
  141. #设置开机启动
  142. systemctl enable docker.service
  143. #启动
  144. systemctl start docker.service
  145. #重启
  146. systemctl daemon-reload
  147. systemctl restart docker
  149. #等待
  150. sleep 30s
  152. #添加docker源
  153. mkdir -p /etc/docker/
  154. touch /etc/docker/daemon.json
  155. cat > /etc/docker/daemon.json <<EOF
  156. {
  157. "registry-mirrors":["https://docker.mirrors.ustc.edu.cn/"],
  158. "exec-opts": ["native.cgroupdriver=systemd"]
  159. }
  160. EOF
  162. #重启docker
  163. systemctl daemon-reload
  164. systemctl restart docker
  166. #等待
  167. sleep 1m
  169. #查看验证docker
  170. docker info
  172. #等待
  173. sleep 1m
  175. #安装Kubeadm、Kubelet、Kubectl
  176. #添加yum源
  177. cat <<EOF | sudo tee /etc/yum.repos.d/kubernetes.repo
  178. [kubernetes]
  179. name=Kubernetes
  180. baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
  181. enabled=1
  182. gpgcheck=0
  183. repo_gpgcheck=0
  184. gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
  185.    http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
  186. exclude=kubelet kubeadm kubectl
  187. EOF
  189. #关闭SELinux
  190. getenforce
  191. sestatus
  192. setenforce 0
  193. getenforce
  194. sestatus
  196. #yum安装kubelet、kubeadm、kubectl
  197. #sudo yum install -y kubelet-1.20.9 kubeadm-1.20.9 kubectl-1.20.9 --disableexcludes=kubernetes
  198. sudo yum install -y kubelet-1.23.3 kubeadm-1.23.3 kubectl-1.23.3 --disableexcludes=kubernetes
  200. #kubelet设置为开机自启
  201. sudo systemctl enable --now kubelet
  203. #检查 kubelet 服务
  204. systemctl status kubelet
  206. #建立虚拟网卡
  207. cat > /etc/sysconfig/network-scripts/ifcfg-eth0:1 <<EOF
  208. BOOTPROTO=static
  209. DEVICE=eth0:1
  210. IPADDR=1.15.230.38
  211. PREFIX=32
  212. TYPE=Ethernet
  213. USERCTL=no
  214. ONBOOT=yes
  215. EOF
  217. #重启网卡,使生效
  218. systemctl restart network
  220. #等待
  221. sleep 30s
  223. ip addr
  225. #重启网卡后,重新开启ipv4
  226. cat /proc/sys/net/ipv4/ip_forward
  227. echo "1" > /proc/sys/net/ipv4/ip_forward
  228. cat /proc/sys/net/ipv4/ip_forward
  230. #修改kubelet启动参数
  231. mkdir -p /usr/lib/systemd/system/kubelet.service.d/
  232. cp /opt/software/k8s/ren-10-kubeadm.conf /usr/lib/systemd/system/kubelet.service.d/
  233. rm -rf /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf
  234. mv /usr/lib/systemd/system/kubelet.service.d/ren-10-kubeadm.conf /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf
  236. #准备kubeadm初始化环境
  237. #编写 kubeadm-config.yaml 文件，准备初始化主节点
  238. cat > /root/kubeadm-config.yaml <<EOF
  239. apiVersion: kubeadm.k8s.io/v1beta2
  240. kind: ClusterConfiguration
  241. kubernetesVersion: v1.23.3
  242. apiServer:
  243.   certSANs:    #填写所有kube-apiserver节点的hostname、IP、VIP
  244.   - ren    #替换为hostname
  245.   - 1.15.230.38   #替换为公网
  246.   - 10.0.4.17  #替换为私网
  247.   - 10.96.0.1   #不要替换，此IP是API的集群地址，部分服务会用到
  248. controlPlaneEndpoint: 1.15.230.38:6443 #替换为公网IP
  249. networking:
  250.   podSubnet: 10.244.0.0/16
  251.   serviceSubnet: 10.96.0.0/12
  252. ---
  253. apiVersion: kubeproxy-config.k8s.io/v1alpha1
  254. kind: KubeProxyConfiguration
  255. featureGates:
  256.   SupportIPVSProxyMode: true
  257. mode: ipvs
  258. EOF
  260. #查看要下载的镜像
  261. kubeadm config images list
  263. #等待
  264. sleep 30s
  266. #编写镜像拉取脚本，准备分发各节点执行
  267. cat >/root/pull_k8s_images.sh << "EOF"
  268. # 内容为
  269. set -o errexit
  270. set -o nounset
  271. set -o pipefail
  273. ##这里定义需要下载的版本
  274. #KUBE_VERSION=v1.20.9
  275. #KUBE_PAUSE_VERSION=3.2
  276. #ETCD_VERSION=3.4.13-0
  277. #DNS_VERSION=1.7.0
  278. KUBE_VERSION=v1.23.3
  279. KUBE_PAUSE_VERSION=3.6
  280. ETCD_VERSION=3.5.1-0
  281. DNS_VERSION=1.8.6
  283. ##这是原来被墙的仓库
  284. GCR_URL=k8s.gcr.io
  286. ##这里就是写你要使用的仓库,也可以使用gotok8s
  287. DOCKERHUB_URL=registry.cn-hangzhou.aliyuncs.com/google_containers
  289. ##这里是镜像列表
  290. images=(
  291. kube-proxy:${KUBE_VERSION}
  292. kube-scheduler:${KUBE_VERSION}
  293. kube-controller-manager:${KUBE_VERSION}
  294. kube-apiserver:${KUBE_VERSION}
  295. pause:${KUBE_PAUSE_VERSION}
  296. etcd:${ETCD_VERSION}
  297. coredns:${DNS_VERSION}
  298. )
  300. ## 这里是拉取和改名的循环语句, 先下载, 再tag重命名生成需要的镜像, 再删除下载的镜像
  301. for imageName in ${images[@]} ; do
  302.   docker pull $DOCKERHUB_URL/$imageName
  303.   docker tag $DOCKERHUB_URL/$imageName $GCR_URL/$imageName
  304.   docker rmi $DOCKERHUB_URL/$imageName
  305. done
  306. EOF
  308. #授权 镜像拉取脚本 执行权限
  309. chmod +x /root/pull_k8s_images.sh
  311. #执行 镜像拉取 脚本
  312. bash /root/pull_k8s_images.sh
  314. #验证镜像拉取
  315. docker images

主节点执行 kubeadm ，进行初始化
kubeadm init —config=kubeadm-config.yaml

yan

yan-os.sh nohup /opt/software/k8s/yan-os.sh > yan-os.log 2>&1 &

  1. cd /opt/software/k8s/
  2. vi yan-os.sh
  1. #设置重启自动加载模块
  2. modprobe br_netfilter
  3. sysctl -p /etc/sysctl.conf
  5. #查看
  6. lsmod | grep br_netfilter
  8. #永久设置
  9. #新建 rc.sysinit
  10. cat > /etc/rc.sysinit <<EOF
  11. #!/bin/bash
  12. for file in /etc/sysconfig/modules/*.modules ; do
  13. [ -x $file ] && $file
  14. done
  15. EOF
  17. #新建 br_netfilter.modules
  18. cat > /etc/sysconfig/modules/br_netfilter.modules <<EOF
  19. modprobe br_netfilter
  20. EOF
  22. #授权br_netfilter.modules文件执行权限
  23. chmod 755 /etc/sysconfig/modules/br_netfilter.modules
  25. #查看
  26. lsmod |grep br_netfilter
  28. #新建k8s网桥配置文件
  29. cat > /root/k8s.conf <<EOF
  30. #开启网桥模式
  31. net.bridge.bridge-nf-call-ip6tables = 1
  32. net.bridge.bridge-nf-call-iptables = 1
  33. #开启转发
  34. net.ipv4.ip_forward = 1
  35. ##关闭ipv6
  36. net.ipv6.conf.all.disable_ipv6=1
  37. EOF
  39. #拷贝k8s网桥配置文件到系统目录下
  40. cp /root/k8s.conf /etc/sysctl.d/k8s.conf
  41. sysctl -p /etc/sysctl.d/k8s.conf
  43. #设置时区
  44. # 设置系统时区为 中国/上海
  45. timedatectl set-timezone Asia/Shanghai
  46. # 将当前的UTC时间写入硬件时钟
  47. timedatectl set-local-rtc 0
  48. # 重启依赖于系统时间的服务
  49. systemctl restart rsyslog
  50. systemctl restart crond
  52. #关闭邮件服务
  53. systemctl stop postfix && systemctl disable postfix
  55. #设置rsyslogd、systemd、journald
  56. mkdir /var/log/journal # 持久化保存日志的目录
  57. mkdir /etc/systemd/journald.conf.d
  59. #新建 journald 配置文件
  60. cat > /etc/systemd/journald.conf.d/99-prophet.conf <<EOF
  61. [Journal]
  62. # 持久化
  63. Storage=persistent
  64. # 压缩历史日志
  65. Compress=yes
  66. SysnIntervalSec=5m
  67. RateLimitInterval=30s
  68. RateLimitBurst=1000
  69. # 最大占用空间 10G
  70. SystemMaxUse=10G
  71. # 单日志文件最大 200M
  72. SystemMaxFileSize=200M
  73. # 日志保存时间 2 周
  74. MaxRetentionSec=2week
  75. # 不将日志转发到 syslog
  76. ForwardToSyslog=no
  77. EOF
  79. #重启 journald 使生效
  80. systemctl restart systemd-journald
  82. #ipvs前置条件准备
  83. modprobe br_netfilter
  84. #新建 ipvs.modules 配置文件
  85. cat > /etc/sysconfig/modules/ipvs.modules <<EOF
  86. #!/bin/bash
  87. modprobe -- ip_vs
  88. modprobe -- ip_vs_rr
  89. modprobe -- ip_vs_wrr
  90. modprobe -- ip_vs_sh
  91. modprobe -- nf_conntrack_ipv4
  92. EOF
  94. #授权 ipvs.modules 文件执行权限
  95. chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules
  97. #查看已载入系统的模块
  98. lsmod | grep -e ip_vs -e nf_conntrack_ipv4
  100. #关闭swap分区
  101. free -lh
  102. #删除 swap 区所有内容
  103. swapoff -a
  104. free -lh
  106. #开启ipv4
  107. cat /proc/sys/net/ipv4/ip_forward
  108. echo "1" > /proc/sys/net/ipv4/ip_forward
  110. #安装docker
  111. cd /opt/software/docker
  112. tar xzvf docker-20.10.7.tgz
  113. chmod +x docker/*
  114. mv docker/* /usr/local/bin/
  116. #创建docker配置文件
  117. echo '[Unit]
  118. Description=Docker Application Container Engine
  119. Documentation=http://docs.docker.io
  120. After=network.target
  121. [Service]
  122. Environment="PATH=/usr/local/bin:/bin:/sbin:/usr/bin:/usr/sbin"
  123. ExecStart=/usr/local/bin/dockerd -H unix:///var/run/docker.sock -H tcp://0.0.0.0:2375
  124. ExecReload=/bin/kill -s HUP $MAINPID
  125. Restart=always
  126. RestartSec=5
  127. TimeoutSec=0
  128. LimitNOFILE=infinity
  129. LimitNPROC=infinity
  130. LimitCORE=infinity
  131. Delegate=yes
  132. KillMode=process
  133. [Install]
  134. WantedBy=multi-user.target
  135. ' >> /etc/systemd/system/docker.service
  137. #重新加载docker配置文件
  138. cd /usr/local/bin
  139. #重新加载配置文件
  140. systemctl daemon-reload
  141. #设置开机启动
  142. systemctl enable docker.service
  143. #启动
  144. systemctl start docker.service
  145. #重启
  146. systemctl daemon-reload
  147. systemctl restart docker
  149. #等待
  150. sleep 30s
  152. #添加docker源
  153. mkdir -p /etc/docker/
  154. touch /etc/docker/daemon.json
  155. cat > /etc/docker/daemon.json <<EOF
  156. {
  157. "registry-mirrors":["https://docker.mirrors.ustc.edu.cn/"],
  158. "exec-opts": ["native.cgroupdriver=systemd"]
  159. }
  160. EOF
  162. #重启docker
  163. systemctl daemon-reload
  164. systemctl restart docker
  166. #等待
  167. sleep 1m
  169. #查看验证docker
  170. docker info
  172. #等待
  173. sleep 1m
  175. #安装Kubeadm、Kubelet、Kubectl
  176. #添加yum源
  177. cat <<EOF | sudo tee /etc/yum.repos.d/kubernetes.repo
  178. [kubernetes]
  179. name=Kubernetes
  180. baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
  181. enabled=1
  182. gpgcheck=0
  183. repo_gpgcheck=0
  184. gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
  185.    http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
  186. exclude=kubelet kubeadm kubectl
  187. EOF
  189. #关闭SELinux
  190. getenforce
  191. sestatus
  192. setenforce 0
  193. getenforce
  194. sestatus
  196. #yum安装kubelet、kubeadm、kubectl
  197. #sudo yum install -y kubelet-1.20.9 kubeadm-1.20.9 kubectl-1.20.9 --disableexcludes=kubernetes
  198. sudo yum install -y kubelet-1.23.3 kubeadm-1.23.3 kubectl-1.23.3 --disableexcludes=kubernetes
  200. #kubelet设置为开机自启
  201. sudo systemctl enable --now kubelet
  203. #检查 kubelet 服务
  204. systemctl status kubelet
  206. #建立虚拟网卡
  207. cat > /etc/sysconfig/network-scripts/ifcfg-eth0:1 <<EOF
  208. BOOTPROTO=static
  209. DEVICE=eth0:1
  210. IPADDR=101.34.64.205
  211. PREFIX=32
  212. TYPE=Ethernet
  213. USERCTL=no
  214. ONBOOT=yes
  215. EOF
  217. #重启网卡,使生效
  218. systemctl restart network
  220. #等待
  221. sleep 30s
  223. ip addr
  225. #重启网卡后,重新开启ipv4
  226. cat /proc/sys/net/ipv4/ip_forward
  227. echo "1" > /proc/sys/net/ipv4/ip_forward
  228. cat /proc/sys/net/ipv4/ip_forward
  230. #修改kubelet启动参数
  231. mkdir -p /usr/lib/systemd/system/kubelet.service.d/
  232. cp /opt/software/k8s/yan-10-kubeadm.conf /usr/lib/systemd/system/kubelet.service.d/
  233. rm -rf /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf
  234. mv /usr/lib/systemd/system/kubelet.service.d/yan-10-kubeadm.conf /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf
  236. #准备kubeadm初始化环境
  237. #编写 kubeadm-config.yaml 文件，准备初始化主节点
  238. cat > /root/kubeadm-config.yaml <<EOF
  239. apiVersion: kubeadm.k8s.io/v1beta2
  240. kind: ClusterConfiguration
  241. kubernetesVersion: v1.23.3
  242. apiServer:
  243.   certSANs:    #填写所有kube-apiserver节点的hostname、IP、VIP
  244.   - ren    #替换为hostname
  245.   - 1.15.230.38   #替换为公网
  246.   - 10.0.4.17  #替换为私网
  247.   - 10.96.0.1   #不要替换，此IP是API的集群地址，部分服务会用到
  248. controlPlaneEndpoint: 1.15.230.38:6443 #替换为公网IP
  249. networking:
  250.   podSubnet: 10.244.0.0/16
  251.   serviceSubnet: 10.96.0.0/12
  252. --- 将默认调度方式改为ipvs
  253. apiVersion: kubeproxy-config.k8s.io/v1alpha1
  254. kind: KubeProxyConfiguration
  255. featureGates:
  256.   SupportIPVSProxyMode: true
  257. mode: ipvs
  258. EOF
  260. #查看要下载的镜像
  261. kubeadm config images list
  263. #等待
  264. sleep 30s
  266. #编写镜像拉取脚本，准备分发各节点执行
  267. cat >/root/pull_k8s_images.sh << "EOF"
  268. # 内容为
  269. set -o errexit
  270. set -o nounset
  271. set -o pipefail
  273. ##这里定义需要下载的版本
  274. #KUBE_VERSION=v1.20.9
  275. #KUBE_PAUSE_VERSION=3.2
  276. #ETCD_VERSION=3.4.13-0
  277. #DNS_VERSION=1.7.0
  278. KUBE_VERSION=v1.23.13
  279. KUBE_PAUSE_VERSION=3.6
  280. ETCD_VERSION=3.5.1-0
  281. DNS_VERSION=1.8.6
  283. ##这是原来被墙的仓库
  284. GCR_URL=k8s.gcr.io
  286. ##这里就是写你要使用的仓库,也可以使用gotok8s
  287. DOCKERHUB_URL=registry.cn-hangzhou.aliyuncs.com/google_containers
  289. ##这里是镜像列表
  290. images=(
  291. kube-proxy:${KUBE_VERSION}
  292. kube-scheduler:${KUBE_VERSION}
  293. kube-controller-manager:${KUBE_VERSION}
  294. kube-apiserver:${KUBE_VERSION}
  295. pause:${KUBE_PAUSE_VERSION}
  296. etcd:${ETCD_VERSION}
  297. coredns:${DNS_VERSION}
  298. )
  300. ## 这里是拉取和改名的循环语句, 先下载, 再tag重命名生成需要的镜像, 再删除下载的镜像
  301. for imageName in ${images[@]} ; do
  302.   docker pull $DOCKERHUB_URL/$imageName
  303.   docker tag $DOCKERHUB_URL/$imageName $GCR_URL/$imageName
  304.   docker rmi $DOCKERHUB_URL/$imageName
  305. done
  306. EOF
  308. #授权 镜像拉取脚本 执行权限
  309. chmod +x /root/pull_k8s_images.sh
  311. #执行 镜像拉取 脚本
  312. bash /root/pull_k8s_images.sh
  314. #验证镜像拉取
  315. docker images

bai

bai-os.sh nohup /opt/software/k8s/bai-os.sh > bai-os.log 2>&1 &

  1. cd /opt/software/k8s/
  2. vi bai-os.sh
  1. #设置重启自动加载模块
  2. modprobe br_netfilter
  3. sysctl -p /etc/sysctl.conf
  5. #查看
  6. lsmod | grep br_netfilter
  8. #永久设置
  9. #新建 rc.sysinit
  10. cat > /etc/rc.sysinit <<EOF
  11. #!/bin/bash
  12. for file in /etc/sysconfig/modules/*.modules ; do
  13. [ -x $file ] && $file
  14. done
  15. EOF
  17. #新建 br_netfilter.modules
  18. cat > /etc/sysconfig/modules/br_netfilter.modules <<EOF
  19. modprobe br_netfilter
  20. EOF
  22. #授权br_netfilter.modules文件执行权限
  23. chmod 755 /etc/sysconfig/modules/br_netfilter.modules
  25. #查看
  26. lsmod |grep br_netfilter
  28. #新建k8s网桥配置文件
  29. cat > /root/k8s.conf <<EOF
  30. #开启网桥模式
  31. net.bridge.bridge-nf-call-ip6tables = 1
  32. net.bridge.bridge-nf-call-iptables = 1
  33. #开启转发
  34. net.ipv4.ip_forward = 1
  35. ##关闭ipv6
  36. net.ipv6.conf.all.disable_ipv6=1
  37. EOF
  39. #拷贝k8s网桥配置文件到系统目录下
  40. cp /root/k8s.conf /etc/sysctl.d/k8s.conf
  41. sysctl -p /etc/sysctl.d/k8s.conf
  43. #设置时区
  44. # 设置系统时区为 中国/上海
  45. timedatectl set-timezone Asia/Shanghai
  46. # 将当前的UTC时间写入硬件时钟
  47. timedatectl set-local-rtc 0
  48. # 重启依赖于系统时间的服务
  49. systemctl restart rsyslog
  50. systemctl restart crond
  52. #关闭邮件服务
  53. systemctl stop postfix && systemctl disable postfix
  55. #设置rsyslogd、systemd、journald
  56. mkdir /var/log/journal # 持久化保存日志的目录
  57. mkdir /etc/systemd/journald.conf.d
  59. #新建 journald 配置文件
  60. cat > /etc/systemd/journald.conf.d/99-prophet.conf <<EOF
  61. [Journal]
  62. # 持久化
  63. Storage=persistent
  64. # 压缩历史日志
  65. Compress=yes
  66. SysnIntervalSec=5m
  67. RateLimitInterval=30s
  68. RateLimitBurst=1000
  69. # 最大占用空间 10G
  70. SystemMaxUse=10G
  71. # 单日志文件最大 200M
  72. SystemMaxFileSize=200M
  73. # 日志保存时间 2 周
  74. MaxRetentionSec=2week
  75. # 不将日志转发到 syslog
  76. ForwardToSyslog=no
  77. EOF
  79. #重启 journald 使生效
  80. systemctl restart systemd-journald
  82. #ipvs前置条件准备
  83. modprobe br_netfilter
  84. #新建 ipvs.modules 配置文件
  85. cat > /etc/sysconfig/modules/ipvs.modules <<EOF
  86. #!/bin/bash
  87. modprobe -- ip_vs
  88. modprobe -- ip_vs_rr
  89. modprobe -- ip_vs_wrr
  90. modprobe -- ip_vs_sh
  91. modprobe -- nf_conntrack_ipv4
  92. EOF
  94. #授权 ipvs.modules 文件执行权限
  95. chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules
  97. #查看已载入系统的模块
  98. lsmod | grep -e ip_vs -e nf_conntrack_ipv4
  100. #关闭swap分区
  101. free -lh
  102. #删除 swap 区所有内容
  103. swapoff -a
  104. free -lh
  106. #开启ipv4
  107. cat /proc/sys/net/ipv4/ip_forward
  108. echo "1" > /proc/sys/net/ipv4/ip_forward
  110. #安装docker
  111. cd /opt/software/docker
  112. tar xzvf docker-20.10.7.tgz
  113. chmod +x docker/*
  114. mv docker/* /usr/local/bin/
  116. #创建docker配置文件
  117. echo '[Unit]
  118. Description=Docker Application Container Engine
  119. Documentation=http://docs.docker.io
  120. After=network.target
  121. [Service]
  122. Environment="PATH=/usr/local/bin:/bin:/sbin:/usr/bin:/usr/sbin"
  123. ExecStart=/usr/local/bin/dockerd -H unix:///var/run/docker.sock -H tcp://0.0.0.0:2375
  124. ExecReload=/bin/kill -s HUP $MAINPID
  125. Restart=always
  126. RestartSec=5
  127. TimeoutSec=0
  128. LimitNOFILE=infinity
  129. LimitNPROC=infinity
  130. LimitCORE=infinity
  131. Delegate=yes
  132. KillMode=process
  133. [Install]
  134. WantedBy=multi-user.target
  135. ' >> /etc/systemd/system/docker.service
  137. #重新加载docker配置文件
  138. cd /usr/local/bin
  139. #重新加载配置文件
  140. systemctl daemon-reload
  141. #设置开机启动
  142. systemctl enable docker.service
  143. #启动
  144. systemctl start docker.service
  145. #重启
  146. systemctl daemon-reload
  147. systemctl restart docker
  149. #等待
  150. sleep 30s
  152. #添加docker源
  153. mkdir -p /etc/docker/
  154. touch /etc/docker/daemon.json
  155. cat > /etc/docker/daemon.json <<EOF
  156. {
  157. "registry-mirrors":["https://docker.mirrors.ustc.edu.cn/"],
  158. "exec-opts": ["native.cgroupdriver=systemd"]
  159. }
  160. EOF
  162. #重启docker
  163. systemctl daemon-reload
  164. systemctl restart docker
  166. #等待
  167. sleep 1m
  169. #查看验证docker
  170. docker info
  172. #等待
  173. sleep 1m
  175. #安装Kubeadm、Kubelet、Kubectl
  176. #添加yum源
  177. cat <<EOF | sudo tee /etc/yum.repos.d/kubernetes.repo
  178. [kubernetes]
  179. name=Kubernetes
  180. baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
  181. enabled=1
  182. gpgcheck=0
  183. repo_gpgcheck=0
  184. gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
  185.    http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
  186. exclude=kubelet kubeadm kubectl
  187. EOF
  189. #关闭SELinux
  190. getenforce
  191. sestatus
  192. setenforce 0
  193. getenforce
  194. sestatus
  196. #yum安装kubelet、kubeadm、kubectl
  197. #sudo yum install -y kubelet-1.20.9 kubeadm-1.20.9 kubectl-1.20.9 --disableexcludes=kubernetes
  198. sudo yum install -y kubelet-1.23.3 kubeadm-1.23.3 kubectl-1.23.3 --disableexcludes=kubernetes
  200. #kubelet设置为开机自启
  201. sudo systemctl enable --now kubelet
  203. #检查 kubelet 服务
  204. systemctl status kubelet
  206. #建立虚拟网卡
  207. cat > /etc/sysconfig/network-scripts/ifcfg-eth0:1 <<EOF
  208. BOOTPROTO=static
  209. DEVICE=eth0:1
  210. IPADDR=106.12.145.172
  211. PREFIX=32
  212. TYPE=Ethernet
  213. USERCTL=no
  214. ONBOOT=yes
  215. EOF
  217. #重启网卡,使生效
  218. systemctl restart network
  220. #等待
  221. sleep 30s
  223. ip addr
  225. #重启网卡后,重新开启ipv4
  226. cat /proc/sys/net/ipv4/ip_forward
  227. echo "1" > /proc/sys/net/ipv4/ip_forward
  228. cat /proc/sys/net/ipv4/ip_forward
  230. #修改kubelet启动参数
  231. mkdir -p /usr/lib/systemd/system/kubelet.service.d/
  232. cp /opt/software/k8s/bai-10-kubeadm.conf /usr/lib/systemd/system/kubelet.service.d/
  233. rm -rf /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf
  234. mv /usr/lib/systemd/system/kubelet.service.d/bai-10-kubeadm.conf /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf
  236. #准备kubeadm初始化环境
  237. #编写 kubeadm-config.yaml 文件，准备初始化主节点
  238. cat > /root/kubeadm-config.yaml <<EOF
  239. apiVersion: kubeadm.k8s.io/v1beta2
  240. kind: ClusterConfiguration
  241. kubernetesVersion: v1.23.3
  242. apiServer:
  243.   certSANs:    #填写所有kube-apiserver节点的hostname、IP、VIP
  244.   - ren    #替换为hostname
  245.   - 1.15.230.38   #替换为公网
  246.   - 10.0.4.17  #替换为私网
  247.   - 10.96.0.1   #不要替换，此IP是API的集群地址，部分服务会用到
  248. controlPlaneEndpoint: 1.15.230.38:6443 #替换为公网IP
  249. networking:
  250.   podSubnet: 10.244.0.0/16
  251.   serviceSubnet: 10.96.0.0/12
  252. --- 将默认调度方式改为ipvs
  253. apiVersion: kubeproxy-config.k8s.io/v1alpha1
  254. kind: KubeProxyConfiguration
  255. featureGates:
  256.   SupportIPVSProxyMode: true
  257. mode: ipvs
  258. EOF
  260. #查看要下载的镜像
  261. kubeadm config images list
  263. #等待
  264. sleep 30s
  266. #编写镜像拉取脚本，准备分发各节点执行
  267. cat >/root/pull_k8s_images.sh << "EOF"
  268. # 内容为
  269. set -o errexit
  270. set -o nounset
  271. set -o pipefail
  273. ##这里定义需要下载的版本
  274. #KUBE_VERSION=v1.20.9
  275. #KUBE_PAUSE_VERSION=3.2
  276. #ETCD_VERSION=3.4.13-0
  277. #DNS_VERSION=1.7.0
  278. KUBE_VERSION=v1.23.13
  279. KUBE_PAUSE_VERSION=3.6
  280. ETCD_VERSION=3.5.1-0
  281. DNS_VERSION=1.8.6
  283. ##这是原来被墙的仓库
  284. GCR_URL=k8s.gcr.io
  286. ##这里就是写你要使用的仓库,也可以使用gotok8s
  287. DOCKERHUB_URL=registry.cn-hangzhou.aliyuncs.com/google_containers
  289. ##这里是镜像列表
  290. images=(
  291. kube-proxy:${KUBE_VERSION}
  292. kube-scheduler:${KUBE_VERSION}
  293. kube-controller-manager:${KUBE_VERSION}
  294. kube-apiserver:${KUBE_VERSION}
  295. pause:${KUBE_PAUSE_VERSION}
  296. etcd:${ETCD_VERSION}
  297. coredns:${DNS_VERSION}
  298. )
  300. ## 这里是拉取和改名的循环语句, 先下载, 再tag重命名生成需要的镜像, 再删除下载的镜像
  301. for imageName in ${images[@]} ; do
  302.   docker pull $DOCKERHUB_URL/$imageName
  303.   docker tag $DOCKERHUB_URL/$imageName $GCR_URL/$imageName
  304.   docker rmi $DOCKERHUB_URL/$imageName
  305. done
  306. EOF
  308. #授权 镜像拉取脚本 执行权限
  309. chmod +x /root/pull_k8s_images.sh
  311. #执行 镜像拉取 脚本
  312. bash /root/pull_k8s_images.sh
  314. #验证镜像拉取
  315. docker images

节点初始化

ren

ren-init.sh nohup /opt/software/k8s/ren-init.sh > ren-init.log 2>&1 &

  1. cd /opt/software/k8s/
  2. vi ren-init.sh
  1. systemctl status kubelet
  2. #Warning: kubelet.service changed on disk. Run 'systemctl daemon-reload' to reload units.
  3. systemctl daemon-reload
  4. #等待
  5. sleep 30s
  6. #初始化
  7. kubeadm init --config=/root/kubeadm-config.yaml
  9. #等待
  10. sleep 3m
  12. mkdir -p $HOME/.kube
  13. sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  14. sudo chown $(id -u):$(id -g) $HOME/.kube/config
  16. export KUBECONFIG=/etc/kubernetes/admin.conf
  18. #修改kube-apiserver参数
  19. cp /opt/software/k8s/kube-apiserver.yaml /etc/kubernetes/manifests/

yan

yan-join.sh nohup /opt/software/k8s/yan-join.sh > yan-join.log 2>&1 &

  1. systemctl status kubelet
  2. #Warning: kubelet.service changed on disk. Run 'systemctl daemon-reload' to reload units.
  3. systemctl daemon-reload
  5. #worker节点加入集群
  6. kubeadm join 1.15.230.38:6443 --token uifpif.fyr2s4f5gtemqgnn \
  7.     --discovery-token-ca-cert-hash sha256:80accd0bf78574cd8e0df8b3d276e2a8c1453277b510eb02507f8e5a0675676e

bai

bai-join.sh nohup /opt/software/k8s/bai-join.sh > bai-join.log 2>&1 &

  1. systemctl status kubelet
  2. #Warning: kubelet.service changed on disk. Run 'systemctl daemon-reload' to reload units.
  3. systemctl daemon-reload
  5. #worker节点加入集群
  6. kubeadm join 1.15.230.38:6443 --token uifpif.fyr2s4f5gtemqgnn \
  7.     --discovery-token-ca-cert-hash sha256:80accd0bf78574cd8e0df8b3d276e2a8c1453277b510eb02507f8e5a0675676e

查看初始化进度

  1. #监听应用启动情况
  2. kubectl get pod -A -w
  3. #或者
  4. watch -n 1 kubectl get pod -A
  5. #检查各节点连接状态
  6. kubectl get pods -o wide --all-namespaces
  7. #或者
  8. watch -n 1 kubectl get pods -o wide --all-namespaces

安装网络插件 flannel

ren

ren-flannel.sh nohup /opt/software/k8s/ren-flannel.sh > ren-flannel.log 2>&1 &

  1. cd /opt/software/k8s/
  2. vi ren-flannel.sh
  1. kubectl get pod -A
  3. #等待
  4. sleep 3m
  6. kubectl get pod -A
  8. kubectl apply -f /opt/software/k8s/kube-flannel.yml
  10. #等待
  11. sleep 11m
  13. kubectl get pod -A

检查网络是否连通

  1. # 检查pod是否都是ready状态
  2. kubectl get pods -o wide --all-namespaces
  3. ...
  5. # 手动创建一个pod
  6. kubectl create deployment nginx --image=nginx
  8. # 查看pod的ip
  9. kubectl get pods -o wide
  11. # 主节点或其它节点，ping一下此ip,看看是否能ping通