https://github.com/matrix-ops/kbi
    kbi.sh
    点击查看【bilibili】
    高清版视频请前往B站

    image.png
    离线yum源文件
    离线k8s二进制文件

    1. #!/bin/bash
    2. # Kubernetes Binarization Installer v0.0.3
    3. # Author Dolphin-Matrix Ops
    4. #部署本地yum源
    5. echo -e "\033[32m========================================================================\033[0m"
    6. echo -e "\033[32mKubernetes Binarization Installer\033[0m"
    7. echo -e "\033[32m欢迎使用KBI(Kubernetes Binarization Installer)\033[0m"
    8. echo -e "\033[32m========================================================================\033[0m"
    9. echo -e "\033[32m本地yum源部署中......\033[0m"
    10. path=`pwd`
    11. cd $path/yum
    12. yum localinstall ./*.rpm -y &>/dev/null
    13. mkdir -p /var/www/html/kbi &> /dev/null
    14. scp -r $path/* /var/www/html/kbi &> /dev/null
    15. createrepo /var/www/html/kbi/pkg/ &>/dev/null
    16. chown -R apache:apache /var/www/html/
    17. systemctl enable --now httpd.service >/dev/null 2>&1 && 
    18. echo -e "\033[32m本地yum源部署完成，请填写集群部署IP......\033[0m"
    19. echo -e "\033[32m请在部署节点执行安装操作，部署节点可以是集群节点中的其中一个,或是任何可以连接至目标K8s集群的节点\033[0m"
    20. echo -e "\033[32m请提前在所有节点上关闭SELinux和Firewalld，并且做好节点之间SSH互信，免密登录\033[0m"
    21. read -p "输入Master节点IP,以空格分割:" -a MasterIP
    22. read -p "输入Node节点IP,以空格分割,默认与Master节点相同:" -a NodeIP
    23. read -p "输入K8s集群VIP:" k8sVIP
    24. read -p "输入Pod网段,以CIDR格式表示,默认172.23.0.0/16(按回车跳过):" podNet
    25. read -p "输入Service网段,以CIDR格式表示,默认10.253.0.0/16(按回车跳过):" serviceNet
    26. read -p "输入Kubernetes版本,默认1.18.10(按回车跳过): " k8sVersion
    27. read -p "输入docker-ce版本,默认最新版(按回车跳过): " dockerVersion
    28. web=${MasterIP[0]}
    29. #Master节点数量
    30. mCount=${#MasterIP[@]}
    31. #Node节点数量
    32. nCount=${#NodeIP[@]}
    33. if [ $nCount -eq 0 ];then
    34.     nodeCount=(${MasterIP[*]})
    35.     NodeIP=(${MasterIP[*]})
    36. else
    37.     nodeCount=(${MasterIP[*]} ${NodeIP[*]})
    38. fi
    39. echo "节点总数:${#nodeCount[@]},Master数量:${#MasterIP[@]},Node数量:${#NodeIP[@]}"
    40. echo "Master节点："
    41. for i in ${MasterIP[*]};do echo $i;done
    42. echo "Node节点:"
    43. for i in ${NodeIP[*]};do echo $i;done
    44. echo
    45. if [ -z "$k8sVersion" ];then
    46.     k8sVersion=1.18.10
    47. fi
    48. if [ -z "$podNet" ];then
    49.     podNet=172.23.0.0/16
    50. fi
    51. if [ -z "$serviceNet" ];then
    52.     serviceNet=10.253.0.0/16
    53. fi
    54. firstServiceIP=$(echo $serviceNet | awk -F'/' '{print $1}' | sed 's/0$/1/')
    55. clusterDnsIP=$(echo $serviceNet | awk -F'/' '{print $1}' | sed 's/0$/2/')
    57. if [[ -e /etc/kubernetes/pki/bootstrap/token.csv ]];then
    58.     bootstrapToken=$(awk -F',' '{print $1}' /etc/kubernetes/pki/bootstrap/token.csv)
    59. else
    60.     bootstrapToken=$(head -c 16 /dev/urandom | od -An -t x | tr -d ' ')
    61. fi
    63. autoSSHCopy(){
    64.     echo -e "\033[32m正在配置各节点SSH互信免密登录..........\033[0m"
    65.     if [ ! -e /root/.ssh/id_rsa ];then
    66.         echo "公钥文件不存在"
    67.         ssh-keygen -t rsa -P '' -f /root/.ssh/id_rsa -q
    68.     fi
    69.     for i in ${nodeCount[*]};do ssh-copy-id $i;done
    70. }
    72. #Preparation
    73. preparation(){
    74. echo -e "\033[32m开始执行部署流程..........\033[0m"
    75. #cat << EOF > /etc/yum.repos.d/docker-ce.repo
    76. ##/etc/yum.repos.d/docker-ce.repo
    77. #[docker-ce-stable]
    78. #name=Docker CE Stable - $basearch
    79. #baseurl=https://mirrors.aliyun.com/docker-ce/linux/centos/7/\$basearch/stable
    80. #enabled=1
    81. #gpgcheck=1
    82. #gpgkey=https://mirrors.aliyun.com/docker-ce/linux/centos/gpg
    83. #EOF
    84. cat << EOF > /etc/yum.repos.d/pkg.repo
    85. #/etc/yum.repos.d/pkg.repo
    86. [PKG]
    87. name=PKG                
    88. baseurl=http://$web/kbi/pkg
    89. enabled=1
    90. gpgcheck=0
    91. EOF
    92. for i in ${MasterIP[*]};do
    93.     ssh $i "mv /etc/yum.repos.d/CentOS* /tmp" >/dev/null 2>&1
    94.     scp /etc/yum.repos.d/pk.repo $i:/etc/yum.repos.d/ >/dev/null 2>&1
    95.     ssh $i "systemctl mask firewalld && setenforce 0 && sed -i 's/SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config"
    96. done
    97. cat << EOF  > /etc/sysctl.d/kubernetes.conf
    98. net.core.netdev_max_backlog=10000
    99. net.core.somaxconn=32768
    100. net.ipv4.tcp_max_syn_backlog=8096
    101. fs.inotify.max_user_instances=8192
    102. fs.file-max=2097152
    103. fs.inotify.max_user_watches=524288
    104. net.core.bpf_jit_enable=1
    105. net.core.bpf_jit_harden=1
    106. net.core.dev_weight_tx_bias=1
    107. net.core.rmem_max=16777216
    108. net.core.wmem_max=16777216
    109. net.ipv4.tcp_rmem=4096 12582912 16777216
    110. net.ipv4.tcp_wmem=4096 12582912 16777216
    111. net.core.rps_sock_flow_entries=8192
    112. net.ipv4.neigh.default.gc_thresh1=2048
    113. net.ipv4.neigh.default.gc_thresh2=4096
    114. net.ipv4.neigh.default.gc_thresh3=8192
    115. net.ipv4.tcp_max_orphans=32768
    116. net.ipv4.tcp_max_tw_buckets=32768
    117. vm.max_map_count=262144
    118. kernel.threads-max=30058
    119. net.ipv4.ip_forward=1
    120. kernel.core_pattern=core
    121. net.bridge.bridge-nf-call-iptables=1
    122. net.bridge.bridge-nf-call-ip6tables=1
    123. net.ipv4.tcp_tw_recycle=0
    124. vm.swappiness=0
    125. vm.overcommit_memory=1
    126. vm.panic_on_oom=0
    127. fs.inotify.max_user_watches=89100
    128. fs.file-max=52706963
    129. fs.nr_open=52706963
    130. net.ipv6.conf.all.disable_ipv6=1
    131. EOF
    133. #复制阿里云yum源配置文件和kubernetes.conf内核参数文件并安装依赖包
    134. if [[ ! -e /usr/local/bin/cfssl || ! -e /usr/local/bin/cfssljson ]];then
    135.     yum install wget -y &> /dev/null
    136.     wget http://$web/kbi/cfssl -O /usr/local/bin/cfssl 
    137.     #
    138.     wget http://$web/kbi/cfssljson -O /usr/local/bin/cfssljson
    139.     #
    140. fi
    141. chmod a+x /usr/local/bin/*
    142. mkdir -p /etc/kubernetes/pki/CA &> /dev/null
    143. #生成CA证书和私钥
    144. echo -e "\033[32m生成CA自签证书和私钥..........\033[0m"
    145. cat << EOF > /etc/kubernetes/pki/CA/ca-config.json
    146. {
    147.     "signing": {
    148.         "default": {
    149.             "expiry": "87600h"
    150.         },
    151.         "profiles": {
    152.             "kubernetes": {
    153.                 "expiry": "87600h",
    154.                 "usages": [
    155.                     "signing",
    156.                     "key encipherment",
    157.                     "server auth",
    158.                     "client auth"
    159.                 ]
    160.             }
    161.         }
    162.     }
    163. } 
    164. EOF
    165. cat << EOF > /etc/kubernetes/pki/CA/ca-csr.json
    166. {
    167.     "CN": "kubernetes",
    168.     "key": {
    169.         "algo": "rsa",
    170.         "size": 2048
    171.     },
    172.     "names": [
    173.         {
    174.             "C": "CN",
    175.             "ST": "GuangDong",
    176.             "L": "GuangZhou",
    177.             "O": "Dolphin",
    178.             "OU": "Ops"
    179.         }
    180.     ]
    181. } 
    182. EOF
    184. cd /etc/kubernetes/pki/CA
    185. if [[ ! -e /etc/kubernetes/pki/CA/ca.pem && ! -e /etc/kubernetes/pki/CA/ca-key.pem ]];then
    186.     cfssl gencert -initca /etc/kubernetes/pki/CA/ca-csr.json | cfssljson -bare ca
    187. fi
    188. cat << EOF > /tmp/daemon.json
    189. {
    190. "max-concurrent-downloads": 3,
    191. "max-concurrent-uploads": 5,
    192. "registry-mirrors": ["https://0bb06s1q.mirror.aliyuncs.com"],
    193. "storage-driver": "overlay2",
    194. "storage-opts": ["overlay2.override_kernel_check=true"],
    195. "log-driver": "json-file",
    196. "log-opts": {"max-size": "100m",
    197. "max-file": "3"}
    198. }
    199. EOF
    201. for i in ${nodeCount[*]};do
    202.     ssh $i "mv /etc/yum.repos.d/CentOS* /tmp" >/dev/null 2>&1
    203.     scp /etc/yum.repos.d/pkg.repo root@$i:/etc/yum.repos.d/ >/dev/null 2>&1
    204.     #scp /etc/yum.repos.d/docker-ce.repo root@$i:/etc/yum.repos.d/
    205.     scp /etc/sysctl.d/kubernetes.conf root@$i:/etc/sysctl.d/
    206.     ssh $i "yum install -y curl unzip sysstat conntrack br_netfilter ipvsadm ipset jq iptables iptables-services libseccomp && modprobe br_netfilter && sysctl -p /etc/sysctl.d/kubernetes.conf && mkdir -p /etc/kubernetes/pki/ &> /dev/null"
    207.     ssh $i "systemctl mask firewalld && setenforce 0 && sed -i 's/SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config"
    208.     if [ -z "$dockerVersion" ];then
    209.         ssh $i yum install docker-ce -y
    210.     else
    211.         ssh $i yum install docker-ce-$dockerVersion -y
    212.     fi
    213.     ssh $i mkdir /etc/kubernetes/pki/CA &> /dev/null
    214.     scp /etc/kubernetes/pki/CA/* $i:/etc/kubernetes/pki/CA
    215.     echo -e "\033[32m节点$i 初始化安装完成\033[0m" 
    216.     echo -e "\033[32m====================\033[0m"
    217.     echo 
    218. done
    220. #iptables
    221. echo -e "\033[32m正在为各节点配置iptables规则..........\033[0m"
    222. cat << EOF > /etc/sysconfig/iptables
    223. *filter
    224. :INPUT ACCEPT [0:0]
    225. :FORWARD ACCEPT [0:0]
    226. :OUTPUT ACCEPT [0:0]
    227. -A INPUT -p tcp -m state --state NEW -m tcp --dport 22 -j ACCEPT
    228. -A INPUT -p tcp -m state --state NEW -m tcp --dport 80 -j ACCEPT
    229. -A INPUT -p tcp -m state --state NEW -m tcp --dport 443 -j ACCEPT
    230. -A INPUT -p tcp -m state --state NEW -m tcp --dport 514 -j ACCEPT
    231. -A INPUT -p tcp -m state --state NEW -m tcp --dport 1080 -j ACCEPT
    232. -A INPUT -p tcp -m state --state NEW -m tcp --dport 2379 -j ACCEPT
    233. -A INPUT -p tcp -m state --state NEW -m tcp --dport 2380 -j ACCEPT
    234. -A INPUT -p tcp -m state --state NEW -m tcp --dport 6443 -j ACCEPT
    235. -A INPUT -p tcp -m state --state NEW -m tcp --dport 8080 -j ACCEPT
    236. -A INPUT -p tcp -m state --state NEW -m tcp --dport 8443 -j ACCEPT
    237. -A INPUT -m pkttype --pkt-type multicast -j ACCEPT
    238. -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT
    239. -A INPUT -p icmp -j ACCEPT
    240. -A INPUT -i lo -j ACCEPT
    241. COMMIT
    242. EOF
    243. for i in ${nodeCount[*]};do
    244.     scp /etc/sysconfig/iptables $i:/etc/sysconfig/iptables
    245.     ssh $i systemctl restart iptables
    246. done
    248. #配置NTP
    249. #将以输入的第一个MasterIP作为NTP服务器
    250. echo -e "\033[32m正在配置NTP服务器，服务器地址为${MasterIP[0]}..........\033[0m"
    251. allowNTP=${MasterIP[0]}
    252. netNTP=$(echo $allowNTP | awk -F'.' '{print $1,$2 }' | sed  's/ /./')
    253. cat << EOF > /tmp/chrony.conf
    254. server ntp.aliyun.com iburst
    255. driftfile /var/lib/chrony/drift
    256. makestep 1.0 3
    257. rtcsync
    258. allow ${netNTP}.0.0/16
    259. logdir /var/log/chrony
    260. EOF
    261. cat << EOF > /tmp/chrony.conf_otherNode
    262. server ${MasterIP[0]} iburst
    263. driftfile /var/lib/chrony/drift
    264. makestep 1.0 3
    265. rtcsync
    266. logdir /var/log/chrony
    267. EOF
    268. scp /etc/chrony.conf ${MasterIP[0]}:/etc/
    269. ssh ${MasterIP[0]} systemctl restart chronyd
    270. echo -e "\033[32mNTP服务器完成..........\033[0m"
    271. }
    273. deployHaproxyKeepalived (){
    274. #生成Haproxy的配置文件，默认使用MasterIP中的前三个节点
    275. cat << EOF >> /tmp/haproxy.cfg 
    276. global
    277.     log /dev/log    local0
    278.     log /dev/log    local1 notice
    279.     chroot /var/lib/haproxy
    280.     stats socket /var/run/haproxy-admin.sock mode 660 level admin
    281.     stats timeout 30s
    282.     user haproxy
    283.     group haproxy
    284.     daemon
    285.     nbproc 1
    287. defaults
    288.     log     global
    289.     timeout connect 5000
    290.     timeout client  10m
    291.     timeout server  10m
    293. listen  admin_stats
    294.     bind 0.0.0.0:10080
    295.     mode http
    296.     log 127.0.0.1 local0 err
    297.     stats refresh 30s
    298.     stats uri /status
    299.     stats realm welcome login\ Haproxy
    300.     stats auth admin:DreamCatcher
    301.     stats hide-version
    302.     stats admin if TRUE
    304. listen kube-master
    305.     bind 0.0.0.0:8443
    306.     mode tcp
    307.     option tcplog
    308.     balance source
    309.     server k8s-master1 ${MasterIP[0]}:6443 check inter 2000 fall 2 rise 2 weight 1
    310.     server k8s-master2 ${MasterIP[1]}:6443 check inter 2000 fall 2 rise 2 weight 1
    311.     server k8s-master3 ${MasterIP[2]}:6443 check inter 2000 fall 2 rise 2 weight 1
    312. EOF
    314. #安装配置Keepalived和Haproxy，并根据节点的不同分别为不同节点的Keepalived设置优先级
    315. weight=1
    316. for i in ${MasterIP[*]};do
    317. ((keepalivedPriority=$weight+100))
    318. ssh $i exec "yum install haproxy keepalived -y && systemctl enable haproxy keepalived"
    319. interfaceName=$(ssh $i "ip a | grep -i $i -B 2 | awk 'NR==1{print \$2}' | sed 's/://'")
    320. cat << EOF > /tmp/keepalived.conf
    321. global_defs {
    322.     router_id k8s-master-$i
    323. }
    325. vrrp_script check-haproxy {
    326.     script "killall -0 haproxy"
    327.     interval 5
    328.     weight -30
    329. }
    331. vrrp_instance VI-kube-master {
    332.     state MASTER
    333.     priority $keepalivedPriority
    334.     dont_track_primary
    335.     interface $interfaceName
    336.     virtual_router_id 68
    337.     advert_int 3
    338.     track_script {
    339.         check-haproxy
    340.     }
    341.     virtual_ipaddress {
    342.         $k8sVIP   
    343.     }
    344. }
    345. EOF
    346. ((weight=$weight+10))
    347. scp /tmp/haproxy.cfg $i:/etc/haproxy/haproxy.cfg
    348. scp /tmp/keepalived.conf $i:/etc/keepalived/
    349. echo -e "\033[32m节点$i 正在启动Haproxy && Keepalived..........\033[0m"
    350. ssh $i "systemctl start haproxy keepalived && systemctl enable haproxy keepalived"
    351. echo -e "\033[32m节点${i} Haproxy && Keepalived启动完成\033[0m"
    352. echo 
    353. done
    354. }
    356. deployETCD(){
    357.     echo -e "\033[32m正在部署etcd..........\033[0m"
    358.     mkdir -p /etc/kubernetes/pki/etcd/ &> /dev/null
    359.     cat << EOF > /etc/kubernetes/pki/etcd/etcd-csr.json
    360.     {
    361.         "CN": "etcd",
    362.         "hosts": [
    363.             "127.0.0.1",
    364.             "${MasterIP[0]}",
    365.             "${MasterIP[1]}",
    366.             "${MasterIP[2]}"
    367.         ],
    368.         "key": {
    369.             "algo": "rsa",
    370.             "size": 2048
    371.         },
    372.         "names": [
    373.             {
    374.                 "C": "CN",
    375.                 "ST": "GuangDong",
    376.                 "L": "GuangZhou",
    377.                 "O": "Dolphin",
    378.                 "OU": "Ops"
    379.             }
    380.         ]
    381.     }
    382. EOF
    384. cd /etc/kubernetes/pki/etcd/
    385. if [[ ! -e /etc/kubernetes/pki/etcd/etcd.pem && ! -e /etc/kubernetes/pki/etcd/etcd-key.pem ]];then
    386. cfssl gencert -ca=/etc/kubernetes/pki/CA/ca.pem \
    387. -ca-key=/etc/kubernetes/pki/CA/ca-key.pem \
    388. -config=/etc/kubernetes/pki/CA/ca-config.json \
    389. -profile=kubernetes etcd-csr.json | cfssljson -bare etcd
    390. fi
    392. if [[ ! -e /tmp/etcd-v3.3.10-linux-amd64.tar.gz ]];then
    393.     wget http://$web/kbi/etcd-v3.3.10-linux-amd64.tar.gz -O /tmp/etcd-v3.3.10-linux-amd64.tar.gz
    394.     tar xf /tmp/etcd-v3.3.10-linux-amd64.tar.gz -C /tmp
    395. fi
    397. index=0
    398. for i in ${MasterIP[*]};do
    399. ssh $i "mkdir /tmp/etcd/ &> /dev/null"
    400. cat << EOF > /tmp/etcd/etcd.conf
    401. ETCD_ARGS="--name=etcd-$index \\
    402.   --cert-file=/etc/kubernetes/pki/etcd/etcd.pem \\
    403.   --key-file=/etc/kubernetes/pki/etcd/etcd-key.pem \\
    404.   --peer-cert-file=/etc/kubernetes/pki/etcd/etcd.pem \\
    405.   --peer-key-file=/etc/kubernetes/pki/etcd/etcd-key.pem \\
    406.   --trusted-ca-file=/etc/kubernetes/pki/CA/ca.pem \\
    407.   --peer-trusted-ca-file=/etc/kubernetes/pki/CA/ca.pem \\
    408.   --initial-advertise-peer-urls=https://${i}:2380 \\
    409.   --listen-peer-urls=https://${i}:2380 \\
    410.   --listen-client-urls=https://${i}:2379,http://127.0.0.1:2379 \\
    411.   --advertise-client-urls=https://${i}:2379 \\
    412.   --initial-cluster-token=etcd-cluster-1 \\
    413.   --initial-cluster=etcd-0=https://${MasterIP[0]}:2380,etcd-1=https://${MasterIP[1]}:2380,etcd-2=https://${MasterIP[2]}:2380 \\
    414.   --initial-cluster-state=new \\
    415.   --data-dir=/var/lib/etcd"
    416. EOF
    417. scp /tmp/etcd/etcd.conf $i:/usr/local/etc/
    418. cat << EOF > /tmp/etcd/etcd.service
    419. [Unit]
    420. Description=Etcd Server
    421. Documentation=https://github.com/coreos
    422. After=network.target
    423. After=network-online.target
    424. Wants=network-online.target
    426. [Service]
    427. Type=notify
    428. WorkingDirectory=/var/lib/etcd/
    429. EnvironmentFile=/usr/local/etc/etcd.conf
    430. ExecStart=/usr/local/bin/etcd \$ETCD_ARGS
    431. Restart=on-failure
    432. RestartSec=5
    433. LimitNOFILE=65536
    435. [Install]
    436. WantedBy=multi-user.target
    437. EOF
    438. scp /tmp/etcd/etcd.service $i:/etc/systemd/system/
    439. scp /tmp/etcd-v3.3.10-linux-amd64/etcd* $i:/usr/local/bin
    440. ssh $i mkdir /etc/kubernetes/pki/etcd/ /var/lib/etcd/  &>/dev/null
    441. scp /etc/kubernetes/pki/etcd/* $i:/etc/kubernetes/pki/etcd/
    442. ((index++))
    443. done
    445. echo -e "\033[32m正在启动etcd.....\033[0m"
    446. ssh ${MasterIP[0]} exec "systemctl enable etcd && systemctl start etcd" &> /dev/null &
    448. for i in ${MasterIP[*]};do
    449.     ssh $i "systemctl start etcd && systemctl enable etcd &"
    450.     echo -e "\033[32m${i} etcd启动成功\033[0m"
    451. done
    452. }
    454. setKubectl(){
    455.     if [[ ! $(which kube-apiserver) ]];then
    456.         wget http://$web/kbi/$k8sVersion/kubernetes-server-linux-amd64.tar.gz -O /opt/kubernetes-server-linux-amd64.tar.gz && tar xvf /opt/kubernetes-server-linux-amd64.tar.gz  -C /opt/&& cd /opt/kubernetes/server/bin && rm -rf *.tar *.docker_tag 
    457.         #如果上述链接失效，请使用如下链接
    458.         #wget http://dl.k8s.io/$k8sVersion/kubernetes-server-linux-amd64.tar.gz -O /opt/kubernetes-server-linux-amd64.tar.gz && tar xvf /opt/kubernetes-server-linux-amd64.tar.gz  && cd /opt/kubernetes/server/bin && rm -rf *.tar *.docker_tag && mv * /usr/local/bin/
    459.         for i in ${nodeCount[*]};do
    460.             scp /opt/kubernetes/server/bin/* $i:/usr/local/bin/
    461.             ssh $i "chmod a+x /usr/local/bin/*"
    462.         done
    463.     else
    464.         echo -e "\033[31m二进制文件已存在，跳过下载和复制Kubernetes v${k8sVersion}二进制文件的步骤\033[0m"
    465.     fi
    467.     mkdir -p /etc/kubernetes/pki/admin
    468.     cd /etc/kubernetes/pki/admin
    469.     cat <<EOF > /etc/kubernetes/pki/admin/admin-csr.json
    470.     {
    471.         "CN": "admin",
    472.         "hosts": [],
    473.         "key": {
    474.             "algo": "rsa",
    475.             "size": 2048
    476.         },
    477.         "names": [
    478.             {
    479.                 "C": "CN",
    480.                 "ST": "GuangZhou",
    481.                 "L": "GuangDong",
    482.                 "O": "system:masters",
    483.                 "OU": "Ops"
    484.             }
    485.         ]
    486.     }
    487. EOF
    489. cfssl gencert -ca=/etc/kubernetes/pki/CA/ca.pem \
    490. -ca-key=/etc/kubernetes/pki/CA/ca-key.pem \
    491. -config=/etc/kubernetes/pki/CA/ca-config.json \
    492. -profile=kubernetes /etc/kubernetes/pki/admin/admin-csr.json | cfssljson -bare admin
    494. kubectl config set-cluster kubernetes \
    495. --certificate-authority=/etc/kubernetes/pki/CA/ca.pem \
    496. --embed-certs=true \
    497. --server=https://${k8sVIP}:8443 \
    498. --kubeconfig=/etc/kubernetes/pki/admin/admin.conf
    499. kubectl config set-credentials admin \
    500. --client-certificate=/etc/kubernetes/pki/admin/admin.pem \
    501. --embed-certs=true \
    502. --client-key=/etc/kubernetes/pki/admin/admin-key.pem \
    503. --kubeconfig=/etc/kubernetes/pki/admin/admin.conf
    504. kubectl config set-context admin@kubernetes \
    505. --cluster=kubernetes \
    506. --user=admin \
    507. --kubeconfig=/etc/kubernetes/pki/admin/admin.conf
    508. kubectl config use-context admin@kubernetes --kubeconfig=/etc/kubernetes/pki/admin/admin.conf
    510. for i in ${MasterIP[*]};do
    511.     ssh $i mkdir -p /etc/kubernetes/pki/admin /root/.kube/ &>/dev/null
    512.     scp /etc/kubernetes/pki/admin/admin* $i:/etc/kubernetes/pki/admin/
    513.     scp /etc/kubernetes/pki/admin/admin.conf $i:/root/.kube/config
    514.     echo -e "\033[32m${i} kubectl配置完成\033[0m"
    515. done
    516. }
    518. deployFlannel(){
    519.     mkdir -p /etc/kubernetes/pki/flannel/
    520.     cd /etc/kubernetes/pki/flannel/
    521.     cat << EOF > /etc/kubernetes/pki/flannel/flannel-csr.json
    522.     {
    523.         "CN": "flanneld",
    524.         "hosts": [],
    525.         "key": {
    526.             "algo": "rsa",
    527.             "size": 2048
    528.         },
    529.         "names": [
    530.             {
    531.                 "C": "CN",
    532.                 "ST": "GuangDong",
    533.                 "L": "GuangZhou",
    534.                 "O": "Dolphin",
    535.                 "OU": "Ops"
    536.             }
    537.         ]
    538.     }
    539. EOF
    541. if [[ ! -e /etc/kubernetes/pki/flannel/flannel.pem && ! -e /etc/kubernetes/pki/flannel/flannel-key.pem ]];then
    542. cfssl gencert -ca=/etc/kubernetes/pki/CA/ca.pem \
    543. -ca-key=/etc/kubernetes/pki/CA/ca-key.pem \
    544. -config=/etc/kubernetes/pki/CA/ca-config.json \
    545. -profile=kubernetes /etc/kubernetes/pki/flannel/flannel-csr.json | cfssljson -bare flannel
    546. fi
    548. etcdctl --endpoints=https://${MasterIP[0]}:2379 \
    549. --ca-file=/etc/kubernetes/pki/CA/ca.pem \
    550. --cert-file=/etc/kubernetes/pki/flannel/flannel.pem \
    551. --key-file=/etc/kubernetes/pki/flannel/flannel-key.pem \
    552. set /kubernetes/network/config  '{"Network":"'${podNet}'", "SubnetLen": 24, "Backend": {"Type": "vxlan"}}'
    554. if [[ ! $(which flanneld) ]];then
    555.    wget http://$web/kbi/flannel-v0.10.0-linux-amd64.tar.gz -O /opt/flannel-v0.10.0-linux-amd64.tar.gz 
    556.    tar xf /opt/flannel-v0.10.0-linux-amd64.tar.gz -C /opt/
    557.    cp /opt/{flanneld,mk-docker-opts.sh} /usr/local/bin/
    558. fi
    560. cat << EOF > /etc/systemd/system/flanneld.service
    561. [Unit]
    562. Description=Flanneld overlay address etcd agent
    563. Documentation=https://github.com/coreos
    564. After=network.target
    565. After=network-online.target
    566. Wants=network-online.target
    567. After=etcd.service
    568. Before=docker.service
    570. [Service]
    571. Type=notify
    572. EnvironmentFile=/usr/local/etc/flanneld.conf
    573. ExecStart=/usr/local/bin/flanneld \$FLANNELD_ARGS
    574. ExecStartPost=/usr/local/bin/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/docker
    575. Restart=on-failure
    577. [Install]
    578. WantedBy=multi-user.target
    579. RequiredBy=docker.service
    580. EOF
    582. cat << EOF > /usr/local/etc/flanneld.conf
    583. FLANNELD_ARGS="-etcd-cafile=/etc/kubernetes/pki/CA/ca.pem \\
    584.   -etcd-certfile=/etc/kubernetes/pki/flannel/flannel.pem \\
    585.   -etcd-keyfile=/etc/kubernetes/pki/flannel/flannel-key.pem \\
    586.   -etcd-endpoints=https://${MasterIP[0]}:2379,https://${MasterIP[1]}:2379,https://${MasterIP[2]}:2379 \\
    587.   -etcd-prefix=/kubernetes/network"
    588. EOF
    590. cat << EOF > /tmp/docker.service
    591. [Unit]
    592. Description=Docker Application Container Engine
    593. Documentation=https://docs.docker.com
    594. BindsTo=containerd.service
    595. After=network-online.target firewalld.service containerd.service
    596. Wants=network-online.target
    597. Requires=docker.socket
    598. [Service]
    599. Type=notify
    600. EnvironmentFile=-/run/flannel/docker
    601. ExecStart=/usr/bin/dockerd -H fd:// \$DOCKER_NETWORK_OPTIONS --containerd=/run/containerd/containerd.sock
    602. ExecReload=/bin/kill -s HUP $MAINPID
    603. TimeoutSec=0
    604. RestartSec=2
    605. Restart=always
    606. StartLimitBurst=3
    607. StartLimitInterval=60s
    608. LimitNOFILE=infinity
    609. LimitNPROC=infinity
    610. LimitCORE=infinity
    611. TasksMax=infinity
    612. Delegate=yes
    613. KillMode=process
    614. [Install]
    615. WantedBy=multi-user.target
    616. EOF
    617. for i in ${nodeCount[*]};do
    618.     ssh $i "mkdir -p /etc/kubernetes/pki/flannel/ /run/flannel && touch /run/flannel/docker"
    619.     scp /opt/{flanneld,mk-docker-opts.sh} $i:/usr/local/bin/
    620.     scp /etc/kubernetes/pki/flannel/flannel* $i:/etc/kubernetes/pki/flannel/
    621.     scp /etc/systemd/system/flanneld.service $i:/etc/systemd/system/flanneld.service
    622.     scp /usr/local/etc/flanneld.conf $i:/usr/local/etc/flanneld.conf
    623.     scp /tmp/docker.service $i:/usr/lib/systemd/system/docker.service
    624.     ssh $i "systemctl daemon-reload && systemctl enable --now docker flanneld"
    625.     scp /tmp/daemon.json $i:/etc/docker/
    626.     ssh $i "systemctl restart flanneld && systemctl restart docker"
    627.     if [ $? ];then
    628.         echo -e "\033[32m $i Flanneld 启动成功\033[0m"
    629.     else
    630.         echo -e "\033[31m $i Flanneld 启动失败\033[0m"
    631.     fi
    632. done
    633. }
    635. deployApiserver(){
    636.     mkdir -p /etc/kubernetes/pki/apiserver/ /etc/kubernetes/pki/bootstrap &> /dev/null
    637. if [[ ! -e /etc/kubernetes/pki/bootstrap/token.csv ]];then
    638.     cat << EOF > /etc/kubernetes/pki/bootstrap/token.csv
    639. ${bootstrapToken},kubelet-bootstrap,10001,"system:kubelet-bootstrap"
    640. EOF
    641. fi
    642.     cd /etc/kubernetes/pki/apiserver/ 
    643.     cat << EOF > /etc/kubernetes/pki/apiserver/apiserver-csr.json
    644. {
    645.     "CN": "kubernetes",
    646.     "hosts": [
    647.       "127.0.0.1",
    648.       "${firstServiceIP}",
    649.       "kubernetes",
    650.       "kubernetes.default",
    651.       "kubernetes.default.svc",
    652.       "kubernetes.default.svc.cluster",
    653.       "kubernetes.default.svc.cluster.local"
    654.     ],
    655.     "key": {
    656.         "algo": "rsa",
    657.         "size": 2048
    658.     },
    659.     "names": [
    660.         {
    661.             "C": "CN",
    662.             "ST": "GuangDong",
    663.             "L": "GuangZhou",
    664.             "O": "Dolphin",
    665.             "OU": "Ops"
    666.         }
    667.     ]
    668. }
    669. EOF
    671. cat << EOF > /etc/systemd/system/kube-apiserver.service
    672. [Unit]
    673. Description=Kubernetes API Server
    674. Documentation=https://github.com/GoogleCloudPlatform/kubernetes
    675. After=network.target
    677. [Service]
    678. User=root
    679. EnvironmentFile=/usr/local/etc/kube-apiserver.conf
    680. ExecStart=/usr/local/bin/kube-apiserver \$KUBE_API_ARGS
    681. Restart=on-failure
    682. RestartSec=5
    683. Type=notify
    684. LimitNOFILE=65536
    686. [Install]
    687. WantedBy=multi-user.target
    688. EOF
    690.     #遍历所有节点,将所有节点的IP写入到csr.json里面的hosts字段
    691.     nIndex=0
    692.     nodeCountLen=${#nodeCount[*]}
    693.     while (( nIndex < nodeCountLen ))
    694.     do
    695.        sed -i "4 a\"${nodeCount[$nIndex]}\"," /etc/kubernetes/pki/apiserver/apiserver-csr.json
    696.        sed  '5s/^/      /' /etc/kubernetes/pki/apiserver/apiserver-csr.json
    697.        ((nIndex++))
    698.     done
    699.     sed -i "4 a\"${k8sVIP}\"," /etc/kubernetes/pki/apiserver/apiserver-csr.json
    700.     sed  '5s/^/      /' /etc/kubernetes/pki/apiserver/apiserver-csr.json
    701.     if [[ ! -e /etc/kubernetes/pki/apiserver.pem && ! -e /etc/kubernetes/pki/apiserver/apiserver-key.pem ]];then
    702.         cfssl gencert -ca=/etc/kubernetes/pki/CA/ca.pem \
    703.         -ca-key=/etc/kubernetes/pki/CA/ca-key.pem \
    704.         -config=/etc/kubernetes/pki/CA/ca-config.json \
    705.         -profile=kubernetes apiserver-csr.json | cfssljson -bare apiserver    
    706.     fi
    708. for i in ${MasterIP[*]};do
    709. cat << EOF > /tmp/kube-apiserver.conf
    710. KUBE_API_ARGS="--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota,NodeRestriction \\
    711.   --advertise-address=$i \\
    712.   --bind-address=$i \\
    713.   --insecure-port=0 \\
    714.   --authorization-mode=Node,RBAC \\
    715.   --runtime-config=rbac.authorization.k8s.io/v1beta1 \\
    716.   --kubelet-https=true \\
    717.   --token-auth-file=/etc/kubernetes/pki/bootstrap/token.csv \\
    718.   --service-cluster-ip-range=${serviceNet} \\
    719.   --service-node-port-range=10000-60000 \\
    720.   --tls-cert-file=/etc/kubernetes/pki/apiserver/apiserver.pem \\
    721.   --tls-private-key-file=/etc/kubernetes/pki/apiserver/apiserver-key.pem \\
    722.   --client-ca-file=/etc/kubernetes/pki/CA/ca.pem \\
    723.   --service-account-key-file=/etc/kubernetes/pki/CA/ca-key.pem \\
    724.   --etcd-cafile=/etc/kubernetes/pki/CA/ca.pem \\
    725.   --etcd-certfile=/etc/kubernetes/pki/apiserver/apiserver.pem \\
    726.   --etcd-keyfile=/etc/kubernetes/pki/apiserver/apiserver-key.pem \\
    727.   --storage-backend=etcd3 \\
    728.   --etcd-servers=https://${MasterIP[0]}:2379,https://${MasterIP[1]}:2379,https://${MasterIP[2]}:2379 \\
    729.   --enable-swagger-ui=true \\
    730.   --allow-privileged=true \\
    731.   --apiserver-count=3 \\
    732.   --audit-log-maxage=30 \\
    733.   --audit-log-maxbackup=3 \\
    734.   --audit-log-maxsize=100 \\
    735.   --audit-log-path=/var/lib/audit.log \\
    736.   --event-ttl=1h \\
    737.   --logtostderr=false \\
    738.   --log-dir=/var/log/kubernetes/apiserver \\
    739.   --v=2  1>>/var/log/kubernetes/apiserver/kube-apiserver.log 2>&1"
    740. EOF
    742.     ssh $i mkdir -p /etc/kubernetes/pki/apiserver/ /etc/kubernetes/pki/bootstrap /var/log/kubernetes/apiserver  &> /dev/null
    743.     scp /etc/kubernetes/pki/bootstrap/token.csv $i:/etc/kubernetes/pki/bootstrap/
    744.     scp /etc/kubernetes/pki/apiserver/apiserver* $i:/etc/kubernetes/pki/apiserver/
    745.     scp /etc/systemd/system/kube-apiserver.service $i:/etc/systemd/system/kube-apiserver.service
    746.     scp /tmp/kube-apiserver.conf $i:/usr/local/etc/kube-apiserver.conf
    747.     ssh $i "systemctl enable kube-apiserver && systemctl start kube-apiserver &>/dev/null"
    748.     if [ $? ];then
    749.         echo -e "\033[32m $i kube-apiserver 启动成功\033[0m"
    750.     else
    751.         echo -e "\033[31m $i kube-apiserver 启动失败，请检查日志文件\033[0m"
    752.     fi
    753. done
    754. }
    756. deployControllerManager(){
    757.     mkdir -p /etc/kubernetes/pki/controller-manager
    758.     cd /etc/kubernetes/pki/controller-manager
    759.     cat << EOF > /etc/kubernetes/pki/controller-manager/controller-manager-csr.json
    760.     {
    761.         "CN": "system:kube-controller-manager",
    762.         "hosts": [
    763.           "${MasterIP[0]}",
    764.           "${MasterIP[1]}",
    765.           "${MasterIP[2]}"
    766.         ],
    767.         "key": {
    768.             "algo": "rsa",
    769.             "size": 2048
    770.         },
    771.         "names": [
    772.             {
    773.                 "C": "CN",
    774.                 "ST": "GuangDong",
    775.                 "L": "GuangZhou",
    776.                 "O": "system:kube-controller-manager",
    777.                 "OU": "Ops"
    778.             }
    779.         ]
    780.     }
    781. EOF
    782.     if [[ ! -e /etc/kubernetes/pki/controller-manager.pem && ! -e /etc/kubernetes/pki/controller-manager-key.pem ]];then
    783.         cfssl gencert -ca=/etc/kubernetes/pki/CA/ca.pem  -ca-key=/etc/kubernetes/pki/CA/ca-key.pem -config=/etc/kubernetes/pki/CA/ca-config.json -profile=kubernetes /etc/kubernetes/pki/controller-manager/controller-manager-csr.json | cfssljson -bare controller-manager
    784.     fi
    786. cat << EOF > /etc/systemd/system/kube-controller-manager.service
    787. [Unit]
    788. Description=Kubernetes Controller Manager
    789. Documentation=https://github.com/GoogleCloudPlatform/kubernetes
    790. After=network.target
    791. After=kube-apiserver.service
    793. [Service]
    794. EnvironmentFile=/usr/local/etc/kube-controller-manager.conf
    795. ExecStart=/usr/local/bin/kube-controller-manager \$KUBE_CONTROLLER_MANAGER_ARGS
    796. Restart=on-failure
    797. RestartSec=5
    799. [Install]
    800. WantedBy=multi-user.target
    801. EOF
    803. cat << EOF > /usr/local/etc/kube-controller-manager.conf
    804. KUBE_CONTROLLER_MANAGER_ARGS="--master=https://${k8sVIP}:8443 \\
    805.   --kubeconfig=/etc/kubernetes/pki/controller-manager/controller-manager.conf \\
    806.   --allocate-node-cidrs=true \\
    807.   --service-cluster-ip-range=${serviceNet} \\
    808.   --cluster-cidr=${podNet} \\
    809.   --cluster-name=kubernetes \\
    810.   --cluster-signing-cert-file=/etc/kubernetes/pki/CA/ca.pem \\
    811.   --cluster-signing-key-file=/etc/kubernetes/pki/CA/ca-key.pem \\
    812.   --service-account-private-key-file=/etc/kubernetes/pki/CA/ca-key.pem \\
    813.   --root-ca-file=/etc/kubernetes/pki/CA/ca.pem \\
    814.   --use-service-account-credentials=true \\
    815.   --controllers=*,bootstrapsigner,tokencleaner \\
    816.   --leader-elect=true \\
    817.   --logtostderr=false \\
    818.   --log-dir=/var/log/kubernetes/controller-manager \\
    819.   --v=2  1>>/var/log/kubernetes/kube-controller-manager.log 2>&1"
    820. EOF
    822. kubectl config set-cluster kubernetes \
    823. --certificate-authority=/etc/kubernetes/pki/CA/ca.pem \
    824. --embed-certs=true \
    825. --server=https://${k8sVIP}:8443 \
    826. --kubeconfig=/etc/kubernetes/pki/controller-manager/controller-manager.conf
    827. kubectl config set-credentials system:kube-controller-manager \
    828. --client-certificate=/etc/kubernetes/pki/controller-manager/controller-manager.pem \
    829. --embed-certs=true \
    830. --client-key=/etc/kubernetes/pki/controller-manager/controller-manager-key.pem \
    831. --kubeconfig=/etc/kubernetes/pki/controller-manager/controller-manager.conf
    832. kubectl config set-context system:kube-controller-manager@kubernetes \
    833. --cluster=kubernetes \
    834. --user=system:kube-controller-manager \
    835. --kubeconfig=/etc/kubernetes/pki/controller-manager/controller-manager.conf
    836. kubectl config use-context system:kube-controller-manager@kubernetes --kubeconfig=/etc/kubernetes/pki/controller-manager/controller-manager.conf
    838. for i in ${MasterIP[*]};do
    839.     ssh $i mkdir -p  /etc/kubernetes/pki/controller-manager /var/log/kubernetes/controller-manager/ &>/dev/null
    840.     scp /etc/kubernetes/pki/controller-manager/* $i:/etc/kubernetes/pki/controller-manager/
    841.     scp /usr/local/etc/kube-controller-manager.conf $i:/usr/local/etc/kube-controller-manager.conf
    842.     scp /etc/systemd/system/kube-controller-manager.service $i:/etc/systemd/system/kube-controller-manager.service
    843.     ssh $i "systemctl enable kube-controller-manager && systemctl start kube-controller-manager &>/dev/null "
    844.     if [ $? ];then
    845.         echo -e "\033[32m $i kube-controller-manager 启动成功\033[0m"
    846.     else
    847.         echo -e "\033[31m $i kube-apiserver 启动失败，请检查日志文件\033[0m"
    848.     fi
    849. done
    850. }
    852. deployScheduler(){
    853.     mkdir -p /etc/kubernetes/pki/scheduler/ /var/log/kubernetes/scheduler &>/dev/null
    854.     cd /etc/kubernetes/pki/scheduler/
    855.     cat << EOF > /etc/kubernetes/pki/scheduler/scheduler-csr.json
    856. {
    857.     "CN": "system:kube-scheduler",
    858.     "hosts": [
    859.       "${MasterIP[0]}",
    860.       "${MasterIP[1]}",
    861.       "${MasterIP[2]}"
    862.     ],
    863.     "key": {
    864.         "algo": "rsa",
    865.         "size": 2048
    866.     },
    867.     "names": [
    868.         {
    869.             "C": "CN",
    870.             "ST": "GuangDong",
    871.             "L": "GuangZhou",
    872.             "O": "system:kube-scheduler",
    873.             "OU": "Ops"
    874.         }
    875.     ]
    876. }
    877. EOF
    879. if [[ ! -e /etc/kubernetes/pki/scheduler/scheduler-key.pem && ! -e /etc/kubernetes/pki/scheduler/scheduler.pem ]];then
    880.     cfssl gencert -ca=/etc/kubernetes/pki/CA/ca.pem \
    881.     -ca-key=/etc/kubernetes/pki/CA/ca-key.pem \
    882.     -config=/etc/kubernetes/pki/CA/ca-config.json \
    883.     -profile=kubernetes /etc/kubernetes/pki/scheduler/scheduler-csr.json | cfssljson -bare scheduler
    884. fi
    886.     kubectl config set-cluster kubernetes \
    887.     --certificate-authority=/etc/kubernetes/pki/CA/ca.pem \
    888.     --embed-certs=true \
    889.     --server=https://${k8sVIP}:8443 \
    890.     --kubeconfig=/etc/kubernetes/pki/scheduler/scheduler.conf
    891.     kubectl config set-credentials system:kube-scheduler \
    892.     --client-certificate=/etc/kubernetes/pki/scheduler/scheduler.pem \
    893.     --embed-certs=true \
    894.     --client-key=/etc/kubernetes/pki/scheduler/scheduler-key.pem \
    895.     --kubeconfig=/etc/kubernetes/pki/scheduler/scheduler.conf
    896.     kubectl config set-context system:kube-scheduler@kubernetes \
    897.     --cluster=kubernetes \
    898.     --user=system:kube-scheduler \
    899.     --kubeconfig=scheduler.conf
    900.     kubectl config use-context system:kube-scheduler@kubernetes --kubeconfig=scheduler.conf 
    902.     cat << EOF > /etc/systemd/system/kube-scheduler.service
    903. [Unit]
    904. Description=Kubernetes Scheduler
    905. Documentation=https://github.com/GoogleCloudPlatform/kubernetes
    906. After=network.target
    907. After=kube-apiserver.service
    909. [Service]
    910. EnvironmentFile=/usr/local/etc/kube-scheduler.conf
    911. ExecStart=/usr/local/bin/kube-scheduler \$KUBE_SCHEDULER_ARGS
    912. Restart=on-failure
    913. RestartSec=5
    915. [Install]
    916. WantedBy=multi-user.target
    917. EOF
    919. cat << EOF > /usr/local/etc/kube-scheduler.conf
    920. KUBE_SCHEDULER_ARGS="--master=https://${k8sVIP}:8443 \
    921.   --kubeconfig=/etc/kubernetes/pki/scheduler/scheduler.conf \
    922.   --leader-elect=true \
    923.   --logtostderr=false \
    924.   --log-dir=/var/log/kubernetes/scheduler \
    925.   --v=2"
    926. EOF
    928. for i in ${MasterIP[*]};do
    929.     ssh $i "mkdir -p /etc/kubernetes/pki/scheduler/ /var/log/kubernetes/scheduler/ &> /dev/null"
    930.     scp /usr/local/etc/kube-scheduler.conf $i:/usr/local/etc/
    931.     scp /etc/kubernetes/pki/scheduler/* $i:/etc/kubernetes/pki/scheduler/
    932.     scp /etc/systemd/system/kube-scheduler.service $i:/etc/systemd/system/
    933.     ssh $i "systemctl enable kube-scheduler && systemctl start kube-scheduler &> /dev/null"
    934.     if [ $? ];then
    935.         echo -e "\033[32m $i kube-scheduler 启动成功\033[0m"
    936.     else
    937.         echo -e "\033[31m $i kube-scheduler 启动失败，请检查日志文件\033[0m"
    938.     fi
    939. done
    940. }
    942. deployKubelet(){
    943.     kubectl create clusterrolebinding kubelet-bootstrap --clusterrole=system:node-bootstrapper --user=kubelet-bootstrap &> /dev/null
    944.     cd /etc/kubernetes/pki/bootstrap/
    945.     echo ${bootstrapToken}
    946.     kubectl config set-cluster kubernetes     --certificate-authority=/etc/kubernetes/pki/CA/ca.pem --embed-certs=true --server=https://${k8sVIP}:8443 --kubeconfig=bootstrap.kubeconfig
    947.     kubectl config set-credentials kubelet-bootstrap --token=${bootstrapToken} --kubeconfig=bootstrap.kubeconfig    
    948.     kubectl config set-context default --cluster=kubernetes --user=kubelet-bootstrap --kubeconfig=bootstrap.kubeconfig    
    949.     kubectl config use-context default --kubeconfig=bootstrap.kubeconfig    
    951. cat << EOF > /etc/systemd/system/kubelet.service
    952. [Unit]
    953. Description=Kubernetes Kubelet
    954. Documentation=https://github.com/GoogleCloudPlatform/kubernetes
    955. After=docker.service
    956. Requires=docker.service
    958. [Service]
    959. WorkingDirectory=/var/lib/kubelet
    960. EnvironmentFile=/usr/local/etc/kubelet.conf
    961. ExecStart=/usr/local/bin/kubelet \$KUBELET_ARGS
    962. Restart=on-failure
    963. RestartSec=5
    965. [Install]
    966. WantedBy=multi-user.target
    967. EOF
    969. for i in ${NodeIP[*]};do
    970. cat << EOF > /tmp/kubelet.conf
    971. KUBELET_ARGS="--address=$i \\
    972.   --hostname-override=$i \\
    973.   --pod-infra-container-image=gcr.io/google_containers/pause-amd64:3.0 \\
    974.   --bootstrap-kubeconfig=/etc/kubernetes/pki/bootstrap/bootstrap.kubeconfig \\
    975.   --kubeconfig=/etc/kubernetes/pki/bootstrap/kubelet.kubeconfig \\
    976.   --cert-dir=/etc/kubernetes/pki/bootstrap \\
    977.   --cluster-dns=${clusterDnsIP} \\
    978.   --cluster-domain=cluster.local. \\
    979.   --serialize-image-pulls=false \\
    980.   --fail-swap-on=false \\
    981.   --logtostderr=false \\
    982.   --log-dir=/var/log/kubernetes/kubelet \\
    983.   --v=2"
    984. EOF
    986.     ssh $i mkdir -p /etc/kubernetes/pki/bootstrap/ /var/lib/kubelet /var/log/kubernetes/kubelet &>/dev/null
    987.     scp /etc/systemd/system/kubelet.service $i:/etc/systemd/system/
    988.     scp /tmp/kubelet.conf $i:/usr/local/etc/
    989.     scp /etc/kubernetes/pki/bootstrap/bootstrap.kubeconfig $i:/etc/kubernetes/pki/bootstrap/
    990.     ssh $i "systemctl enable kubelet && systemctl start kubelet"
    991.     if [ $? ];then
    992.         echo -e "\033[32m $i kubelet 启动成功\033[0m"
    993.     else
    994.         echo -e "\033[31m $i kubelet 启动失败，请检查日志文件\033[0m"
    995.     fi
    996. done
    998. #确保在所有节点都发出了CSR之后再进行approve操作
    999. sleep 10
    1000. for i in $(kubectl get csr | awk 'NR>1{print $1}' );do kubectl certificate approve $i ;done
    1001. wget http://$web/kbi/pause-amd64-3.0.tar.gz -O /tmp/pause-amd64-3.0.tar.gz
    1002. for i in ${NodeIP[*]};do
    1003.     scp /tmp/pause-amd64-3.0.tar.gz $i:/tmp
    1004.     ssh $i "docker image load -i /tmp/pause-amd64-3.0.tar.gz"
    1005. done
    1006. }
    1008. deployKubeProxy(){
    1009.     mkdir -p /etc/kubernetes/pki/proxy        
    1010.     cd /etc/kubernetes/pki/proxy
    1011.     cat << EOF > proxy-csr.json
    1012. {
    1013.     "CN": "system:kube-proxy",
    1014.     "hosts": [],
    1015.     "key": {
    1016.         "algo": "rsa",
    1017.         "size": 2048
    1018.     },
    1019.     "names": [
    1020.         {
    1021.             "C": "CN",
    1022.             "ST": "GuangDong",
    1023.             "L": "GuangZhou",
    1024.             "O": "system:kube-proxy",
    1025.             "OU": "Ops"
    1026.         }
    1027.     ]
    1028. }
    1029. EOF
    1031.     if [[ ! -e /etc/kubernetes/pki/proxy/proxy.pem && ! -e /etc/kubernetes/pki/proxy/proxy-key.pem ]];then
    1032.         cfssl gencert -ca=/etc/kubernetes/pki/CA/ca.pem -ca-key=/etc/kubernetes/pki/CA/ca-key.pem -config=/etc/kubernetes/pki/CA/ca-config.json -profile=kubernetes proxy-csr.json | cfssljson -bare proxy    
    1033.     fi
    1035.     kubectl config set-cluster kubernetes \
    1036.     --certificate-authority=/etc/kubernetes/pki/CA/ca.pem \
    1037.     --embed-certs=true \
    1038.     --server=https://${k8sVIP}:8443 \
    1039.     --kubeconfig=proxy.kubeconfig
    1040.     kubectl config set-credentials system:kube-proxy \
    1041.     --client-certificate=/etc/kubernetes/pki/proxy/proxy.pem \
    1042.     --embed-certs=true \
    1043.     --client-key=/etc/kubernetes/pki/proxy/proxy-key.pem \
    1044.     --kubeconfig=proxy.kubeconfig
    1045.      kubectl config set-context system:kube-proxy@kubernetes \
    1046.     --cluster=kubernetes \
    1047.     --user=system:kube-proxy \
    1048.     --kubeconfig=proxy.kubeconfig
    1049.     kubectl config use-context system:kube-proxy@kubernetes --kubeconfig=proxy.kubeconfig
    1051. cat << EOF > /etc/systemd/system/kube-proxy.service
    1052. [Unit]
    1053. Description=Kubernetes Kube-Proxy Server
    1054. Documentation=https://github.com/GoogleCloudPlatform/kubernetes
    1055. After=network.target
    1057. [Service]
    1058. WorkingDirectory=/var/lib/kube-proxy
    1059. EnvironmentFile=/usr/local/etc/kube-proxy.conf
    1060. ExecStart=/usr/local/bin/kube-proxy \$KUBE_PROXY_ARGS
    1061. Restart=on-failure
    1062. RestartSec=5
    1063. LimitNOFILE=65536
    1065. [Install]
    1066. WantedBy=multi-user.target
    1067. EOF
    1069. for i in ${NodeIP[*]};do
    1070. cat << EOF > /tmp/kube-proxy.conf
    1071. KUBE_PROXY_ARGS="--bind-address=$i \\
    1072.   --hostname-override=$i \\
    1073.   --cluster-cidr=${serviceNet} \\
    1074.   --kubeconfig=/etc/kubernetes/pki/proxy/proxy.kubeconfig \\
    1075.   --logtostderr=false \\
    1076.   --log-dir=/var/log/kubernetes/proxy \\
    1077.   --v=2"
    1078. EOF
    1079.     ssh $i mkdir -p /etc/kubernetes/pki/proxy/  /var/log/kubernetes/proxy  /var/lib/kube-proxy  &> /dev/null
    1080.     scp /etc/systemd/system/kube-proxy.service $i:/etc/systemd/system/
    1081.     scp /etc/kubernetes/pki/proxy/* $i:/etc/kubernetes/pki/proxy/
    1082.     scp /tmp/kube-proxy.conf $i:/usr/local/etc/
    1083.     ssh $i "systemctl enable kube-proxy && systemctl start kube-proxy "
    1084.     if [ $? ];then
    1085.         echo -e "\033[32m $i kube-proxy 启动成功\033[0m"
    1086.     else
    1087.         echo -e "\033[31m $i kube-proxy 启动失败，请检查日志文件\033[0m"
    1088.     fi
    1089. done
    1090. }
    1092. deployIngressController(){
    1093.     echo -e "\033[32m 正在部署nginx-ingress-controller.. \033[0m"
    1094.     if [ ! -e /tmp/nginx-ingress-controller-0.27.1.tar.gz ];then
    1095.         wget http://$web/kbi/nginx-ingress-controller-0.27.1.tar.gz -O /tmp/nginx-ingress-controller-0.27.1.tar.gz
    1096.     fi
    1097.     wget http://$web/kbi/nginx-ingress-controller-mandatory.yaml -O /tmp/nginx-ingress-controller-mandatory.yaml
    1098.     wget http://$web/kbi/nginx-ingress-controller-service.yaml -O /tmp/nginx-ingress-controller-service.yaml
    1099.     for i in ${NodeIP[*]};do
    1100.         scp /tmp/nginx-ingress-controller-0.27.1.tar.gz /tmp/nginx-ingress-controller-mandatory.yaml $i:/tmp/
    1101.         ssh $i exec  "docker image load -i /tmp/nginx-ingress-controller-0.27.1.tar.gz"
    1102.     done
    1103.     kubectl apply -f /tmp/nginx-ingress-controller-mandatory.yaml
    1104.     kubectl apply -f /tmp/nginx-ingress-controller-service.yaml
    1105.     sleep 5
    1106.     kubectl scale deploy -n ingress-nginx nginx-ingress-controller --replicas=${#NodeIP[@]}
    1107. }
    1109. deployCoreDNS(){
    1110.    echo
    1111.    echo -e "\033[32m 正在部署CoreDNS..... \033[0m"
    1112.    if [ ! -e /tmp/coredns-deployment-1.8.0.tar.gz ];then
    1113.         wget http://$web/kbi/coredns-deployment-1.8.0.tar.gz -O /tmp/coredns-deployment-1.8.0.tar.gz
    1114.         tar xf /tmp/coredns-deployment-1.8.0.tar.gz -C /tmp
    1115.    fi
    1116.    if [ ! -e /tmp/coredns-image-1.8.0.tar.gz ];then
    1117.         wget http://$web/kbi/coredns-image-1.8.0.tar.gz -O /tmp/coredns-image-1.8.0.tar.gz
    1118.    fi
    1120.    for i in ${NodeIP[*]};do
    1121.         scp /tmp/coredns-image-1.8.0.tar.gz $i:/tmp/ 
    1122.         ssh $i exec "docker image load -i /tmp/coredns-image-1.8.0.tar.gz"
    1123.    done
    1124.    bash /tmp/deployment-master/kubernetes/deploy.sh -i ${clusterDnsIP} -s | kubectl apply -f -
    1125.    sleep 5
    1126.    kubectl scale deploy -n kube-system coredns --replicas=${#NodeIP[@]}
    1127. }
    1128. autoSSHCopy
    1129. preparation
    1130. deployHaproxyKeepalived
    1131. deployETCD
    1132. setKubectl
    1133. deployFlannel
    1134. deployApiserver
    1135. deployControllerManager
    1136. deployScheduler
    1137. deployKubelet
    1138. deployKubeProxy
    1139. deployIngressController
    1140. deployCoreDNS
    1141. echo "source <(kubectl completion bash)" >> ~/.bashrc
    1142. kubectl get nodes
    1143. kubectl get cs
    1144. kubectl cluster-info
    1145. echo -e "\033[32m Deployment Complete 高可用K8S集群部署完成\033[0m"