1 RBAC
切换 contextkubectl create ns app-team1kubectl create serviceaccount cicd-token -n app-team1kubectl create clusterrole deployment-clusterrole --verb=create --resource=deployment,statefulset,daemonsetkubectl create rolebinding cicd-clusterrole --clusterrole=deployment-clusterrole --serviceaccount=app-team1:cicd-token
2 设置节点不可用
切换 contextkubectl cordon ek8s-node-1kubectl drain ek8s-node-1 --ignore-daemonsets --delete-local-data --force
3 升级kubeadm
apt-cache show kubeadm|grep kubeadm=1.20.1-00
kubectl version
切换 contextkubectl get nodesssh mk8s-master-0kubectl cordon mk8s-master-0kubectl drain mk8s-master-0 --ignore-daemonsetsapt-mark unhold kubeadm kubectl kubeletapt-get update && apt-get install -y kubeadm=1.20.1-00 kubelet=1.20.1-00kubectl=1.20.1-00apt-mark hold kubeadm kubectl kubeletsystemctl daemon-reloadsystemctl restart kubelet.servicekubeadm upgrade plankubeadm upgrade apply v1.20.1 --etcd-upgrade=falsekubectl -n kube-system rollout history deployment corednskubectl rollout undo deployment coredns -n kube-systemkubectl uncordon mk8s-master-0
4 备份还原etcd
实验环境配置
export ETCDCTL_API=3
[rancher@rmaster01 ~]$ docker ps -a |grep etcdc93210bbcacc rancher/rke-tools:v0.1.72 "/docker-entrypoint.…" 3 weeks ago Up 6 days etcd-rolling-snapshotse073d4c5266b rancher/coreos-etcd:v3.4.13-rancher1 "/usr/local/bin/etcd…" 3 weeks ago Up 6 days etcd[rancher@rmaster01 ~]$ docker cp e073d4c5266b:/usr/local/bin/etcdctl /usr/local/binopen /usr/local/bin/etcdctl: permission denied[rancher@rmaster01 ~]$ exitlogout[root@rmaster01 ~]# docker cp e073d4c5266b:/usr/local/bin/etcdctl /usr/local/bin
备份:export ETCDCTL_API=3ETCDCTL_API=3 etcdctl --endpoints https://172.0.0.1:2379 --cacert=/opt/KUIN00601/ca.crt --cert=/opt/KUIN00601/etcd-client.crt --key=/opt/KUIN00601/etcd-client.key snapshot save /var/lib/backup/etcd-snapshot.db查看备份ETCDCTL_API=3 etcdctl --write-out=table snapshot status /var/lib/backup/etcd-snapshot.db还原:systemctl stop kubeletmv /var/lib/etcd/ /var/lib/bak_etcdETCDCTL_API=3 etcdctl --endpoints https://172.0.0.1:2379 --cacert=/opt/KUIN00601/ca.crt --cert=/opt/KUIN00601/etcd-client.crt --key=/opt/KUIN00601/etcd-client.key --name k8s-master --data-dir="/var/lib/etcd/" --skip-hash-check --initial-advertise-peer-urls=https://127.0.0.1:2380 --initial-cluster k8s-master=https://127.0.0.1:2380 snapshot restore /var/lib/backup/etcd-snapshot-previous.db#systemctl start kubelet
5 配置网络策略
https://kubernetes.io/docs/concepts/services-networking/network-policies/
apiVersion: networking.k8s.io/v1kind: NetworkPolicymetadata:name: test-network-policynamespace: defaultspec:podSelector:matchLabels:role: dbpolicyTypes:- Ingress- Egressingress:- from:- ipBlock:cidr: 172.17.0.0/16except:- 172.17.1.0/24- namespaceSelector:matchLabels:project: myproject- podSelector:matchLabels:role: frontendports:- protocol: TCPport: 6379egress:- to:- ipBlock:cidr: 10.0.0.0/24ports:- protocol: TCPport: 5978
在命名空间 fubar 中创建网络策略 allow-port-from-namespace只允许 ns my-app 中的 pod 连上 fubar 中 pod 的 80 端口注意: 这里有 2 个 ns ,一个为 fubar(目标pod的ns),另外一个为 my-app(访问源pod的ns)
apiVersion: networking.k8s.io/v1kind: NetworkPolicymetadata:name: allow-port-from-namespacenamespace: fubarspec:podSelector:matchLabels: {}policyTypes:- Ingressingress:- from:- namespaceSelector:matchLabels:my-app-key: my-app-value- podSelector:matchLabels: {}ports:- protocol: TCPport: 80
6 创建 service
1)edit front-end ,在containers 中添加如下内容kubectl edit deployment front-endports:- name: httpprotocol: TCPcontainerPort: 802)expose 对应端口kubectl expose deployment front-end --type=NodePort --port=80 --target-port=80 --name=front-end-svc
7 创建 ingress
https://kubernetes.io/docs/concepts/services-networking/ingress/
apiVersion: networking.k8s.io/v1kind: Ingressmetadata:name: minimal-ingressannotations:nginx.ingress.kubernetes.io/rewrite-target: /spec:rules:- http:paths:- path: /testpathpathType: Prefixbackend:service:name: testport:number: 80
apiVersion: networking.k8s.io/v1kind: Ingressmetadata:name: pingnamespace: ing-internalannotations:nginx.ingress.kubernetes.io/rewrite-target: /spec:rules:- http:paths:- path: /hellopathType: Prefixbackend:service:name: helloport:number: 5678
8 扩容deployment
kubectl scale deployment --replicas=6 guestbook
9 调度pod 到指定节点
https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/
apiVersion: v1kind: Podmetadata:name: nginxlabels:env: testspec:containers:- name: nginximage: nginximagePullPolicy: IfNotPresentnodeSelector:disktype: ssd
切换 contextapiVersion: v1kind: Podmetadata:name: nginx-kusc0041spec:containers:- name: nginximage: nginxnodeSelector:disk: ssd
【】
[root@master ~]# kubectl label nodes node01 disk=ssdnode/node01 labeled[root@master ~]# kubectl get nodes --show-labelsNAME STATUS ROLES AGE VERSION LABELSmaster Ready master 28d v1.18.17 beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=master,kubernetes.io/os=linux,node-role.kubernetes.io/master=node01 Ready <none> 28d v1.18.0 beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,disk=ssd,kubernetes.io/arch=amd64,kubernetes.io/hostname=node01,kubernetes.io/os=linuxnode02 Ready <none> 28d v1.18.0 beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=node02,kubernetes.io/os=linux[root@master ~]#[root@master ~]# kubectl run nginx-kusc0041 --image=nginx --dry-run=client -oyaml > 9.yaml[root@master ~]#修改yaml文件、添加nodeSelectornodeSelector:disk: ssd[root@master ~]# cat 9.yamlapiVersion: v1kind: Podmetadata:creationTimestamp: nulllabels:run: nginx-kusc0041name: nginx-kusc0041spec:containers:- image: nginxname: nginx-kusc0041resources: {}nodeSelector:disk: ssddnsPolicy: ClusterFirstrestartPolicy: Alwaysstatus: {}[root@master ~]#[root@master ~]# kubectl get pod -o wideNAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATESmyapp-687598b8b4-9w6m6 1/1 Running 0 61m 172.16.196.132 node01 <none> <none>myapp-687598b8b4-j7b54 1/1 Running 0 61m 172.16.196.133 node01 <none> <none>myapp-687598b8b4-wrkm9 1/1 Running 0 61m 172.16.196.134 node01 <none> <none>nginx-kusc0041 1/1 Running 0 64s 172.16.196.136 node01 <none> <none>[root@master ~]#删除标签[root@master ~]# kubectl get nodes node01 --show-labelsNAME STATUS ROLES AGE VERSION LABELSnode01 Ready <none> 28d v1.18.0 beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,disk=ssd,kubernetes.io/arch=amd64,kubernetes.io/hostname=node01,kubernetes.io/os=linux[root@master ~]# kubectl label nodes node01 disk-node/node01 labeled[root@master ~]#设置标签给node02 再次验证[root@master ~]# kubectl label nodes node02 disk=ssdnode/node02 labeled[root@master ~]# kubectl get podNAME READY STATUS RESTARTS AGEmyapp-687598b8b4-9w6m6 1/1 Running 0 66mmyapp-687598b8b4-j7b54 1/1 Running 0 66mmyapp-687598b8b4-wrkm9 1/1 Running 0 66mnginx-kusc0041 1/1 Running 0 5m49s[root@master ~]# kubectl delete pod nginx-kusc0041pod "nginx-kusc0041" deleted[root@master ~]# kubectl get pod -o wideNAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATESmyapp-687598b8b4-9w6m6 1/1 Running 0 67m 172.16.196.132 node01 <none> <none>myapp-687598b8b4-j7b54 1/1 Running 0 67m 172.16.196.133 node01 <none> <none>myapp-687598b8b4-wrkm9 1/1 Running 0 67m 172.16.196.134 node01 <none> <none>[root@master ~]# kubectl apply -f 9.yamlpod/nginx-kusc0041 created[root@master ~]# kubectl get pod -o wideNAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATESmyapp-687598b8b4-9w6m6 1/1 Running 0 67m 172.16.196.132 node01 <none> <none>myapp-687598b8b4-j7b54 1/1 Running 0 67m 172.16.196.133 node01 <none> <none>myapp-687598b8b4-wrkm9 1/1 Running 0 67m 172.16.196.134 node01 <none> <none>nginx-kusc0041 1/1 Running 0 4s 172.16.140.76 node02 <none> <none>[root@master ~]#
10 统计 ready 状态节点数量
切换 contextkubectl get nodeskubectl describe nodes | grep -i taint | grep NoSchedule两者数据相减,echo number > /path/file
11 配置多容器
https://kubernetes.io/docs/concepts/workloads/pods/
apiVersion: v1kind: Podmetadata:name: kucc1spec:containers:- name: nginximage: nginx- name: redisimage: redis
【】
[root@master ~]# kubectl run kucc1 --image=nginx --dry-run=client -oyamlapiVersion: v1kind: Podmetadata:creationTimestamp: nulllabels:run: kucc1name: kucc1spec:containers:- image: nginxname: kucc1resources: {}dnsPolicy: ClusterFirstrestartPolicy: Alwaysstatus: {}[root@master ~]#
apiVersion: v1kind: PersistentVolumemetadata:name: task-pv-volumelabels:type: localspec:storageClassName: manualcapacity:storage: 10GiaccessModes:- ReadWriteOncehostPath:path: "/mnt/data"
apiVersion: v1kind: PersistentVolumemetadata:name: app-configspec:capacity:storage: 2GiaccessModes:- ReadWriteManyhostPath:path: /srv/app-config
【】
[root@master ~]# kubectl apply -f 12.yamlpersistentvolume/app-config created[root@master ~]# kubectl get pvNAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGEapp-config 2Gi RWX Retain Available 4s[root@master ~]#
apiVersion: v1kind: PersistentVolumeClaimmetadata:name: pv-volumespec:accessModes:- ReadWriteOnceresources:requests:storage: 10MistorageClassName: csi-hostpath-sc
apiVersion: v1kind: Podmetadata:name: web-serverspec:containers:- name: nginximage: nginxvolumeMounts:- mountPath: "/usr/share/nginx/html"name: mypvvolumes:- name: mypvpersistentVolumeClaim:claimName: pv-volume
kubectl edit pvc pv-volume --record
14 监控 pod 日志
切换 contextkubectl logs foobar | grep unable-to-access-website > /opt/KUTR00101/foobar
15 添加 sidecar 容器并输出日志
https://kubernetes.io/docs/concepts/cluster-administration/logging/
kubectl get pod big-corp-app -oyaml >15.yaml
volumeMounts:- name: varlogmountPath: /var/log- name: count-log-1image: busyboxargs: [/bin/sh, -c, 'tail -n+1 -f /var/log/1.log']volumeMounts:- name: varlogmountPath: /var/log- name: count-log-2image: busyboxargs: [/bin/sh, -c, 'tail -n+1 -f /var/log/2.log']volumeMounts:- name: varlogmountPath: /var/logvolumes:- name: varlogemptyDir: {}
volumeMounts:- name: varlogmountPath: /var/log- name: sidecarimage: busyboxargs: [/bin/sh, -c, 'tail -n+1 -f /var/log/11-factor-app.log']volumeMounts:- name: varlogmountPath: /var/logvolumes:- name: varlogemptyDir: {}
【】
-name: logsmountPath:/var/log-name: busyboximage: busybox args:[/bin/sh,-c,' tail -n+1-f/var/log/big-corp-app. log]volumeMounts:-name: logs mountPath:/var/log
- name: varlogemptyDir: {}
16 查看cpu 使用率最高的pod
切换 contextkubectl top pod -l name=cpu-loader -A --sort-by=cpuecho podName >> /opt/KUTR00401/KUTR00401.txt
17 排查集群中故障节点
切换 contextkubectl get nodesssh wk8s-node-0sudo -isystemctl status kubeletsystemctl enable kubeletsystemctl restart kubeletsystemctl status kubelet再次 get nodes, 确保节点恢复 Ready 状态
