本文基于H3CloudOS平台KaaS集群使用NFS动态卷供应配置指导文档

准备工作

1、NFS存储服务器配置

1.1 安装nfs-utils 软件包并设置启动

[root@clc-ops-nfs ~]# yum install -y nfs-utils 
[root@clc-ops-nfs ~]# systemctl start nfs-server 
[root@clc-ops-nfs ~]# systemctl enable nfs-server

1.2 配置NFS共享目录

[root@clc-ops-nfs ~]# mkdir /nfs-data
[root@clc-ops-nfs ~]# vi /etc/exports
/nfs-data   *(rw,async,no_root_squash)
[root@clc-ops-nfs ~]# exportfs -avr
exporting *:/nfs-data

2、KaaS集群配置NFS动态卷供应

2.1 配置k8s集群kube-apiserver.yaml (以 clc.kaas.ops的KaaS集群为例)

编辑kube-apiserver.yaml文件的24行 添加 - —feature-gates=RemoveSelfLink=false

[root@clc-kaas-ops-master-01-e3490 ~]# vi /etc/kubernetes/manifests/kube-apiserver.yaml
apiVersion: v1
kind: Pod
metadata:
  annotations:
    kubeadm.kubernetes.io/kube-apiserver.advertise-address.endpoint: 10.1.24.11:6443
  creationTimestamp: null
  labels:
    component: kube-apiserver
    tier: control-plane
  name: kube-apiserver
  namespace: kube-system
spec:
  containers:
  - command:
    - kube-apiserver
    - --advertise-address=10.1.24.11
    - --allow-privileged=true
    - --anonymous-auth=True
    - --apiserver-count=1
    - --authorization-mode=Node,RBAC
    - --bind-address=0.0.0.0
    - --client-ca-file=/etc/kubernetes/ssl/ca.crt
    - --feature-gates=RemoveSelfLink=false

修改完成后重启master节点的kubelet

[root@clc-test-master-38cad ~]# systemctl restart kubelet

2.2 上传nfs动态卷配置文件到K8S集群master节点

动态卷供应需要使用nfs-client-provisioner.tar的docker镜像文件以及配置相关管deploymet的yaml

• nfs-client-provisioner.tar —-镜像
• external-storage-master.zip —-yaml合集

  [root@clc-test-master-38cad ~]# ll 
  total 56832 
  -rw-r--r-- 1 root root 11662350 Apr 19 05:40 external-storage-master.zip 
  -rw-r--r-- 1 root root 46082048 Apr 19 05:39 nfs-client-provisioner.tar
  

  2.3 Harbor上传镜像

  镜像load

  [root@clc-kaas-ops-master-01-e3490 ~]#  docker load -i nfs-client-provisioner.tar
  8dfad2055603: Loading layer [=======================================>]  4.284MB/4.284MB 
  a17ae64bae4f: Loading layer [=======================================>]  2.066MB/2.066MB 
  bd01fa00617b: Loading layer [=======================================>]  39.72MB/39.72MB 
  Loaded image: quay.io/external_storage/nfs-client-provisioner:latest
  

  镜像tag

  [root@clc-kaas-ops-master-01-e3490 ~]# docker images |grep nfs-client-provisioner
  quay.io/external_storage/nfs-client-provisioner                                                       latest                 16d2f904b0d8        3 years ago         45.5MB
  [root@clc-kaas-ops-master-01-e3490 ~]# docker tag quay.io/external_storage/nfs-client-provisioner:latest os-harbor-svc.default.svc.cloudos:11443/clc2ops/nfs-client-provisioner:latest
  

  Harbor上并推入镜像

  [root@clc-kaas-ops-master-01-e3490 ~]#  docker push  os-harbor-svc.default.svc.cloudos:11443/clc2ops/nfs-client-provisioner:latest
  

  2.4 配置nfs动态卷配置文件到K8S集群master节点

  解压external-storage-master.zip文件

  [root@clc-kaas-ops-master-01-e3490 ~]# unzip external-storage-master.zip 
  [root@clc-kaas-ops-master-01-e3490 ~]# cd /root/external-storage-master/nfs-client/deploy
  [root@clc-kaas-ops-master-01-e3490 deploy]# ll
  total 24
  -rw-r--r-- 1 root root  221 May 14 18:45 class.yaml
  -rw-r--r-- 1 root root 1030 Apr 22  2020 deployment-arm.yaml
  -rw-r--r-- 1 root root 1046 May 13 21:17 deployment.yaml
  drwxr-xr-x 2 root root  214 Apr 22  2020 objects
  -rw-r--r-- 1 root root 1824 May 13 21:07 rbac.yaml
  -rw-r--r-- 1 root root  241 Apr 22  2020 test-claim.yaml
  -rw-r--r-- 1 root root  424 Apr 22  2020 test-pod.yaml
  [root@clc-kaas-ops-master-01-e3490 deploy]#
  

  创建用于创建nfs动态卷deploy的namespace 本次以clc2-ops为例
  针对clc-nfs-volume namespace修改rbac.yaml文件(替换default namespace为目标namespace)并部署 ```shell [root@clc-kaas-ops-master-01-e3490 deploy]# sed -i ‘s/namespace: default/namespace: clc2-ops/g’ rbac.yaml [root@clc-kaas-ops-master-01-e3490 deploy]# kubectl apply -f rbac.yaml serviceaccount/nfs-client-provisioner unchanged clusterrole.rbac.authorization.k8s.io/nfs-client-provisioner-runner unchanged clusterrolebinding.rbac.authorization.k8s.io/run-nfs-client-provisioner configured role.rbac.authorization.k8s.io/leader-locking-nfs-client-provisioner unchanged rolebinding.rbac.authorization.k8s.io/leader-locking-nfs-client-provisioner unchanged [root@clc-kaas-ops-master-01-e3490 deploy]#

针对clc-nfs-volume namespace修改deployment.yaml文件
```shell
[root@clc-kaas-ops-master-01-e3490 deploy]# sed -i 's/namespace: default/namespace: clc2-ops/g' deployment.yaml
[root@clc-kaas-ops-master-01-e3490 deploy]#

[root@clc-test-master-38cad deploy]# vi deployment.yaml

• 修改image镜像： os-harbor-svc.default.svc.cloudos:11443/clc2ops/nfs-client-provisioner:latest
• 修改后端nfs server
  • ip： 10.1.24.192
  • 共享目录：/nfs-data

[root@clc-kaas-ops-master-01-e3490 deploy]# vi deployment.yaml
[root@clc-kaas-ops-master-01-e3490 deploy]#
apiVersion: apps/v1
kind: Deployment
metadata:
     name: nfs-client-provisioner
  labels:
    app: nfs-client-provisioner
  # replace with namespace where provisioner is deployed
  namespace: clc2-ops                    #确认ns
spec:
  replicas: 1
  strategy:
    type: Recreate
  selector:
    matchLabels:
      app: nfs-client-provisioner
  template:
    metadata:
      labels:
        app: nfs-client-provisioner
    spec:
      serviceAccountName: nfs-client-provisioner
      containers:
        - name: nfs-client-provisioner
          image: quay.io/external_storage/nfs-client-provisioner:latest
          volumeMounts:
            - name: nfs-client-root
              mountPath: /persistentvolumes
          env:
            - name: PROVISIONER_NAME
              value: fuseim.pri/ifs      #此处名称需要和sc的名称保持一致
            - name: NFS_SERVER
              value: 10.1.24.192         #nfs server IP
            - name: NFS_PATH
              value: /nfs-data           #nfs server 共享目录
      volumes:
        - name: nfs-client-root
          nfs:
            server: 10.1.24.192          #nfs server IP
            path: /nfs-data              #nfs server 共享目录
~

[root@clc-kaas-ops-master-01-e3490 deploy]# kubectl apply -f deployment.yaml
deployment.apps/nfs-client-provisioner created
[root@clc-kaas-ops-master-01-e3490 deploy]#

编辑class.yaml 创建storageclass

[root@clc-test-master-38cad deploy]# vi class.yaml 
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: clc2ops-nfs-400g-storage
provisioner: fuseim.pri/ifs # or choose another name, must match deployment's env PROVISIONER_NAME'
parameters:
archiveOnDelete: "false"
reclaimPolicy: Retain
volumeBindingMode: Immediate
~
[root@clc-kaas-ops-master-01-e3490 deploy]# kubectl apply -f class.yaml
storageclass.storage.k8s.io/clc2ops-nfs-400g-storage created
[root@clc-kaas-ops-master-01-e3490 deploy]#
[root@clc-kaas-ops-master-01-e3490 deploy]# kubectl get sc
NAME           PROVISIONER      RECLAIMPOLICY   VOLUMEBINDINGMODE   ALLOWVOLUMEEXPANSION   AGE
clc2ops-400g   fuseim.pri/ifs   Retain          Immediate           true                   16m
[root@clc-kaas-ops-master-01-e3490 deploy]#

3、动态卷供应测试

3.1 创建PVC测试

[root@clc-kaas-ops-master-01-e3490 deploy]# vi pvc.yaml
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: test-claim
  namespace: clc2-ops
  annotations:
    volume.beta.kubernetes.io/storage-class: "clc2ops-400g"
spec:
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 1Mi
[root@clc-kaas-ops-master-01-e3490 deploy]# kubectl apply -f test-claim.yaml
persistentvolumeclaim/mypvc created
[root@clc-kaas-ops-master-01-e3490 deploy]# kubectl get pvc -n clc2-ops
NAME         STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
test-claim   Bound    pvc-ef823402-c339-4078-b4cd-d2bf85ccbe38   1Mi        RWX            clc2ops-400g   25m
[root@clc-kaas-ops-master-01-e3490 deploy]#

3.2 pod中引用pvc

apiVersion: v1
kind: Pod
metadata:
  name: test-pod
  namespace: clc2-ops
spec:
  containers:
    - name: myfrontend
      image: os-harbor-svc.default.svc.cloudos:11443/clc2ops/nginx:latest
      volumeMounts:
      - mountPath: "/var/www/html"
        name: mypd
  volumes:
    - name: mypd
      persistentVolumeClaim:
        claimName: test-claim