1.Kubernetes ceph rbd 基于 storageclass动态生成pv
1、创建rbd的供应商 provisioner ceph.com/rbd（环境变量里面的值）
#把rbd-provisioner.tar.gz上传，手动解压，这里面封装的是镜像
[root@master ceph]# docker load -i rbd-provisioner.tar.gz
1d31b5806ba4: Loading layer [==================================================>] 208.3MB/208.3MB
499d93e0e038: Loading layer [==================================================>] 164.1MB/164.1MB
7c9bb3d61493: Loading layer [==================================================>] 44.52MB/44.52MB
Loaded image: quay.io/xianchao/external_storage/rbd-provisioner:v1
[root@master rbd-provisioner]# cat rbd-provisioner.yaml
kind: ClusterRole #定义了一个ClusterRole，可以对哪些资源做操作
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: rbd-provisioner
rules:
- apiGroups: [“”]
resources: [“persistentvolumes”]
verbs: [“get”, “list”, “watch”, “create”, “delete”]
- apiGroups: [“”]
resources: [“persistentvolumeclaims”]
verbs: [“get”, “list”, “watch”, “update”]
- apiGroups: [“storage.k8s.io”]
resources: [“storageclasses”]
verbs: [“get”, “list”, “watch”]
- apiGroups: [“”]
resources: [“events”]
verbs: [“create”, “update”, “patch”]
- apiGroups: [“”]
resources: [“services”]
resourceNames: [“kube-dns”,”coredns”]
verbs: [“list”, “get”]
—-
kind: ClusterRoleBinding #定义了一个clusterrolebinding，将下面的serviceAccount: rbd-provisioner 绑定
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: rbd-provisioner
subjects:
- kind: ServiceAccount
name: rbd-provisioner
namespace: default
roleRef:
kind: ClusterRole
name: rbd-provisioner
apiGroup: rbac.authorization.k8s.io
—-
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: rbd-provisioner
rules:
- apiGroups: [“”]
resources: [“secrets”]
verbs: [“get”]
- apiGroups: [“”]
resources: [“endpoints”]
verbs: [“get”, “list”, “watch”, “create”, “update”, “patch”]
—-
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: rbd-provisioner
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: rbd-provisioner
subjects:
- kind: ServiceAccount
name: rbd-provisioner
namespace: default
—-
apiVersion: apps/v1
kind: Deployment
metadata:
name: rbd-provisioner
spec:
selector:
matchLabels:
app: rbd-provisioner
replicas: 1
strategy:
type: Recreate
template:
metadata:
labels:
app: rbd-provisioner
spec:
containers:
- name: rbd-provisioner
image: quay.io/xianchao/external_storage/rbd-provisioner:v1
imagePullPolicy: IfNotPresent
env:
- name: PROVISIONER_NAME #供应商名字
value: ceph.com/rbd
serviceAccount: rbd-provisioner
—-
apiVersion: v1
kind: ServiceAccount
metadata:
name: rbd-provisioner
[root@master rbd-provisioner]# kubectl get pod
NAME READY STATUS RESTARTS AGE
rbd-provisioner-6bbc95cd74-g6lcd 1/1 Running 0 9s
2、创建ceph-secret
#创建pool池
[root@master1-admin ceph]# ceph osd pool create k8stest1 56
pool ‘k8stest1’ created
[root@master1-admin ceph]# ceph osd pool ls
rbd
cephfs_data
cephfs_metadata
k8srbd1
k8stest
k8stest1
You have new mail in /var/spool/mail/root
[root@master1-admin ~]# ceph auth get-key client.admin | base64
QVFDOWF4eGhPM0UzTlJBQUJZZnVCMlZISVJGREFCZHN0UGhMc3c9PQ==
[root@master rbd-provisioner]# cat ceph-secret-1.yaml
apiVersion: v1
kind: Secret
metadata:
name: ceph-secret-1
type: “ceph.com/rbd”
data:
key: QVFDOWF4eGhPM0UzTlJBQUJZZnVCMlZISVJGREFCZHN0UGhMc3c9PQ==
[root@master rbd-provisioner]# kubectl get secret
NAME TYPE DATA AGE
ceph-secret Opaque 1 19h
ceph-secret-1 ceph.com/rbd 1 2m33s
default-token-cwbdx kubernetes.io/service-account-token 3 91d
nfs-client-provisioner-token-plww9 kubernetes.io/service-account-token 3 19d
qingcloud kubernetes.io/dockerconfigjson 1 91d
rbd-provisioner-token-82bql kubernetes.io/service-account-token 3 10m
[root@master ceph]# cat rbd-provisioner/storageclass.yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: k8s-rbd
provisioner: ceph.com/rbd
parameters:
monitors: 192.168.0.5:6789,192.168.0.6:6789,192.168.0.7:6789
adminId: admin
adminSecretName: ceph-secret-1
pool: k8stest1
userId: admin
userSecretName: ceph-secret-1
fsType: xfs
imageFormat: “2”
imageFeatures: “layering”
# pool: k8stest1 这个pool池子是之前创建的
[root@master rbd-provisioner]# kubectl apply -f storageclass.yaml
storageclass.storage.k8s.io/k8s-rbd created
[root@master rbd-provisioner]# kubectl get sc
NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE
k8s-rbd ceph.com/rbd Delete Immediate false 7s
这个时候存储类在申请pv的时候就会找到ceph.com/rbd供应商，然后通过该供应商从集群里面去划分出来一个pv。
4、创建pvc
invalid AccessModes [ReadWriteMany]: only AccessModes [ReadWriteOnce ReadOnlyMany] are supported(可以看到rbd不支持[ReadWriteMany)
[root@master rbd-provisioner]# kubectl get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
ceph-pvc Bound ceph-pv 1Gi RWX 18h
example-local-claim Bound example-pv 5Gi RWO local-storage 63d
rbd-pvc Pending k8s-rbd 5s
[root@master rbd-provisioner]# kubectl describe pvc rbd-pvc
Name: rbd-pvc
Namespace: default
StorageClass: k8s-rbd
Status: Pending
Volume:
Labels:
Annotations: volume.beta.kubernetes.io/storage-provisioner: ceph.com/rbd
Finalizers: [kubernetes.io/pvc-protection]
Capacity:
Access Modes:
VolumeMode: Filesystem
Mounted By:
Events:
Type Reason Age From Message
—— ——— —— —— ———-
Normal Provisioning 10s (x2 over 25s) ceph.com/rbd_rbd-provisioner-6bbc95cd74-g6lcd_ee50d24f-015f-11ec-a6a6-9e54668c01e4 External provisioner is provisioning volume for claim “default/rbd-pvc”
Warning ProvisioningFailed 10s (x2 over 25s) ceph.com/rbd_rbd-provisioner-6bbc95cd74-g6lcd_ee50d24f-015f-11ec-a6a6-9e54668c01e4 failed to provision volume with StorageClass “k8s-rbd”: invalid AccessModes [ReadWriteMany]: only AccessModes [ReadWriteOnce ReadOnlyMany] are supported
Normal ExternalProvisioning 10s (x3 over 25s) persistentvolume-controller waiting for a volume to be created, either by external provisioner “ceph.com/rbd” or manually created by system administrator
[root@master rbd-provisioner]# kubectl delete -f
ceph-secret-1.yaml rbd-provisioner.yaml rbd-pvc.yaml storageclass.yaml
[root@master rbd-provisioner]# kubectl delete -f rbd-pvc.yaml
persistentvolumeclaim “rbd-pvc” deleted
[root@master rbd-provisioner]# vim rbd-pvc.yaml
[root@master rbd-provisioner]# cat rbd-pvc.yaml
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: rbd-pvc
spec:
accessModes:
- ReadWriteOnce
volumeMode: Filesystem
resources:
requests:
storage: 1Gi
storageClassName: k8s-rbd
[root@master rbd-provisioner]# kubectl apply -f rbd-pvc.yaml
persistentvolumeclaim/rbd-pvc created
[root@master rbd-provisioner]# kubectl get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
ceph-pvc Bound ceph-pv 1Gi RWX 19h
example-local-claim Bound example-pv 5Gi RWO local-storage 63d
rbd-pvc Bound pvc-5eef6286-c89f-421a-88d1-1a6593423087 1Gi RWO k8s-rbd 6s
5、创建pod，挂载pvc
[root@master rbd-provisioner]# cat pod-sto.yaml
apiVersion: v1
kind: Pod
metadata:
labels:
test: rbd-pod
name: ceph-rbd-pod
spec:
containers:
- name: ceph-rbd-nginx
image: nginx
imagePullPolicy: IfNotPresent
volumeMounts:
- name: ceph-rbd
mountPath: /mnt
readOnly: false
volumes:
- name: ceph-rbd
persistentVolumeClaim:
claimName: rbd-pvc
[root@master rbd-provisioner]# kubectl get pod
NAME READY STATUS RESTARTS AGE
ceph-rbd-pod 1/1 Running 0 26s
2.Kubernetes ceph fs 基于 storageclass动态生成pv
1. 实验环境简述:
本实验主要演示了将现有CephFS文件系统存储用作k8s 动态创建持久性存储（pv)的示例。假设您的环境已经建立了一个工作的Ceph集群。
因为上一篇文档中描述的使用RBD模式创建的pvc不支持RWM(readwriteMany)，只支持 RWO(ReadWriteOnce)和ROM(ReadOnlyMany)
k8s 不支持跨节点挂载同一Ceph RBD，支持跨节点挂载 CephFS，让所有的任务都调度到指定node上执行，来保证针对RBD的操作在同一节点上。
2. 配置步骤:
1. 在ceph服务器上面创建ceph fs文件系统
ceph osd pool create fs_kube_data 128
ceph osd pool create fs_kube_metadata 128
ceph fs new cephfs fs_kube_metadata fs_kube_data
2. 测试挂载CephFS文件系统
cephfs有两种mount方式：内核（即mount命令）、用户态（ceph-fuse）
- 内核态挂载命令:
mount -t ceph 10.83.32.224:/ /mnt/cephfs -o name=admin,secret=AQDvBadczen5HRAAD9G3IIU4v9IUtQC6Qf0g5w==
# 这里的secret，是需要在ceph主机上面查看的
[root@k8sdemo-ceph1 ~]# ceph auth print-key client.admin
AQDvBadczen5HRAAD9G3IIU4v9IUtQC6Qf0g5w==[root@k8sdemo-ceph1 ~]#
# 在ceph服务器上面查看secret的内容
- 用户态挂载命令：
cephfs还支持用户态mount，方法是使用ceph-fuse命令。
scp -r /etc/ceph/ceph.client.admin.keyring root@node-01:/etc/ceph/
# 首先是将ceph存储上面的认证文件拷贝到挂载主机
yum -y install ceph-fuse #安装ceph-fuse软件，需要有ceph.repo仓库文件，可从ceph服务器拷贝
ceph-fuse -m 10.83.32.224:6789 /tmp/mount/
- 用户态挂载可以限制目录的大小容量
需要两步：
1. 设置用户待使用目录的attr:
[root@node-01 mount]# mkdir -p prometheus
[root@node-01 mount]# mkdir -p grafana
[root@node-01 mount]# ll
total 1
-rw-r—r—. 1 root root 0 Apr 9 16:57 chenbin
-rw-r—r—. 1 root root 0 Apr 9 16:57 docker
-rw-r—r—. 1 root root 0 Apr 9 16:57 file
-rw-r—r—. 1 root root 0 Apr 9 16:57 gaoyang
drwxr-xr-x. 1 root root 0 Apr 9 19:55 grafana
-rw-r—r—. 1 root root 0 Apr 9 16:57 mk
drwxr-xr-x. 1 root root 0 Apr 9 19:55 prometheus
-rw-r—r—. 1 root root 0 Apr 9 16:34 test
[root@node-01 mount]#
yum -y install attr #安装attr 设置属性的命令
setfattr -n ceph.quota.max_bytes -v 100000000 /tmp/mount/grafana/
# 设置目录的容量限制为100Mb
# 在另外一台k8s node2节点安装ceph-fuse，拷贝ceph密钥文件，挂载cephfs目录，并且设置容量限制
ls /etc/yum.repos.d/ceph.repo
yum install -y ceph-fuse
mkdir -p /tmp/cephfs
scp -r /etc/ceph/ceph.client.admin.keyring root@node-02:/etc/ceph/
ceph-fuse -m 10.83.32.224:6789 -r /grafana /tmp/cephfs —client-quota
cd /tmp/CephFS
dd if=/dev/zero of=test2 bs=1M count=10
# 模拟写入文件
[root@node-02 cephfs]# df -h|grep cephfs
ceph-fuse 92M 92M 0 100% /tmp/cephfs
# 发现使用率已经超过100%
[root@node-02 cephfs]# dd if=/dev/zero of=test2 bs=1M count=100
dd: error writing ‘test2’: Disk quota exceeded
1+0 records in
0+0 records out
0 bytes (0 B) copied, 0.00127417 s, 0.0 kB/s
# 不能写入文件了
3. 测试pod使用CephFS文件系统
vim ceph-test.yaml
apiVersion: v1
kind: Pod
metadata:
name: cephfs2
namespace: kube-system
spec:
containers:
- name: cephfs-rw
image: nginx
volumeMounts:
- mountPath: “/mnt/cephfs”
name: cephfs
volumes:
- name: cephfs
cephfs:
monitors:
- 10.83.32.224:6789
user: “admin”
secretRef:
name: ceph-secret
readOnly: true
kubectl apply -f ceph-test.yaml
[root@master-01 cephfs_provisioner]# kubectl exec -it -n kube-system cephfs2 — mount|grep cephfs
10.83.32.224:6789:/ on /mnt/cephfs type ceph (ro,relatime,name=admin,secret=
[root@master-01 cephfs_provisioner]#
# 可以查看到cephfs2的pod已经挂载了cephfs
4. 测试pvc使用CephFS文件系统
创建一个pv,这个pv调用了cephfs文件系统，其中需要设置cephfs mds服务器的地址和secret。这里发现一个问题就是刚开始创建的pvc，总是自动关联storagecalss,后面设置了一个storageclass： “”cai keyi g
[root@master-01 cephfs_provisioner]# cat ceph-pvc-test.yaml
—-
apiVersion: v1
kind: PersistentVolume
metadata:
name: cephfs-pv3
namespace: kube-system
spec:
capacity:
storage: 1Gi
accessModes:
- ReadWriteOnce
cephfs:
monitors:
- 10.83.32.225:6789
path: /grafana
user: admin
secretRef:
name: ceph-secret
readOnly: false
persistentVolumeReclaimPolicy: Recycle
—-
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: cephfs-pv3
namespace: kube-system
spec:
accessModes:
- ReadWriteOnce
storageClassName: “”
resources:
requests:
storage: 1Gi
[root@master-01 cephfs_provisioner]# kubectl get pv -n kube-system cephfs-pv3
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE
cephfs-pv3 1Gi RWO Recycle Bound kube-system/cephfs-pv3
[root@master-01 cephfs_provisioner]# kubectl get pvc -n kube-system cephfs-pv3
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
cephfs-pv3 Bound cephfs-pv3 1Gi RWO 19s
5. 测试cephfs用作storage class
官方支持storage class的存储类型里，明明没有cephfs呀。目前k8s（v1.10）的确是不支持cephfs用作storage class的，但是我们已经有了ceph集群，而RBD又不支持 ReadWriteMany，这样就没办法做共享存储类型的Volume了，使用场景会受一些限制。而为了这个场景再单独去维护一套glusterfs，又很不划算，当然是能用cephfs做storage class最完美啦。
社区其实是有项目做这个的，就是 external storage，只是现在还在孵化器里。所谓external storage其实就是一个 controller，它会去监听 apiserver 的storage class api的变化，当发现有cephfs的请求，它会拿来处理：根据用户的请求，创建PV，并将PV的创建请求发给api server；PV创建后再将其与PVC绑定。
同时，controller的 cephfs-provisioner会调用 cephfs_provisioner.py 脚本去cephfs集群上去创建目录，目录名规则为 “kubernetes-dynamic-pvc-%s”, uuid.NewUUID()。 也就是说，external-storage作为ceph集群的一个用户，自己维护了不同PVC和cephfs目录的对应关系。
git clone https://github.com/kubernetes-incubator/external-storage.git
cd /data/cephfs_provisioner/external-storage/ceph/cephfs/deploy
[root@master-01 deploy]# ll
total 4
drwxr-xr-x. 2 root root 29 Apr 9 17:51 non-rbac
drwxr-xr-x. 2 root root 152 Apr 9 18:16 rbac
-rw-r—r—. 1 root root 1270 Apr 9 17:51 README.md
[root@master-01 deploy]#
# 下载external-storage cephfs storageclass git路径
NAMESPACE=kube-system # 设置命名空间，修改yaml文件里面的命名空间为kube-system
sed -r -i “s/namespace: [^ ]+/namespace: $NAMESPACE/g” ./rbac/.yaml
sed -r -i “N;s/(name: PROVISIONER_SECRET_NAMESPACE.\n[[:space:]])value:./\1value: $NAMESPACE/“ ./rbac/deployment.yaml
kubectl -n kube-system apply -f ./rbac/
[root@master-01 deploy]# kubectl get pods -n kube-system cephfs-provisioner-79d97b7bdf-s48fv
NAME READY STATUS RESTARTS AGE
cephfs-provisioner-79d97b7bdf-s48fv 1/1 Running 0 59m
# 等 cephfs-provisioner 程序启动起来之后，就可以配置cephfs的storageclass呢
[root@master-01 cephfs_provisioner]# cat cephfs-sc.yaml
kind: StorageClass
apiVersion: storage.k8s.io/v1
metadata:
name: cephfs
namespace: kube-system
provisioner: ceph.com/cephfs
parameters:
monitors: 10.83.32.225:6789
adminId: admin
adminSecretName: ceph-secret
adminSecretNamespace: “kube-system”
# 注意这里的 adminSecretNamespace的值一定要是kube-system，因为我们实验中的Secret是在kube-system创建的；
[root@master-01 cephfs_provisioner]#
[root@master-01 cephfs_provisioner]# kubectl get sc -n kube-system cephfs
NAME PROVISIONER AGE
cephfs ceph.com/cephfs 2m18s
[root@master-01 cephfs_provisioner]#
# 配置好了cephfs的storageclass之后，就是配置pvc，调用cephfs的sc
[root@master-01 cephfs_provisioner]# cat cephfs-pvc.yaml
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: claim
annotations:
volume.beta.kubernetes.io/storage-class: “cephfs”
spec:
accessModes:
- ReadWriteMany
resources:
requests:
storage: 1Gi
—-
kind: Pod
apiVersion: v1
metadata:
name: test-pod
spec:
containers:
- name: test-pod
image: nginx
volumeMounts:
- name: pvc
mountPath: “/mnt”
restartPolicy: “Never”
volumes:
- name: pvc
persistentVolumeClaim:
claimName: claim
[root@master-01 cephfs_provisioner]#
[root@master-01 ceph]# kubectl get pods -n kube-system test-pod
NAME READY STATUS RESTARTS AGE
test-pod 1/1 Running 0 22m
# 测试pod是运行起来了
[root@master-01 ceph]# kubectl exec -it -n kube-system test-pod — mount|grep mnt
10.83.32.225:6789:/volumes/kubernetes/kubernetes/kubernetes-dynamic-pvc-4227a096-5b38-11e9-b7f4-0275c3b3f98a on /mnt type ceph (rw,relatime,name=kubernetes-dynamic-user-4227a0c4-5b38-11e9-b7f4-0275c3b3f98a,secret=
[root@master-01 ceph]# kubectl exec -it -n kube-system test-pod — touch /mnt/testgy.txt
# 查看pod里面的挂载是有的，并且创建了一个文件到pod的挂载目录
[root@node-01 kubernetes-dynamic-pvc-4227a096-5b38-11e9-b7f4-0275c3b3f98a]# ls
testgy.txt
[root@node-01 kubernetes-dynamic-pvc-4227a096-5b38-11e9-b7f4-0275c3b3f98a]# pwd
/tmp/mount/volumes/kubernetes/kubernetes/kubernetes-dynamic-pvc-4227a096-5b38-11e9-b7f4-0275c3b3f98a
[root@node-01 kubernetes-dynamic-pvc-4227a096-5b38-11e9-b7f4-0275c3b3f98a]# df -h|grep ceph
10.83.32.224:/ 3.8T 36G 3.7T 1% /mnt/cephfs
ceph-fuse 3.8T 36G 3.7T 1% /tmp/mount
ceph-fuse 3.8T 36G 3.7T 1% /var/lib/kubelet/pods/d5eb3a8d-5ad4-11e9-811e-480fcf482f56/volumes/kubernetes.io~cephfs/pvc-d5cabe11-5ad4-11e9-811e-480fcf482f56
[root@node-01 kubernetes-dynamic-pvc-4227a096-5b38-11e9-b7f4-0275c3b3f98a]#
# 我们在另外一台机器上面可以查看到刚才pod创建的文件，其实就是cephfs把pvc映射了一个目录。
若有收获，就点个赞吧
0 人点赞
