1、获取ETCD证书

对于etcd集群，在搭建的时候我们就采用了https证书认证的方式，所以这里如果想用Prometheus访问到etcd集群的监控数据，就需要添加证书

2、systemctl status etcd查看证书路径

  1. vim /usr/lib/systemd/system/etcd.service
  2. [Unit]
  3. Description=Etcd Server
  4. After=network.target
  5. After=network-online.target
  6. Wants=network-online.target
  7. Documentation=https://github.com/coreos
  9. [Service]
  10. Type=notify
  11. WorkingDirectory=/var/lib/etcd
  12. ExecStart=/usr/local/bin/etcd \
  13. --name=master1 \
  14. --data-dir=/data1/etcd/var/lib/etcd \
  15. --cert-file=/etc/etcd/pki/server.pem \
  16. --key-file=/etc/etcd/pki/server-key.pem \
  17. --trusted-ca-file=/etc/etcd/pki/ca.pem \
  18. --peer-cert-file=/etc/etcd/pki/peer.pem \
  19. --peer-key-file=/etc/etcd/pki/peer-key.pem \
  20. --peer-trusted-ca-file=/etc/etcd/pki/ca.pem \
  21. --listen-peer-urls=https://172.31.243.137:2380 \
  22. --initial-advertise-peer-urls=https://172.31.243.137:2380 \
  23. --listen-client-urls=https://172.31.243.137:2379,http://127.0.0.1:2379 \
  24. --advertise-client-urls=https://172.31.243.137:2379 \
  25. --initial-cluster-token=etcd-cluster-0 \
  26. --initial-cluster=master1=https://172.31.243.137:2380,master2=https://172.31.243.232:2380,master3=https://172.31.243.253:2380 \
  27. --initial-cluster-state=new \
  28. --heartbeat-interval=250 \
  29. --election-timeout=2000
  30. Restart=on-failure
  31. RestartSec=5
  32. LimitNOFILE=65536
  34. [Install]
  35. WantedBy=multi-user.target

3、创建secret

  1. #第一种方法
  2. kubectl create secret generic etcd-certs --from-file=/etc/kubernetes/cert/ca.pem --from-file=/etc/etcd/cert/etcd.pem --from-file=/etc/etcd/cert/etcd-key.pem -n monitoring
  3. #第二种方法
  4. 编辑 etcd-secret.yaml 
  5. apiVersion: v1
  6. data:
  7.   ca.pem: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURFakNDQWZxZ0F3SUJBZ0lVYkV2bVJ2SndDUTZPREVXNGUvZ3ZISVN6Zms0d0RRWUpLb1pJaHZjTkFRRUwKQlFBd0R6RU5NQXNHQTFVRUF4TUVaWFJqWkRBZUZ3MHlNREEyTVRrd05qTXdNREJhRncweU5UQTJNVGd3TmpNdwpNREJhTUE4eERUQUxCZ05WQkFNVEJHVjBZMlF3Z2dFaU1BMEdDU3FHU0liM0RRRUJBUVVBQTRJQkR3QXdnZ0VLCkFvSUJBUUM4dmVwMWgwejdJaGpPZHZpWXRweXFJcDNEVDZGaklIYXArU2pzSDQzN1JyVHVmdXluNzZ2U2VjTVEKclQ3c0JOaC9qdCtCNlQzQU5OYWQwcTM5WjRXckxCajlTQUNIb3NaNmh4UDZNMzY1VHNiZkVJYittdXR4QXZLQwo4M2E2a0Y3dUkwUGJXV2N2b1JEbGxZV1VFOEVQbzNseXFJbU5NUFNEV3hwQStPUUY3NHJ0ZzRuSWZSYWlDWU9YClVIMlJoZTM3UGNLYWJacFpXeFNaQUhRWlQ5NTBpOG8yOWlUdHp2eXlQOXBVY0VlNmZCcEljNDYzMXNoQ2g2S0YKWnNEaDJTT3dvdWcxYTBZU2tZY0F6ZU1pTktZVFR1VXFnc0h1YlZBTS8va0tjVTJCa3pQVmN2ZFdlNWU0ZHc4NAoyS29RMngzdFpOeFZIbnE1TjdqemxMZnorQUxYQWdNQkFBR2paakJrTUE0R0ExVWREd0VCL3dRRUF3SUJCakFTCkJnTlZIUk1CQWY4RUNEQUdBUUgvQWdFQ01CMEdBMVVkRGdRV0JCU1drRnc4LzFhK2I1U29POXhod1BFT3owc3oKZHpBZkJnTlZIU01FR0RBV2dCU1drRnc4LzFhK2I1U29POXhod1BFT3owc3pkekFOQmdrcWhraUc5dzBCQVFzRgpBQU9DQVFFQWFieEdxRFU3VmIwcjA4eWZYNFlsK2p6SDUxZ1hKZ0NVSkVrOGVxOFg1RlJ3Ynd1dXN4Qm45dUVQCkZTcFQ4ZTF0YUNTRkdQSVFFRFloQ3lWVHNIcHBGdzN2WHZ2eldlU1FNM05UeVdiR0pxREZ3MVNCd3dGa3JHcncKK3l0RFVkVE9tK2VDWVhGc0ZJMWVUSFhWNjZJSW9WUFFJRU1Ma3lGVzRvMm5IaWFNb2NCTk5yTVJiRWF5SkFwRwppbFRZODc2dVd6Y0Z3cDhWQTdMZ2pCN1M3cXNUOG1YM3RxVDZJQ1NBcy9FSnF1c3BFeW9oV1NQbzhreWhUdjVlCnhMd0VrWHFsdUREQmQvWTA1ZVhZd1NxZTErTVVQR3IyNzV4aXJNNnVEWmU1WWNtS2pqVWlEZEhOQ3B4UDdxcmcKZktCa01pV2dJUThZNWJNQVhRVnlCUFg4UFFvVFhBPT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
  8.   server-key.pem: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUtKRUE1dkVVL0JVVk1KaElOeXMyNG1TSFZ4eUszUnNCL0VQc0dzTVFMOTlvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFN1lLV3RnVXRlZkRtdk1ESmtrZWUrYVkxZGJGcDMza3JYbVFlRGp1KzRsMXRpS2tTZmI0UAoyZC9md0NnTkdhU2NvV0NnSklickhRUXp2c2pKeDJ6UjNnPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo=
  9.   server.pem: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUNyRENDQVpTZ0F3SUJBZ0lVRUNGS1psemtBWDR4dWtDTFNabHdyaldGcDlJd0RRWUpLb1pJaHZjTkFRRUwKQlFBd0R6RU5NQXNHQTFVRUF4TUVaWFJqWkRBZUZ3MHlNREEyTVRrd05qTTFNREJhRncweU5UQTJNVGd3TmpNMQpNREJhTURZeEN6QUpCZ05WQkFZVEFrTk9NUXN3Q1FZRFZRUUlFd0pUU0RFTE1Ba0dBMVVFQnhNQ1UwZ3hEVEFMCkJnTlZCQU1UQkdWMFkyUXdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CQndOQ0FBVHRncGEyQlMxNThPYTgKd01tU1I1NzVwalYxc1duZmVTdGVaQjRPTzc3aVhXMklxUko5dmcvWjM5L0FLQTBacEp5aFlLQWtodXNkQkRPKwp5TW5IYk5IZW80R2pNSUdnTUE0R0ExVWREd0VCL3dRRUF3SUZvREFkQmdOVkhTVUVGakFVQmdnckJnRUZCUWNECkFRWUlLd1lCQlFVSEF3SXdEQVlEVlIwVEFRSC9CQUl3QURBZEJnTlZIUTRFRmdRVSt0anFJWEJBQVNsZXVuemEKZVhjZHhaVllQVk13SHdZRFZSMGpCQmd3Rm9BVWxwQmNQUDlXdm0rVXFEdmNZY0R4RHM5TE0zY3dJUVlEVlIwUgpCQm93R0ljRWZ3QUFBWWNFckIvemlZY0VyQi96NkljRXJCL3ovVEFOQmdrcWhraUc5dzBCQVFzRkFBT0NBUUVBCkcvaDNkY3NzRklEYlpndlVoeHVZeG0zL08xN2F5ZUZaSTNJTW4zUThyYmhzejhrTlZXUjNTOXdqMEVza0ZTay8Kb0c3ekpOaW91QjgrUFo5SE84b2pTYmNkT3BRMi85UjVlMkUrQTg1YU5XTEl6VmZaWDQwNGJoZmJoU0g1QlpjZQpDNDZETmovRktRVWRGQkYvSHp3ODM4ZDM0ckNaREFzVUlKN3JXNjFoenFsU1ZsUUZPb3hDQWwwSW05eGIxL2FHCmFhQ2d0M1dQN1RqNTZPUi9YYnZYeEtxQUkwUURsdHU4R0FoRk9BTVZUU0pvVnlGT0dmcVptTzg3dmlVWmQ4WVYKTE1Rc3J0a0EyT1FvSWRIMW9sc1NTb0NCeGk3K2JIUVo1Y3Mwb21kcHhNV1FQSG5CWFIxZ2x1WW1PRnJSVGQxcQpsa3dYNVNGcnI5THNlQnpVYkZPbEVnPT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
  10. kind: Secret
  11. metadata:
  12.   name: etcd-certs
  13.   namespace: monitoring
  14. type: Opaque
  16. #查看
  17. [root@hf-aipaas-172-31-243-137 etcd]# kubectl describe secrets -n monitoring etcd-certs
  18. Name:         etcd-certs
  19. Namespace:    monitoring
  20. Labels:       <none>
  21. Annotations:  
  22. Type:         Opaque
  24. Data
  25. ====
  26. server.pem:      989 bytes
  27. ca.pem:          1127 bytes
  28. server-key.pem:  227 bytes
  30. #查看prometheus相关配置
  32. [root@hf-aipaas-172-31-243-137 etcd]# kubectl exec -it -n monitoring prometheus-k8s-0 /bin/sh
  33. Defaulting container name to prometheus.
  34. Use 'kubectl describe pod/prometheus-k8s-0 -n monitoring' to see all of the containers in this pod.
  35. /prometheus $ ls /etc/prometheus/secrets/etcd-ssl/
  36. ls: /etc/prometheus/secrets/etcd-ssl/: No such file or directory
  37. /prometheus $ ls /etc/prometheus/secrets/etcd-certs/
  38. ca.pem          server-key.pem  server.pem

4、将etcd-ssl secret配置放到prometheus资源对象中

修改prometheus-prometheus.yaml 文件

  1. secrets:   #增加etcd配置
  2.  - etcd-certs
  3.  ruleSelector:
  4.    matchLabels:
  5.      prometheus: k8s
  6.      role: alert-rules

5、编辑etcd-service.yaml

  1. apiVersion: v1
  2. kind: Service
  3. metadata:
  4.   name: etcd-k8s
  5.   namespace: kube-system
  6.   labels:
  7.     k8s-app: etcd
  8. spec:
  9.   type: ClusterIP
  10.   clusterIP: None
  11.   ports:
  12.   - name: port
  13.     port: 2379
  14.     protocol: TCP
  16. ---
  17. apiVersion: v1
  18. kind: Endpoints
  19. metadata:
  20.   name: etcd-k8s
  21.   namespace: kube-system
  22.   labels:
  23.     k8s-app: etcd
  24. subsets:
  25. - addresses:
  26.   - ip: 172.31.243.137
  27.   - ip: 172.31.243.232
  28.   - ip: 172.31.243.254
  29.   ports:
  30.   - name: port
  31.     port: 2379
  32.     protocol: TCP

6、编辑etcd-servicemonitor.yaml

  1. apiVersion: monitoring.coreos.com/v1
  2. kind: ServiceMonitor
  3. metadata:
  4.   name: etcd-k8s
  5.   namespace: monitoring
  6.   labels:
  7.     k8s-app: etcd-k8s
  8.     monitor: k8s
  9. spec:
  10.   jobLabel: k8s-app
  11.   endpoints:
  12.   - port: port
  13.     interval: 30s
  14.     scheme: https
  15.     tlsConfig:
  16.       caFile: /etc/prometheus/secrets/etcd-certs/ca.pem
  17.       certFile: /etc/prometheus/secrets/etcd-certs/server.pem
  18.       keyFile: /etc/prometheus/secrets/etcd-certs/server-key.pem
  19.       insecureSkipVerify: true
  20.   selector:
  21.     matchLabels:
  22.       k8s-app: etcd
  23.   namespaceSelector:
  24.     matchNames:
  25.     - kube-system

7、查看prometheus界面target

image.png