1. import json
  2. from kubernetes import client, config
  3. config.kube_config.load_kube_config(config_file="./kubeconfig.yaml")
  4. api_client = client.ApiClient()
  6. namesapce = '???'
  8. url = '/apis/metrics.k8s.io/v1beta1/namespaces/' + namesapce + '/pods'
  9. ret_metrics = api_client.call_api(url, 'GET', auth_settings=['BearerToken'], response_type='json', _preload_content=False)
  10. response = ret_metrics[0].data.decode('utf-8')
  11. res = json.loads(response)
  12. for i in res['items']:
  13.     for j in i['containers']:
  14.         ## j['name']   ## 'le-15ccaef4-b63d-4e23-ae0c-418ab706d23b'
  15.         ## j['usage']['cpu']   ## '1434540n'
  16.         print('## memory: ',j['usage']['memory'])   ## 'memory': '52308Ki'
  18. ## memory:  628128Ki
  19. ## memory:  6844156Ki
  20. ## memory:  52352Ki
  21. ## memory:  6208008Ki
  22. podname = '?????'
  23. url_pod = '/apis/metrics.k8s.io/v1beta1/namespaces/' + namesapce + '/pods/' + podname
  24. ret_metric = api_client.call_api(url_pod, 'GET', auth_settings=['BearerToken'], response_type='json', _preload_content=False)
  25. response = ret_metric[0].data.decode('utf-8')
  26. res = json.loads(response)
  27. for j in res['containers']:
  28.     print('## memory: ',j['usage']['memory'])
  2. _token="XXXX"
  3. """
  4. ## /etc/lico/kube_server.csv
  5. ##
  6. #name,display_name,kube_cluster_addr,ingress_ctrl_addr,gpu_resource_name,prometheus_server
  7. mykube,My k8s,https://10.240.208.162:6443,http://10.240.208.94:31484,nvidia.com/gpu,http://10.240.208.94:31893
  8. """
  9. _pvc="zhouyj-pvc"
  10. _namespace="zhouyj"
  11. _job_name = "-e37d851f9b86"
  12. _stateful_name = "e37d851f9b86"
  13. _service_url="https://10.240.208.162:6443"
  14. gpu_resource_name="nvidia.com/gpu"
  16. from scheduler.adapter.kube.kuberesource.kube_client import KubeClient
  17. from scheduler.adapter.scheduler_factory import create_kubernetes_scheduler
  18. from client.auth.dataclass import RequireUserContext
  19. from scheduler.adapter.kube.kube_config import SchedulerConfig as KubeConfig
  21. _scheduler = create_kubernetes_scheduler(service_url=_service_url, namespace=_namespace, token=_token, pvc=_pvc, config=KubeConfig(gres_resource_dict={gpu_resource_name: "gpu"}))
  23. _scheduler._kube_client.list_job_pods(_namespace, _job_name)
  24. _scheduler._kube_client.list_stateful_pods(_namespace, _stateful_name)
  25. _get_mem(_scheduler._kube_client.api_client, _namespace, )
  26. # _kube_client = KubeClient(_token, _service_url)
  27. # _kube_client.list_job_pods(_namespace, _job_name)
  28. # _kube_client.list_stateful_pods(_namespace, _stateful_name)
  31. def _get_mem(api_client, namesapce, podname):
  32.     import json
  33.     url_pod = '/apis/metrics.k8s.io/v1beta1/namespaces/' + namesapce + '/pods/' + podname
  34.     ret_metric = api_client.call_api(url_pod, 'GET', auth_settings=['BearerToken'], response_type='json', _preload_content=False)
  35.     response = ret_metric[0].data.decode('utf-8')
  36.     res = json.loads(response)
  37.     for j in res['containers']:
  38.         # print('## memory: ',j['usage']['memory'])
  39.         return j['usage']['memory']
  41. _get_mem(_kube_client.api_client, _namespace, )
  44. _service_url="https://10.240.208.162:6443"
  45. _kube_client = KubeClient(_token, _service_url);_kube_client.list_job_pods(_namespace, _job_name)
  2. _get_mem(_scheduler._kube_client.api_client, _namespace, 'e37d851f9b86-gfk2b')
  3. /usr/lib/python3.6/site-packages/urllib3/connectionpool.py:847: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings
  4.   InsecureRequestWarning)
  5. Traceback (most recent call last):
  6.   File "<console>", line 1, in <module>
  7.   File "<console>", line 4, in _get_mem
  8.   File "/usr/lib/python3.6/site-packages/kubernetes/client/api_client.py", line 353, in call_api
  9.     _preload_content, _request_timeout, _host)
  10.   File "/usr/lib/python3.6/site-packages/kubernetes/client/api_client.py", line 184, in __call_api
  11.     _request_timeout=_request_timeout)
  12.   File "/usr/lib/python3.6/site-packages/kubernetes/client/api_client.py", line 377, in request
  13.     headers=headers)
  14.   File "/usr/lib/python3.6/site-packages/kubernetes/client/rest.py", line 243, in GET
  15.     query_params=query_params)
  16.   File "/usr/lib/python3.6/site-packages/kubernetes/client/rest.py", line 233, in request
  17.     raise ApiException(http_resp=r)
  18. kubernetes.client.exceptions.ApiException: (403)
  19. Reason: Forbidden
  20. HTTP response headers: HTTPHeaderDict({'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'X-Content-Type-Options': 'nosniff', 'Date': 'Sun, 25 Apr 2021 11:09:46 GMT', 'Content-Length': '438'})
  21. HTTP response body: b'{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pods.metrics.k8s.io \\"8d73-e37d851f9b86-gfk2b\\" is forbidden: User \\"system:serviceaccount:zhou:default\\" cannot get resource \\"pods\\" in API group \\"metrics.k8s.io\\" in the namespace \\"zhou\\"","reason":"Forbidden","details":{"name":"e37d851f9b86-gfk2b","group":"metrics.k8s.io","kind":"pods"},"code":403}\n'
  23. >>>

问题

is forbidden

  1. [root@k8s-master k8s_api]# kubectl get apiservices
  2. NAME                                   SERVICE                      AVAILABLE   AGE
  3. v1.                                    Local                        True        75d
  4. v1.admissionregistration.k8s.io        Local                        True        75d
  5. v1.apiextensions.k8s.io                Local                        True        75d
  6. v1.apps                                Local                        True        75d
  7. v1.authentication.k8s.io               Local                        True        75d
  8. v1.authorization.k8s.io                Local                        True        75d
  9. v1.autoscaling                         Local                        True        75d
  10. v1.batch                               Local                        True        75d
  11. v1.certificates.k8s.io                 Local                        True        60d
  12. v1.coordination.k8s.io                 Local                        True        75d
  13. v1.events.k8s.io                       Local                        True        60d
  14. v1.networking.k8s.io                   Local                        True        75d
  15. v1.rbac.authorization.k8s.io           Local                        True        75d
  16. v1.scheduling.k8s.io                   Local                        True        75d
  17. v1.storage.k8s.io                      Local                        True        75d
  18. v1beta1.admissionregistration.k8s.io   Local                        True        75d
  19. v1beta1.apiextensions.k8s.io           Local                        True        75d
  20. v1beta1.authentication.k8s.io          Local                        True        75d
  21. v1beta1.authorization.k8s.io           Local                        True        75d
  22. v1beta1.batch                          Local                        True        75d
  23. v1beta1.certificates.k8s.io            Local                        True        75d
  24. v1beta1.coordination.k8s.io            Local                        True        75d
  25. v1beta1.discovery.k8s.io               Local                        True        75d
  26. v1beta1.events.k8s.io                  Local                        True        75d
  27. v1beta1.extensions                     Local                        True        75d
  28. v1beta1.metrics.k8s.io                 kube-system/metrics-server   True        59d
  29. v1beta1.networking.k8s.io              Local                        True        75d
  30. v1beta1.node.k8s.io                    Local                        True        75d
  31. v1beta1.policy                         Local                        True        75d
  32. v1beta1.rbac.authorization.k8s.io      Local                        True        75d
  33. v1beta1.scheduling.k8s.io              Local                        True        75d
  34. v1beta1.storage.k8s.io                 Local                        True        75d
  35. v2beta1.autoscaling                    Local                        True        75d
  36. v2beta2.autoscaling                    Local                        True        75d
  39. [root@k8s-master k8s_api]# kubectl get svc metrics-server  -n kube-system -o yaml > metrics-server.yaml
  40. [root@k8s-master k8s_api]# cat metrics-server.yaml
  42. apiVersion: v1
  43. kind: Service
  44. metadata:
  45.   annotations:
  46.     kubectl.kubernetes.io/last-applied-configuration: |
  47.       {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{},"labels":{"addonmanager.kubernetes.io/mode":"Reconcile","kubernetes.io/cluster-service":"true","kubernetes.io/name":"Metrics-server"},"name":"metrics-server","namespace":"kube-system"},"spec":{"ports":[{"nodePort":30731,"port":443,"protocol":"TCP","targetPort":"https"}],"selector":{"k8s-app":"metrics-server"},"type":"NodePort"}}
  48.   creationTimestamp: "2021-02-25T03:27:04Z"
  49.   labels:
  50.     addonmanager.kubernetes.io/mode: Reconcile
  51.     kubernetes.io/cluster-service: "true"
  52.     kubernetes.io/name: Metrics-server
  53.   managedFields:
  54.   - apiVersion: v1
  55.     fieldsType: FieldsV1
  56.     fieldsV1:
  57.       f:metadata:
  58.         f:annotations:
  59.           .: {}
  60.           f:kubectl.kubernetes.io/last-applied-configuration: {}
  61.         f:labels:
  62.           .: {}
  63.           f:addonmanager.kubernetes.io/mode: {}
  64.           f:kubernetes.io/cluster-service: {}
  65.           f:kubernetes.io/name: {}
  66.       f:spec:
  67.         f:externalTrafficPolicy: {}
  68.         f:ports:
  69.           .: {}
  70.           k:{"port":443,"protocol":"TCP"}:
  71.             .: {}
  72.             f:nodePort: {}
  73.             f:port: {}
  74.             f:protocol: {}
  75.             f:targetPort: {}
  76.         f:selector:
  77.           .: {}
  78.           f:k8s-app: {}
  79.         f:sessionAffinity: {}
  80.         f:type: {}
  81.     manager: kubectl-client-side-apply
  82.     operation: Update
  83.     time: "2021-03-12T09:46:17Z"
  84.   name: metrics-server
  85.   namespace: kube-system
  86.   resourceVersion: "6879542"
  87.   selfLink: /api/v1/namespaces/kube-system/services/metrics-server
  88.   uid: 1858e3f9-28fa-4583-be32-7d7948a0042c
  89. spec:
  90.   clusterIP: 10.1.77.113
  91.   externalTrafficPolicy: Cluster
  92.   ports:
  93.   - nodePort: 30731
  94.     port: 443
  95.     protocol: TCP
  96.     targetPort: https
  97.   selector:
  98.     k8s-app: metrics-server
  99.   sessionAffinity: None
  100.   type: NodePort
  101. status:
  102.   loadBalancer: {}
  105. [root@k8s-master k8s_api]# kubectl get  apiservices v1.batch -o yaml >  batch.yaml
  106. [root@k8s-master k8s_api]# cat batch.yaml 
  107. apiVersion: apiregistration.k8s.io/v1
  108. kind: APIService
  109. metadata:
  110.   creationTimestamp: "2021-02-09T06:46:33Z"
  111.   labels:
  112.     kube-aggregator.kubernetes.io/automanaged: onstart
  113.   name: v1.batch
  114.   resourceVersion: "20"
  115.   selfLink: /apis/apiregistration.k8s.io/v1/apiservices/v1.batch
  116.   uid: 8bc5f1de-31b6-4d0e-92e4-cfd929f3827b
  117. spec:
  118.   group: batch
  119.   groupPriorityMinimum: 17400
  120.   version: v1
  121.   versionPriority: 15
  122. status:
  123.   conditions:
  124.   - lastTransitionTime: "2021-02-09T06:46:33Z"
  125.     message: Local APIServices are always available
  126.     reason: Local
  127.     status: "True"
  128.     type: Available
  131.  [root@k8s-master k8s_api]# kubectl describe apiservices  v1beta1.metrics.k8s.io
  132. Name:         v1beta1.metrics.k8s.io
  133. Namespace:    
  134. Labels:       addonmanager.kubernetes.io/mode=Reconcile
  135.               kubernetes.io/cluster-service=true
  136. Annotations:  <none>
  137. API Version:  apiregistration.k8s.io/v1
  138. Kind:         APIService
  139. Metadata:
  140.   Creation Timestamp:  2021-02-25T03:26:44Z
  141.   Resource Version:    3510481
  142.   Self Link:           /apis/apiregistration.k8s.io/v1/apiservices/v1beta1.metrics.k8s.io
  143.   UID:                 e7718cf6-7ef2-41d5-979f-f0a74900f0d9
  144. Spec:
  145.   Group:                     metrics.k8s.io
  146.   Group Priority Minimum:    100
  147.   Insecure Skip TLS Verify:  true
  148.   Service:
  149.     Name:            metrics-server
  150.     Namespace:       kube-system
  151.     Port:            443
  152.   Version:           v1beta1
  153.   Version Priority:  100
  154. Status:
  155.   Conditions:
  156.     Last Transition Time:  2021-02-25T05:58:09Z
  157.     Message:               all checks passed
  158.     Reason:                Passed
  159.     Status:                True
  160.     Type:                  Available
  161. Events:                    <none>
  164. [root@k8s-master k8s_api]# kubectl describe apiservices v1.batch
  165. Name:         v1.batch
  166. Namespace:    
  167. Labels:       kube-aggregator.kubernetes.io/automanaged=onstart
  168. Annotations:  <none>
  169. API Version:  apiregistration.k8s.io/v1
  170. Kind:         APIService
  171. Metadata:
  172.   Creation Timestamp:  2021-02-09T06:46:33Z
  173.   Resource Version:    20
  174.   Self Link:           /apis/apiregistration.k8s.io/v1/apiservices/v1.batch
  175.   UID:                 8bc5f1de-31b6-4d0e-92e4-cfd929f3827b
  176. Spec:
  177.   Group:                   batch
  178.   Group Priority Minimum:  17400
  179.   Version:                 v1
  180.   Version Priority:        15
  181. Status:
  182.   Conditions:
  183.     Last Transition Time:  2021-02-09T06:46:33Z
  184.     Message:               Local APIServices are always available
  185.     Reason:                Local
  186.     Status:                True
  187.     Type:                  Available
  188. Events:

solution

RABC 绑定

  2. kind: Role
  3. apiVersion: rbac.authorization.k8s.io/v1
  4. metadata:
  5.   namespace: default
  6.   name: metrics-reader
  7. rules:
  8. - apiGroups: ["metrics.k8s.io"]
  9.   resources: ["pods"]
  10.   verbs: ["get", "watch", "list"]
  11. - apiGroups: ["metrics.k8s.io"]
  12.   resources: ["nodes"]
  13.   verbs: ["get", "watch", "list"]
  15. ---
  17. kind: RoleBinding
  18. apiVersion: rbac.authorization.k8s.io/v1
  19. metadata:
  20.   name: read-pods
  21.   namespace: default
  22. subjects:
  23. - kind: User
  24.   name: system:serviceaccount:zhouyj:default
  25.   apiGroup: rbac.authorization.k8s.io
  26. roleRef:
  27.   kind: Role
  28.   name: metrics-reader
  29.   apiGroup: rbac.authorization.k8s.io
  33. ---
  34. kind: ClusterRole
  35. apiVersion: rbac.authorization.k8s.io/v1
  36. metadata:
  37.   name: metrics-reader
  38. rules:
  39. - apiGroups: ["metrics.k8s.io"]
  40.   resources: ["pods"]
  41.   verbs: ["get", "watch", "list"]
  42. - apiGroups: ["metrics.k8s.io"]
  43.   resources: ["nodes"]
  44.   verbs: ["get", "watch", "list"]
  45. ---
  46. apiVersion: rbac.authorization.k8s.io/v1
  47. kind: ClusterRoleBinding
  48. metadata:
  49.   name: metrics
  50. roleRef:
  51.   apiGroup: rbac.authorization.k8s.io
  52.   kind: ClusterRole
  53.   name: metrics-reader
  54. subjects:
  55. - kind: User
  56.   name: system:serviceaccount:zhouyj:default
  57.   apiGroup: rbac.authorization.k8s.io

end

参考： https://zhuanlan.zhihu.com/p/138995000