自动发现配置

为解决上面的问题，Prometheus Operator 为我们提供了一个额外的抓取配置的来解决这个问题，我们可以通过添加额外的配置来进行服务发现进行自动监控。和前面自定义的方式一样，我们想要在 Prometheus Operator 当中去自动发现并监控具有prometheus.io/scrape=true这个 annotations 的 Service，之前我们定义的 Prometheus 的配置如下：

  1.     - job_name: kubernetes-service-endpoints
  2.       kubernetes_sd_configs:
  3.       - role: endpoints
  4.       relabel_configs:
  5.       - action: keep
  6.         regex: true
  7.         source_labels:
  8.         - __meta_kubernetes_service_annotation_prometheus_io_scrape
  9.       - action: replace
  10.         regex: (https?)
  11.         source_labels:
  12.         - __meta_kubernetes_service_annotation_prometheus_io_scheme
  13.         target_label: __scheme__
  14.       - action: replace
  15.         regex: (.+)
  16.         source_labels:
  17.         - __meta_kubernetes_service_annotation_prometheus_io_path
  18.         target_label: __metrics_path__
  19.       - action: replace
  20.         regex: ([^:]+)(?::\d+)?;(\d+)
  21.         replacement: $1:$2
  22.         source_labels:
  23.         - __address__
  24.         - __meta_kubernetes_service_annotation_prometheus_io_port
  25.         target_label: __address__
  26.       - action: labelmap
  27.         regex: __meta_kubernetes_service_label_(.+)
  28.       - action: replace
  29.         source_labels:
  30.         - __meta_kubernetes_namespace
  31.         target_label: kubernetes_namespace
  32.       - action: replace
  33.         source_labels:
  34.         - __meta_kubernetes_service_name
  35.         target_label: kubernetes_name
  38.     - job_name: kubernetes-pods
  39.       kubernetes_sd_configs:
  40.       - role: pod
  41.       relabel_configs:
  42.       - action: keep
  43.         regex: true
  44.         source_labels:
  45.         - __meta_kubernetes_pod_annotation_prometheus_io_scrape
  46.       - action: replace
  47.         regex: (.+)
  48.         source_labels:
  49.         - __meta_kubernetes_pod_annotation_prometheus_io_path
  50.         target_label: __metrics_path__
  51.       - action: replace
  52.         regex: ([^:]+)(?::\d+)?;(\d+)
  53.         replacement: $1:$2
  54.         source_labels:
  55.         - __address__
  56.         - __meta_kubernetes_pod_annotation_prometheus_io_port
  57.         target_label: __address__
  58.       - action: labelmap
  59.         regex: __meta_kubernetes_pod_label_(.+)
  60.       - action: replace
  61.         source_labels:
  62.         - __meta_kubernetes_namespace
  63.         target_label: kubernetes_namespace
  64.       - action: replace
  65.         source_labels:
  66.         - __meta_kubernetes_pod_name
  67.         target_label: kubernetes_pod_name

要想自动发现集群中的 Service，就需要我们在 Service 的annotation区域添加：prometheus.io/scrape=true的声明要想自动发现集群中的 pod，也需要我们在 pod 的annotation区域添加：prometheus.io/scrape=true的声明例如我们把之前的redis监控给修改一下，做成pod和svc，自动发现

  1. annotations:
  2.   prometheus.io/scrape: "true"
  3.   prometheus.io/port: "9121"

configmap实例

  1. apiVersion: v1
  2. kind: ConfigMap
  3. metadata:
  4.   name: prometheus-config
  5.   namespace: kube-system 
  6.   labels:
  7.     kubernetes.io/cluster-service: "true"
  8.     addonmanager.kubernetes.io/mode: EnsureExists
  9. data:
  10.   # 存放prometheus配置文件
  11.   prometheus.yml: |
  12.     # 配置采集目标
  13.     scrape_configs:
  14.     - job_name: kubernetes-nodes
  15.       static_configs:
  16.       - targets:
  17.         # 采集自身
  18.         - 192.168.1.110:9100
  19.         - 192.168.1.111:9100
  21.     # 采集：Apiserver 生存指标
  22.     # 创建的job name 名称为 kubernetes-apiservers
  23.     - job_name: kubernetes-apiservers
  24.       # 基于k8s的服务发现
  25.       kubernetes_sd_configs:
  26.       - role: endpoints
  27.       # 使用通信标记标签
  28.       relabel_configs:
  29.       # 保留正则匹配标签
  30.       - action: keep
  31.         # 已经包含
  32.         regex: default;kubernetes;https
  33.         source_labels:
  34.         - __meta_kubernetes_namespace
  35.         - __meta_kubernetes_service_name
  36.         - __meta_kubernetes_endpoint_port_name
  37.       # 使用方法为https、默认http
  38.       scheme: https
  39.       tls_config:
  40.         # promethus访问Apiserver使用认证
  41.         ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
  42.         # 跳过https认证
  43.         insecure_skip_verify: true
  44.       # promethus访问Apiserver使用认证
  45.       bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
  47.     # 采集：Kubelet 生存指标
  48.     - job_name: kubernetes-nodes-kubelet
  49.       kubernetes_sd_configs:
  50.       # 发现集群中所有的Node
  51.       - role: node
  52.       relabel_configs:
  53.       # 通过regex获取关键信息
  54.       - action: labelmap
  55.         regex: __meta_kubernetes_node_label_(.+)
  56.       scheme: https
  57.       tls_config:
  58.         ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
  59.         insecure_skip_verify: true
  60.       bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
  62.     # 采集：nodes-cadvisor 信息
  63.     - job_name: kubernetes-nodes-cadvisor
  64.       kubernetes_sd_configs:
  65.       - role: node
  66.       relabel_configs:
  67.       - action: labelmap
  68.         regex: __meta_kubernetes_node_label_(.+)
  69.       # 重命名标签
  70.       - target_label: __metrics_path__
  71.         replacement: /metrics/cadvisor
  72.       scheme: https
  73.       tls_config:
  74.         ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
  75.         insecure_skip_verify: true
  76.       bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
  78.     # 采集：service-endpoints 信息
  79.     - job_name: kubernetes-service-endpoints
  80.       # 选定指标
  81.       kubernetes_sd_configs:
  82.       - role: endpoints
  83.       relabel_configs:
  84.       - action: keep
  85.         regex: true
  86.         # 指定源标签
  87.         source_labels:
  88.         - __meta_kubernetes_service_annotation_prometheus_io_scrape
  89.       - action: replace
  90.         regex: (https?)
  91.         source_labels:
  92.         - __meta_kubernetes_service_annotation_prometheus_io_scheme
  93.         # 重命名标签采集
  94.         target_label: __scheme__
  95.       - action: replace
  96.         regex: (.+)
  97.         source_labels:
  98.         - __meta_kubernetes_service_annotation_prometheus_io_path
  99.         target_label: __metrics_path__
  100.       - action: replace
  101.         regex: ([^:]+)(?::\d+)?;(\d+)
  102.         replacement: $1:$2
  103.         source_labels:
  104.         - __address__
  105.         - __meta_kubernetes_service_annotation_prometheus_io_port
  106.         target_label: __address__
  107.       - action: labelmap
  108.         regex: __meta_kubernetes_service_label_(.+)
  109.       - action: replace
  110.         source_labels:
  111.         - __meta_kubernetes_namespace
  112.         target_label: kubernetes_namespace
  113.       - action: replace
  114.         source_labels:
  115.         - __meta_kubernetes_service_name
  116.         target_label: kubernetes_name
  118.     # 采集：kubernetes-services 服务指标
  119.     - job_name: kubernetes-services
  120.       kubernetes_sd_configs:
  121.       - role: service
  122.       # 黑盒探测，探测IP与端口是否可用
  123.       metrics_path: /probe
  124.       params:
  125.         module:
  126.         - http_2xx
  127.       relabel_configs:
  128.       - action: keep
  129.         regex: true
  130.         source_labels:
  131.         - __meta_kubernetes_service_annotation_prometheus_io_probe
  132.       - source_labels:
  133.         - __address__
  134.         target_label: __param_target
  135.       # 使用 blackbox进行黑盒探测
  136.       - replacement: blackbox
  137.         target_label: __address__
  138.       - source_labels:
  139.         - __param_target
  140.         target_label: instance
  141.       - action: labelmap
  142.         regex: __meta_kubernetes_service_label_(.+)
  143.       - source_labels:
  144.         - __meta_kubernetes_namespace
  145.         target_label: kubernetes_namespace
  146.       - source_labels:
  147.         - __meta_kubernetes_service_name
  148.         target_label: kubernetes_name
  150.     # 采集： kubernetes-pods 信息
  151.     - job_name: kubernetes-pods
  152.       kubernetes_sd_configs:
  153.       - role: pod
  154.       relabel_configs:
  155.       - action: keep
  156.         regex: true
  157.         source_labels:
  158.         # 只保留采集的信息
  159.         - __meta_kubernetes_pod_annotation_prometheus_io_scrape
  160.       - action: replace
  161.         regex: (.+)
  162.         source_labels:
  163.         - __meta_kubernetes_pod_annotation_prometheus_io_path
  164.         target_label: __metrics_path__
  165.       - action: replace
  166.         regex: ([^:]+)(?::\d+)?;(\d+)
  167.         replacement: $1:$2
  168.         source_labels:
  169.         # 采集地址
  170.         - __address__
  171.         # 采集端口 
  172.         - __meta_kubernetes_pod_annotation_prometheus_io_port
  173.         target_label: __address__
  174.       - action: labelmap
  175.         regex: __meta_kubernetes_pod_label_(.+)
  176.       - action: replace
  177.         source_labels:
  178.         - __meta_kubernetes_namespace
  179.         target_label: kubernetes_namespace
  180.       - action: replace
  181.         source_labels:
  182.         - __meta_kubernetes_pod_name
  183.         target_label: kubernetes_pod_name
  184.     alerting:
  185.       # 告警配置文件
  186.       alertmanagers:
  187.       - kubernetes_sd_configs:
  188.           # 采用动态获取
  189.           - role: pod
  190.         tls_config:
  191.           ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
  192.         bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
  193.         relabel_configs:
  194.         - source_labels: [__meta_kubernetes_namespace]
  195.           regex: kube-system 
  196.           action: keep
  197.         - source_labels: [__meta_kubernetes_pod_label_k8s_app]
  198.           regex: alertmanager
  199.           action: keep
  200.         - source_labels: [__meta_kubernetes_pod_container_port_number]
  201.           regex:
  202.           action: drop