1 下载

官网： https://prometheus.io/download/
wget https://github.com/prometheus/prometheus/releases/download/v2.22.2/prometheus-2.22.2.linux-amd64.tar.gz

2 解压安装

tar zxf prometheus-2.22.2.linux-amd64.tar.gz
cd prometheus-2.22.2.linux-amd64

3 写进systemd服务中

  1. vim /usr/lib/systemd/system/prometheus.service
  3. [Unit]
  4. Description=Prometheus
  5. Documentation=https://prometheus.io/
  6. After=network.target
  8. [Service]
  9. Type=simple
  10. ExecStart=/data01/prometheus-download/prometheus-2.22.2.linux-amd64/prometheus --config.file=/data01/prometheus-download/prometheus-2.22.2.linux-amd64/prometheus.yml --storage.tsdb.path=/data01/prometheus-download/prometheus-2.22.2.linux-amd64/data
  11. ExecReload=/bin/kill -HUP $MAINPID
  12. ExecStop=/bin/kill -KILL $MAINPID
  13. KillMode=control-group
  14. Restart=on-failure
  16. [Install]
  17. WantedBy=multi-user.target

4 启动

  1. systemctl enable prometheus
  2. systemctl start prometheus
  3. systemctl stop prometheus
  4. systemctl reload prometheus

5 设置prometheus的配置

  1. prometheus.yml
  2. # my global config
  3. global:
  4.   scrape_interval:     15s # 设定抓取数据的周期，默认为1min
  5.   evaluation_interval: 15s # 设定更新rules文件的周期，默认为1min
  6.   # scrape_timeout is set to the global default (10s).
  8. # Alertmanager configuration
  9. alerting:
  10.   alertmanagers:
  11.   - static_configs:
  12.     - targets:  # 设定alertmanager和prometheus交互的接口，即alertmanager监听的ip地址和端口
  13.       - 127.0.0.1:9093
  15. # rule配置，首次读取默认加载，之后根据evaluation_interval设定的周期加载
  16. rule_files:
  17.   - "first_rules.yml"
  18.   # - "second_rules.yml"
  20. # A scrape configuration containing exactly one endpoint to scrape:
  21. # Here it's Prometheus itself.
  22. scrape_configs:
  23.   # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
  24.   - job_name: 'prometheus' # job_name默认写入timeseries的labels中，可以用于查询使用
  25.     scrape_interval: 15s # 抓取周期，默认采用global配置
  26.     static_configs:      # 静态配置
  27.     - targets: ['127.0.0.1:9090']  # prometheus所要抓取数据的地址，即instance实例项
  29.   - job_name: 'node_export'     #node_export配置
  30.     static_configs:
  31.       - targets: 
  32.         - 127.0.0.1:9100
  34.   - job_name: 'blackbox_http_2xx'
  35.     scrape_interval: 45s
  36.     metrics_path: /probe
  37.     params:
  38.       module: [http_2xx]  # Look for a HTTP 200 response.
  40.  ###配置文件 
  41.  #   file_sd_configs: 
  42.  #   - refresh_interval: 1m
  43.  #     files: 
  44.  #     - "/home/prometheus/conf/service_post.yml"
  45.     static_configs:
  46.         - targets:
  47.           - https://www.baidu.com
  48.     relabel_configs:
  49.         - source_labels: [__address__]
  50.           target_label: __param_target
  51.         - source_labels: [__param_target]
  52.           target_label: instance
  53.         - target_label: __address__
  54.           replacement: 127.0.0.1:9115
  56.   - job_name: "blackbox_telnet_port"
  57.     scrape_interval: 5s
  58.     metrics_path: /probe
  59.     params:
  60.       module: [tcp_connect]
  61.     static_configs:
  62.         - targets: [ '127.0.0.1:3306' ]
  63.           labels:
  64.             group: '数据库监控'
  65.         - targets: [ '10.240.0.7:6379' ]
  66.           labels:
  67.             group: '中文官网redis监控'
  68.         - targets: [ '10.240.0.7:6380' ]
  69.           labels:
  70.             group: '英文官网redis监控'
  71.     relabel_configs:
  72.         - source_labels: [__address__]
  73.           target_label: __param_target
  74.         - source_labels: [__param_target]
  75.           target_label: instance
  76.         - target_label: __address__
  77.           replacement: 127.0.0.1:9115
  79.   - job_name: 'blackbox_ssl_expiry'
  80.     metrics_path: /probe
  81.     params:
  82.       module: [http_2xx]  # Look for a HTTP 200 response.
  83.     static_configs:
  84.       - targets:
  85.         - www.baidu.com.cn  # Targent to probe
  86.     relabel_configs:
  87.       - source_labels: [__address__]
  88.         target_label: __param_target
  89.       - source_labels: [__param_target]
  90.         target_label: instance
  91.       - target_label: __address__
  92.         replacement: 127.0.0.1:9115  # Blackbox exporter.

6 报警项设置

  1. first_rules.yml #报警项配置文件，和路径是在prometheus.yml文件中配置
  3. groups:
  4.   - name: blackbox_network_stats
  5.     rules:
  6.     - alert: blackbox_network_stats
  7.       expr: probe_success == 0
  8.       for: 1m
  9.       labels:
  10.         severity: critical
  11.       annotations:
  12.         summary: "Instance {{ $labels.instance }}  is down"
  13.         description: "This requires immediate action!"
  15.   - name: ssl_expiry.rules 
  16.     rules: 
  17.     - alert: SSLCertExpiringSoon 
  18.       expr: (probe_ssl_earliest_cert_expiry{job="blackbox_ssl_expiry"} - time())/86400 < 30 
  19.       for: 10m
  20.       labels:
  21.         severity: warn
  22.       annotations:
  23.         summary: "ssl证书过期警告"
  24.         description: '域名{{$labels.instance}}的证书还有{{ printf "%.1f" $value }}天就过期了,请尽快更新证书'
  26.   - name: 主机状态-监控告警
  27.     rules:
  28.     - alert: 主机状态
  29.       expr: up == 0
  30.       for: 1m
  31.       labels:
  32.         severity: critical
  33.       annotations:
  34.         summary: "{{$labels.instance}}:服务器宕机"
  35.         description: "{{$labels.instance}}:服务器延时超过5分钟"
  37.     - alert: CPU使用情况
  38.       expr: 100-(avg(irate(node_cpu_seconds_total{mode="idle"}[5m])) by(instance)* 100) > 60
  39.       for: 1m
  40.       labels:
  41.         severity: warn
  42.       annotations:
  43.         summary: "{{$labels.mountpoint}} CPU使用率过高！"
  44.         description: "{{$labels.mountpoint }} CPU使用大于60%(目前使用:{{$value}}%)"
  46.     - alert: 内存使用
  47.       expr: 100 -(node_memory_MemTotal_bytes -node_memory_MemFree_bytes+node_memory_Buffers_bytes+node_memory_Cached_bytes ) / node_memory_MemTotal_bytes * 100> 80
  48.       for: 1m
  49.       labels:
  50.         severity: warn
  51.       annotations:
  52.         summary: "{{$labels.mountpoint}} 内存使用率过高！"
  53.         description: "{{$labels.mountpoint }} 内存使用大于80%(目前使用:{{$value}}%)"
  54.     - alert: IO性能
  55.       expr: 100-(avg(irate(node_disk_io_time_seconds_total[1m])) by(instance)* 100) < 60
  56.       for: 1m
  57.       labels:
  58.         severity: warn
  59.       annotations:
  60.         summary: "{{$labels.mountpoint}} 流入磁盘IO使用率过高！"
  61.         description: "{{$labels.mountpoint }} 流入磁盘IO大于60%(目前使用:{{$value}})"
  63.     - alert: 网络
  64.       expr: ((sum(rate (node_network_transmit_bytes_total{device!~'tap.*|veth.*|br.*|docker.*|virbr*|lo*'}[5m])) by (instance)) / 100) > 102400
  65.       for: 1m
  66.       labels:
  67.         severity: warn
  68.       annotations:
  69.         summary: "{{$labels.mountpoint}} 流出网络带宽过高！"
  70.         description: "{{$labels.mountpoint }}流出网络带宽持续2分钟高于100M. RX带宽使用率{{$value}}"
  72.     - alert: TCP会话
  73.       expr: node_netstat_Tcp_CurrEstab > 1000
  74.       for: 1m
  75.       labels:
  76.         severity: critical
  77.       annotations:
  78.         summary: "{{$labels.mountpoint}} TCP_ESTABLISHED过高！"
  79.         description: "{{$labels.mountpoint }} TCP_ESTABLISHED大于1000%(目前使用:{{$value}}%)"
  81.     - alert: 磁盘容量
  82.       expr: 100-(node_filesystem_free_bytes{fstype=~"ext4|xfs"}/node_filesystem_size_bytes {fstype=~"ext4|xfs"}*100) > 80
  83.       for: 1m
  84.       labels:
  85.         severity: warn
  86.       annotations:
  87.         summary: "{{$labels.mountpoint}} 磁盘分区使用率过高！"
  88.         description: "{{$labels.mountpoint }} 磁盘分区使用大于80%(目前使用:{{$value}}%)"

7 检查语法是否正确

  1. #!/bin/bash
  2. # 用于检查 prometheus的配置文件和规则文件是否有错误
  3. # 需要根据自己的部署和配置文件路径修改下面的脚本部分内容
  4. kubectl exec -ti -n monitoring prometheus-stateful-0 -c prometheus-server  -- /bin/sh -c "cd /etc/prometheus/ && /bin/promtool check config prometheus.yml" | grep -A 2  FAILED
  5. kubectl exec -ti -n monitoring prometheus-stateful-0 -c prometheus-server  -- /bin/sh -c "cd /etc/prometheus/ && /bin/promtool check rules rules/*.yaml" | grep -A 2  FAILED
  6. echo
  7. echo "========================================================================================="
  8. echo "如果全部正确，没有返回值。"
  9. echo "如果有报错，会返回错误的行数(指的是pod里面配置文件的对应的行数)或报错的原因"
  10. echo "========================================================================================="

8 热重启
直接重启prometheus即可。因为部署prometheus时开启了热重启 直接
然后热重启：curl -XPOST http://localhost:9090/-/reload
prometheus热重启
prometheus启动命令添加参数 —web.enable-lifecycle