Linux系统监控

  1. groups:
  2.   - name: node-alert
  3.     rules:
  4.     - alert: NodeDown
  5.       expr: up{job="node"} == 0
  6.       for: 5m
  7.       labels:
  8.         severity: critical
  9.         instance: "{{ $labels.instance }}"
  10.       annotations:
  11.         summary: "instance: {{ $labels.instance }} down"
  12.         description: "Instance: {{ $labels.instance }} 已经宕机 5分钟"
  13.         value: "{{ $value }}"
  15.     - alert: NodeCpuHigh
  16.       expr: (1 - avg by (instance) (irate(node_cpu_seconds_total{job="node",mode="idle"}[5m]))) * 100 > 80
  17.       for: 5m
  18.       labels:
  19.         severity: warning
  20.         instance: "{{ $labels.instance }}"
  21.       annotations:
  22.         summary: "instance: {{ $labels.instance }} cpu使用率过高"
  23.         description: "CPU 使用率超过 80%"
  24.         value: "{{ $value }}"
  26.     - alert: NodeCpuIowaitHigh
  27.       expr: avg by (instance) (irate(node_cpu_seconds_total{job="node",mode="iowait"}[5m])) * 100 > 50
  28.       for: 5m
  29.       labels:
  30.         severity: warning
  31.         instance: "{{ $labels.instance }}"
  32.       annotations:
  33.         summary: "instance: {{ $labels.instance }} cpu iowait 使用率过高"
  34.         description: "CPU iowait 使用率超过 50%"
  35.         value: "{{ $value }}"
  37.     - alert: NodeLoad5High
  38.       expr: node_load5 > (count by (instance) (node_cpu_seconds_total{job="node",mode='system'})) * 1.2
  39.       for: 5m
  40.       labels:
  41.         severity: warning
  42.         instance: "{{ $labels.instance }}"
  43.       annotations:
  44.         summary: "instance: {{ $labels.instance }} load(5m) 过高"
  45.         description: "Load(5m) 过高，超出cpu核数 1.2倍"
  46.         value: "{{ $value }}"
  48.     - alert: NodeMemoryHigh
  49.       expr: (1 - node_memory_MemAvailable_bytes{job="node"} / node_memory_MemTotal_bytes{job="node"}) * 100 > 90
  50.       for: 5m
  51.       labels:
  52.         severity: warning
  53.         instance: "{{ $labels.instance }}"
  54.       annotations:
  55.         summary: "instance: {{ $labels.instance }} memory 使用率过高"
  56.         description: "Memory 使用率超过 90%"
  57.         value: "{{ $value }}"
  59.     - alert: NodeDiskRootHigh
  60.       expr: (1 - node_filesystem_avail_bytes{job="node",fstype=~"ext.*|xfs",mountpoint ="/"} / node_filesystem_size_bytes{job="node",fstype=~"ext.*|xfs",mountpoint ="/"}) * 100 > 90
  61.       for: 10m
  62.       labels:
  63.         severity: warning
  64.         instance: "{{ $labels.instance }}"
  65.       annotations:
  66.         summary: "instance: {{ $labels.instance }} disk(/ 分区) 使用率过高"
  67.         description: "Disk(/ 分区) 使用率超过 90%"
  68.         value: "{{ $value }}"
  70.     - alert: NodeDiskBootHigh
  71.       expr: (1 - node_filesystem_avail_bytes{job="node",fstype=~"ext.*|xfs",mountpoint ="/boot"} / node_filesystem_size_bytes{job="node",fstype=~"ext.*|xfs",mountpoint ="/boot"}) * 100 > 80
  72.       for: 10m
  73.       labels:
  74.         severity: warning
  75.         instance: "{{ $labels.instance }}"
  76.       annotations:
  77.         summary: "instance: {{ $labels.instance }} disk(/boot 分区) 使用率过高"
  78.         description: "Disk(/boot 分区) 使用率超过 80%"
  79.         value: "{{ $value }}"
  81.     - alert: NodeDiskReadHigh
  82.       expr: irate(node_disk_read_bytes_total{job="node"}[5m]) > 20 * (1024 ^ 2)
  83.       for: 5m
  84.       labels:
  85.         severity: warning
  86.         instance: "{{ $labels.instance }}"
  87.       annotations:
  88.         summary: "instance: {{ $labels.instance }} disk 读取字节数 速率过高"
  89.         description: "Disk 读取字节数 速率超过 20 MB/s"
  90.         value: "{{ $value }}"
  92.     - alert: NodeDiskWriteHigh
  93.       expr: irate(node_disk_written_bytes_total{job="node"}[5m]) > 20 * (1024 ^ 2)
  94.       for: 5m
  95.       labels:
  96.         severity: warning
  97.         instance: "{{ $labels.instance }}"
  98.       annotations:
  99.         summary: "instance: {{ $labels.instance }} disk 写入字节数 速率过高"
  100.         description: "Disk 写入字节数 速率超过 20 MB/s"
  101.         value: "{{ $value }}"
  103.     - alert: NodeDiskReadRateCountHigh
  104.       expr: irate(node_disk_reads_completed_total{job="node"}[5m]) > 3000
  105.       for: 5m
  106.       labels:
  107.         severity: warning
  108.         instance: "{{ $labels.instance }}"
  109.       annotations:
  110.         summary: "instance: {{ $labels.instance }} disk iops 每秒读取速率过高"
  111.         description: "Disk iops 每秒读取速率超过 3000 iops"
  112.         value: "{{ $value }}"
  114.     - alert: NodeDiskWriteRateCountHigh
  115.       expr: irate(node_disk_writes_completed_total{job="node"}[5m]) > 3000
  116.       for: 5m
  117.       labels:
  118.         severity: warning
  119.         instance: "{{ $labels.instance }}"
  120.       annotations:
  121.         summary: "instance: {{ $labels.instance }} disk iops 每秒写入速率过高"
  122.         description: "Disk iops 每秒写入速率超过 3000 iops"
  123.         value: "{{ $value }}"
  125.     - alert: NodeInodeRootUsedPercentHigh
  126.       expr: (1 - node_filesystem_files_free{job="node",fstype=~"ext4|xfs",mountpoint="/"} / node_filesystem_files{job="node",fstype=~"ext4|xfs",mountpoint="/"}) * 100 > 80
  127.       for: 10m
  128.       labels:
  129.         severity: warning
  130.         instance: "{{ $labels.instance }}"
  131.       annotations:
  132.         summary: "instance: {{ $labels.instance }} disk(/ 分区) inode 使用率过高"
  133.         description: "Disk (/ 分区) inode 使用率超过 80%"
  134.         value: "{{ $value }}"
  136.     - alert: NodeInodeBootUsedPercentHigh
  137.       expr: (1 - node_filesystem_files_free{job="node",fstype=~"ext4|xfs",mountpoint="/boot"} / node_filesystem_files{job="node",fstype=~"ext4|xfs",mountpoint="/boot"}) * 100 > 80
  138.       for: 10m
  139.       labels:
  140.         severity: warning
  141.         instance: "{{ $labels.instance }}"
  142.       annotations:
  143.         summary: "instance: {{ $labels.instance }} disk(/boot 分区) inode 使用率过高"
  144.         description: "Disk (/boot 分区) inode 使用率超过 80%"
  145.         value: "{{ $value }}"
  147.     - alert: NodeFilefdAllocatedPercentHigh
  148.       expr: node_filefd_allocated{job="node"} / node_filefd_maximum{job="node"} * 100 > 80
  149.       for: 10m
  150.       labels:
  151.         severity: warning
  152.         instance: "{{ $labels.instance }}"
  153.       annotations:
  154.         summary: "instance: {{ $labels.instance }} filefd 打开百分比过高"
  155.         description: "Filefd 打开百分比 超过 80%"
  156.         value: "{{ $value }}"
  158.     - alert: NodeNetworkNetinBitRateHigh
  159.       expr: avg by (instance) (irate(node_network_receive_bytes_total{device=~"eth0|eth1|ens33|ens37"}[1m]) * 8) > 20 * (1024 ^ 2) * 8
  160.       for: 3m
  161.       labels:
  162.         severity: warning
  163.         instance: "{{ $labels.instance }}"
  164.       annotations:
  165.         summary: "instance: {{ $labels.instance }} network 接收比特数 速率过高"
  166.         description: "Network 接收比特数 速率超过 20MB/s"
  167.         value: "{{ $value }}"
  169.     - alert: NodeNetworkNetoutBitRateHigh
  170.       expr: avg by (instance) (irate(node_network_transmit_bytes_total{device=~"eth0|eth1|ens33|ens37"}[1m]) * 8) > 20 * (1024 ^ 2) * 8
  171.       for: 3m
  172.       labels:
  173.         severity: warning
  174.         instance: "{{ $labels.instance }}"
  175.       annotations:
  176.         summary: "instance: {{ $labels.instance }} network 发送比特数 速率过高"
  177.         description: "Network 发送比特数 速率超过 20MB/s"
  178.         value: "{{ $value }}"
  180.     - alert: NodeNetworkNetinPacketErrorRateHigh
  181.       expr: avg by (instance) (irate(node_network_receive_errs_total{device=~"eth0|eth1|ens33|ens37"}[1m])) > 15
  182.       for: 3m
  183.       labels:
  184.         severity: warning
  185.         instance: "{{ $labels.instance }}"
  186.       annotations:
  187.         summary: "instance: {{ $labels.instance }} 接收错误包 速率过高"
  188.         description: "Network 接收错误包 速率超过 15个/秒"
  189.         value: "{{ $value }}"
  191.     - alert: NodeNetworkNetoutPacketErrorRateHigh
  192.       expr: avg by (instance) (irate(node_network_transmit_packets_total{device=~"eth0|eth1|ens33|ens37"}[1m])) > 15
  193.       for: 3m
  194.       labels:
  195.         severity: warning
  196.         instance: "{{ $labels.instance }}"
  197.       annotations:
  198.         summary: "instance: {{ $labels.instance }} 发送错误包 速率过高"
  199.         description: "Network 发送错误包 速率超过 15个/秒"
  200.         value: "{{ $value }}"
  202.     - alert: NodeProcessBlockedHigh
  203.       expr: node_procs_blocked{job="node"} > 10
  204.       for: 10m
  205.       labels:
  206.         severity: warning
  207.         instance: "{{ $labels.instance }}"
  208.       annotations:
  209.         summary: "instance: {{ $labels.instance }} 当前被阻塞的任务的数量过多"
  210.         description: "Process 当前被阻塞的任务的数量超过 10个"
  211.         value: "{{ $value }}"
  213.     - alert: NodeTimeOffsetHigh
  214.       expr: abs(node_timex_offset_seconds{job="node"}) > 3 * 60
  215.       for: 2m
  216.       labels:
  217.         severity: info
  218.         instance: "{{ $labels.instance }}"
  219.       annotations:
  220.         summary: "instance: {{ $labels.instance }} 时间偏差过大"
  221.         description: "Time 节点的时间偏差超过 3m"
  222.         value: "{{ $value }}"

windows系统监控

  1. groups:
  2.   - name: windows-alert
  3.     rules:
  4.     - alert: 节点WODN状态
  5.       expr: up{job="windows"} == 0
  6.       for: 5m
  7.       labels:
  8.         severity: emergency
  9.         instance: "{{ $labels.instance }}"
  10.       annotations:
  11.         summary: "instance: {{ $labels.instance }} down"
  12.         description: "告警节点: {{ $labels.instance }} 告警内容:已经宕机5分钟"
  13.         value: "{{ $value }}"
  14.     - alert: CPU使用率高
  15.       expr: 100 - (avg by (instance) (irate(windows_cpu_time_total{job="windows",mode="idle"}[2m])) * 100) > 80
  16.       for: 5m
  17.       labels:
  18.         severity: warning
  19.         instance: "{{ $labels.instance }}"
  20.       annotations:
  21.         summary: "instance: {{ $labels.instance }} CPU使用率高"
  22.         description: "告警节点: {{ $labels.instance }} 告警内容: CUP使用率超过80%(当前值:{{ $value}})"
  23.         value: "{{ $value }}"
  24.     - alert: 内存使用率高
  25.       expr: 100.0 - 100 * windows_os_physical_memory_free_bytes{job="windows"} / windows_cs_physical_memory_bytes{job="windows"} > 40
  26.       for: 5m
  27.       labels:
  28.         severity: warning
  29.         instance: "{{ $labels.instance }}"
  30.       annotations:
  31.         summary: "instance: {{ $labels.instance }} 内存使用率高"
  32.         description: "告警节点: {{ $labels.instance }} 告警内容:内存使用率超过70%(当前值:{{ $value}})"
  33.         value: "{{ $value }}"
  34.     - alert: C盘使用率高
  35.       expr: (1 - (windows_logical_disk_free_bytes{job="windows",volume=~"C:"}/windows_logical_disk_size_bytes{job="windows",volume=~"C:"}))*100 > 85
  36.       for: 5m
  37.       labels:
  38.         severity: critical
  39.         instance: "{{ $labels.instance }}"
  40.       annotations:
  41.         summary: "instance: {{ $labels.instance }} C盘使用率高"
  42.         description: "告警节点: {{ $labels.instance }} 告警内容:C盘使用率超过85%(当前值:{{ $value}})"
  43.         value: "{{ $value }}"
  44.     - alert: 磁盘使用率高
  45.       expr: (max by (instance) (1-windows_logical_disk_free_bytes{job="windows"}/windows_logical_disk_size_bytes{job="windows"}))*100 > 85
  46.       for: 5m
  47.       labels:
  48.         severity: critical
  49.         instance: "{{ $labels.instance }}"
  50.       annotations:
  51.         summary: "instance: {{ $labels.instance }} 磁盘使用率高"
  52.         description: "告警节点: {{ $labels.instance }} 告警内容:磁盘使用率超过85%(当前值:{{ $value}})"
  53.         value: "{{ $value }}"
  54.     - alert: 网络使用率高
  55.       expr: (irate(windows_net_bytes_total{job="windows",nic!~'isatap.*|VPN.*'}[5m]) * 8 / windows_net_current_bandwidth{job="windows",nic!~'isatap.*|VPN.*'}) * 100 > 0.5
  56.       for: 5m
  57.       labels:
  58.         severity: warning
  59.         instance: "{{ $labels.instance }}"
  60.       annotations:
  61.         summary: "instance: {{ $labels.instance }} 网络使用率高"
  62.         description: "告警节点: {{ $labels.instance }} 告警内容:网络使用率超过50%(当前值:{{ $value}})"
  63.         value: "{{ $value }}"