参考自： https://gitlab.com/gitlab-com/runbooks/blob/master/rules/node.yml

    1. groups:
    2. - name: CPU rules
    3.   interval: 1m
    4.   rules:
    5.   # The count of CPUs per node, useful for getting CPU time as a percent of total.
    6.   - record: instance:node_cpus:count
    7.     expr: >
    8.       count without (cpu, mode) (
    9.         node_cpu_seconds_total{mode="idle"}
    10.       )
    11.   # CPU in use by CPU.
    12.   - record: instance_cpu:node_cpu_seconds_not_idle:rate1m
    13.     expr: >
    14.       sum without (mode) (
    15.         1 - rate(node_cpu_seconds_total{mode="idle"}[1m])
    16.       )
    17.   # CPU in use by mode.
    18.   # Split recording for iowait to avoid reset bugs.
    19.   - record: instance_mode:node_cpu_seconds:rate1m
    20.     expr: >
    21.       sum without (cpu) (
    22.         rate(node_cpu_seconds_total{mode!="iowait"}[1m])
    23.       )
    24.   - record: instance_mode:node_cpu_seconds:deriv1m
    25.     expr: >
    26.       sum without (cpu) (
    27.         deriv(node_cpu_seconds_total{mode="iowait"}[1m]) > 0
    28.       )
    29.   # CPU in use ratio.
    30.   - record: instance:node_cpu_utilization:ratio
    31.     expr: >
    32.       avg without (cpu) (
    33.         instance_cpu:node_cpu_seconds_not_idle:rate1m
    34.       )
    35.   # CPU summaries
    36.   - record: job:node_cpu_utilization:min_ratio
    37.     expr: >
    38.       min without (fqdn,instance,node,pod) (
    39.         instance:node_cpu_utilization:ratio
    40.       )
    41.   - record: job:node_cpu_utilization:avg_ratio
    42.     expr: >
    43.       avg without (fqdn,instance,node,pod) (
    44.         instance:node_cpu_utilization:ratio
    45.       )
    46.   - record: job:node_cpu_utilization:max_ratio
    47.     expr: >
    48.       max without (fqdn,instance,node,pod) (
    49.         instance:node_cpu_utilization:ratio
    50.       )
    52.   # CPU Alerts
    53.   - alert: HighCPU
    54.     expr: instance:node_cpu_utilization:ratio > 0.95
    55.     for: 2h
    56.     labels:
    57.       pager: pagerduty
    58.       severity: s1
    59.       alert_type: cause
    60.     annotations:
    61.       runbook: docs/uncategorized/node_cpu.md
    62.       title: CPU use percent is extremely high on {{ if $labels.fqdn }}{{ $labels.fqdn
    63.         }}{{ else }}{{ $labels.instance }}{{ end }} for the past 2 hours.
    65. # Rules for calculating and alerting on long-term node utilization issues.
    66. - name: Utilization
    67.   interval: 300s
    68.   rules:
    69.   - record: instance:cpu_utilization:ratio_max
    70.     expr: max_over_time(instance:node_cpu_utilization:ratio[300s])
    71.   - record: instance:cpu_utilization:ratio_avg
    72.     expr: avg_over_time(instance:node_cpu_utilization:ratio[300s])
    73.   - record: instance:cpu_utilization:ratio_q95
    74.     expr: quantile_over_time(0.95, instance:node_cpu_utilization:ratio[300s])
    75.   - record: instance:memory_utilization:ratio_max
    76.     expr: max_over_time(instance:node_memory_utilization:ratio[300s])
    77.   - record: instance:memory_utilization:ratio_avg
    78.     expr: avg_over_time(instance:node_memory_utilization:ratio[300s])
    79.   - record: instance:memory_utilization:ratio_q95
    80.     expr: quantile_over_time(0.95, instance:node_memory_utilization:ratio[300s])
    81. # TODO(bjk): This is a proposal for an alert. We should send this to a webhook that opens an issue.
    82. #  - alert: NodeUnderUtilized
    83. #    expr: >
    84. #      (quantile_over_time(0.95, instance:cpu_utilization:ratio_q95[1d]) * 100 < 10)
    85. #        and
    86. #      (quantile_over_time(0.95, instance:memory_utilization:ratio_q95[1d]) * 100 < 10)
    87. #    for: 7d
    89. - name: Node memory
    90.   rules:
    91.   - record: instance:node_memory_available:ratio
    92.     expr: >
    93.       (
    94.         node_memory_MemAvailable_bytes or
    95.         (
    96.           node_memory_Buffers_bytes +
    97.           node_memory_Cached_bytes +
    98.           node_memory_MemFree_bytes +
    99.           node_memory_Slab_bytes
    100.         )
    101.       ) /
    102.       node_memory_MemTotal_bytes
    103.   - record: instance:node_memory_utilization:ratio
    104.     expr: 1 - instance:node_memory_available:ratio
    106. - name: Node filesystem rules
    107.   rules:
    108.   - record: instance:node_filesystem_avail:ratio
    109.     expr: node_filesystem_avail_bytes{device=~"(/dev/.+|tank/dataset)"} / node_filesystem_size_bytes{device=~"(/dev/.+|tank/dataset)"}
    110.   - record: instance:node_disk_writes_completed:irate1m
    111.     expr: sum(irate(node_disk_writes_completed_total{device=~"sd.*"}[1m])) WITHOUT (device)
    112.   - record: instance:node_disk_reads_completed:irate1m
    113.   #  expr: sum(irate(node_disk_reads_completed_total{device=~"sd.*"}[1m])) WITHOUT (device)
    114.     expr: |-
    115.       avg by (node) (
    116.         irate(node_disk_io_time_seconds_total{job="node-exporter",device=~"nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+"}[1m])
    117.       * on (namespace, pod) group_left(node)
    118.         node_namespace_pod:kube_pod_info:
    119.       )
    120.   - record: node:node_disk_utilisation:avg_irate
    121.     expr: avg(irate(node_disk_io_time_weighted_seconds_total{job="node-exporter",device=~"nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+"}[1m]))
    122.   - record: ':node_disk_saturation:avg_irate'
    123.     expr: |-
    124.       avg by (node) (
    125.         irate(node_disk_io_time_weighted_seconds_total{job="node-exporter",device=~"nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+"}[1m])
    126.       * on (namespace, pod) group_left(node)
    127.         node_namespace_pod:kube_pod_info:
    128.       )
    129.   - record: node:node_disk_saturation:avg_irate
    130.     expr: |-
    131.       max by (instance, namespace, pod, device) ((node_filesystem_size_bytes{fstype=~"ext[234]|btrfs|xfs|zfs"}
    132.       - node_filesystem_avail_bytes{fstype=~"ext[234]|btrfs|xfs|zfs"})
    133.       / node_filesystem_size_bytes{fstype=~"ext[234]|btrfs|xfs|zfs"})
    134.   - record: 'node:node_filesystem_usage:'
    135.     expr: max by (instance, namespace, pod, device) (node_filesystem_avail_bytes{fstype=~"ext[234]|btrfs|xfs|zfs"} / node_filesystem_size_bytes{fstype=~"ext[234]|btrfs|xfs|zfs"})
    136.   #  - record: 'node:node_filesystem_avail:'
    137.   - alert: FilesystemFullSoon
    138.     expr: predict_linear(node_filesystem_avail_bytes{fstype=~"(ext.|xfs|zfs)"}[6h], 24 * 3600) < 0
    139.     for: 2h
    140.     labels:
    141.       severity: s4
    142.       alert_type: cause
    143.     annotations:
    144.       title: 'Filesystem will be full SOON'
    145.       description: >
    146.         The filesystem is predicted to be full in will be full in the next 24 hours.
    147.       runbook: docs/monitoring/filesystem_alerts.md
    148.       instance: '{{ $labels.instance }}'
    149.       device: '{{ $labels.device }}'
    150.       mountpoint: '{{ $labels.mountpoint }}'
    151. - name: Misc rules
    152.   rules:
    153.   - record: instance:up:count
    154.     expr: count(up{job="node"} == 1) WITHOUT (instance, fqdn)
    155.   - alert: FleetSizeChanged
    156.     expr: changes(instance:up:count[2m]) >= 1
    157.     for: 1m
    158.     labels:
    159.       severity: s4
    160.       alert_type: cause
    161.     annotations:
    162.       description: The {{ $labels.type }} fleet has changed, this can be due to having
    163.         more or less, if it's the latter it can be because nodes went down silently
    164.       title: The fleet size has changed in the last 5 minutes
    165.   - alert: HighMemoryPressure
    166.     expr: instance:node_memory_available:ratio * 100 < 10 and rate(node_vmstat_pgmajfault[1m]) > 1000
    167.     for: 1m
    168.     labels:
    169.       severity: s4
    170.       alert_type: cause
    171.     annotations:
    172.       description: The node is under heavy memory pressure.  The available memory is under 5% and
    173.         there is a high rate of major page faults.
    174.       runbook: docs/monitoring/node_memory_alerts.md
    175.       value: 'Available memory {{ $value | printf "%.2f" }}%'
    176.       title: Node is under heavy memory pressure
    177.   - alert: CPUStalls
    178.     expr: rate(syslog_rcu_sched_stalls_total[1m]) > 0 or rate(rcu_sched_stalls_total[1m]) > 0
    179.     for: 10m
    180.     labels:
    181.       severity: s4
    182.       alert_type: cause
    183.     annotations:
    184.       description: The node is encountering RCU CPU stall warnings, which may cause the node to lock up occasionally.
    185.         Check `/var/log/kern.log` for more details. You may need to contact the cloud provider and possibly redeploy the VM.
    186.       title: CPU stall warnings have been detected on {{ if $labels.fqdn }}{{ $labels.fqdn }}
    187.         {{ else }}{{ $labels.instance }}{{ end }} for the past 10 minutes.
    189. # TODO eventually, once all kernels are upgraded to support
    190. # node_vmstat_oom_kill, we can deprecate the syslog_oom_events metric and remove
    191. # it from our mtail configuration.
    192. - name: OOM kills detected
    193.   rules:
    194.   - alert: TooManyOOMKills
    195.     expr: >
    196.       increase(node_vmstat_oom_kill[1h]) > 2
    197.       or
    198.       increase(syslog_oom_kills_total[1h]) > 2
    199.     labels:
    200.       severity: s3
    201.       alert_type: cause
    202.     annotations:
    203.       title: Several OOM kills detected on {{$labels.fqdn}} recently
    204.       description: >
    205.         Find out which process by running `dmesg | grep -i oom`, and continue
    206.         debugging.