基于docker 方式部署
    组件介绍：

    • loki是主服务器，负责存储日志和处理查询。
    • promtail是代理，负责收集日志并将其发送给 loki 。
    • Grafana用于 UI 展示。

    部署
    loki

    1. mkdir     /etc/loki && cd /etc/loki
    2. wget https://raw.githubusercontent.com/grafana/loki/v2.4.1/cmd/loki/loki-local-config.yaml -O loki-config.yaml
    3. mv loki-config.yaml local-config.yaml
    4. [root@k8s-node02 loki]# vim local-config.yaml 
    6.   http_listen_port: 3100
    7.   grpc_listen_port: 9096
    9. common:
    10.   path_prefix: /loki
    11.   storage:
    12.     filesystem:
    13.       chunks_directory: /loki/chunks
    14.       rules_directory: /loki/rules
    15.   replication_factor: 1
    16.   ring:
    17.     instance_addr: 127.0.0.1
    18.     kvstore:
    19.       store: inmemory
    21. schema_config:
    22.   configs:
    23.     - from: 2020-10-24
    24.       store: boltdb-shipper
    25.       object_store: filesystem
    26.       schema: v11
    27.       index:
    28.         prefix: index_
    29.         period: 24h
    31. ruler:
    32.   alertmanager_url: http://192.168.100.14:9093
    33.   enable_api: true
    34.   ring:
    35.     kvstore:
    36.       store: inmemory
    37.   #storage:
    38.   #  type: local
    39.   #  local:
    40.   #    directory: /loki/rules
    41.   #flush_period: 1m
    44. 创建告警规则文件
    45. mkdir -p /etc/loki/rules/fake
    46. [root@k8s-node02 fake]# cat rules.yaml 
    47. groups:
    48.   - name: testlog
    49.     rules:
    50.       - alert: test_log_line number
    51.         #expr: count_over_time({filename="/data/logs/nginx/a.log"}|="ERROR"[5s]) > 0
    52.         #expr: sum by (filename) (count_over_time({job="nginxlog"}|="ERROR"[5s])) > 0
    53.         expr: sum by (filename,message,host) (count_over_time({job="test-log"}|~"ERROR"|regexp "(?P<message>.*)$"[5s])) > 0
    54.         for: 0
    55.         labels:
    56.             severity: "critical"
    57.         annotations:
    58.           summary:  "{{ $labels.host }}"
    59.           description: "{{ $labels.message }}"
    60.           filename: "{{ $labels.filename }}"
    61.   docker 运行
    62.   docker run --name loki -v /etc/loki/local-config.yaml:/etc/loki/local-config.yaml -v /etc/loki/rules/:/loki/rules/  -d  -p 3100:3100 grafana/loki:2.4.1

    promtail

    1. mkdir /etc/promtail && cd /etc/promtail
    2. wget https://raw.githubusercontent.com/grafana/loki/v2.4.1/clients/cmd/promtail/promtail-docker-config.yaml -O promtail-config.yaml
    3. mv promtail-config.yaml config.yml
    4. 修改配置文件
    5. [root@k8s-node02 promtail]# cat config.yml 
    6. server:
    7.   http_listen_port: 9080
    8.   grpc_listen_port: 0
    10. positions:
    11.   filename: /tmp/positions.yaml
    13. clients:
    14.   - url: http://192.168.100.14:3100/loki/api/v1/push
    16. scrape_configs:
    17.   - job_name: linux
    18.     static_configs:
    19.       - targets:
    20.           - localhost
    21.         labels:
    22.           job: messages
    23.           host: localhost
    24.           __path__: /var/log/messages*
    25.   - job_name: nginx
    26.     static_configs:
    27.       - targets:
    28.           - localhost
    29.         labels:
    30.           job: test-log
    31.           host: k8s-node02
    32.           __path__: /data/logs/nginx/*.log
    34. docker 运行
    35. docker run --name promtail -v /etc/promtail/config.yml:/etc/promtail/config.yml -d  -v /data/logs:/data/logs grafana/promtail:2.4.1

    grafana

    1. docker run -d  -p 3000:3000  --name=grafana  grafana/grafana:7.3.7

    仪表盘配置
    image.png
    仪表盘上查看数据
    image.png
    alertmanager

    1. mkdir -p /data/monitor/alertmanager && cd /data/monitor/alertmanager
    2. [root@k8s-node02 promtail]# cat /data/monitor/alertmanager/alertmanager.yml 
    4. global:
    5.   smtp_smarthost: 'smtp.exmail.qq.com:465'
    6.   smtp_from: 'noreply@marketin.cn'
    7.   smtp_auth_username: 'noreply@marketin.cn'
    8.   smtp_auth_password: 'mI2bjgi5ni'
    10.   resolve_timeout: 5m
    12. route:
    13.   group_by: ['alertname', 'instance']
    14.   group_wait: 5s
    15.   group_interval: 10s
    16.   repeat_interval: 5m  
    17.   receiver: "dingtalk"
    19. inhibit_rules:
    20. - source_match:
    21.     severity: 'critical'
    22.   target_match_re:
    23.     severity: ^(warning)$
    24.   equal: ['alertname', 'instance']
    26. receivers:
    28. - name: dingtalk
    29.   webhook_configs:
    30.   - url: 'http://192.168.100.14:8060/dingtalk/webhook/send'
    31.     send_resolved: true
    32.  运行docker
    33. docker run -d -p 9093:9093 --name=alertmanager  \
    34.   -v /data/monitor/alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml \
    35.   prom/alertmanager:latest

    配置钉钉告警

    1. [root@k8s-node02 prometheus-webhook-dingtalk-1.4.0.linux-amd64]# cat /root/prometheus-webhook-dingtalk-1.4.0.linux-amd64/config.yml 
    2. ## Request timeout
    3. # timeout: 5s
    5. ## Customizable templates path
    6. templates:
    7.   - contrib/templates/legacy/template.tmpl
    9. ## You can also override default template using `default_message`
    10. ## The following example to use the 'legacy' template from v0.3.0
    11. # default_message:
    12. #   title: '{{ template "legacy.title" . }}'
    13. #   text: '{{ template "legacy.content" . }}'
    15. ## Targets, previously was known as "profiles"
    16. targets:
    17.   webhook:
    18.     url: https://oapi.dingtalk.com/robot/send?access_token=67a51455f433f3dd9bfa9ee9655ca26021cd92a02258cfc82e0ff7ddca1b2bd8
    19.     #message:
    20.     #  text: '{{ template "ding.link.content" . }}'
    22. 配置发送至钉钉内容：
    23. [root@k8s-node02 prometheus-webhook-dingtalk-1.4.0.linux-amd64]# cat /root/prometheus-webhook-dingtalk-1.4.0.linux-amd64/contrib/templates/legacy/template.tmpl
    24.  {{ define "ding.link.title" }}[日志报警]{{ end }}
    25. {{ define "ding.link.content" -}}
    26. {{- if gt (len .Alerts.Firing) 0 -}}
    27.   {{ range $i, $alert := .Alerts.Firing }}
    28.     [dangqian]:{{ index $alert.Labels "severity" }}
    29.     [告警主机]:{{ index $alert.Annotations "summary"}}
    30.     [日志文件]:{{ index $alert.Annotations "filename"}}
    31.     [告警时间]:{{ dateInZone "2006.01.02 15:04:05" (.StartsAt) "Asia/Shanghai" }}
    32.     [告警内容]:{{ index $alert.Annotations "description" }}
    33.   {{ end }}{{- end }}
    34. {{- end }}