收集Kubernetes中应用的日志信息并输出到Elasticsearch中
    新增 vi filebeat-kubernetes-7.3.yaml 文件
    部署类型为DaemonSet确保在Kubernetes中每个节点都有一份部署。

    1. ---
    2. apiVersion: v1
    3. kind: ConfigMap
    4. metadata:
    5.   name: filebeat-config
    6.   namespace: kube-system
    7.   labels:
    8.     k8s-app: filebeat
    9. data:
    10.   filebeat.yml: |-
    11.     filebeat.inputs:
    12.     - type: container
    13.       paths:
    14.         - /var/log/containers/ruoyi-admin-*.log
    15.       processors:
    16.         - add_kubernetes_metadata:
    17.             in_cluster: true
    18.             host: ${NODE_NAME}
    19.             matchers:
    20.             - logs_path:
    21.                 logs_path: "/var/log/containers/"
    23.     # To enable hints based autodiscover, remove `filebeat.inputs` configuration and uncomment this:
    24.     #filebeat.autodiscover:
    25.     #  providers:
    26.     #    - type: kubernetes
    27.     #      host: ${NODE_NAME}
    28.     #      hints.enabled: true
    29.     #      hints.default_config:
    30.     #        type: container
    31.     #        paths:
    32.     #          - /var/log/containers/*${data.kubernetes.container.id}.log
    34.     # https://www.elastic.co/guide/en/beats/filebeat/7.3/filtering-and-enhancing-data.html
    35.     processors:
    36.       - decode_json_fields:
    37.           fields: ["message","logger_name","thread_name","level","level_value"]
    38.           process_array: false
    39.           max_depth: 1
    40.           target: ""
    41.           overwrite_keys: false
    42.       - drop_fields:
    43.           fields: ["@version"]
    44.     #  - add_cloud_metadata:
    45.     #  - add_host_metadata:
    47.     #cloud.id: ${ELASTIC_CLOUD_ID}
    48.     #cloud.auth: ${ELASTIC_CLOUD_AUTH}
    50.     # https://www.elastic.co/guide/en/beats/filebeat/7.3/elasticsearch-output.html
    51.     output.elasticsearch:
    52.       hosts: ['http://10.144.104.148:9200','http://10.144.66.152:9200']
    53.       #username: ${ELASTICSEARCH_USERNAME}
    54.       #password: ${ELASTICSEARCH_PASSWORD}
    55.       indices:
    56.         - index: "ruoyi-admin-%{+yyyy.MM.dd}"
    57.         # - index: "ruoyi-admin-info-%{+yyyy.MM.dd}"
    58.         #   when.contains:
    59.         #     message: "info"
    60.         # - index: "ruoyi-admin-warn-%{+yyyy.MM.dd}"
    61.         #   when.contains:
    62.         #     message: "warn"
    63.         # - index: "ruoyi-admin-error-%{+yyyy.MM.dd}"
    64.         #   when.contains:
    65.         #     message: "error"
    67. ---
    68. apiVersion: apps/v1
    69. kind: DaemonSet
    70. metadata:
    71.   name: filebeat
    72.   namespace: kube-system
    73.   labels:
    74.     k8s-app: filebeat
    75. spec:
    76.   selector:  
    77.     matchLabels:  
    78.       k8s-app: filebeat
    79.   template:
    80.     metadata:
    81.       labels:
    82.         k8s-app: filebeat
    83.     spec:
    84.       serviceAccountName: filebeat
    85.       terminationGracePeriodSeconds: 30
    86.       hostNetwork: true
    87.       dnsPolicy: ClusterFirstWithHostNet
    88.       containers:
    89.       - name: filebeat
    90.         image: 192.168.28.150:8001/elastic/docker.elastic.co/beats/filebeat:7.3.2
    91.         args: [
    92.           "-c", "/etc/filebeat.yml",
    93.           "-e",
    94.         ]
    95.         env:
    96.         - name: ELASTICSEARCH_HOST
    97.           value: elasticsearch
    98.         - name: ELASTICSEARCH_PORT
    99.           value: "9200"
    100.         - name: ELASTICSEARCH_USERNAME
    101.           value: elastic
    102.         - name: ELASTICSEARCH_PASSWORD
    103.           value: changeme
    104.         - name: ELASTIC_CLOUD_ID
    105.           value:
    106.         - name: ELASTIC_CLOUD_AUTH
    107.           value:
    108.         - name: NODE_NAME
    109.           valueFrom:
    110.             fieldRef:
    111.               fieldPath: spec.nodeName
    112.         securityContext:
    113.           runAsUser: 0
    114.           # If using Red Hat OpenShift uncomment this:
    115.           #privileged: true
    116.         resources:
    117.           limits:
    118.             memory: 200Mi
    119.           requests:
    120.             cpu: 100m
    121.             memory: 100Mi
    122.         volumeMounts:
    123.         - name: config
    124.           mountPath: /etc/filebeat.yml
    125.           readOnly: true
    126.           subPath: filebeat.yml
    127.         - name: data
    128.           mountPath: /usr/share/filebeat/data
    129.         - name: varlibdockercontainers
    130.           mountPath: /var/lib/docker/containers
    131.           readOnly: true
    132.         - name: varlog
    133.           mountPath: /var/log
    134.           readOnly: true
    135.       volumes:
    136.       - name: config
    137.         configMap:
    138.           defaultMode: 0600
    139.           name: filebeat-config
    140.       - name: varlibdockercontainers
    141.         hostPath:
    142.           path: /var/lib/docker/containers
    143.       - name: varlog
    144.         hostPath:
    145.           path: /var/log
    146.       # data folder stores a registry of read status for all files, so we don't send everything again on a Filebeat pod restart
    147.       - name: data
    148.         hostPath:
    149.           path: /var/lib/filebeat-data
    150.           type: DirectoryOrCreate
    151. ---
    152. apiVersion: rbac.authorization.k8s.io/v1
    153. kind: ClusterRoleBinding
    154. metadata:
    155.   name: filebeat
    156. subjects:
    157. - kind: ServiceAccount
    158.   name: filebeat
    159.   namespace: kube-system
    160. roleRef:
    161.   kind: ClusterRole
    162.   name: filebeat
    163.   apiGroup: rbac.authorization.k8s.io
    164. ---
    165. apiVersion: rbac.authorization.k8s.io/v1
    166. kind: ClusterRole
    167. metadata:
    168.   name: filebeat
    169.   labels:
    170.     k8s-app: filebeat
    171. rules:
    172. - apiGroups: [""] # "" indicates the core API group
    173.   resources:
    174.   - namespaces
    175.   - pods
    176.   verbs:
    177.   - get
    178.   - watch
    179.   - list
    180. ---
    181. apiVersion: v1
    182. kind: ServiceAccount
    183. metadata:
    184.   name: filebeat
    185.   namespace: kube-system
    186.   labels:
    187.     k8s-app: filebeat
    188. ---

    应用部署文件

    kubectl apply -f filebeat-kubernetes-7.3.yaml