1 删除日志的命令

我们知道日志都存储在elastic集群中，且日志每天被分割成一个index，例如：

  1. / # curl elasticsearch-logging:9200/_cat/indices?v
  2. health status index               uuid                   pri rep docs.count docs.deleted store.size pri.store.size
  3. green  open   logstash-2019.04.29 ejMBlRcJQvqK76xIerenYg   5   1      69864            0     65.9mb         32.9mb
  4. green  open   logstash-2019.04.28 hacNCuQVTQCUL62Sl8avOA   5   1      17558            0     21.3mb         10.6mb
  5. green  open   .kibana_1           MVjF8lQeRDeKfoZcDhA93A   1   1          2            0     30.1kb           15kb
  6. green  open   logstash-2019.05.05 m2aD8X9RQ3u48DvVq18x_Q   5   1      31218            0     34.4mb         17.2mb
  7. green  open   logstash-2019.05.01 66OjwM5wT--DZaVfzUdXYQ   5   1      50610            0     54.6mb         27.1mb
  8. green  open   logstash-2019.04.30 L3AH165jT6izjHHa5L5g0w   5   1      56401            0     55.5mb         27.8mb
  9. ...

因此 EFK 中的日志自动清理，只要定时去删除 es 中的 index 即可，如下命令

$ curl -X DELETE elasticsearch-logging:9200/logstash-xxxx.xx.xx

基于 alpine:3.8 创建镜像es-index-rotator 查看Dockerfile(下面)，然后创建一个cronjob去完成清理任务

$ kubectl apply -f /etc/ansible/manifests/efk/es-index-rotator/

2 验证日志清理

  • 查看 cronjob

    $ kubectl get cronjob -n kube-system 
NAME               SCHEDULE      SUSPEND   ACTIVE   LAST SCHEDULE   AGE
es-index-rotator   3 1 */1 * *   False     0        19h             20h

  • 查看日志清理情况

    $ kubectl get pod -n kube-system |grep es-index-rotator
es-index-rotator-1557507780-7xb89             0/1     Completed   0          19h
# 查看日志，可以了解日志清理情况
$ kubectl logs -n kube-system es-index-rotator-1557507780-7xb89 es-index-rotator

3 k8s中定时任务的创建yaml文件

apiVersion: batch/v1beta1
kind: CronJob
metadata:
  name: es-index-rotator
  namespace: kube-system
spec:
  # 每天1点3分执行
  schedule: "3 1 */1 * *"
  jobTemplate:
    spec:
      template:
        spec:
          containers:
          - name: es-index-rotator
            image: easzlab/es-index-rotator:0.2.1
            # 保留最近10天日志
            command:
            - /bin/rotate.sh
            - "10"
            - "logstash"  # fluented 默认创建的index形如'logstash-2020.01.01'
          restartPolicy: OnFailure
  concurrencyPolicy: Forbid
  successfulJobsHistoryLimit: 2
  failedJobsHistoryLimit: 1

4 es-index-rotator:0.2.1镜像的dockerfile文件

FROM alpine:3.9

COPY rotate.sh /bin/rotate.sh

RUN echo "===> Installing essential tools..."   && \
    apk --update add bash curl coreutils        && \
    echo "===> Cleaning up cache..."            && \
    rm -rf /var/cache/apk/*                     && \
    chmod +x /bin/rotate.sh

CMD ["/bin/rotate.sh"]

5 rotate.sh脚本文件

#!/bin/bash
#--------------------------------------------------
# Rotate the indices in elastic of the EFK deployment
#
# @author:  gjmzj
# @usage:   ./rotator.sh <max_days_of_log> [<index_prefix1> ...] 
# @repo:    https://github.com/kubeasz/mirrorepo/es-index-rotator
# @ref:     https://github.com/easzlab/kubeasz/tree/master/manifests/efk/es-index-rotator/rotator.yaml

set -o nounset
set -o errexit
#set -o xtrace

[[ "$#" -gt 1 && $1 =~ ^[1-9][0-9]{0,2}$ ]] || \
{ echo 'Usage: ./rotator.sh <max_days_of_log> [<index_prefix1> <index_prefix2> ...]'; exit 1; }

max_days_of_log="$1"

echo -e "\n[INFO] rotate job starts, try to keep $max_days_of_log days of logs."

curl -s elasticsearch-master:9200/_cat/indices > /tmp/indices || \
{ echo "[ERROR] Can not connect to elastic!"; exit 1; }

for index_prefix in "${@:2}";do
        cat /tmp/indices|grep "$index_prefix"|wc -l > /tmp/lines
        curr_days_of_log=$(cat /tmp/lines)
        curr_days_of_log=$((${curr_days_of_log}-2))

        if [[ "$max_days_of_log" -gt "$curr_days_of_log" ]];then
        echo "[WARN] No need to rotate the ES indices: $index_prefix-*!"
        else
        first_day=$(date -d "$max_days_of_log days ago" +'%Y.%m.%d')
                cat /tmp/indices|grep "$index_prefix"|cut -d' ' -f3|sed "s/$index_prefix-//g"|sed "s/-/\./g" > /tmp/index
        rotate=$(cat /tmp/index|sort|sed -n "1,/$first_day/"p)
        for day in $rotate;do
                curl -s -X DELETE "elasticsearch-master:9200/$index_prefix-$day"
         day=$(echo $day|sed 's/\./-/g')
         curl -s -X DELETE "elasticsearch-master:9200/$index_prefix-$day"
        done
         echo -e "\n[INFO] Success to rotate the ES indices: $index_prefix-*!"
        fi
done

exit 0

6 构建镜像

docker build -t kangkangluma/es-index-rotator:0.2.1
docker login -u kangkangluma -p Welcome12 docker.io
docker push kangkangluma/es-index-rotator:0.2.1

https://github.com/easzlab/kubeasz/blob/master/manifests/efk/es-index-rotator/rotator.yaml