logstash(7.9.3)安装参考

1、拉取镜像

docker pull logstash:7.9.3

2、挂载卷

  1. mkdir /home/vvvv/docker/dockerVolumes/logstash/config
  2. mkdir /home/vvvv/docker/dockerVolumes/logstash/pipeline

2.1、config文件配置

vim logstash.yml

  1. config:
  2.   reload:
  3.     automatic: true
  4.     interval: 3s
  5. xpack:
  6.   management.enabled: false
  7.   monitoring.enabled: false

vim pipelines.yml

  1.  - pipeline.id: logstash_dev # ！！！"-"前后都有空格！！！
  2.    path.config: "/usr/share/logstash/pipeline/logstash_dev.conf" # path是容器内的路径！！！

2.2、pipelines文件配置

vim logstash_dev.conf

  1. input{
  2.     kafka {
  3.         bootstrap_servers => "10.6.62.211:9092"
  4.         topics => ["DIDLIST_caesar_dynamic_data"]
  5.         client_id => "logstash-0-0"
  6.         group_id => "logstash"
  7.         codec => "json"
  8.         add_field => {
  9.             "s_index" => "caesar_dynamic_data"
  10.         }
  11.     }
  13.     kafka {
  14.         bootstrap_servers => "10.6.62.211:9092"
  15.         topics => ["DIDLIST_caesar_data_bank_make_ios"]
  16.         client_id => "logstash-0-4"
  17.         group_id => "logstash"
  18.         codec => "json"
  19.         add_field => {
  20.             "s_index" => "caesar_data_bank_make_ios"
  21.             "[@metadata][b_submeter]" => "true"
  22.         }
  23.     }
  24. }
  26. filter {
  27.     date {
  28.         match => ["message","UNIX_MS"]
  29.         target => "@timestamp"
  30.     }
  31.     ruby {
  32.         code => "event.set('timestamp', event.get('@timestamp').time.localtime + 8*60*60)"
  33.     }
  34.     ruby {
  35.         code => "event.set('@timestamp',event.get('timestamp'))"
  36.     }
  37.     mutate {
  38.         convert => ["timestamp", "string"]
  39.         gsub => ["timestamp", "T([\S\s]*?)Z", ""]
  40.         gsub => ["timestamp", "-", "."]
  41.     }
  42. }
  44. output {
  45.     if [@metadata][b_submeter] == "true" {
  46.         elasticsearch {
  47.             hosts => ["10.6.62.212:9400"]
  48.             document_id => "%{id}"
  49.             index => "%{s_index}_%{timestamp}"
  50.         }
  51.     } else {
  52.         elasticsearch {
  53.             hosts => ["10.6.62.212:9400"]
  54.             document_id => "%{id}"
  55.             index => "%{s_index}"
  56.         }
  57.     }
  58. }

3、部署容器

  1. # restart：只要docker启动，容器就会跟随启动
  2. # privileged=true：解决Docker挂载主机目录Docker访问出现cannot open directory.:Permission denied
  3. # -d 后台启动
  4. docker run -d --restart=always --privileged=true --name=logstash \
  5. -p 5047:5047 -p 9600:9600 \
  6. -v /home/vvvv/docker/dockerVolumes/logstash/config/:/usr/share/logstash/config \
  7. -v /home/vvvv/docker/dockerVolumes/logstash/pipeline/:/usr/share/logstash/pipeline fc2df485e5c5

4、查看日志

docker logs -tf logtstash