部署准备

  1. wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.3.2-linux-x86_64.tar.gz
  2. wget https://artifacts.elastic.co/downloads/logstash/logstash-7.3.2.tar.gz
  3. wget https://artifacts.elastic.co/downloads/kibana/kibana-7.3.2-linux-x86_64.tar.gz
  4. wget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.3.2-linux-x86_64.tar.gz
  9. # mkdir /home/elk
  11. # ll /home/elk
  12. -rw-r--r--. 1 root root 285050383 十一 23  2019 elasticsearch-7.3.2-linux-x86_64.tar.gz
  13. -rw-r--r--. 1 root root  25274202 十一 23  2019 filebeat-7.3.2-linux-x86_64.tar.gz
  14. -rw-r--r--. 1 root root 236654252 十一 23  2019 kibana-7.3.2-linux-x86_64.tar.gz
  15. -rw-r--r--. 1 root root 171783584 十一 23  2019 logstash-7.3.2.tar.gz

elastic

  1. # tar -zxvf  elasticsearch-7.3.2-linux-x86_64.tar.gz
  2. 修改配置文件
  3. [root@dev-app-60 elasticsearch-7.3.2]# vim config/elasticsearch.yml 
  4. ——————————————————————————————————————————————————————————————————————————————————————————————————————————————————
  5. #ES监听地址任意IP都可访问，也可以是自己服务器的IP
  6. network.host: 0.0.0.0
  7. http.port: 9200
  8. ——————————————————————————————————————————————————————————————————————————————————————————————————————————————————
  10. 优化类配置
  11. vi /etc/sysctl.conf
  12. fs.file-max=65536
  13. vm.max_map_count=262144
  14. # sysctl -p
  16. vi /etc/security/limits.conf
  17. *               soft    nofile          65536
  18. *               hard    nofile          65536
  19. *               soft    nproc           65536
  20. *               hard    nproc           65536
  23. #############################################添加用户和组#############################################################
  24. 添加组
  25. groupadd elkgroup   
  27. 在elkgroup下添加elkuser用户，并设密码
  28. useradd elkuser -g elkgroup -p elkuser
  31. 文件目录权限修改
  32. [root@dev-app-60 elk]# chown  elkuser.  /home/elk  -R
  34. # ll
  35. total 0
  36. drwxr-xr-x. 10 elkuser elkgroup 183 三月 10 18:05 elasticsearch-7.3.2
  37. drwxr-xr-x.  5 elkuser elkgroup 212 三月 11 09:21 filebeat-7.3.2-linux-x86_64
  38. drwxr-xr-x. 14 elkuser elkgroup 271 三月 11 09:21 kibana-7.3.2-linux-x86_64
  39. drwxr-xr-x. 12 elkuser elkgroup 255 三月 10 18:08 logstash-7.3.2
  42. 定位到elasticsearch安装目录下为elkuser用户设置访问权限
  43. # chown -R  elkuser  /home/elk/elasticsearch-7.3.2
  45. 启动
  46. # su  elkuser 
  47. $ cd /home/elk/elasticsearch-7.3.2
  48. nohup  ./bin/elasticsearch &
  51. 检测是否启动
  52. # netstat -tanp|grep  9200
  53. tcp6       0      0 :::9200                 :::*                    LISTEN      9090/java     
  55. $ curl 127.0.0.1:9200
  56. {
  57.   "name" : "dev-app-60",
  58.   "cluster_name" : "elasticsearch",
  59.   "cluster_uuid" : "_na_",
  60.   "version" : {
  61.     "number" : "7.3.2",
  62.     "build_flavor" : "default",
  63.     "build_type" : "tar",
  64.     "build_hash" : "1c1faf1",
  65.     "build_date" : "2019-09-06T14:40:30.409026Z",
  66.     "build_snapshot" : false,
  67.     "lucene_version" : "8.1.0",
  68.     "minimum_wire_compatibility_version" : "6.8.0",
  69.     "minimum_index_compatibility_version" : "6.0.0-beta1"
  70.   },
  71.   "tagline" : "You Know, for Search"
  72. }
  77. elastic添加密码
  78. [elkuser@dev-app-60 elasticsearch-7.3.2]$ vim config/elasticsearch.yml 
  79. http.cors.enabled: true
  80. http.cors.allow-origin: "*"
  81. http.cors.allow-headers: Authorization
  82. xpack.security.enabled: true
  83. xpack.security.transport.ssl.enabled: true
  84. xpack.security.authc.accept_default_password: true
  89. +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  90. 指定密码比较复杂的时候，可以随机 密码 
  91. [elkuser@dev-app-60 elasticsearch-7.3.2]$ ./bin/elasticsearch-setup-passwords auto
  92. +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  95. 执行设置用户名和密码的命令,这里需要为4个用户分别设置密码，elastic,apm_system, kibana, logstash_system,betas_system,remote_monitoring_user
  96. [elkuser@dev-app-60 elasticsearch-7.3.2]$ ./bin/elasticsearch-setup-passwords interactive
  103.  ./bin/elasticsearch-setup-passwords interactive
  104. future versions of Elasticsearch will require Java 11; your Java version from [/home/jdk/jre] does not meet this requirement
  106. Failed to determine the health of the cluster running at http://10.2.204.60:9200
  107. Unexpected response code [503] from calling GET http://10.2.204.60:9200/_cluster/health?pretty
  108. Cause: master_not_discovered_exception
  110. It is recommended that you resolve the issues with your cluster before running elasticsearch-setup-passwords.
  111. It is very likely that the password changes will fail when run against an unhealthy cluster.
  113. Do you want to continue with the password setup process [y/N]y
  115. Initiating the setup of passwords for reserved users elastic,apm_system,kibana,logstash_system,beats_system,remote_monitoring_user.
  116. You will be prompted to enter passwords as the process progresses.
  117. Please confirm that you would like to continue [y/N]y
  120. Enter password for [elastic]: 
  121. Reenter password for [elastic]: 
  122. Enter password for [apm_system]: 
  123. Reenter password for [apm_system]: 
  124. Enter password for [kibana]: 
  125. Reenter password for [kibana]: 
  126. Enter password for [logstash_system]: 
  127. Reenter password for [logstash_system]: 
  128. Enter password for [beats_system]: 
  129. Reenter password for [beats_system]: 
  130. Enter password for [remote_monitoring_user]: 
  131. Reenter password for [remote_monitoring_user]: 
  132. Changed password for user [apm_system]
  133. Changed password for user [kibana]
  134. Changed password for user [logstash_system]
  135. Changed password for user [beats_system]
  136. Changed password for user [remote_monitoring_user]
  137. Changed password for user [elastic]

logstash

  1. #测试
  2. curl -H "Content-Type:application/json" -XPOST -u elastic 'http://127.0.0.1:9200/_xpack/security/user/elastic/_password' -d '{ "password" : "123456" }'
  4. # cd /home/elk
  5. # tar -zxvf logstash-7.3.2.tar.gz
  6. # cd logstash-7.3.2/
  8. 配置文件
  9. vim  pipelines.yml
  10. path.config: "/home/elk/logstash-7.3.2/config/conf.d/*.conf"
  11. [root@dev-app-60 config]# mkdir  conf.d
  12. [root@dev-app-60 config]# cd conf.d/
  16. #+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++#
  17. 配置文件
  18. [elkuser@dev-app-60 logstash-7.3.2]$ cat config/conf.d/app.conf 
  19. input {
  20.   beats {
  21.     port => 5044
  22.   }
  23. }
  25. filter {
  26.   if "dev-app-allocation" in [tags] {
  27.     grok {
  28.       match => { "message" => "\[%{TIMESTAMP_ISO8601:log_timestamp}\]%{GREEDYDATA:log_info}" }
  29.       remove_field => ["log_info","agent","ecs.version","log.flags","log.offset"]
  30.       }
  31.     mutate { 
  32.       gsub => [ "log_info", "\\n", "\n\r" ] 
  33.       }
  34.     date { 
  35.       match => [ "log_timestamp" , "yyyy-MM-dd HH:mm:ss,SSS" ]
  36.      }
  37.    }
  40. output {
  41.   if "dev-app" in [tags] {
  42.     elasticsearch {
  43.       hosts => ["http://10.2.204.60:9200"]
  44.       index => "dev-app%{+YYYY.MM.dd}"
  45.       user => "elastic"
  46.       password => "elastic密码"
  47.      }
  48.    }
  49.     #stdout { codec => rubydebug }
  50.   }
  51. }  
  52. #+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++#
  54. 启动
  55. [elkuser@dev-app-60 logstash-7.3.2]$ nohup  ./bin/logstash &

logstash限制内存

  1. [elkuser@dev-app-60 config]$ cat /home/elk/logstash-7.3.2/config/jvm.options

排错记录

  1. [elkuser@dev-app-60 elasticsearch-7.3.2]$ ./bin/elasticsearch-setup-passwords interactive
  2. ERROR: Failed to set password for user [apm_system].
  5. 注释：
  6. discovery.seed_hosts
  7. cluster.initial_master_nodes
  8. 添加
  9. discovery.type: single-node
  10. 重启elastic
  15. [elkuser@dev-app-60 logstash-7.3.2]$ ./bin/logstash -e'input {stdin {}} output {stdout {}}'
  16. Thread.exclusive is deprecated, use Thread::Mutex
  17. Sending Logstash logs to /home/elk/logstash-7.3.2/logs which is now configured via log4j2.properties
  18. [2021-03-11T11:57:21,225][WARN ][logstash.config.source.multilocal] Ignoring the 'pipelines.yml' file because modules or command line options are specified
  19. [2021-03-11T11:57:21,240][INFO ][logstash.runner          ] Starting Logstash {"logstash.version"=>"7.3.2"}
  20. [2021-03-11T11:57:21,267][INFO ][logstash.agent           ] No persistent UUID file found. Generating new UUID {:uuid=>"6426ad87-6c81-4c91-823e-1b44e06139a8", :path=>"/home/elk/logstash-7.3.2/data/uuid"}
  21. [2021-03-11T11:57:22,581][INFO ][org.reflections.Reflections] Reflections took 110 ms to scan 1 urls, producing 19 keys and 39 values 
  22. [2021-03-11T11:57:23,968][WARN ][org.logstash.instrument.metrics.gauge.LazyDelegatingGauge] A gauge metric of an unknown type (org.jruby.RubyArray) has been create for key: cluster_uuids. This may result in invalid serialization.  It is recommended to log an issue to the responsible developer/development team.
  23. [2021-03-11T11:57:23,972][INFO ][logstash.javapipeline    ] Starting pipeline {:pipeline_id=>"main", "pipeline.workers"=>8, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>50, "pipeline.max_inflight"=>1000, :thread=>"#<Thread:0x287ba869 run>"}
  24. [2021-03-11T11:57:24,028][INFO ][logstash.javapipeline    ] Pipeline started {"pipeline.id"=>"main"}
  25. The stdin plugin is now waiting for input:
  26. [2021-03-11T11:57:24,161][INFO ][logstash.agent           ] Pipelines running {:count=>1, :running_pipelines=>[:main], :non_running_pipelines=>[]}
  27. [2021-03-11T11:57:24,401][INFO ][logstash.agent           ] Successfully started Logstash API endpoint {:port=>9600}########出现此处则输入要输出的内容
  28. xlxtest
  29. /home/elk/logstash-7.3.2/vendor/bundle/jruby/2.5.0/gems/awesome_print-1.7.0/lib/awesome_print/formatters/base_formatter.rb:31: warning: constant ::Fixnum is deprecated
  30. {
  31.     "@timestamp" => 2021-03-11T03:57:45.394Z,
  32.           "host" => "dev-app-60",
  33.        "message" => "xlxtest",
  34.       "@version" => "1"
  35. }
  37. {
  38.     "@timestamp" => 2021-03-11T03:57:48.869Z,
  39.           "host" => "dev-app-60",
  40.        "message" => "",
  41.       "@version" => "1"
  42. }
  44. 根据配置文件调试

filebeat

  1. # tar -zxvf   filebeat-7.3.2-linux-x86_64.tar.gz 
  2. [root@dev-app-60 elk]# cd logstash-7.3.2/config/
  3. [root@dev-app-60 config]# su elkuser
  4. [elkuser@dev-app-60 filebeat-7.3.2-linux-x86_64]$ cp filebeat.yml   filebeat.yml_bak.0311 
  5. [elkuser@dev-app-60 filebeat-7.3.2-linux-x86_64]$ vim filebeat.yml
  8. +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  9. [elkuser@dev-app-60 filebeat-7.3.2-linux-x86_64]$ cat filebeat.yml
  10. filebeat.inputs:
  12. - type: log
  13.   enabled: true
  14.   paths:
  15.     - /home/output/logs/allocation/*.log
  16.   tags: ["dev-app"]
  17.   multiline.pattern: '^\[\d{4}-\d{1,2}-\d{1,2}\s\d{1,2}:\d{1,2}:\d{1,2}'
  18.   multiline.negate: true
  19.   multiline.match: after
  21. #============================= Filebeat modules ===============================
  23. filebeat.config.modules:
  24.   # Glob pattern for configuration loading
  25.   path: ${path.config}/modules.d/*.yml
  27.   # Set to true to enable config reloading
  28.   reload.enabled: false
  30.   # Period on which files under path should be checked for changes
  31.   #reload.period: 10s
  33. #==================== Elasticsearch template setting ==========================
  35. setup.template.settings:
  36.   index.number_of_shards: 1
  37.   #index.codec: best_compression
  38.   #_source.enabled: false
  39. #================================ Outputs =====================================
  42. #----------------------------- Logstash output --------------------------------
  43. output.logstash:
  44.   # The Logstash hosts
  45.   hosts: ["10.2.204.60:5044"]  #  需要在 logstash 配置文件中配置
  46. +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  48. 启动
  49. # su  elkuser
  50. [elkuser@dev-app-60 filebeat-7.3.2-linux-x86_64]$ nohup   /home/elk/filebeat-7.3.2-linux-x86_64/filebeat  &

kibana

  1. # cd /home/elk/kibana-7.3.2-linux-x86_64/config
  2. [elkuser@dev-app-60 config]$ cat kibana.yml|grep -v "#"|grep -v ^$
  3. server.host: "0.0.0.0"
  4. elasticsearch.username: "elastic"
  5. elasticsearch.password: "elastic密码"
  7. 启动
  8. [elkuser@dev-app-60 elk]$ nohup /home/elk/kibana-7.3.2-linux-x86_64/bin/kibana &

kibana查询通配符

  1. 通配符
  3. ? 匹配单个字符
  4. * 匹配0到多个字符
  6. 示例：kiba?a, el*search
  7. ? * 不能用作第一个字符，例如：?text *text

kibana-转义特殊字符

  1. 转义特殊字符
  2. + - = && || > < ! ( ) { } [ ] ^ " ~ * ? : \ /
  4. 以上字符当作值搜索的时候需要用\转义
  6. 1\+  用来查询1+

kibana—web界面

  1. http://10.2.204.60:5601/login?next=%2F#?_g=()
  2. elastic/elastic密码