Json的好处
    1. 原生日志需要做正则匹配，比较麻烦
    2. Json格式的日志不需要正则能直接分段采集

    Nginx使用Json格式日志
    [root@server12 ~]# vim /usr/local/nginx/conf/nginx.conf
    log_format json ‘{“@timestamp”:”$time_iso8601”,’
    ‘“clientip”:”$remote_addr”,’
    ‘“status”:$status,’
    ‘“bodysize”:$body_bytes_sent,’
    ‘“referer”:”$http_referer”,’
    ‘“ua”:”$http_user_agent”,’
    ‘“handletime”:$request_time,’
    ‘“url”:”$uri”}’;
    access_log logs/access.log;
    access_log logs/access.json.log json;
    [root@server12 ~]# nginx -t
    nginx: the configuration file /usr/local/nginx/conf/nginx.conf syntax is ok
    nginx: configuration file /usr/local/nginx/conf/nginx.conf test is successful
    [root@server12 ~]# nginx -s reload
    [root@server12 ~]# tail -f /usr/local/nginx/logs/access.log
    192.168.10.1 - - [13/Mar/2022:15:35:49 +0800] “GET / HTTP/1.1” 304 0 “-“ “Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36”
    192.168.10.1 - - [13/Mar/2022:15:35:49 +0800] “GET / HTTP/1.1” 304 0 “-“ “Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36”
    192.168.10.1 - - [13/Mar/2022:15:35:49 +0800] “GET / HTTP/1.1” 304 0 “-“ “Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36”
    192.168.10.1 - - [13/Mar/2022:15:35:49 +0800] “GET / HTTP/1.1” 304 0 “-“ “Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36”
    192.168.10.1 - - [13/Mar/2022:15:35:50 +0800] “GET / HTTP/1.1” 304 0 “-“ “Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36”
    192.168.10.1 - - [13/Mar/2022:15:35:50 +0800] “GET / HTTP/1.1” 304 0 “-“ “Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36”
    192.168.10.1 - - [13/Mar/2022:15:35:50 +0800] “GET / HTTP/1.1” 304 0 “-“ “Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36”
    192.168.10.1 - - [13/Mar/2022:15:35:52 +0800] “GET / HTTP/1.1” 304 0 “-“ “Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36”
    192.168.10.1 - - [13/Mar/2022:15:35:52 +0800] “GET / HTTP/1.1” 304 0 “-“ “Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36”
    192.168.10.1 - - [13/Mar/2022:15:35:53 +0800] “GET / HTTP/1.1” 304 0 “-“ “Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36”
    ^C
    [root@server12 ~]# curl 192.168.10.12
    <!DOCTYPE html>






    Welcome to nginx!


    If you see this page, the nginx web server is successfully installed and
    working. Further configuration is required.



    For online documentation and support please refer to
    nginx.org.
    ](http://nginx.org/">nginx.org.
    )
    Commercial support is available at
    nginx.com.

    ](http://nginx.com/">nginx.com.

    )

    Thank you for using nginx.




    [root@server12 ~]# ls /usr/local/nginx/logs/
    access.json.log access.log error.log nginx.pid
    [root@server12 ~]# tail -f /usr/local/nginx/logs/access.json.log
    {“@timestamp”:”2022-03-13T15:53:18+08:00”,”clientip”:”192.168.10.12”,”status”:200,”bodysize”:612,”referer”:”-“,”ua”:”curl/7.29.0”,”handletime”:0.000,”url”:”/index.html”}
    ^C
    [root@server12 ~]#
    部署服务介绍
    3. 192.168.10.11 Kibana ES
    4. 192.168.10.12 Logstash Filebeat

    Filebeat采集Json格式的日志
    [root@server12 ~]# vim /usr/local/filebeat-6.6.0/filebeat.yml
    filebeat.inputs:
    - type: log
    tail_files: true
    backoff: “1s”
    paths:
    - /usr/local/nginx/logs/access.json.log
    output:
    logstash:
    hosts: [“192.168.10.12:5044”]

    重启filebeat
    [root@server12 ~]# ps aux | grep filebeat
    root 8674 0.0 0.9 435920 18300 pts/0 Sl 14:53 0:00 /usr/local/filebeat-6.6.0/filebeat -e -c /usr/local/filebeat-6.6.0/filebeat.yml
    root 8968 0.0 0.0 112652 960 pts/0 R+ 15:57 0:00 grep —color=auto filebeat
    [root@server12 ~]# kill 8674
    [root@server12 ~]# ps aux | grep filebeat
    root 8970 0.0 0.0 112652 960 pts/0 R+ 15:57 0:00 grep —color=auto filebeat
    [1]+ Done nohup /usr/local/filebeat-6.6.0/filebeat -e -c /usr/local/filebeat-6.6.0/filebeat.yml
    [root@server12 ~]# ps aux | grep filebeat
    root 8973 0.0 0.0 112652 960 pts/0 R+ 15:57 0:00 grep —color=auto filebeat
    [root@server12 ~]# nohup /usr/local/filebeat-6.6.0/filebeat -e -c /usr/local/filebeat-6.6.0/filebeat.yml >> /tmp/filebeat.log &
    [1] 8974
    [root@server12 ~]# tail -f /tmp/filebeat.log
    DELETE /logstash-2022.03.13
    1.png11.png


    刷新nginx页面
    2.png
    GET /_cat/indices?v
    3.png
    4.png
    Logstash正则提取的配置备份
    filter {
    grok {
    match => {
    “message” => ‘(?[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}) - - [(?[^ ]+ +[0-9]+)] “(?[A-Z]+) (?[^ ]+) HTTP/\d.\d” (?[0-9]+) (?[0-9]+) “[^”]+” “(?[^”]+)”‘
    }
    remove_field => [“message”,”@version”,”path”,”beat”,”input”,”log”,”offset”,”prospector”,”source”,”tags”]
    }
    date {
    match => [“requesttime”, “dd/MMM/yyyy:HH:mm:ss Z”]
    target => “@timestamp”
    }
    }

    Logstash解析Json日志

    [root@server12 ~]# vim /usr/local/logstash-6.6.0/config/logstash.conf
    input {
    beats {
    host => ‘0.0.0.0’
    port => 5044
    }
    }
    filter {
    json {
    source => “message”
    #remove_field => [“message”,”@version”,”path”,”beat”,”input”,”log”,”offset”,”prospector”,”source”,”tags”]
    }
    }
    output {
    elasticsearch {
    hosts => [“http://192.168.10.11:9200“]
    }
    }

    DELETE /logstash-2022.03.13
    5.png

    GET /_cat/indices?v
    6.png

    刷新nginx页面
    7.png


    GET /_cat/indices?v
    8.png
    GET /logstash-2022.03.13/_search?q=
    9.png

    [root@server12 ~]# vim /usr/local/logstash-6.6.0/config/logstash.conf
    input {
    beats {
    host => ‘0.0.0.0’
    port => 5044
    }
    }
    filter {
    json {
    source => “message”
    remove_field => [“message”,”@version”,”path”,”beat”,”input”,”log”,”offset”,”prospector”,”source”,”tags”]
    }
    }
    output {
    elasticsearch {
    hosts => [“http://192.168.10.11:9200“]
    }
    }

    GET /_cat/indices?v

    DELETE /logstash-2022.03.13

    GET /logstash-2022.03.13/_search?q=

    10.png