ELK 企业级日志分析系统 - 21 增加一台Filebeat收集httpd日志 - 《运维-自动化部署》

增加一台Filebeat收集httpd日志
Logstash收集日志
1. 依赖于Java环境，用来收集日志比较重，占用内存和CPU
2. Filebeat相对轻量，占用服务器资源小
3. 一般选用Filebeat来进行日志收集

Filebeat的安装
4. 下载二进制文件
5. 解压移到对应的目录完成安装/usr/local/

Filebeat的二进制安装
tar xvf filebeat-6.6.0-linux-x86_64.tar.gz
mv filebeat-6.6.0-linux-x86_64 /usr/local/filebeat-6.6.0
httpd的安装和基本操作
yum install -y httpd
[root@server13 ~]# systemctl start httpd
[root@server13 ~]# curl 127.0.0.1
[root@server13 ~]# find / -name access_log
/var/log/httpd/access_log
[root@server13 ~]# cat /var/log/httpd/access_log
127.0.0.1 - - [15/Mar/2022:10:52:26 -0400] “GET / HTTP/1.1” 403 3985 “-“ “curl/7.29.0”
[root@server13 ~]# find / -name Shanghai
/usr/share/zoneinfo/Asia/Shanghai
/usr/share/zoneinfo/posix/Asia/Shanghai
/usr/share/zoneinfo/right/Asia/Shanghai
[root@server13 ~]# cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
cp: overwrite ‘/etc/localtime’? y
[root@server13 ~]# date
Tue Mar 15 22:53:12 CST 2022
[root@server13 ~]# curl 127.0.0.1
[root@server13 ~]# cat /var/log/httpd/access_log
127.0.0.1 - - [15/Mar/2022:10:52:26 -0400] “GET / HTTP/1.1” 403 3985 “-“ “curl/7.29.0”
127.0.0.1 - - [15/Mar/2022:10:53:16 -0400] “GET / HTTP/1.1” 403 3985 “-“ “curl/7.29.0”
[root@server13 ~]# date
Tue Mar 15 22:53:34 CST 2022
[root@server13 ~]# systemctl restart httpd
[root@server13 ~]# curl 127.0.0.1
[root@server13 ~]# cat /var/log/httpd/access_log
127.0.0.1 - - [15/Mar/2022:10:52:26 -0400] “GET / HTTP/1.1” 403 3985 “-“ “curl/7.29.0”
127.0.0.1 - - [15/Mar/2022:10:53:16 -0400] “GET / HTTP/1.1” 403 3985 “-“ “curl/7.29.0”
::1 - - [15/Mar/2022:10:53:48 -0400] “OPTIONS HTTP/1.0” 200 - “-“ “Apache/2.4.6 (Red Hat Enterprise Linux) (internal dummy connection)”
::1 - - [15/Mar/2022:10:53:48 -0400] “OPTIONS HTTP/1.0” 200 - “-“ “Apache/2.4.6 (Red Hat Enterprise Linux) (internal dummy connection)”
::1 - - [15/Mar/2022:10:53:48 -0400] “OPTIONS HTTP/1.0” 200 - “-“ “Apache/2.4.6 (Red Hat Enterprise Linux) (internal dummy connection)”
::1 - - [15/Mar/2022:10:53:48 -0400] “OPTIONS HTTP/1.0” 200 - “-“ “Apache/2.4.6 (Red Hat Enterprise Linux) (internal dummy connection)”
::1 - - [15/Mar/2022:10:53:48 -0400] “OPTIONS HTTP/1.0” 200 - “-“ “Apache/2.4.6 (Red Hat Enterprise Linux) (internal dummy connection)”
127.0.0.1 - - [15/Mar/2022:22:53:53 +0800] “GET / HTTP/1.1” 403 3985 “-“ “curl/7.29.0”
[root@server13 ~]# curl 127.0.0.1
[root@server13 ~]# cat /var/log/httpd/access_log
127.0.0.1 - - [15/Mar/2022:10:52:26 -0400] “GET / HTTP/1.1” 403 3985 “-“ “curl/7.29.0”
127.0.0.1 - - [15/Mar/2022:10:53:16 -0400] “GET / HTTP/1.1” 403 3985 “-“ “curl/7.29.0”
::1 - - [15/Mar/2022:10:53:48 -0400] “OPTIONS HTTP/1.0” 200 - “-“ “Apache/2.4.6 (Red Hat Enterprise Linux) (internal dummy connection)”
::1 - - [15/Mar/2022:10:53:48 -0400] “OPTIONS HTTP/1.0” 200 - “-“ “Apache/2.4.6 (Red Hat Enterprise Linux) (internal dummy connection)”
::1 - - [15/Mar/2022:10:53:48 -0400] “OPTIONS HTTP/1.0” 200 - “-“ “Apache/2.4.6 (Red Hat Enterprise Linux) (internal dummy connection)”
::1 - - [15/Mar/2022:10:53:48 -0400] “OPTIONS HTTP/1.0” 200 - “-“ “Apache/2.4.6 (Red Hat Enterprise Linux) (internal dummy connection)”
::1 - - [15/Mar/2022:10:53:48 -0400] “OPTIONS HTTP/1.0” 200 - “-“ “Apache/2.4.6 (Red Hat Enterprise Linux) (internal dummy connection)”
127.0.0.1 - - [15/Mar/2022:22:53:53 +0800] “GET / HTTP/1.1” 403 3985 “-“ “curl/7.29.0”
127.0.0.1 - - [15/Mar/2022:22:54:08 +0800] “GET / HTTP/1.1” 403 3985 “-“ “curl/7.29.0”

部署服务介绍
6. 192.168.10.11 Kibana ES
7. 192.168.10.13 Filebeat

Filebeat发送日志到ES配置/usr/local/filebeat-6.6.0/filebeat.yml
[root@server13 ~]# mv /usr/local/filebeat-6.6.0/filebeat.yml /usr/local/filebeat-6.6.0/filebeat.yml.bak
[root@server13 ~]# vim /usr/local/filebeat-6.6.0/filebeat.yml
filebeat.inputs:
- type: log
tail_files: true
backoff: “1s”
paths:
- /var/log/httpd/access_log

output:
elasticsearch:
hosts: [“192.168.10.11:9200”]

启动Filebeat
8. 前台启动： /usr/local/filebeat-6.6.0/filebeat -e -c /usr/local/filebeat-6.6.0/filebeat.yml
9. 后台启动：nohup /usr/local/filebeat-6.6.0/filebeat -e -c /usr/local/filebeat-6.6.0/filebeat.yml >/tmp/filebeat.log 2>&1 &
[root@server13 ~]# nohup /usr/local/filebeat-6.6.0/filebeat -e -c /usr/local/filebeat-6.6.0/filebeat.yml >/tmp/filebeat.log 2>&1 &
[2] 4445
[root@server13 ~]# curl 192.168.10.13

Kibana上查看日志数据
10. GET /xxx/_search?q=

11. 创建索引观察

Filebeat -> ES -> Kibana
12. 适合查看日志
13. 不适合具体日志的分析

Filebeat -> Logstash -> ES -> Kibana
Filebeat配置发往Logstash
[root@server13 ~]# vim /usr/local/filebeat-6.6.0/filebeat.yml
filebeat.inputs:
- type: log
tail_files: true
backoff: “1s”
paths:
- /var/log/httpd/access_log

output:
logstash:
hosts: [“192.168.10.12:5044”]
[root@server13 ~]# ps aux | grep filebeat
root 4445 0.0 0.8 432748 15780 pts/0 Sl 22:58 0:00 /usr/local/filebeat-6.6.0/filebeat -e -c /usr/local/filebeat-6.6.0/filebeat.yml
root 4473 0.0 0.0 112648 960 pts/0 R+ 23:10 0:00 grep —color=auto filebeat
[root@server13 ~]# kill -9 4445
[root@server13 ~]# nohup /usr/local/filebeat-6.6.0/filebeat -e -c /usr/local/filebeat-6.6.0/filebeat.yml >/tmp/filebeat.log 2>&1 &
[1] 4474

Logstash过滤Filebeat数据再发到ES
[root@server12 ~]# vim /usr/local/logstash-6.6.0/config/logstash.conf
input {
beats {
host => ‘0.0.0.0’
port => 5044
}
}

filter {
grok {
match => {
“message” => ‘(?[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}) - - [(?[^ ]+ +[0-9]+)] “(?[A-Z]+) (?[^ ]+) HTTP/\d.\d” (?[0-9]+) (?[0-9]+) “[^”]+” “(?[^”]+)”‘
}
remove_field => [“message”,”@version”,”path”,”beat”,”input”,”log”,”offset”,”prospector”,”source”,”tags”]
}
date {
match => [“requesttime”, “dd/MMM/yyyy:HH:mm:ss Z”]
target => “@timestamp”
}
}

output{
elasticsearch {
hosts => [“http://192.168.10.11:9200“]
}
}
[root@server12 ~]# /usr/local/logstash-6.6.0/bin/logstash -f /usr/local/logstash-6.6.0/config/logstash.conf
[root@server13 ~]# curl 192.168.10.13

Kibana上查看数据

1. GET /xxx/_search?q=

创建索引查看数据