之前架构
Filebeat(多台)
Filebeat(多台) -> Logstash(正则) -> Elasticsearch(入库) -> Kibana展现
Filebeat(多台)
架构存在的问题
1. Logstash性能不足的时候
2. 扩容Logstash,Filebeat的配置可能会不一致
架构优化
Filebeat(多台) Logstash
Filebeat(多台) -> Redis、Kafka -> Logstash(正则) -> Elasticsearch(入库) -> Kibana展现
部署服务介绍
3. 192.168.10.11 Kibana ES
4. 192.168.10.12 Logstash Filebeat Redis
Redis服务器搭建
yum install -y wget net-tools gcc gcc-c++ make tar openssl openssl-devel cmake
cd /usr/local/src
wget http://download.redis.io/releases/redis-4.0.9.tar.gz
tar xvf redis-4.0.9.tar.gz
cd redis-4.0.9
make
mkdir -pv /usr/local/redis/conf /usr/local/redis/bin
cp src/redis* /usr/local/redis/bin/
cat redis.conf | grep -v “#” | grep -v “^$” > /usr/local/redis/conf/redis.conf
验证Redis服务器
5. 更改Redis配置(daemon、dir、requirepass)
6. 密码设置为jaking
7. 验证set、get操作
[root@server12 redis-4.0.9]# vim /usr/local/redis/conf/redis.conf
bind 127.0.0.1
protected-mode yes
port 6379
tcp-backlog 511
timeout 0
tcp-keepalive 300
daemonize yes
supervised no
pidfile /var/run/redis_6379.pid
loglevel notice
logfile “”
databases 16
always-show-logo yes
save 900 1
save 300 10
save 60 10000
stop-writes-on-bgsave-error yes
rdbcompression yes
rdbchecksum yes
dbfilename dump.rdb
dir /tmp
requirepass jaking
slave-serve-stale-data yes
slave-read-only yes
repl-diskless-sync no
repl-diskless-sync-delay 5
repl-disable-tcp-nodelay no
slave-priority 100
lazyfree-lazy-eviction no
lazyfree-lazy-expire no
lazyfree-lazy-server-del no
slave-lazy-flush no
appendonly no
appendfilename “appendonly.aof”
appendfsync everysec
no-appendfsync-on-rewrite no
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb
aof-load-truncated yes
aof-use-rdb-preamble no
lua-time-limit 5000
slowlog-log-slower-than 10000
slowlog-max-len 128
latency-monitor-threshold 0
notify-keyspace-events “”
hash-max-ziplist-entries 512
hash-max-ziplist-value 64
list-max-ziplist-size -2
list-compress-depth 0
set-max-intset-entries 512
zset-max-ziplist-entries 128
zset-max-ziplist-value 64
hll-sparse-max-bytes 3000
activerehashing yes
client-output-buffer-limit normal 0 0 0
client-output-buffer-limit slave 256mb 64mb 60
client-output-buffer-limit pubsub 32mb 8mb 60
hz 10
aof-rewrite-incremental-fsync yes
Redis的启动命令
/usr/local/redis/bin/redis-server /usr/local/redis/conf/redis.conf
Redis的简单操作
8. /usr/local/redis/bin/redis-cli
9. auth jaking
10. set name jaking
11. get name
[root@server12 redis-4.0.9]# /usr/local/redis/bin/redis-server /usr/local/redis/conf/redis.conf
18311:C 14 Mar 23:29:59.441 # oO0OoO0OoO0Oo Redis is starting oO0OoO0OoO0Oo
18311:C 14 Mar 23:29:59.442 # Redis version=4.0.9, bits=64, commit=00000000, modified=0, pid=18311, just started
18311:C 14 Mar 23:29:59.442 # Configuration loaded
[root@server12 redis-4.0.9]# /usr/local/redis/bin/redis-cli
127.0.0.1:6379> info
NOAUTH Authentication required.
127.0.0.1:6379> auth jaking
OK
127.0.0.1:6379> info
# Server
redis_version:4.0.9
redis_git_sha1:00000000
redis_git_dirty:0
redis_build_id:3455642c9da3b028
redis_mode:standalone
os:Linux 3.10.0-514.el7.x86_64 x86_64
arch_bits:64
multiplexing_api:epoll
atomicvar_api:atomic-builtin
gcc_version:4.8.5
process_id:18312
run_id:08544775f766a7485b86869123590deea5c92dfe
tcp_port:6379
uptime_in_seconds:51
uptime_in_days:0
hz:10
lru_clock:3104682
executable:/usr/local/redis/bin/redis-server
config_file:/usr/local/redis/conf/redis.conf
# Clients
connected_clients:1
client_longest_output_list:0
client_biggest_input_buf:0
blocked_clients:0
# Memory
used_memory:849472
used_memory_human:829.56K
used_memory_rss:7704576
used_memory_rss_human:7.35M
used_memory_peak:849472
used_memory_peak_human:829.56K
used_memory_peak_perc:100.12%
used_memory_overhead:836238
used_memory_startup:786608
used_memory_dataset:13234
used_memory_dataset_perc:21.05%
total_system_memory:2985820160
total_system_memory_human:2.78G
used_memory_lua:37888
used_memory_lua_human:37.00K
maxmemory:0
maxmemory_human:0B
maxmemory_policy:noeviction
mem_fragmentation_ratio:9.07
mem_allocator:jemalloc-4.0.3
active_defrag_running:0
lazyfree_pending_objects:0
# Persistence
loading:0
rdb_changes_since_last_save:0
rdb_bgsave_in_progress:0
rdb_last_save_time:1647271799
rdb_last_bgsave_status:ok
rdb_last_bgsave_time_sec:-1
rdb_current_bgsave_time_sec:-1
rdb_last_cow_size:0
aof_enabled:0
aof_rewrite_in_progress:0
aof_rewrite_scheduled:0
aof_last_rewrite_time_sec:-1
aof_current_rewrite_time_sec:-1
aof_last_bgrewrite_status:ok
aof_last_write_status:ok
aof_last_cow_size:0
# Stats
total_connections_received:1
total_commands_processed:1
instantaneous_ops_per_sec:0
total_net_input_bytes:71
total_net_output_bytes:73
instantaneous_input_kbps:0.00
instantaneous_output_kbps:0.00
rejected_connections:0
sync_full:0
sync_partial_ok:0
sync_partial_err:0
expired_keys:0
expired_stale_perc:0.00
expired_time_cap_reached_count:0
evicted_keys:0
keyspace_hits:0
keyspace_misses:0
pubsub_channels:0
pubsub_patterns:0
latest_fork_usec:0
migrate_cached_sockets:0
slave_expires_tracked_keys:0
active_defrag_hits:0
active_defrag_misses:0
active_defrag_key_hits:0
active_defrag_key_misses:0
# Replication
role:master
connected_slaves:0
master_replid:6aa262b99b2dad59775123f557e5c0c6036d605c
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:0
second_repl_offset:-1
repl_backlog_active:0
repl_backlog_size:1048576
repl_backlog_first_byte_offset:0
repl_backlog_histlen:0
# CPU
used_cpu_sys:0.04
used_cpu_user:0.01
used_cpu_sys_children:0.00
used_cpu_user_children:0.00
# Cluster
cluster_enabled:0
# Keyspace
127.0.0.1:6379>
127.0.0.1:6379> set name jaking
OK
127.0.0.1:6379> get name
“jaking”
127.0.0.1:6379>
127.0.0.1:6379> exit
[root@server12 redis-4.0.9]# ps aux | grep redis
root 18312 0.1 0.2 145256 7524 ? Ssl 23:29 0:00 /usr/local/redis/bin/redis-server 127.0.0.1:6379
root 18318 0.0 0.0 112652 956 pts/0 R+ 23:32 0:00 grep —color=auto redis
[root@server12 redis-4.0.9]#