Architecture.pngserver_list.png

前置知识

  1. 后台启动容器：docker-compose up -d
  2. 查看容器运行情况：docker-compose ps
  3. 停止容器：docker-compose stop
  4. 启动容器：docker-compose start
  5. 停止并删除容器：docker-compose down
  6. 停止并删除容器并删除volume：docker-compose down --volumes
  8. rm -rf /data/es/{es01,es02,es03}/data && rm -rf /data/es/{es01,es02,es03}/log

1、防止JVM报错

  1. 修改宿主机配置：
  2. vi /etc/sysctl.conf
  4. 末尾追加：
  5. vm.max_map_count=262145
  7. 保存后执行(最好重启服务器)：
  8. sysctl  -p

2、搭建es集群

  1. 1、创建映射目录
  2. mkdir -p /data/es/{es01,es02,es03}/data && \
  3. mkdir -p /data/es/{es01,es02,es03}/log && \
  4. mkdir -p /data/es/plugins/elasticsearch-analysis-ik-7.2.0
  6. 下载分词器：
  7. https://github.com/medcl/elasticsearch-analysis-ik/releases/download/v7.2.0/elasticsearch-analysis-ik-7.2.0.zip
  8. 解压到:
  9. /data/es/plugins/elasticsearch-analysis-ik-7.2.0
  11. 授权（因为es集群是使用非root用户启动的）：
  12. chmod -R 777 /data/es
  14. 2、创建docker-compose.yml
  15. version: "3.4"
  16. services:
  17.   es01:
  18.     image: docker.elastic.co/elasticsearch/elasticsearch:7.2.0
  19.     container_name: es01
  20.     restart: always
  21.     environment:
  22.       - node.name=es01
  23.       - cluster.name=spartacus
  24.       - node.master=true
  25.       - node.data=true
  26.       - discovery.seed_hosts=es01,es02,es03
  27.       - cluster.initial_master_nodes=es01,es02,es03
  28.       - http.cors.enabled=true
  29.       - http.cors.allow-origin="*"
  30.       - xpack.security.enabled=true
  31.       - xpack.security.transport.ssl.enabled=true
  32.       - xpack.security.transport.ssl.keystore.type=PKCS12
  33.       - xpack.security.transport.ssl.verification_mode=certificate
  34.       - xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
  35.       - xpack.security.transport.ssl.truststore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
  36.       - xpack.security.transport.ssl.truststore.type=PKCS12
  37.       - bootstrap.memory_lock=true
  38.       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
  39.       - "TZ=Asia/Shanghai"
  40.     ulimits:
  41.       memlock:
  42.         soft: -1
  43.         hard: -1
  44.       nofile:
  45.         soft: 65536
  46.         hard: 65536
  47.     ports:
  48.       - "9200:9200"
  49.       - "9300:9300"
  50.     volumes:
  51.       - ./elastic-certificates.p12:/usr/share/elasticsearch/config/elastic-certificates.p12
  52.       - /data/es/es01/data:/usr/share/elasticsearch/data
  53.       - /data/es/es01/log:/usr/share/elasticsearch/log
  54.       - /data/es/plugins:/usr/share/elasticsearch/plugins
  55.     networks:
  56.       - net-es
  57.   es02:
  58.     image: docker.elastic.co/elasticsearch/elasticsearch:7.2.0
  59.     container_name: es02
  60.     restart: always
  61.     environment:
  62.       - node.name=es02
  63.       - cluster.name=spartacus
  64.       - node.master=true
  65.       - node.data=true
  66.       - discovery.seed_hosts=es01,es02,es03
  67.       - cluster.initial_master_nodes=es01,es02,es03
  68.       - http.cors.enabled=true
  69.       - http.cors.allow-origin="*"
  70.       - xpack.security.enabled=true
  71.       - xpack.security.transport.ssl.enabled=true
  72.       - xpack.security.transport.ssl.keystore.type=PKCS12
  73.       - xpack.security.transport.ssl.verification_mode=certificate
  74.       - xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
  75.       - xpack.security.transport.ssl.truststore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
  76.       - xpack.security.transport.ssl.truststore.type=PKCS12
  77.       - bootstrap.memory_lock=true
  78.       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
  79.       - "TZ=Asia/Shanghai"
  80.     ulimits:
  81.       memlock:
  82.         soft: -1
  83.         hard: -1
  84.       nofile:
  85.         soft: 65536
  86.         hard: 65536
  87.     ports:
  88.       - "9201:9200"
  89.       - "9301:9300"
  90.     volumes:
  91.       - ./elastic-certificates.p12:/usr/share/elasticsearch/config/elastic-certificates.p12
  92.       - /data/es/es02/data:/usr/share/elasticsearch/data
  93.       - /data/es/es02/log:/usr/share/elasticsearch/log
  94.       - /data/es/plugins:/usr/share/elasticsearch/plugins
  95.     networks:
  96.       - net-es
  97.   es03:
  98.     image: docker.elastic.co/elasticsearch/elasticsearch:7.2.0
  99.     container_name: es03
  100.     restart: always
  101.     environment:
  102.       - node.name=es03
  103.       - cluster.name=spartacus
  104.       - node.master=true
  105.       - node.data=true
  106.       - discovery.seed_hosts=es01,es02,es03
  107.       - cluster.initial_master_nodes=es01,es02,es03
  108.       - http.cors.enabled=true
  109.       - http.cors.allow-origin="*"
  110.       - xpack.security.enabled=true
  111.       - xpack.security.transport.ssl.enabled=true
  112.       - xpack.security.transport.ssl.keystore.type=PKCS12
  113.       - xpack.security.transport.ssl.verification_mode=certificate
  114.       - xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
  115.       - xpack.security.transport.ssl.truststore.path=/usr/share/elasticsearch/config/elastic-certificates.p12
  116.       - xpack.security.transport.ssl.truststore.type=PKCS12
  117.       - bootstrap.memory_lock=true
  118.       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
  119.       - "TZ=Asia/Shanghai"
  120.     ulimits:
  121.       memlock:
  122.         soft: -1
  123.         hard: -1
  124.       nofile:
  125.         soft: 65536
  126.         hard: 65536
  127.     ports:
  128.       - "9202:9200"
  129.       - "9302:9300"
  130.     volumes:
  131.       - ./elastic-certificates.p12:/usr/share/elasticsearch/config/elastic-certificates.p12
  132.       - /data/es/es03/data:/usr/share/elasticsearch/data
  133.       - /data/es/es03/log:/usr/share/elasticsearch/log
  134.       - /data/es/plugins:/usr/share/elasticsearch/plugins
  135.     networks:
  136.       - net-es
  137. networks:
  138.   net-es:
  139.     driver: bridge
  141. 3、生成证书文件，用于es各节点之间进行ssl通信
  142. #启一个临时容器，用于创建证书用
  143. docker run -d --name es -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" elasticsearch:7.2.0
  144. #进入容器
  145. docker exec -it es /bin/bash
  146. #生成ca: elastic-stack-ca.p12（enter键跳过设置证书名称、密码）
  147. ./bin/elasticsearch-certutil ca
  148. #再生成cert: elastic-certificates.p12（enter键跳过设置证书名称、密码）
  149. #退出容器，拷贝证书至宿主机（与docker-compose.yml同目录）
  150. docker cp es:/usr/share/elasticsearch/elastic-certificates.p12 .
  151. #销毁临时es容器
  152. docker stop es
  153. docker rm es
  155. 4、授权
  156. chmod 777 elastic-certificates.p12
  157. chmod 777 docker-compose.yml
  159. 5、后台启动
  160. docker-compose up -d
  162. 6、设置密码（多个节点只需要设置其中一个即可）
  163. 进入任意一个容器：
  164. docker exec -it --user root es01 /bin/bash
  165. 设置密码：
  166. ./bin/elasticsearch-setup-passwords interactive
  168. #控制台交互....
  170. 温馨提示：建议所有密码都设置一样，方便记忆，比如Pwd@123

3、搭建Logstash

  1. 1、创建logstash配置文件（/apps/logstash/logstash.conf）
  2. input {
  3.   tcp {
  4.     mode => "server"
  5.     host => "0.0.0.0" #监听指定的IP，此处为不限制来源IP
  6.     port => 4560
  7.     codec => json_lines
  8.   }
  9. }
  11. filter {
  12.     #增加一个字段，计算 timestamp 8小时
  13.     ruby {
  14.         code => "event.set('timestamp', event.get('@timestamp').time.utc+8*60*60)"
  15.     }
  17.     #用 mutate 插件先转换为 string 类型，gsub只处理string类型的数据
  18.     #再用正则匹配，最终得到想要的日期
  19.     mutate {
  20.         convert => ["timestamp", "string"]
  21.         gsub => ["timestamp", "T([\S\s]*?)Z", ""]
  22.         gsub => ["timestamp", "-", "."]
  23.    }
  24. }
  26. output {
  27.   if "spartacus-discovery" == [applicationName] {
  28.           elasticsearch {
  29.           hosts => ["10.0.0.5:9200","10.0.0.5:9201","10.0.0.5:9202"]
  30.           action => "index"
  31.           codec => json
  32.           index => "spartacus-discovery-logs-%{timestamp}"
  33.           user => elastic
  34.           password => "Pwd@123"
  35.       }
  36.    }
  38.    if "spartacus-article" == [applicationName] {
  39.           elasticsearch {
  40.           hosts => ["10.0.0.5:9200","10.0.0.5:9201","10.0.0.5:9202"]
  41.           action => "index"
  42.           codec => json
  43.           index => "spartacus-article-logs-%{timestamp}"
  44.           user => elastic
  45.           password => "Pwd@123"
  46.       }
  47.    }
  49.    if "spartacus-auth" == [applicationName] {
  50.           elasticsearch {
  51.           hosts => ["10.0.0.5:9200","10.0.0.5:9201","10.0.0.5:9202"]
  52.           action => "index"
  53.           codec => json
  54.           index => "spartacus-auth-logs-%{timestamp}"
  55.           user => elastic
  56.           password => "Pwd@123"
  57.       }
  58.    }
  60.    if "spartacus-chat" == [applicationName] {
  61.           elasticsearch {
  62.           hosts => ["10.0.0.5:9200","10.0.0.5:9201","10.0.0.5:9202"]
  63.           action => "index"
  64.           codec => json
  65.           index => "spartacus-chat-logs-%{timestamp}"
  66.           user => elastic
  67.           password => "Pwd@123"
  68.       }
  69.    }
  71.    if "spartacus-comment" == [applicationName] {
  72.           elasticsearch {
  73.           hosts => ["10.0.0.5:9200","10.0.0.5:9201","10.0.0.5:9202"]
  74.           action => "index"
  75.           codec => json
  76.           index => "spartacus-comment-logs-%{timestamp}"
  77.           user => elastic
  78.           password => "Pwd@123"
  79.       }
  80.    }
  82.    if "spartacus-datasyner" == [applicationName] {
  83.           elasticsearch {
  84.           hosts => ["10.0.0.5:9200","10.0.0.5:9201","10.0.0.5:9202"]
  85.           action => "index"
  86.           codec => json
  87.           index => "spartacus-datasyner-logs-%{timestamp}"
  88.           user => elastic
  89.           password => "Pwd@123"
  90.       }
  91.    }
  93.    if "spartacus-gateway" == [applicationName] {
  94.           elasticsearch {
  95.           hosts => ["10.0.0.5:9200","10.0.0.5:9201","10.0.0.5:9202"]
  96.           action => "index"
  97.           codec => json
  98.           index => "spartacus-gateway-logs-%{timestamp}"
  99.           user => elastic
  100.           password => "Pwd@123"
  101.       }
  102.    }
  104.    if "spartacus-monitor" == [applicationName] {
  105.           elasticsearch {
  106.           hosts => ["10.0.0.5:9200","10.0.0.5:9201","10.0.0.5:9202"]
  107.           action => "index"
  108.           codec => json
  109.           index => "spartacus-monitor-logs-%{timestamp}"
  110.           user => elastic
  111.           password => "Pwd@123"
  112.       }
  113.    }
  115.    if "spartacus-friday" == [applicationName] {
  116.           elasticsearch {
  117.           hosts => ["10.0.0.5:9200","10.0.0.5:9201","10.0.0.5:9202"]
  118.           action => "index"
  119.           codec => json
  120.           index => "spartacus-friday-logs-%{timestamp}"
  121.           user => elastic
  122.           password => "Pwd@123"
  123.       }
  124.    }
  126.    if "spartacus-sunday" == [applicationName] {
  127.           elasticsearch {
  128.           hosts => ["10.0.0.5:9200","10.0.0.5:9201","10.0.0.5:9202"]
  129.           action => "index"
  130.           codec => json
  131.           index => "spartacus-sunday-logs-%{timestamp}"
  132.           user => elastic
  133.           password => "Pwd@123"
  134.       }
  135.    }
  137.    if "spartacus-resource" == [applicationName] {
  138.           elasticsearch {
  139.           hosts => ["10.0.0.5:9200","10.0.0.5:9201","10.0.0.5:9202"]
  140.           action => "index"
  141.           codec => json
  142.           index => "spartacus-resource-logs-%{timestamp}"
  143.           user => elastic
  144.           password => "Pwd@123"
  145.       }
  146.    }
  148.    if "spartacus-system" == [applicationName] {
  149.           elasticsearch {
  150.           hosts => ["10.0.0.5:9200","10.0.0.5:9201","10.0.0.5:9202"]
  151.           action => "index"
  152.           codec => json
  153.           index => "spartacus-system-logs-%{timestamp}"
  154.           user => elastic
  155.           password => "Pwd@123"
  156.       }
  157.    }
  158. }
  161. 注：
  162. input中的tcp输入块可配置多个，表示logstash后端会起多个线程监听多个输入源（可利用这个特点变相实现单节点logstash的高可用）
  164. input、output配置项可参考：
  165. https://www.elastic.co/guide/en/logstash/current/plugins-inputs-tcp.html#plugins-inputs-tcp-host
  168. 2、创建docker-compose.yml
  169. version: "3.4"
  170. services:
  171.   logstash:
  172.     image: logstash:7.2.0
  173.     container_name: logstash
  174.     restart: always
  175.     volumes:
  176.       - /apps/logstash/logstash.conf:/usr/share/logstash/pipeline/logstash.conf
  177.     ports:
  178.       - 4560:4560
  179.     networks:
  180.       - net-lt
  181. networks:
  182.   net-lt:
  183.     driver: bridge
  185. 3、授权
  186. chmod 777 docker-compose.yml
  188. 4、后台启动logstash
  189. docker-compose up -d
  191. 5、安装json_lines插件
  192. # 进入logstash容器
  193. docker exec -it logstash /bin/bash
  194. # 进入bin目录
  195. cd /bin/
  196. # 安装插件
  197. logstash-plugin install logstash-codec-json_lines
  198. # 退出容器
  199. exit
  200. # 重启logstash服务
  201. docker restart logstash

4、搭建kibana

  1. 1、创建docker-compose.yml
  2. version: "3.4"
  3. services:
  4.   kibana:
  5.     image: kibana:7.2.0
  6.     container_name: kibana
  7.     restart: always
  8.     environment:
  9.       - SERVER_HOST="0.0.0.0"
  10.       - ELASTICSEARCH_HOSTS=["http://10.0.0.5:9200","http://10.0.0.5:9201","http://10.0.0.5:9202"]
  11.       - ELASTICSEARCH_USERNAME="elastic"
  12.       - ELASTICSEARCH_PASSWORD=Pwd@123"
  13.     ports:
  14.       - "5601:5601"
  15.     networks:
  16.       - net-kb
  17. networks:
  18.   net-kb:
  19.     driver: bridge
  21. 2、授权
  22. chmod 777 docker-compose.yml
  24. 3、后台启动kibana
  25. docker-compose up -d
  28. 注意：如果创建索引模式报错： POST 403 (forbidden)
  30. 查看索引状态：
  31. GET _cat/indices
  33. 查看全局配置
  34. GET .kibana/_settings
  36. 重点关注一个指标：read_only_allow_delete是否为true，如果是则置为false:
  37. PUT _settings
  38. {
  39.   "index": {
  40.     "blocks": {
  41.       "read_only_allow_delete": "false"
  42.     }
  43.   }
  44. }

5、云服务器安全组

开放云服务器5601端口（kibana控制台访问端口）：
image.png

然后访问kibana控制台：
http://云主机公网IP:5601

账号密码即是 步骤2 中为各个账号设置的密码（如果认真操作了肯定知道我在说什么）！

如果搞不定，请加群讨论，扫码关注，发送“加群”
mp_qrcode.jpg