12.ES 6.8.9集群安装 - 《这就是运维》

环境准备
下载es包
解压es包
创建es用户
修改配置文件
修改jvm
创建es启动脚本
启动es集群
配置X-PACK
配置head插件
安装kibana

环境准备

所有机器做如下准备

基本yum源

yum install -y epel-release 
yum install -y wget bash-com*  git
yum update -y

yum -y install  gcc bc gcc-c++ ncurses ncurses-devel cmake elfutils-libelf-devel openssl-devel flex* bison* autoconf automake zlib* fiex* libxml* ncurses-devel libmcrypt* libtool-ltdl-devel* make cmake  pcre pcre-devel openssl openssl-devel   jemalloc-devel tlc libtool vim unzip wget lrzsz bash-comp* ipvsadm ipset jq sysstat conntrack libseccomp conntrack-tools socat curl wget git conntrack-tools psmisc nfs-utils tree bash-completion conntrack libseccomp net-tools crontabs sysstat iftop nload strace bind-utils tcpdump htop telnet lsof

关闭防火墙，swap，selinux

#关闭防火墙
systemctl disable --now firewalld

#关闭swap
swapoff -a
sed -ri '/^[^#]*swap/s@^@#@' /etc/fstab

#关闭selinux
setenforce 0
sed -ri '/^[^#]*SELINUX=/s#=.+$#=disabled#' /etc/selinux/config

主机

hostname	ip	安装组件
master01	10.140.0.6	es+x-pack,head,kibana
master02	10.140.0.9	es+x-pack
master03	10.140.0.10	es+x-pack

配置系统参数

cat <<EOF > /etc/sysctl.d/es.conf
net.ipv6.conf.all.disable_ipv6 = 1           #禁用ipv6
net.ipv6.conf.default.disable_ipv6 = 1       #禁用ipv6
net.ipv6.conf.lo.disable_ipv6 = 1            #禁用ipv6
net.ipv4.neigh.default.gc_stale_time = 120   #决定检查过期多久邻居条目
net.ipv4.conf.all.rp_filter = 0              #关闭反向路由校验
net.ipv4.conf.default.rp_filter = 0          #关闭反向路由校验
net.ipv4.conf.default.arp_announce = 2       #始终使用与目标IP地址对应的最佳本地IP地址作为ARP请求的源IP地址
net.ipv4.conf.lo.arp_announce = 2            #始终使用与目标IP地址对应的最佳本地IP地址作为ARP请求的源IP地址
net.ipv4.conf.all.arp_announce = 2           #始终使用与目标IP地址对应的最佳本地IP地址作为ARP请求的源IP地址
net.ipv4.ip_forward = 1                      #启用ip转发功能
net.ipv4.tcp_max_tw_buckets = 5000           #表示系统同时保持TIME_WAIT套接字的最大数量
net.ipv4.tcp_syncookies = 1                  #表示开启SYN Cookies。当出现SYN等待队列溢出时，启用cookies来处理
net.ipv4.tcp_max_syn_backlog = 1024          #接受SYN同包的最大客户端数量
net.ipv4.tcp_synack_retries = 2              #活动TCP连接重传次数
net.bridge.bridge-nf-call-ip6tables = 1      #要求iptables对bridge的数据进行处理
net.bridge.bridge-nf-call-iptables = 1       #要求iptables对bridge的数据进行处理
net.bridge.bridge-nf-call-arptables = 1      #要求iptables对bridge的数据进行处理
net.netfilter.nf_conntrack_max = 2310720     #修改最大连接数
fs.inotify.max_user_watches=89100            #同一用户同时可以添加的watch数目
fs.may_detach_mounts = 1                     #允许文件卸载
fs.file-max = 52706963                       #系统级别的能够打开的文件句柄的数量
fs.nr_open = 52706963                        #单个进程可分配的最大文件数
vm.overcommit_memory=1                       #表示内核允许分配所有的物理内存，而不管当前的内存状态如何
vm.panic_on_oom=0                            #内核将检查是否有足够的可用内存供应用进程使用
vm.swappiness = 0                            #关闭swap
net.ipv4.tcp_keepalive_time = 600            #修复ipvs模式下长连接timeout问题,小于900即可
net.ipv4.tcp_keepalive_intvl = 30            #探测没有确认时，重新发送探测的频度
net.ipv4.tcp_keepalive_probes = 10           #在认定连接失效之前，发送多少个TCP的keepalive探测包
vm.max_map_count=524288                      #定义了一个进程能拥有的最多的内存区域
EOF

sysctl --system

cat>/etc/security/limits.d/es.conf<<EOF
*       soft    nproc   131072
*       hard    nproc   131072
*       soft    nofile  131072
*       hard    nofile  131072
*       soft    memlock unlimited
*       hard    memlock unlimited
root    soft    nproc   131072
root    hard    nproc   131072
root    soft    nofile  131072
root    hard    nofile  131072
EOF

安装jdk 1.8

yum -y install java-1.8.0-openjdk java-1.8.0-openjdk-devel

下载es包

wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.8.9.tar.gz

elasticsearch-6.8.9.tar.gz

解压es包

3个机器都需要部署

tar xvf elasticsearch-6.8.9.tar.gz

mkdir /data
mv elasticsearch-6.8.9 /data/elasticsearch
mkdir /data/elasticsearch/startlogs

创建es用户

3个机器都需要创建es用户

groupadd es && useradd -d /home/es -g es -m es
chown -R es:es /data/elasticsearch

修改配置文件

3个机器都得需要修改配置文件

vim /data/elasticsearch/config/elasticsearch.yml

master01的配置

cluster.name: escluster
node.name: master01
node.master: true
node.data: true
path.data: /data/elasticsearch/data
path.logs: /data/elasticsearch/logs
bootstrap.memory_lock: true
bootstrap.system_call_filter: false
http.port: 9200
network.host: 0.0.0.0
discovery.zen.minimum_master_nodes: 2
discovery.zen.ping_timeout: 5s
discovery.zen.ping.unicast.hosts: ["10.140.0.6:9300","10.140.0.9:9300","10.140.0.10:9300"]
http.cors.enabled: true
http.cors.allow-origin: "*"
http.cors.allow-headers: "Authorization,X-Requested-With,Content-Length,Content-Type"

master02的配置

cluster.name: escluster
node.name: master02
node.master: true
node.data: true
path.data: /data/elasticsearch/data
path.logs: /data/elasticsearch/logs
bootstrap.memory_lock: true
bootstrap.system_call_filter: false
http.port: 9200
network.host: 0.0.0.0
discovery.zen.minimum_master_nodes: 2
discovery.zen.ping_timeout: 5s
discovery.zen.ping.unicast.hosts: ["10.140.0.6:9300","10.140.0.9:9300","10.140.0.10:9300"]
http.cors.enabled: true
http.cors.allow-origin: "*"
http.cors.allow-headers: "Authorization,X-Requested-With,Content-Length,Content-Type"

master03的配置

cluster.name: escluster
node.name: master03
node.master: true
node.data: true
path.data: /data/elasticsearch/data
path.logs: /data/elasticsearch/logs
bootstrap.memory_lock: true
bootstrap.system_call_filter: false
http.port: 9200
network.host: 0.0.0.0
discovery.zen.minimum_master_nodes: 2
discovery.zen.ping_timeout: 5s
discovery.zen.ping.unicast.hosts: ["10.140.0.6:9300","10.140.0.9:9300","10.140.0.10:9300"]
http.cors.enabled: true
http.cors.allow-origin: "*"
http.cors.allow-headers: "Authorization,X-Requested-With,Content-Length,Content-Type"

修改jvm

三个机器都要修改

vim /data/elasticsearch/config/jvm.options

-Xms2g
-Xmx2g

创建es启动脚本

vim /etc/init.d/elasticsearch

#!/bin/bash
ROOT=/data
USER=es
NAME=elasticsearch

DIR=${ROOT}/${NAME}
BIN=${DIR}/bin/${NAME}
PID=${DIR}/${NAME}.pid
LOG=${DIR}/startlogs/${NAME}.log

start(){
  ps -ef |grep ${DIR}|grep -v grep
  PROC_STAT=$?
  if [ ${PROC_STAT} != 0 ]
        then
        su - ${USER} -c "${BIN} > ${LOG} 2>&1 &"
        sleep 1
        PROC_PID=`ps -ef|grep ${DIR}|grep -v grep|awk '{print $2}'| head -n1`
        su - ${USER} -c "echo ${PROC_PID} > ${PID}"
  else
        echo "${NAME} start fail,please check it!"
  fi
}

stop(){
  if [ -f $PID ]
        then
        kill $(cat $PID)
        rm -rf ${PID}
  else
        PROC_PID=`ps -ef |grep ${DIR}|grep -v grep|awk '{print $2}'|head -n1`
        if [ -n ${PROC_PID} ]
                then
                kill ${PROC_PID}
                echo "${NAME} had close."
        else
                echo "${NAME} maybe not run ,please check it!"
        fi
  fi
}
status(){
  ps -ef |grep ${DIR}|grep -v grep
}

case "$1" in
  start)
        start
  ;;
  stop)
        stop
  ;;
  status)
        status
  ;;
  restart)
        stop
        sleep 5
        start
  ;;
  *)
  echo "Usage: /etc/init.d/${NAME} {start|stop|restart}"
  exit 1
esac
exit 0

chmod +x /etc/init.d/elasticsearch

启动es集群

三台机器其次启动

service elasticsearch start

curl 127.0.0.1:9200/_cat/nodes?v

配置X-PACK

三个机器都配置x-pack

su - es
cd /data/elasticsearch/bin
./elasticsearch-keystore create

./elasticsearch-certutil ca --pem

一路回车

unzip elastic-stack-ca.zip
./elasticsearch-certutil cert --ca-cert ca/ca.crt --ca-key ca/ca.key --pem

一路回车

unzip certificate-bundle.zip
mkdir ../config/x-pack
mv ca/  ../config/x-pack/
mv instance/  ../config/x-pack/

添加配置

xpack.security.enabled: true
xpack.security.transport.ssl.enabled: true
xpack.ssl.key: x-pack/instance/instance.key
xpack.ssl.certificate: x-pack/instance/instance.crt
xpack.ssl.certificate_authorities: x-pack/ca/ca.crt
xpack.ssl.verification_mode: none
xpack.ssl.client_authentication: required

重启es

service elasticsearch stop
service elasticsearch start

设置密码
我这边密码都是统一为elastic

su - es
cd /data/elasticsearch/bin/
./elasticsearch-setup-passwords interactive

curl --user elastic:elastic '127.0.0.1:9200/_cat/nodes?v'

配置head插件

只需要在一台机器上部署就行

wget https://github.com/mobz/elasticsearch-head/archive/master.zip  
unzip master.zip

yum -y install bzip2 gcc-c++ make
curl --silent --location https://rpm.nodesource.com/setup_10.x | bash -
yum install -y nodejs
cd elasticsearch-head-master
npm install -g grunt-cli
npm install phantomjs-prebuilt@2.1.16 --ignore-scripts
npm install --registry=https://registry.npm.taobao.org

vim Gruntfile.js 
##添加一行
hostname: '*'

启动插件

nohup grunt server &

#访问
http://ip:9100
#认证访问
http://ip:9100/?auth_user=elastic&auth_password=密码

安装kibana

wget https://artifacts.elastic.co/downloads/kibana/kibana-6.8.9-linux-x86_64.tar.gz

解压

tar xvf kibana-6.8.9-linux-x86_64.tar.gz

编辑配置文件

vim kibana-6.8.9-linux-x86_64/config/kibana.yml

server.port: 5601
server.host: "0.0.0.0"
elasticsearch.hosts: ["http://10.140.0.6:9200"]
elasticsearch.username: "elastic"
elasticsearch.password: "elastic"
xpack.security.encryptionKey: encryptionKeysecurity12345678909876543210
xpack.reporting.encryptionKey: encryptionKeyreporting12345678909876543210

nohup kibana-6.8.9-linux-x86_64/bin/kibana &

访问kibana