Docker部署cdh集群(2)ClouderaManager节点

初始化节点环境

docker exec -it cm.hadoop /bin/sh
#安装常见工具
yum install -y kde-l10n-Chinese telnet reinstall glibc-common vim wget ntp net-tools
#配置中文环境变量
(
cat <<EOF
export LC_ALL=zh_CN.utf8
export LANG=zh_CN.utf8
export LANGUAGE=zh_CN.utf8
EOF
) >> ~/.bashrc \
&& localedef -c -f UTF-8 -i zh_CN zh_CN.utf8 \
&& source ~/.bashrc \
&& echo $LANG
#配置ntp服务
vim /etc/ntp.conf #注释原来的server,添加国内的server

server 0.cn.pool.ntp.org server 1.cn.pool.ntp.org server 2.cn.pool.ntp.org server 3.cn.pool.ntp.org

systemctl start ntpd && \
systemctl enable ntpd && \
ntpdate -u 0.cn.pool.ntp.org && \
hwclock --systohc && \
timedatectl set-timezone Asia/Shanghai && \  #配置时区
date
#开启ssh
yum install openssh-clients.x86_64 && \
yum install openssh-server.x86_64 && \
systemctl enable sshd && \
systemctl start sshd
#按提示修改root密码 所有节点保持一致
passwd 
#关闭transparent_hugepage  关闭内存按2M页大小分配
echo never > /sys/kernel/mm/transparent_hugepage/defrag && \
echo never > /sys/kernel/mm/transparent_hugepage/enabled
vim /etc/rc.local

echo never > /sys/kernel/mm/transparent_hugepage/defrag echo never > /sys/kernel/mm/transparent_hugepage/enabled

sysctl vm.swappiness=10 #默认是60 即物理内存用到40%就开始使用swap虚拟内存
echo "vm.swappiness=10" >> /etc/sysctl.conf
sysctl -p #立刻生效

安装Mysql

#下载mysql安装包
mkdir -p /root/hadoop_CHD/mysql \
&& wget -O /root/hadoop_CHD/mysql/mysql-5.7.27-1.el7.x86_64.rpm-bundle.tar \
https://dev.mysql.com/get/Downloads/MySQL-5.7/mysql-5.7.27-1.el7.x86_64.rpm-bundle.tar \
&& ls /root/hadoop_CHD/mysql
#下载mysql-jdbc
mkdir -p /root/hadoop_CHD/mysql-jdbc \
&& wget -O /root/hadoop_CHD/mysql-jdbc/mysql-connector-java-5.1.48.tar.gz \
https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-5.1.48.tar.gz \
&& ls /root/hadoop_CHD/mysql-jdbc
#安装mysql
cd /root/hadoop_CHD/mysql/;tar -xvf mysql-5.7.27-1.el7.x86_64.rpm-bundle.tar \
&& yum install -y libaio numactl \
&& rpm -ivh mysql-community-common-5.7.27-1.el7.x86_64.rpm \
&& rpm -ivh mysql-community-libs-5.7.27-1.el7.x86_64.rpm \
&& rpm -ivh mysql-community-client-5.7.27-1.el7.x86_64.rpm \
&& rpm -ivh mysql-community-server-5.7.27-1.el7.x86_64.rpm \
&& rpm -ivh mysql-community-libs-compat-5.7.27-1.el7.x86_64.rpm \
&& echo character-set-server=utf8 >> /etc/my.cnf 
#配置mysql-jdbc
mkdir -p /usr/share/java/ \
&& cd /root/hadoop_CHD/mysql-jdbc/;tar -zxvf mysql-connector-java-5.1.48.tar.gz \
&& cp /root/hadoop_CHD/mysql-jdbc/mysql-connector-java-5.1.48/mysql-connector-java-5.1.48-bin.jar /usr/share/java/mysql-connector-java.jar \

配置元数据

(
cat <<EOF
set password for root@localhost = password('123456Aa.');
grant all privileges on *.* to 'root'@'%' identified by '123456Aa.';
flush privileges;
CREATE DATABASE scm DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE amon DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE rman DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE hue DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE metastore DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE sentry DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE nav DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE navms DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE oozie DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON scm.* TO 'scm'@'%' IDENTIFIED BY '123456Aa.';
GRANT ALL ON amon.* TO 'amon'@'%' IDENTIFIED BY '123456Aa.';
GRANT ALL ON rman.* TO 'rman'@'%' IDENTIFIED BY '123456Aa.';
GRANT ALL ON hue.* TO 'hue'@'%' IDENTIFIED BY '123456Aa.';
GRANT ALL ON metastore.* TO 'hive'@'%' IDENTIFIED BY '123456Aa.';
GRANT ALL ON sentry.* TO 'sentry'@'%' IDENTIFIED BY '123456Aa.';
GRANT ALL ON nav.* TO 'nav'@'%' IDENTIFIED BY '123456Aa.';
GRANT ALL ON navms.* TO 'navms'@'%' IDENTIFIED BY '123456Aa.';
GRANT ALL ON oozie.* TO 'oozie'@'%' IDENTIFIED BY '123456Aa.';
SHOW DATABASES;
EOF
) >> /root/c.sql
#获取mysql数据库的密码
#如果启动mysql失败,用journalctl -u mysqld 查看报错信息
systemctl start mysqld && grep password /var/log/mysqld.log | sed 's/.*\(............\)$/\1/'
#执行上面创建的脚本
mysql -uroot -p
mysql> source /root/c.sql

安装Cloudera-Manager

#下载Cloudera-Manager相关安装包
mkdir -p /root/hadoop_CHD/cloudera-repos \
&& wget -O /root/hadoop_CHD/cloudera-repos/allkeys.asc \
https://archive.cloudera.com/cm6/6.2.1/allkeys.asc \
&& wget -O /root/hadoop_CHD/cloudera-repos/cloudera-manager-agent-6.2.1-1426065.el7.x86_64.rpm \
https://archive.cloudera.com/cm6/6.2.1/redhat7/yum/RPMS/x86_64/cloudera-manager-agent-6.2.1-1426065.el7.x86_64.rpm \
&& wget -O /root/hadoop_CHD/cloudera-repos/cloudera-manager-daemons-6.2.1-1426065.el7.x86_64.rpm \
https://archive.cloudera.com/cm6/6.2.1/redhat7/yum/RPMS/x86_64/cloudera-manager-daemons-6.2.1-1426065.el7.x86_64.rpm \
&& wget -O /root/hadoop_CHD/cloudera-repos/cloudera-manager-server-6.2.1-1426065.el7.x86_64.rpm \
https://archive.cloudera.com/cm6/6.2.1/redhat7/yum/RPMS/x86_64/cloudera-manager-server-6.2.1-1426065.el7.x86_64.rpm \
&& wget -O /root/hadoop_CHD/cloudera-repos/cloudera-manager-server-db-2-6.2.1-1426065.el7.x86_64.rpm \
https://archive.cloudera.com/cm6/6.2.1/redhat7/yum/RPMS/x86_64/cloudera-manager-server-db-2-6.2.1-1426065.el7.x86_64.rpm \
&& wget -O /root/hadoop_CHD/cloudera-repos/enterprise-debuginfo-6.2.1-1426065.el7.x86_64.rpm \
https://archive.cloudera.com/cm6/6.2.1/redhat7/yum/RPMS/x86_64/enterprise-debuginfo-6.2.1-1426065.el7.x86_64.rpm \
&& wget -O /root/hadoop_CHD/cloudera-repos/oracle-j2sdk1.8-1.8.0+update181-1.x86_64.rpm \
https://archive.cloudera.com/cm6/6.2.1/redhat7/yum/RPMS/x86_64/oracle-j2sdk1.8-1.8.0+update181-1.x86_64.rpm
#下载Parcel包
mkdir -p /root/hadoop_CHD/parcel \
&& wget -O /root/hadoop_CHD/parcel/CDH-6.2.1-1.cdh6.2.1.p0.1425774-el7.parcel \
https://archive.cloudera.com/cdh6/6.2.1/parcels/CDH-6.2.1-1.cdh6.2.1.p0.1425774-el7.parcel \
&& wget -O /root/hadoop_CHD/parcel/manifest.json \
https://archive.cloudera.com/cdh6/6.2.1/parcels/manifest.json \
&& ll /root/hadoop_CHD/parcel
#搭建本地yum源
yum -y install httpd createrepo \
&& systemctl start httpd \
&& systemctl enable httpd \
&& cd /root/hadoop_CHD/cloudera-repos/ && createrepo . \
&& mv /root/hadoop_CHD/cloudera-repos /var/www/html/
#安装JDK
cd /var/www/html/cloudera-repos/;rpm -ivh oracle-j2sdk1.8-1.8.0+update181-1.x86_64.rpm
#安装Cloudera-Manager
(
cat <<EOF
[cloudera-manager]
name=Cloudera Manager 6.2.1
baseurl=http://172.10.0.2/cloudera-repos/
gpgcheck=0
enabled=1
EOF
) >> /etc/yum.repos.d/cloudera-manager.repo \
&& yum clean all \
&& yum makecache \
&& yum install -y cloudera-manager-daemons cloudera-manager-agent cloudera-manager-server 
#配置parcel库
cd /opt/cloudera/parcel-repo/;mv /root/hadoop_CHD/parcel/* ./ \
&& sha1sum CDH-6.2.1-1.cdh6.2.1.p0.1425774-el7.parcel| awk '{ print $1 }' >CDH-6.2.1-1.cdh6.2.1.p0.1425774-el7.parcel.sha \
&& chown -R cloudera-scm:cloudera-scm /opt/cloudera/parcel-repo/* 
#初始化scm库
/opt/cloudera/cm/schema/scm_prepare_database.sh mysql scm scm 123456Aa.
#启动服务
systemctl start cloudera-scm-server 
#检查是否成功
tail -f /var/log/cloudera-scm-server/cloudera-scm-server.log | grep "INFO WebServerImpl:com.cloudera.server.cmf.WebServerImpl: Started Jetty server"

浏览器访问http://172.10.0.2:7180 用户名/密码:admin/admin