0. 目标

  1. 基于 hadoop docker 集群搭建 ，在 hd01 搭建安装 flume 服务端

    1. 文件准备

  2. 更新 dockerfileHDFS 增加 flume

  3. 更新 docker-compose.yml 增加 flume

  4. 文件 element/configure/ 增加 flume-1.9.0/ 目录

    1. configure/
      1. flume-1.9.0/
        1. dokshare/
        2. job/
        3. lib/
    2. images/
      1. apache-flume-1.9.0-bin.tar.gz

        1.1 dockerfileHDFS

  5. 增加安装 nc 命令

  6. 增加 flume ```dockerfile FROM centos:7

SSH + net-tools + snappy

RUN yum install -y openssh-server sudo net-tools snappy.x86_64 nc RUN sed -i ‘s/UsePAM yes/UsePAM no/g’ /etc/ssh/sshd_config RUN echo “PermitRootLogin yes” >> /etc/ssh/sshd_config RUN yum install -y openssh-clients

RUN echo “root:123456” | chpasswd RUN echo “root ALL=(ALL) ALL” >> /etc/sudoers RUN ssh-keygen -t dsa -f /etc/ssh/ssh_host_dsa_key RUN ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key

RUN mkdir /var/run/sshd

JAVA

ADD element/images/jdk-8u141-linux-x64.tar.gz /usr/local/ RUN mv /usr/local/jdk1.8.0_141 /usr/local/jdk1.8 ENV JAVA_HOME /usr/local/jdk1.8 ENV PATH $JAVA_HOME/bin:$PATH

HADOOP

ADD element/images/hadoop-3.3.0.tar.gz /usr/local RUN mv /usr/local/hadoop-3.3.0 /usr/local/hadoop ENV HADOOP_HOME /usr/local/hadoop ENV PATH $HADOOP_HOME/bin:$PATH

HIVE

ADD element/images/apache-hive-3.1.2-bin.tar.gz /usr/local RUN mv /usr/local/apache-hive-3.1.2-bin /usr/local/hive-3.1.2 ENV HIVE_HOME /usr/local/hive-3.1.2 ENV PATH $HIVE_HOME/bin:$PATH

FLUME

ADD element/images/apache-flume-1.9.0-bin.tar.gz /usr/local RUN mv /usr/local/apache-flume-1.9.0-bin /usr/local/flume-1.9.0

RUN yum install -y which sudo

RUN mkdir /mysh

EXPOSE 22 CMD [“/usr/sbin/sshd”, “-D”]

  1. <a name="Mnd4i"></a>
  2. ## 1.2 docker-compose.yml
  4. 1. 增加 flume 目录映射
  5. ```yaml
  6. version: '3.5'
  7. services:
  8. hd01:
  9. image: my-hadoop:3.3.0
  10. container_name: hd01
  11. hostname: hd01
  12. extra_hosts:
  13. - "hd-mysql57-01:172.24.0.6"
  14. - "hd02:172.24.0.12"
  15. - "hd03:172.24.0.13"
  16. networks:
  17. hd-network:
  18. ipv4_address: 172.24.0.11
  19. volumes:
  20. - ${PWD}/element/configure/hadoop/etc-hd01:/usr/local/hadoop/etc
  21. - ${PWD}/element/configure/hadoop/dokshare:/usr/local/hadoop/dokshare
  22. - ${PWD}/element/configure/hive-3.1.2/conf:/usr/local/hive-3.1.2/conf
  23. - ${PWD}/element/configure/hive-3.1.2/lib:/usr/local/hive-3.1.2/lib
  24. - ${PWD}/element/configure/hive-3.1.2/dokshare:/usr/local/hive-3.1.2/dokshare
  25. - ${PWD}/element/mysh:/mysh
  26. - ${PWD}/element/configure/flume-1.9.0/job:/usr/local/flume-1.9.0/job
  27. - ${PWD}/element/configure/flume-1.9.0/lib:/usr/local/flume-1.9.0/lib
  28. - ${PWD}/element/configure/flume-1.9.0/dokshare:/usr/local/flume-1.9.0/dokshare
  29. environment:
  30. - HDFS_NAMENODE_USER=root
  31. - HDFS_DATANODE_USER=root
  32. - HDFS_SECONDARYNAMENODE_USER=root
  33. - YARN_RESOURCEMANAGER_USER=root
  34. - YARN_NODEMANAGER_USER=root
  35. hd02:
  36. image: my-hadoop:3.3.0
  37. container_name: hd02
  38. hostname: hd02
  39. extra_hosts:
  40. - "hd-mysql57-01:172.24.0.6"
  41. - "hd01:172.24.0.11"
  42. - "hd03:172.24.0.13"
  43. networks:
  44. hd-network:
  45. ipv4_address: 172.24.0.12
  46. volumes:
  47. - ${PWD}/element/configure/hadoop/etc-hd02:/usr/local/hadoop/etc
  48. - ${PWD}/element/configure/hadoop/dokshare:/usr/local/hadoop/dokshare
  49. - ${PWD}/element/mysh:/mysh
  50. environment:
  51. - HDFS_NAMENODE_USER=root
  52. - HDFS_DATANODE_USER=root
  53. - HDFS_SECONDARYNAMENODE_USER=root
  54. - YARN_RESOURCEMANAGER_USER=root
  55. - YARN_NODEMANAGER_USER=root
  56. hd03:
  57. image: my-hadoop:3.3.0
  58. container_name: hd03
  59. hostname: hd03
  60. extra_hosts:
  61. - "hd-mysql57-01:172.24.0.6"
  62. - "hd01:172.24.0.11"
  63. - "hd02:172.24.0.12"
  64. networks:
  65. hd-network:
  66. ipv4_address: 172.24.0.13
  67. volumes:
  68. - ${PWD}/element/configure/hadoop/etc-hd03:/usr/local/hadoop/etc
  69. - ${PWD}/element/configure/hadoop/dokshare:/usr/local/hadoop/dokshare
  70. - ${PWD}/element/mysh:/mysh
  71. environment:
  72. - HDFS_NAMENODE_USER=root
  73. - HDFS_DATANODE_USER=root
  74. - HDFS_SECONDARYNAMENODE_USER=root
  75. - YARN_RESOURCEMANAGER_USER=root
  76. - YARN_NODEMANAGER_USER=root
  77. hd-mysql57-01:
  78. image: mysql:5.7
  79. container_name: hd-mysql57-01
  80. hostname: hd-mysql57-01
  81. networks:
  82. hd-network:
  83. ipv4_address: 172.24.0.6
  84. environment:
  85. #最好使用此设定时区，其它静像也可以使用
  86. - TZ=CST-8
  87. - MYSQL_ROOT_PASSWORD=123456
  88. - MYSQL_DATABASE=hive
  89. - MYSQL_USER=my_user
  90. - MYSQL_PASSWORD=my_pw
  91. #可以加--default-time-zone='+8:00'设定时区
  92. command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci
  93. volumes:
  94. #本地文件目录
  95. - ${PWD}/element/mysql57/data/:/var/lib/mysql
  96. networks:
  97. hd-network:
  98. name: hd-network
  99. ipam:
  100. config:
  101.         - subnet: 172.24.0.0/24

1.3 element/ 目录

1.3.1 element/configure/flume-1.9.0/ 目录

  1. dokshare/ 目录

这个目录主要作用是共享文件，共享后续操作使用hive需要用到的资料。

  1. job/ 目录

共享每个 agent 配置文件

  1. lib/ 目录
    1. 由于 flume 和 hadoop 使用的 lib 不同，会出现问题，所以通过 docker 映射 lib/ 目录，并删除 lib/ 目录下的文件 guava-11.0.2.jar

      1.3.2 element/images/ 目录

      下载文件：apache-flume-1.9.0-bin.tar.gz

      2. 启动

      案例：https://www.yuque.com/weghp/bigdata/rwnd0i