kafka - kafka - docker - 《Big Data》

镜像

镜像

[制作基础镜像 - centos]
FROM scratch
ADD centos-7.4.1708-docker.tar.xz /
//
FROM scratch
ADD centos-7.7-x86_64-docker.tar.xz /
LABEL org.label-schema.schema-version="1.0" \
    org.label-schema.name="CentOS Base Image" \
    org.label-schema.vendor="CentOS" \
    org.label-schema.license="GPLv2" \
    org.label-schema.build-date="20191024"
CMD ["/bin/bash"]
[]
# 设置时区
RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
[]
    # alpinelinux
    : https://alpinelinux.org/downloads/

[准备资料数据]

    # 安装包
    : jdk-8u211-linux-x64.tar.gz & zookeeper-3.4.14.tar.gz & kafka_2.12-2.2.0.tgz

    # 启动脚本
  : run.sh

[Dockerfile]

FROM centos:7
MAINTAINER xknower <xknower@126.com>

RUN mkdir -p /opt && chown root.root /opt && chmod 700 /opt
WORKDIR /opt

ADD ./jdk-8u211-linux-x64.tar.gz /opt

ADD ./zookeeper-3.4.14.tar.gz /opt
WORKDIR /opt/zookeeper-3.4.14
RUN cp conf/zoo_sample.cfg /etc/zoo.cfg

ADD ./kafka_2.12-2.2.0.tgz /opt
WORKDIR /opt/kafka_2.12-2.2.0
RUN mkdir -p /etc/kafka
RUN cp /opt/kafka_2.12-2.2.0/config/* /etc/kafka/

ADD ./run.sh /opt/
RUN chmod a+x /opt/run.sh

EXPOSE 9092
EXPOSE 2181

#WORKDIR /root
#RUN yum  update -y
#Run yum install vim net-tools -y

WORKDIR /opt
CMD ["/opt/run.sh"]


[运行脚本]

#!/bin/sh
# ENV
export JAVA_HOME=/opt/jdk1.8.0_211 && export PATH=$PATH:$JAVA_HOME/bin
export ZK_HOME=/opt/zookeeper-3.4.14 && export PATH=$PATH:$ZK_HOME/bin
export KAFKA_HOME=/opt/kafka_2.12-2.2.0 && export PATH=$PATH:$KAFKA_HOME/bin

# CONFIG
ls
echo "text" > /opt/run.log

#
zkServer.sh start /etc/zoo.cfg
kafka-server-start.sh /opt/kafka_2.12-2.2.0/config/server.properties

# suspend main
read -p "press any key to continue."

[BUILD & RUN]

    : docker build -t ka:v1 .

    # 
    : docker run --rm -it ka:v1

  # 后台运行
  : docker run \
    --name ka-zk \
    -p 2181:2181 \
    -p 9092:9092 \
    -t -d ka:v1

[]

    # 查看容器网络信息
    : docker inspect container-id

    # 打包成镜像文件导出
    : docker save -o 要保存的文件名  要保存的镜像

    # 导入打包的镜像文件
    : docker load --input 文件 / docker load < 文件名

[]

wurstmeister/zookeeper & wurstmeister/kafka

[RUN]

    : docker run \
    --name zookeeper \
    -p 2181:2181 \
    -t -d wurstmeister/zookeeper

       : docker run \
    --name kafka \
    -p 9092:9092 \
    -e KAFKA_BROKER_ID=0 \
    -e KAFKA_ZOOKEEPER_CONNECT=47.110.253.133:2181 \
    -e KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://47.110.253.133:9092 \
    -e KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9092 \
    -t -d wurstmeister/kafka

    # KAFKA_ZOOKEEPER_CONNECT=192.168.204.128:2181                 宿主机IP
    # KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://192.168.204.128:9092  宿主机IP

// bin/kafka-topics.sh --create --zookeeper 192.168.204.128:2181 --replication-factor 2 --partitions 2 --topic partopic
# 创建Replication为2，Partition为2的topic

// bin/kafka-topics.sh --describe --zookeeper 192.168.204.128:2181 --topic partopic
# 查看topic的状态

[集群搭建]

    // 使用docker命令可快速在同一台机器搭建多个kafka，只需要改变brokerId和端口
    docker run \
    --name kafka1 \
    -p 9093:9093 \
    -e KAFKA_BROKER_ID=1 \
    -e KAFKA_ZOOKEEPER_CONNECT=192.168.204.128:2181 \
    -e KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://192.168.204.128:9093 \
    -e KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9093 \
    -t -d wurstmeister/kafka

    []

    : registry.cn-beijing.aliyuncs.com/jsnail/kubernetes-zookeeper:1.0-3.4.10

sh -c start-zookeeper
  --servers=3
  --data_dir=/var/lib/zookeeper/data
  --data_log_dir=/var/lib/zookeeper/data/log
  --conf_dir=/opt/zookeeper/conf
  --client_port=2181
  --election_port=3888
  --server_port=2888
  --tick_time=2000
  --init_limit=10
  --sy

    : fastop/kafka:v2

landoop/fast-data-dev

[]

    # web - 127.0.0.1:3030
  : docker run --rm -it \
    -p 2181:2181 -p 3030:3030 -p 8081:8081 \
    -p 8082:8082 -p 8083:8083 -p 9092:9092 \
    -e ADV_HOST=172.16.3.10 \
    -e advertised.host.name=172.16.3.10 \
        -e advertised.port=9092 \
    landoop/fast-data-dev

  # create topic
  : kafka-topics --zookeeper 127.0.0.1:2181 \
  --create --topic my_topic --partitions 3 --replication-factor 1

    # 生产数据
    : kafka-console-producer --broker-list 127.0.0.1:9092 --topic my_topic

  # 消费数据
  : kafka-console-consumer \
  --bootstrap-server 127.0.0.1:9092 \
  --topic my_topic --from-beginning

  # bin/kafka-topics.sh --zookeeper localhost:2181 --list
  # docker exec zookeeper bin/zkCli.sh ls /brokers/ids

[wurstmeister/kafka]

    # wurstmeister/zookeeper, 只包含了Kafka, 因此需要另行提供ZooKeeper


// docker-compose运行一个只有一个ZooKeeper node和一个Kafka broker的开发环境
version: '2'

services:
  zoo1:
    image: wurstmeister/zookeeper
    restart: unless-stopped
    hostname: zoo1
    ports:
      - "2181:2181"
    container_name: zookeeper

  # kafka version: 1.1.0
  # scala version: 2.12
  kafka1:
    image: wurstmeister/kafka
    ports:
      - "9092:9092"
    environment:
      KAFKA_ADVERTISED_HOST_NAME: localhost
      KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"
      KAFKA_BROKER_ID: 1
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_CREATE_TOPICS: "stream-in:1:1,stream-out:1:1"
    depends_on:
      - zoo1
    container_name: kafka

//
# ZooKeeper部分不变

  kafka1:
    image: wurstmeister/kafka
    ports:
      - "9092:9092"
    environment:
      KAFKA_ADVERTISED_HOST_NAME: 192.168.1.2
      KAFKA_ADVERTISED_PORT: 9092
      KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"
      KAFKA_BROKER_ID: 1
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_CREATE_TOPICS: "stream-in:2:1,stream-out:2:1"
    depends_on:
      - zoo1
    container_name: kafka1


  kafka2:
    image: wurstmeister/kafka
    ports:
      - "9093:9092"
    environment:
      KAFKA_ADVERTISED_HOST_NAME: {ipconfig getifaddr en0指令的结果}
      KAFKA_ADVERTISED_PORT: 9093
      KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"
      KAFKA_BROKER_ID: 2
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
    depends_on:
      - zoo1
    container_name: kafka2