（安装docker、docker-compose 略过，请参考docker安装文档）

pull相关服务镜像

  1. docker pull canal/canal-server 
  2. docker pull canal/canal-admin
  4. docker pull zookeeper
  5. docker pull sheepkiller/kafka-manager
  7. docker pull wurstmeister/kafka 
  9. docker pull portainer/portainer 
  11. docker pull rabbitmq
  13. docker pull rabbitmq:management

创建docker-compose文件夹

  1. mkdir -p /data/docker-containers/zk-kafka-server
  3. mkdir /data/docker-containers/zk-kafka-server/kafka{1,2,3}/logs -p 
  4. mkdir -p /data/docker-containers/zk-kafka-server/zoo{1,2,3}/data
  5. mkdir -p /data/docker-containers/zk-kafka-server/zoo{1,2,3}/datalog

创建YAML配置文件

  2. cd /data/docker-containers/zk-kafka-server
  3. vim docker-compose.yaml
  5. ---
  6. ## by author:albert-dong
  7. version: '3.6'
  9. services:
  11.   zoo1:
  12.     image: zookeeper
  13.     restart: always
  14.     hostname: zoo1
  15.     container_name: zoo1
  16.     ports:
  17.         - 2181:2181
  18.     volumes:
  19.         - "$PWD/zoo1/zk-conf/zoo.cfg:/conf/zoo.cfg"
  20.         - "./zoo1/data:/data"
  21.         - "./zoo1/datalog:/datalog"
  22.     environment:
  23.       ZOO_MY_ID: 1
  24.       ZOO_SERVERS: server.1=0.0.0.0:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888
  25.     networks:
  26.       default:
  27.         ipv4_address: 172.27.0.11
  29.   zoo2:
  30.     image: zookeeper
  31.     restart: always
  32.     hostname: zoo2
  33.     container_name: zoo2
  34.     ports:
  35.         - 2180:2181
  36.     volumes:
  37.         - "$PWD/zoo1/zk-conf/zoo.cfg:/conf/zoo.cfg"
  38.         - "./zoo2/data:/data"
  39.         - "./zoo2/datalog:/datalog"
  40.     environment:
  41.       ZOO_MY_ID: 2
  42.       ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=0.0.0.0:2888:3888 server.3=zoo3:2888:3888
  43.     networks:
  44.       default:
  45.         ipv4_address: 172.27.0.12
  47.   zoo3:
  48.     image: zookeeper
  49.     restart: always
  50.     hostname: zoo3
  51.     container_name: zoo3
  52.     ports:
  53.         - 2179:2181
  54.     volumes:
  55.         - "$PWD/zoo1/zk-conf/zoo.cfg:/conf/zoo.cfg"
  56.         - "./zoo3/data:/data"
  57.         - "./zoo3/datalog:/datalog"
  58.         - "/usr/share/zoneinfo/Asia/Shanghai:/etc/localtime"
  59.     environment:
  60.         ZOO_MY_ID: 3
  61.         ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=0.0.0.0:2888:3888
  62.         #KAFKA_ADVERTISED_HOST_NAME: 192.168.199.207
  63.     networks:
  64.       default:
  65.         ipv4_address: 172.27.0.13
  68.   kafka1:
  69.     image: wurstmeister/kafka
  70.     restart: always
  71.     container_name: kafka1
  72.     hostname: kafka1
  73.     ports:
  74.         - 9092:9092
  75.         - 9999:9999
  76.     environment:
  77.       #KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://192.168.161.62:9092 # 暴露在外的地址
  78.       KAFKA_ADVERTISED_HOST_NAME: kafka1 # 
  79.       KAFKA_HOST_NAME: kafka1
  80.       KAFKA_ZOOKEEPER_CONNECT: zoo1:2181,zoo2:2181,zoo3:2181
  81.       KAFKA_ADVERTISED_PORT: 9092 # 暴露在外的端口
  82.       KAFKA_BROKER_ID: 0 # 
  83.       KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
  84.       JMX_PORT: 9999 # jmx
  85.     volumes:
  86.         - "/usr/share/zoneinfo/Asia/Shanghai:/etc/localtime"
  87.         - "$PWD/kafka1/logs:/kafka"
  88.     links:
  89.         - zoo1
  90.         - zoo2
  91.         - zoo3
  92.     networks:
  93.       default:
  94.         ipv4_address: 172.27.0.16
  97.   kafka2:
  98.     image: wurstmeister/kafka
  99.     restart: always
  100.     container_name: kafka2
  101.     hostname: kafka2
  102.     ports:
  103.         - 9093:9092
  104.         - 9998:9999
  105.     environment:
  106.           #KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://192.168.161.62:9093 # 暴露在外的地址
  107.           KAFKA_ADVERTISED_HOST_NAME: kafka2 # 
  108.           KAFKA_HOST_NAME: kafka2
  109.           KAFKA_ZOOKEEPER_CONNECT: zoo1:2181,zoo2:2181,zoo3:2181
  110.           KAFKA_ADVERTISED_PORT: 9093 # 暴露在外的端口
  111.           KAFKA_BROKER_ID: 0 # 
  112.           KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9093
  113.           JMX_PORT: 9999 # jmx
  114.     volumes:
  115.         - "/usr/share/zoneinfo/Asia/Shanghai:/etc/localtime"
  116.         - "$PWD/kafka2/logs:/kafka"
  117.     links:
  118.         - zoo1
  119.         - zoo2
  120.         - zoo3
  121.     networks:
  122.       default:
  123.         ipv4_address: 172.27.0.17
  126.   kafka3:
  127.     image: wurstmeister/kafka
  128.     restart: always
  129.     container_name: kafka3
  130.     hostname: kafka3
  131.     ports:
  132.         - 9094:9092
  133.         - 9997:9999
  134.     environment:
  135.           #KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://192.168.161.62:9094 # 暴露在外的地址
  136.           KAFKA_ADVERTISED_HOST_NAME: kafka3 # 
  137.           KAFKA_HOST_NAME: kafka3
  138.           KAFKA_ZOOKEEPER_CONNECT: zoo1:2181,zoo2:2181,zoo3:2181
  139.           KAFKA_ADVERTISED_PORT: 9094 # 暴露在外的端口
  140.           KAFKA_BROKER_ID: 0 # 
  141.           KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9094
  142.           JMX_PORT: 9999 # jmx
  143.     volumes:
  144.         - "/usr/share/zoneinfo/Asia/Shanghai:/etc/localtime"
  145.         - "$PWD/kafka3/logs:/kafka"
  146.     links:
  147.         - zoo1
  148.         - zoo2
  149.         - zoo3
  150.     networks:
  151.       default:
  152.         ipv4_address: 172.27.0.18
  158.   kafka-manager:
  159.     image: sheepkiller/kafka-manager
  160.     restart: always
  161.     hostname: kafka-manager
  162.     container_name: kafka-manager1
  163.     ports:
  164.         - 9500:9000
  165.     links:
  166.         - kafka1
  167.         - kafka2
  168.         - kafka3
  169.         - zoo1
  170.         - zoo2
  171.         - zoo3
  172.     volumes:
  173.         - "/usr/share/zoneinfo/Asia/Shanghai:/etc/localtime"
  174.     #    - "./zoo3/data:/data"
  175.     #    - "./zoo3/datalog:/datalog"
  176.     environment:
  177.         ZK_HOSTS: zoo1:2181,zoo2:2181,zoo3:2181
  178.         KAFKA_BROKERS: kafka1:9092,kafka2:9093,kafka3:9094
  179.         APPLICATION_SECRET: letmein
  180.         KAFKA_MANAGER_AUTH_ENABLED: "true" # 开启验证
  181.         KAFKA_MANAGER_USERNAME: "admin" # 用户名
  182.         KAFKA_MANAGER_PASSWORD: "admin" # 密码
  183.         KM_ARGS: -Djava.net.preferIPv4Stack=true
  184.     networks:
  185.       default:
  186.         ipv4_address: 172.27.0.15
  188. #docker-compose scale kafka=3 
  190. networks:
  191.   default:
  192.     external:
  193.       name: zookeeper_network

配置文件测试及服务启动


docker-compose config

docker-compose up -d

初始化集群

#### 初始化zk kafka集群
cd /data/docker-containers/zk-kafka-server/
docker-compose stop &&echo y| docker-compose rm&&docker-compose up -d

ls -laskht zoo{1,2,3}/data/
ls -laskht zoo{1,2,3}/datalog/
rm zoo{1,2,3}/data/* -rf
rm zoo{1,2,3}/datalog/* -rf

ls -lakht kafka{1,2,3}/logs/
rm kafka{1,2,3}/logs/* -rf

查看kafka集群状态 # mode 为leader或follower正常

docker exec -it zoo1 bash bin/zkServer.sh status
docker exec -it zoo2 bash bin/zkServer.sh status
docker exec -it zoo3 bash bin/zkServer.sh status

创建topic 测试

docker exec -it kafka1 bash
kafka-topics.sh --create --zookeeper zoo1:2181 --replication-factor 1 --partitions 3 --topic test001
kafka-topics.sh --list --zookeeper zoo1:2181
kafka-topics.sh --list --zookeeper zoo2:2181
kafka-topics.sh --list --zookeeper zoo3:2181

生产消息

kafka-console-producer.sh --broker-list kafka1:9092,kafka2:9093,kafka3:9094 --topic test001

消费消息

kafka-console-consumer.sh --bootstrap-server kafka1:9092,kafka2:9093,kafka3:9094 --topic test001 --from-beginning