利用docker-compose 来构建Kafka集群操作即遇到的问题

操作

https://www.cnblogs.com/Rostov/p/13530423.html

https://blog.csdn.net/weixin_42831855/article/details/91980398

compose.yml 文件解析 https://www.cnblogs.com/Rostov/p/13528918.html

附上yml文件

version: '2'

services:
  zoo1:
    image: zookeeper
    restart: always
    hostname: zoo1
    container_name: zoo1
    ports:
      - 2184:2181
    volumes:
      - "/Users/sun9/dockerKafkaService/volume/zkcluster/zoo1/data:/data"
      - "/Users/sun9/dockerKafkaService/volume/zkcluster/zoo1/datalog:/datalog"
      - "/Users/sun9/dockerKafkaService/volume/zkcluster/zoo1/conf:/conf"
    environment:
      ZOO_MY_ID: 1
      ZOO_SERVERS: server.1=0.0.0.0:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181
    networks:
      zookeeper_kafka:
        ipv4_address: 172.19.0.11

  zoo2:
    image: zookeeper
    restart: always
    hostname: zoo2
    container_name: zoo2
    ports:
      - 2185:2181
    volumes:
      - "/Users/sun9/dockerKafkaService/volume/zkcluster/zoo2/data:/data"
      - "/Users/sun9/dockerKafkaService/volume/zkcluster/zoo2/datalog:/datalog"
      - "/Users/sun9/dockerKafkaService/volume/zkcluster/zoo2/conf:/conf"
    environment:
      ZOO_MY_ID: 2
      ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=0.0.0.0:2888:3888;2181 server.3=zoo3:2888:3888;2181
    networks:
      zookeeper_kafka:
        ipv4_address: 172.19.0.12

  zoo3:
    image: zookeeper
    restart: always
    hostname: zoo3
    container_name: zoo3
    ports:
      - 2186:2181
    volumes:
      - "/Users/sun9/dockerKafkaService/volume/zkcluster/zoo3/data:/data"
      - "/Users/sun9/dockerKafkaService/volume/zkcluster/zoo3/datalog:/datalog"
      - "/Users/sun9/dockerKafkaService/volume/zkcluster/zoo3/conf:/conf"
    environment:
      ZOO_MY_ID: 3
      ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=0.0.0.0:2888:3888;2181
    networks:
      zookeeper_kafka:
        ipv4_address: 172.19.0.13

  kafka1:
    image: wurstmeister/kafka
    restart: always
    hostname: kafka1
    container_name: kafka1
    ports:
      - 9092:9092
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ADVERTISED_HOST_NAME: kafka1
      KAFKA_ADVERTISED_PORT: 9092
      KAFKA_ZOOKEEPER_CONNECT: zoo1:2181,zoo2:2181,zoo3:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka1:9092
      KAFKA_LISTENERS: PLAINTEXT://kafka1:9092
    volumes:
      - /Users/sun9/dockerKafkaService/volume/kfkluster/kafka1/logs:/kafka
    external_links:
      - zoo1
      - zoo2
      - zoo3
    networks:
      zookeeper_kafka:
        ipv4_address: 172.19.0.14
    depends_on:
      - zoo1
      - zoo2
      - zoo3

  kafka2:
    image: wurstmeister/kafka
    restart: always
    hostname: kafka2
    container_name: kafka2
    ports:
      - 9093:9093
    environment:
      KAFKA_BROKER_ID: 2
      KAFKA_ADVERTISED_HOST_NAME: kafka2
      KAFKA_ADVERTISED_PORT: 9093
      KAFKA_ZOOKEEPER_CONNECT: zoo1:2181,zoo2:2181,zoo3:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka2:9093
      KAFKA_LISTENERS: PLAINTEXT://kafka2:9093
    volumes:
      - /Users/sun9/dockerKafkaService/volume/kfkluster/kafka2/logs:/kafka

    external_links:
      - zoo1
      - zoo2
      - zoo3
    networks:
      zookeeper_kafka:
        ipv4_address: 172.19.0.15
    depends_on:
      - zoo1
      - zoo2
      - zoo3

  kafka3:
    image: wurstmeister/kafka
    restart: always
    hostname: kafka3
    container_name: kafka3
    ports:
      - 9094:9094
    environment:
      KAFKA_BROKER_ID: 3
      KAFKA_ADVERTISED_HOST_NAME: kafka3
      KAFKA_ADVERTISED_PORT: 9094
      KAFKA_ZOOKEEPER_CONNECT: zoo1:2181,zoo2:2181,zoo3:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka3:9094
      KAFKA_LISTENERS: PLAINTEXT://kafka3:9094
    volumes:
      - /Users/sun9/dockerKafkaService/volume/kfkluster/kafka3/logs:/kafka
    external_links:
      - zoo1
      - zoo2
      - zoo3
    networks:
      zookeeper_kafka:
        ipv4_address: 172.19.0.16
    depends_on:
      - zoo1
      - zoo2
      - zoo3

networks:
  zookeeper_kafka:
    external:
      name: zookeeper_kafka

网络环境配置：

docker network create --subnet 172.19.0.0/16 --gateway 172.19.0.1 zookeeper_kafka

踩过的shi坑

1.zookeeper集群connect refused

java.net.ConnectException: Connection refused (Connection refused)

进入容器，调用zkServer.sh status 查看状态，发现没有找到对应的配置

最好通过控制变量法，不停的调试，终于是找到了问题点，在yml文件中需要额外加上

最后成果可以发现，已经可以启动成功

ps: 当指定容器数据卷/conf也要映射到宿主机的时候，zkServer.sh start 命令打印的日志会很清爽

但是原因目前还不清楚？初步估计会优先采用容器数据卷中的配置，不行，才会去其他地方查找

2.kafka集群连接zookeeper连接不上

kafka.common.InconsistentClusterIdException: The Cluster ID kg34qdAcRIWxF77tuPLZ_w doesn’t match stored clusterId Some(RiWfOLDaR2KmNYeern_1ow) in meta.properties. The broker is trying to join the wrong cluster. Configured zookeeper.connect may be wrong.

参考链接： https://lixiaogang5.blog.csdn.net/article/details/105679680?utm_medium=distribute.pc_relevant_t0.none-task-blog-BlogCommendFromMachineLearnPai2-1.channel_param&depth_1-utm_source=distribute.pc_relevant_t0.none-task-blog-BlogCommendFromMachineLearnPai2-1.channel_param

其主要原因就是volume 挂载的目录没有及时清除，（这是我感觉docker设计上很shi的一点，既然它的生命周期的都已经结束了，为什么不顺手把它删除掉）

3.kafka 启动时找不到配置文件

其原因上面也讲过了，将config文件夹挂载了出去，然后发现原本满满当当的config文件夹只剩下一个server.properties了，其他文件全部被删掉了，这个原因也是找不到

正常应该是

update: 原因已经找到，在官网里的解释是这样的

如果使用绑定挂载的方式挂载非空目录，容器内的目录会被主机上的目录覆盖掉，至于为什么会剩下一个server.properties,应该是先覆盖后根据配置文件生成的

要想避免这种情况，可以采用volume 挂载，https://docs.docker.com/storage/volumes/ ，这种方式是不会覆盖的，但缺点就是主机上的目录对用户是不可见的，只适合持久化存储数据