Kontener dockerowy na cluster Kafka z 3 brokerami

Co zrobiłem źle?

version: '3.8'
services:

  zookeeper-1:
    image: confluentinc/cp-zookeeper:latest
    container_name: zookeeper-1
    ports:
      - '12181:2181'
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000

  zookeeper-2:
    image: confluentinc/cp-zookeeper:latest
    container_name: zookeeper-2
    ports:
      - '22181:2181'
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
      
  zookeeper-3:
    image: confluentinc/cp-zookeeper:latest
    container_name: zookeeper-3
    ports:
      - '32181:3181'
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
      
  kafka-1:
    image: confluentinc/cp-kafka:latest
    container_name: kafka-1
    ports:
      - '19092:19092'
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-1:9092,PLAINTEXT_HOST://localhost:19092
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
    depends_on:
      - zookeeper-1
      - zookeeper-2
      - zookeeper-3
      
  kafka-2:
    image: confluentinc/cp-kafka:latest
    container_name: kafka-2
    ports:
      - '29092:29092'
    environment:
      KAFKA_BROKER_ID: 2
      KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-2:9092,PLAINTEXT_HOST://localhost:29092
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
    depends_on:
      - zookeeper-1
      - zookeeper-2
      - zookeeper-3
      
  kafka-3:
    image: confluentinc/cp-kafka:latest
    container_name: kafka-3
    ports:
      - '39092:39092'
    environment:
      KAFKA_BROKER_ID: 3
      KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-3:9092,PLAINTEXT_HOST://localhost:39092
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
    depends_on:
      - zookeeper-1
      - zookeeper-2
      - zookeeper-3

błąd, gdy próbuję utworzyć topic z 3 relipkami:

kafka-1        | org.apache.kafka.common.errors.InvalidReplicationFactorException: Replication factor: 3 larger than available brokers: 2.

Czemu podnosi 2 brokery a nie 3?

hmn... dodałem każdemu

BOOTSTRAP_SERVERS: kafka1:19092,kafka2:29092,kafka3:39092

i chyba działa, czemu wczesniej byly 2 to nie rozumiem

Jednak dalej nie działa... ech...

a jak się łączę z takim clustrem to muszę wskazać adresy wszystkich brokerów czy wystarczy 1? w tym wypadku np.localhost:19092?

Obczaj to: https://github.com/conduktor/kafka-stack-docker-compose/blob/master/zk-multiple-kafka-multiple.yml

Charles_Ray napisał(a):

Obczaj to: https://github.com/conduktor/kafka-stack-docker-compose/blob/master/zk-multiple-kafka-multiple.yml

to jest coś popierdolone
KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL

a ja chcę by inne kontenery docker miały dostęp do tego i z localhosta też. U mnie to działa, a tu się wykrzacza.

Poprawiłem kilka literówek:

version: '3.8'
services:

  zookeeper-1:
    image: confluentinc/cp-zookeeper:latest
    container_name: zookeeper-1
    ports:
      - '12181:2181'
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000

  zookeeper-2:
    image: confluentinc/cp-zookeeper:latest
    container_name: zookeeper-2
    ports:
      - '22181:2181'
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
      
  zookeeper-3:
    image: confluentinc/cp-zookeeper:latest
    container_name: zookeeper-3
    ports:
      - '32181:2181'    
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
      
  kafka-1:
    image: confluentinc/cp-kafka:latest
    container_name: kafka-1
    ports:
      - '19092:19092'
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-1:9092,PLAINTEXT_HOST://localhost:19092
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      BOOTSTRAP_SERVERS: kafka1:19092,kafka2:29092,kafka3:39092
      KAFKA_LOG_MESSAGE_TIMESTAMP_TYPE: LogAppendTime
    depends_on:
      - zookeeper-1
      - zookeeper-2
      - zookeeper-3
      
  kafka-2:
    image: confluentinc/cp-kafka:latest
    container_name: kafka-2
    ports:
      - '29092:29092'
    environment:
      KAFKA_BROKER_ID: 2
      KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-2:9093,PLAINTEXT_HOST://localhost:29092
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      BOOTSTRAP_SERVERS: kafka1:19092,kafka2:29092,kafka3:39092
      KAFKA_LOG_MESSAGE_TIMESTAMP_TYPE: LogAppendTime
    depends_on:
      - zookeeper-1
      - zookeeper-2
      - zookeeper-3
      
  kafka-3:
    image: confluentinc/cp-kafka:latest
    container_name: kafka-3
    ports:
      - '39092:39092'
    environment:
      KAFKA_BROKER_ID: 3
      KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-3:9094,PLAINTEXT_HOST://localhost:39092
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      BOOTSTRAP_SERVERS: kafka1:19092,kafka2:29092,kafka3:39092
      KAFKA_LOG_MESSAGE_TIMESTAMP_TYPE: LogAppendTime
    depends_on:
      - zookeeper-1
      - zookeeper-2
      - zookeeper-3

działa tylko pojedynczy broker