Pomoc w konfiguracji klastra Kafki

Ofiaruję darowiznę 0,1 monero.

Potrzebuję pomocy w dokonfigurowaniu clustra kafkowego:

version: '3.8'
services:

  zookeeper-1:
    image: confluentinc/cp-zookeeper:latest
    container_name: zookeeper-1
    ports:
      - '12181:2181'
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000

  zookeeper-2:
    image: confluentinc/cp-zookeeper:latest
    container_name: zookeeper-2
    ports:
      - '22181:2181'
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000

  zookeeper-3:
    image: confluentinc/cp-zookeeper:latest
    container_name: zookeeper-3
    ports:
      - '32181:2181'    
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000

  kafka-1:
    image: confluentinc/cp-kafka:latest
    container_name: kafka-1
    ports:
      - '19092:19092'
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-1:9092,PLAINTEXT_HOST://localhost:19092
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      BOOTSTRAP_SERVERS: kafka1:19092,kafka2:29092,kafka3:39092
      KAFKA_LOG_MESSAGE_TIMESTAMP_TYPE: LogAppendTime
    depends_on:
      - zookeeper-1
      - zookeeper-2
      - zookeeper-3

  kafka-2:
    image: confluentinc/cp-kafka:latest
    container_name: kafka-2
    ports:
      - '29092:29092'
    environment:
      KAFKA_BROKER_ID: 2
      KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-2:9093,PLAINTEXT_HOST://localhost:29092
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      BOOTSTRAP_SERVERS: kafka1:19092,kafka2:29092,kafka3:39092
      KAFKA_LOG_MESSAGE_TIMESTAMP_TYPE: LogAppendTime
    depends_on:
      - zookeeper-1
      - zookeeper-2
      - zookeeper-3

  kafka-3:
    image: confluentinc/cp-kafka:latest
    container_name: kafka-3
    ports:
      - '39092:39092'
    environment:
      KAFKA_BROKER_ID: 3
      KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-3:9094,PLAINTEXT_HOST://localhost:39092
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      BOOTSTRAP_SERVERS: kafka1:19092,kafka2:29092,kafka3:39092
      KAFKA_LOG_MESSAGE_TIMESTAMP_TYPE: LogAppendTime
    depends_on:
      - zookeeper-1
      - zookeeper-2
      - zookeeper-3

przy odpaleniu powyzszego i łączeniu się z localhost:19092,localhost:29092,localhost:39092 dostaje błąd:

kafka-1 | org.apache.kafka.common.errors.InvalidReplicationFactorException: Replication factor: 3 larger than available brokers: 2.

albo

kafka-1 | org.apache.kafka.common.errors.InvalidReplicationFactorException: Replication factor: 3 larger than available brokers: 1.

U mnie działa bez żadnych błędów ten docker-compose.yml. Stworzyłem testowo jeden topic i przesyła wiadomości jak należy.

vapemil napisał(a):

U mnie działa bez żadnych błędów ten docker-compose.yml. Stworzyłem testowo jeden topic i przesyła wiadomości jak należy.

stwórz topic z 3 replikami to zawoła Ci tym błędem, że ma tylko 2 brokery albo 1 brokera

Nie mam czasu się wczytywać, ale zaczął bym od sprawdzenia logów zookeepera żeby zobaczyć które instancje się zarejestrowały. Warto też przejrzeć logi kafki w kontenerze.

Przyczyny problemów u noobów:

• Docker ma za mało zasobów
• Port jest zajęty lub niedostępny w systemie

Nie pamiętam też ustawień kafki o 07:30 ale jest darmowa książka od Confluent'a: https://www.confluent.io/resources/kafka-the-definitive-guide/

działający cluster:

version: '3.8'
services:

  zookeeper-1:
    image: confluentinc/cp-zookeeper:latest
    container_name: zookeeper-1
    ports:
      - '2181:2181'
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000

  zookeeper-2:
    image: confluentinc/cp-zookeeper:latest
    container_name: zookeeper-2
    ports:
      - '2182:2182'
    environment:
      ZOOKEEPER_CLIENT_PORT: 2182
      ZOOKEEPER_TICK_TIME: 2000

  zookeeper-3:
    image: confluentinc/cp-zookeeper:latest
    container_name: zookeeper-3
    ports:
      - '2183:2183'
    environment:
      ZOOKEEPER_CLIENT_PORT: 2183
      ZOOKEEPER_TICK_TIME: 2000


  kafka-1:
    image: confluentinc/cp-kafka:latest
    container_name: kafka-1
    ports:
      - '19092:19092'
      - '9092:9092'
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_INTERNAL:PLAINTEXT,LISTENER_EXTERNAL:PLAINTEXT
      KAFKA_LISTENERS: LISTENER_INTERNAL://kafka-1:19092,LISTENER_EXTERNAL://kafka-1:9092
      KAFKA_ADVERTISED_LISTENERS: LISTENER_INTERNAL://kafka-1:19092,LISTENER_EXTERNAL://kafka-1:9092
      KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_INTERNAL
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_LOG_MESSAGE_TIMESTAMP_TYPE: LogAppendTime
    depends_on:
      - zookeeper-1
      - zookeeper-2
      - zookeeper-3

  kafka-2:
    image: confluentinc/cp-kafka:latest
    container_name: kafka-2
    ports:
      - '29092:29092'
      - '9093:9093'
    environment:
      KAFKA_BROKER_ID: 2
      KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_INTERNAL:PLAINTEXT,LISTENER_EXTERNAL:PLAINTEXT
      KAFKA_LISTENERS: LISTENER_INTERNAL://kafka-2:29092,LISTENER_EXTERNAL://kafka-2:9093
      KAFKA_ADVERTISED_LISTENERS: LISTENER_INTERNAL://kafka-2:29092,LISTENER_EXTERNAL://kafka-2:9093
      KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_INTERNAL
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_LOG_MESSAGE_TIMESTAMP_TYPE: LogAppendTime
    depends_on:
      - zookeeper-1
      - zookeeper-2
      - zookeeper-3

  kafka-3:
    image: confluentinc/cp-kafka:latest
    container_name: kafka-3
    ports:
      - '9094:9094'
      - '39092:39092'
    environment:
      KAFKA_BROKER_ID: 3
      KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_INTERNAL:PLAINTEXT,LISTENER_EXTERNAL:PLAINTEXT
      KAFKA_LISTENERS: LISTENER_INTERNAL://kafka-3:39092,LISTENER_EXTERNAL://kafka-3:9094
      KAFKA_ADVERTISED_LISTENERS: LISTENER_INTERNAL://kafka-3:39092,LISTENER_EXTERNAL://kafka-3:9094
      KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_INTERNAL
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_LOG_MESSAGE_TIMESTAMP_TYPE: LogAppendTime
    depends_on:
      - zookeeper-1
      - zookeeper-2
      - zookeeper-3