Different Kafka Setups

#Regular Setup

  • 2 different clients
  • Zookeper & Kafka Cluster
//  - docker-compose.yaml - 
version: "3"

services:

  zookeeper: &zookeeper
    image: confluentinc/cp-zookeeper:6.2.0
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_SERVER_ID: "1"


  kafka0:
    image: confluentinc/cp-kafka:6.2.0
    ports: [ "9090:9090"]
    environment:
      KAFKA_BROKER_ID: 0
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_LISTENERS: EXT://:9090,INT://:19092,
      KAFKA_ADVERTISED_LISTENERS: EXT://localhost:9090,INT://kafka0:19092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: EXT:PLAINTEXT,INT:PLAINTEXT
      ALLOW_PLAINTEXT_LISTENER: "yes"
      KAFKA_INTER_BROKER_LISTENER_NAME: INT
    depends_on:
    - "zookeeper"
    volumes:
      - ./data/kafka0:/var/lib/kafka

  kafka1:
    image: confluentinc/cp-kafka:6.2.0
    ports: [ "9091:9091"]
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_LISTENERS: EXT://:9091,INT://:29092,
      KAFKA_ADVERTISED_LISTENERS: EXT://localhost:9091,INT://kafka1:29092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: EXT:PLAINTEXT,INT:PLAINTEXT
      ALLOW_PLAINTEXT_LISTENER: "yes"
      KAFKA_INTER_BROKER_LISTENER_NAME: INT
    depends_on:
    - "zookeeper"
    volumes:
      - ./data/kafka1:/var/lib/kafka

  kafka2:
    image: confluentinc/cp-kafka:6.2.0
    ports: [ "9092:9092"]
    environment:
      KAFKA_BROKER_ID: 2
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_LISTENERS: EXT://:9092,INT://:39092,
      KAFKA_ADVERTISED_LISTENERS: EXT://localhost:9092,INT://kafka2:39092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: EXT:PLAINTEXT,INT:PLAINTEXT
      ALLOW_PLAINTEXT_LISTENER: "yes"
      KAFKA_INTER_BROKER_LISTENER_NAME: INT
    depends_on:
    - "zookeeper"
    volumes:
      - ./data/kafka3:/var/lib/kafka

  kafdrop:
    image: obsidiandynamics/kafdrop:latest
    depends_on:
      - kafka0
      - kafka1
      - kafka2
    ports:
      - 9001:9000
    environment:
      KAFKA_BROKERCONNECT: kafka0:19092


  kafka-ui:
    image: provectuslabs/kafka-ui:latest
    ports:
    - 9000:8080
    environment:
      KAFKA_CLUSTERS_0_NAME: local
      KAFKA_CLUSTERS_0_ZOOKEEPER: zookeeper:2181
      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka0:19092
    depends_on:
    - kafka0
    - kafka1
    - kafka2

#No Zookeeper’s Cluster

  • akhq client
  • no zookipers!
  • 3 repeated kafka insances
  • schmea registry
//  - docker-compose-nozoo.yaml - 
version: "3"

services:

  schema-registry:
    image: confluentinc/cp-schema-registry:7.5.2
    restart: "on-failure"
    environment:
      SCHEMA_REGISTRY_HOST_NAME: schema-registry
      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: "kafka-1:9092,kafka-2:9092,kafka-3:9092"
    healthcheck:
      <<: *x-healthchecks
      test: curl --fail 0.0.0.0.:8081/subjects
    depends_on:
      kafka-1: {condition: service_healthy}
      kafka-2: {condition: service_healthy}
      kafka-3: {condition: service_healthy}

  kafka-1: &kafka
    image: "bitnami/kafka:3.6.0"
    restart: "on-failure"
    environment: &kafka-environment
      ALLOW_ANONYMOUS_LOGIN: yes
      KAFKA_ENABLE_KRAFT: yes
      KAFKA_CFG_PROCESS_ROLES: broker,controller
      KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER
      KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: kraft:PLAINTEXT,CONTROLLER:PLAINTEXT,INTERNAL:PLAINTEXT
      KAFKA_BROKER_ID: 1
      ALLOW_PLAINTEXT_LISTENER: yes
      BITNAMI_DEBUG: yes
      KAFKA_KRAFT_CLUSTER_ID: OTMwNzFhYTY1ODNiNGE5OT
      KAFKA_CFG_KRAFT_REPLICATION_FACTOR: 3
      KAFKA_CFG_DEFAULT_REPLICATION_FACTOR: 3
      KAFKA_CFG_NODE_ID: 1
      KAFKA_CFG_ADVERTISED_LISTENERS: kraft://:9093,INTERNAL://kafka-1:9092
      KAFKA_CFG_LISTENERS: kraft://:9093,CONTROLLER://kafka-1:9094,INTERNAL://:9092
      KAFKA_CFG_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 1@kafka-1:9094,2@kafka-2:9094,3@kafka-3:9094
    volumes:
    - kafka-1:/var/lib/kafka/data/
    healthcheck:
      <<: *x-healthchecks
      test: kafka-cluster.sh cluster-id --bootstrap-server localhost:9092 || exit 1
      retries: 100

  kafka-2:
    <<: *kafka
    environment:
      <<: *kafka-environment
      KAFKA_BROKER_ID: 2
      KAFKA_CFG_NODE_ID: 2
      KAFKA_CFG_ADVERTISED_LISTENERS: kraft://:9093,INTERNAL://kafka-2:9092
      KAFKA_CFG_LISTENERS: kraft://:9093,CONTROLLER://kafka-2:9094,INTERNAL://:9092
    volumes:
    - kafka-2:/var/lib/kafka/data/

  kafka-3:
    <<: *kafka
    environment:
      <<: *kafka-environment
      KAFKA_BROKER_ID: 3
      KAFKA_CFG_NODE_ID: 3
      KAFKA_CFG_ADVERTISED_LISTENERS: kraft://:9093,INTERNAL://kafka-3:9092
      KAFKA_CFG_LISTENERS: kraft://:9093,CONTROLLER://kafka-3:9094,INTERNAL://:9092
    volumes:
    - kafka-3:/var/lib/kafka/data/

  kafka-setup:
    image: linkedin/datahub-kafka-setup:v0.12.1
    hostname: kafka-setup
    environment:
      DATAHUB_PRECREATE_TOPICS: ${DATAHUB_PRECREATE_TOPICS:-false}
      KAFKA_BOOTSTRAP_SERVER: kafka-1:9092,kafka-2:9092,kafka-3:9092
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      USE_CONFLUENT_SCHEMA_REGISTRY: TRUE
    depends_on:
      kafka-1:         {condition: service_healthy}
      kafka-2:         {condition: service_healthy}
      kafka-3:         {condition: service_healthy}
      schema-registry: {condition: service_healthy}
    labels:
      datahub_setup_job: true

  kafka-ui:
    image: tchiotludo/akhq
    depends_on:
      kafka-1:         {condition: service_healthy}
      kafka-2:         {condition: service_healthy}
      kafka-3:         {condition: service_healthy}
      schema-registry: {condition: service_healthy}
    ports :
      - 9100:8080
    environment:
      AKHQ_CONFIGURATION: |
        akhq:
          connections:
            docker-kafka-server:
              properties:
                bootstrap.servers: "kafka-1:9092,kafka-2:9092,kafka-3:9092"