docker-compose.yml

networks:
  app-network:
    driver: bridge

volumes:
  kafka-data:
  grafana-data:
  clickhouse-01-data:
  clickhouse-02-data:
  localstack-data:

services:
  # Kafka Broker with KRaft (no Zookeeper) - Official Apache Kafka
  kafka:
    image: apache/kafka:4.1.1
    container_name: kafka
    hostname: kafka
    ports:
      - "9092:9092"      # External client access
      - "9094:9094"      # Controller port
    environment:
      # KRaft settings
      KAFKA_NODE_ID: 1
      KAFKA_PROCESS_ROLES: broker,controller
      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka:9094
      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
      CLUSTER_ID: MkU3OEVBNTcwNTJENDM2Qk
      
      # Listener configuration
      KAFKA_LISTENERS: PLAINTEXT://:9093,CONTROLLER://:9094,EXTERNAL://:9092
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9093,EXTERNAL://localhost:9092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT

      # Internal topics
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
            
      # Performance tuning
      KAFKA_LOG_RETENTION_HOURS: 168
      KAFKA_LOG_SEGMENT_BYTES: 1073741824
      KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS: 300000
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0

      # Message size limits (20MB for large blocks)
      KAFKA_MESSAGE_MAX_BYTES: 20971521
      KAFKA_REPLICA_FETCH_MAX_BYTES: 20971521

    volumes:
      - kafka-data:/var/lib/kafka/data
    networks:
      - app-network
    healthcheck:
      test: ["CMD-SHELL", "/opt/kafka/bin/kafka-broker-api-versions.sh --bootstrap-server localhost:9093 || exit 1"]
      interval: 10s
      timeout: 10s
      retries: 5
      start_period: 30s

  # Kafka UI (Provectus)
  kafka-ui:
    image: provectuslabs/kafka-ui:latest
    container_name: kafka-ui
    hostname: kafka-ui
    ports:
      - "8080:8080"
    environment:
      # Dynamic config enabled for editing clusters in UI
      DYNAMIC_CONFIG_ENABLED: "true"
      
      # Pre-configured local Kafka cluster
      KAFKA_CLUSTERS_0_NAME: local-kafka
      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka:9093
      KAFKA_CLUSTERS_0_READONLY: "false"
      
      # UI settings
      AUTH_TYPE: "DISABLED"
      SERVER_SERVLET_CONTEXT_PATH: /
    depends_on:
      kafka:
        condition: service_healthy
    networks:
      - app-network
    healthcheck:
      test: ["CMD", "wget", "--spider", "-q", "http://localhost:8080/actuator/health"]
      interval: 10s
      timeout: 5s
      retries: 5
      start_period: 30s

  prometheus:
    image: prom/prometheus:latest
    ports:
      - "9091:9090"
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
    extra_hosts:
      - "host.docker.internal:host-gateway"
    networks:
      - app-network

  grafana:
    image: grafana/grafana:latest
    ports:
      - "3000:3000"
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=admin
    volumes:
      - grafana-data:/var/lib/grafana
    networks:
      - app-network

  # ZooKeeper (required for ClickHouse replication)
  zookeeper:
    image: zookeeper:3.8
    container_name: zookeeper
    hostname: zookeeper
    ports:
      - "2181:2181"
    networks:
      - app-network

  # ClickHouse Server 1
  clickhouse-01:
    image: clickhouse/clickhouse-server:latest
    container_name: clickhouse-01
    hostname: clickhouse-01
    ports:
      - "8123:8123"
      - "9000:9000"
    environment:
      CLICKHOUSE_DB: default
      CLICKHOUSE_USER: default
      CLICKHOUSE_PASSWORD: ""
      CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
    volumes:
      - clickhouse-01-data:/var/lib/clickhouse
      - ./clickhouse-config/cluster.xml:/etc/clickhouse-server/config.d/cluster.xml
      - ./clickhouse-config/macros-01.xml:/etc/clickhouse-server/config.d/macros.xml
    depends_on:
      - zookeeper
    healthcheck:
      test: ["CMD", "wget", "--spider", "-q", "http://localhost:8123/ping"]
      interval: 5s
      timeout: 3s
      retries: 10
      start_period: 10s
    networks:
      - app-network

  # ClickHouse Server 2
  clickhouse-02:
    image: clickhouse/clickhouse-server:latest
    container_name: clickhouse-02
    hostname: clickhouse-02
    ports:
      - "8124:8123"
      - "9001:9000"
    environment:
      CLICKHOUSE_DB: default
      CLICKHOUSE_USER: default
      CLICKHOUSE_PASSWORD: ""
      CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
    volumes:
      - clickhouse-02-data:/var/lib/clickhouse
      - ./clickhouse-config/cluster.xml:/etc/clickhouse-server/config.d/cluster.xml
      - ./clickhouse-config/macros-02.xml:/etc/clickhouse-server/config.d/macros.xml
    depends_on:
      - zookeeper
    healthcheck:
      test: ["CMD", "wget", "--spider", "-q", "http://localhost:8123/ping"]
      interval: 5s
      timeout: 3s
      retries: 10
      start_period: 10s
    networks:
      - app-network

  # Tabix UI for ClickHouse - connects to clickhouse-01
  tabix:
    image: spoonest/clickhouse-tabix-web-client
    platform: linux/amd64
    container_name: tabix
    ports:
      - "8082:80"
    environment:
      CH_NAME: local-clickhouse
      CH_HOST: http://localhost:8123
      CH_LOGIN: default
      CH_PASSWORD: ""
    depends_on:
      clickhouse-01:
        condition: service_healthy
    networks:
      - app-network

  # LocalStack for local AWS services (DynamoDB)
  localstack:
    image: localstack/localstack:4.14.0
    container_name: localstack
    hostname: localstack
    ports:
      - "4566:4566"
    environment:
      SERVICES: dynamodb
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
      - localstack-data:/var/lib/localstack
    networks:
      - app-network