docker-compose.yml

version: "3"
services:
  kafka_b:
    image: bitnami/kafka:latest
    hostname: kafka_b
    ports:
      - "9092:9092"
      - "9094:9094"
    volumes:
      - kafka_data:/bitnami
    environment:
      - KAFKA_ENABLE_KRAFT=yes
      - KAFKA_CFG_PROCESS_ROLES=broker,controller
      - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
      - KAFKA_CFG_LISTENERS=PLAINTEXT:https://:9092,CONTROLLER:https://:9093,EXTERNAL:https://:9094
      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT
      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT:https://127.0.0.1:9092,EXTERNAL:https://kafka_b:9094
      - KAFKA_BROKER_ID=1
      - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@127.0.0.1:9093
      - ALLOW_PLAINTEXT_LISTENER=yes
      - KAFKA_CFG_NODE_ID=1
      - KAFKA_AUTO_CREATE_TOPICS_ENABLE=true
      - KAFKA_CFG_NUM_PARTITIONS=3
    healthcheck:
      test: ["CMD", "sh", "-c", "unset JMX_PORT; kafka-topics.sh --bootstrap-server localhost:9092 --list"]
      interval: 10s
      timeout: 10s
      retries: 5
    networks:
      - gutenberg-network
  init-topics:
    image: bitnami/kafka:latest
    container_name: init_topics
    command: >
      sh -c "
        /opt/bitnami/kafka/bin/kafka-topics.sh --create --if-not-exists --bootstrap-server kafka_b:9094 --replication-factor 1 --partitions 3 --topic ${OUTPUT_TOPIC} &&
        /opt/bitnami/kafka/bin/kafka-topics.sh --create --if-not-exists --bootstrap-server kafka_b:9094 --replication-factor 1 --partitions 3 --topic ${GITHUB_TOPIC}  &&
        /opt/bitnami/kafka/bin/kafka-topics.sh --create --if-not-exists --bootstrap-server kafka_b:9094 --replication-factor 1 --partitions 3 --topic ${VECTORDB_TOPIC_NAME} &&
        /opt/bitnami/kafka/bin/kafka-topics.sh --create --if-not-exists --bootstrap-server kafka_b:9094 --replication-factor 1 --partitions 3 --topic ${PDF_INPUT} &&
        /opt/bitnami/kafka/bin/kafka-topics.sh --create --if-not-exists --bootstrap-server kafka_b:9094 --replication-factor 1 --partitions 3 --topic ${PROCESSED_TOPIC} &&
        /opt/bitnami/kafka/bin/kafka-topics.sh --create --if-not-exists --bootstrap-server kafka_b:9094 --replication-factor 1 --partitions 3 --topic ${RESOURCE_TOPIC}
      "
    depends_on:
      kafka_b:
        condition: service_healthy
    networks:
      - gutenberg-network
  fastapi:
    container_name:   fastapi
    image: jgcapworh92/gutenberg-ingestion-api:latest
    environment:
      smtp_username: "${smtp_username}"
      smtp_password: "${smtp_password}"
      postgres_user: "${POSTGRES_USER}"
      postgres_password: "${POSTGRES_PASSWORD}"
      postgres_port: "${POSTGRES_PORT}"
      postgres_db: "${POSTGRES_DB}"
      postgres_server: "${POSTGRES_HOSTNAME}"
    ports:
      - 8000:8000
    depends_on:
      postgres:
        condition: service_healthy
    restart: always
    networks:
      - gutenberg-network
  gateway_service:
    build: .
    depends_on:
      init-topics:
        condition: service_completed_successfully
    environment:
      APP_NAME: "gateway_service"
      BROKERS: "${BROKERS}"
      INPUT_TOPIC: "${INPUT_TOPIC}"
      RESOURCE_TOPIC: "${RESOURCE_TOPIC}"
      CONSUMER_CONFIG: "${CONSUMER_CONFIG}"
      PRODUCER_CONFIG: "${PRODUCER_CONFIG}"
    volumes:
      - ./recovery/gateway_service:/bytewax/recovery/gateway_service
      -  ./logs:/bytewax/logs
    networks:
      - gutenberg-network
    command: ["python", "-m", "bytewax.run", "-r", "/bytewax/recovery/gateway_service", "-s", "3", "-b", "0", "-w4", "dataflows/gateway_service"]

  github_listener:
    build: .
    container_name: github_retriever
    depends_on:
      init-topics:
        condition: service_completed_successfully
    environment:
      APP_NAME: "github_retriever"
      BROKERS: "${BROKERS}"
      INPUT_TOPIC: "${INPUT_TOPIC}"
      OUTPUT_TOPIC: "${OUTPUT_TOPIC}"
      GITHUB_TOKEN: "${GITHUB_TOKEN}"
      VECTORDB_TOPIC_NAME: "${VECTORDB_TOPIC_NAME}"
      CONSUMER_CONFIG: "${CONSUMER_CONFIG}"
      PRODUCER_CONFIG: "${PRODUCER_CONFIG}"
    volumes:
      - ./recovery/github_listener:/bytewax/recovery/github_listener
      -  ./logs:/bytewax/logs
    networks:
      - gutenberg-network
    command: ["python", "-m", "bytewax.run", "-r", "/bytewax/recovery/github_listener", "-s", "3", "-b", "0", "-w4", "dataflows/github_commit_processing"]

  llm_summary_service:
    build: .
    container_name: llm_summary_service
    depends_on:
      init-topics:
        condition: service_completed_successfully
    environment:
      APP_NAME: "llm_summary_service"
      BROKERS: "${BROKERS}"
      OUTPUT_TOPIC: "${OUTPUT_TOPIC}"
      GITHUB_TOKEN: "${GITHUB_TOKEN}"
      PROCESSED_TOPIC: "${PROCESSED_TOPIC}"
      CONSUMER_CONFIG: "${CONSUMER_CONFIG}"
      PRODUCER_CONFIG: "${PRODUCER_CONFIG}"
      MODEL_PROVIDER: "${MODEL_PROVIDER}"
      VECTORDB_TOPIC_NAME: "${VECTORDB_TOPIC_NAME}"
      TEMPLATE: "${TEMPLATE}"
      OPENAI_API_KEY: "${OPENAI_API_KEY}"
      LOCAL_LLM_URL: "${LOCAL_LLM_URL}"
    volumes:
      - ./recovery/commit_summary_service:/bytewax/recovery/commit_summary_service
      -  ./logs:/bytewax/logs
    networks:
      - gutenberg-network
    command: ["python", "-m", "bytewax.run", "-r", "/bytewax/recovery/commit_summary_service", "-s", "3", "-b", "0", "-w4", "dataflows/commit_summary_service"]
  pdf_summary_service:
    build: .
    container_name: pdf_summary_service
    depends_on:
      init-topics:
        condition: service_completed_successfully
    environment:
      APP_NAME: "pdf_summary_service"
      BROKERS: "${BROKERS}"
      OUTPUT_TOPIC: "${OUTPUT_TOPIC}"
      GITHUB_TOKEN: "${GITHUB_TOKEN}"
      PROCESSED_TOPIC: "${PROCESSED_TOPIC}"
      CONSUMER_CONFIG: "${CONSUMER_CONFIG}"
      PRODUCER_CONFIG: "${PRODUCER_CONFIG}"
      MODEL_PROVIDER: "${MODEL_PROVIDER}"
      VECTORDB_TOPIC_NAME: "${VECTORDB_TOPIC_NAME}"
      PDF_INPUT: "${PDF_INPUT}"
      TEMPLATE: "${TEMPLATE}"
      OPENAI_API_KEY: "${OPENAI_API_KEY}"
      LOCAL_LLM_URL: "${LOCAL_LLM_URL}"
    volumes:
      - ./recovery/pdf_service:/bytewax/recovery/pdf_service
    networks:
      - gutenberg-network
    command: ["python", "-m", "bytewax.run", "-r", "/bytewax/recovery/pdf_service", "-s", "3", "-b", "0", "-w4", "dataflows/pdfProcessing"]
  qdrant:
    image: qdrant/qdrant
    ports:
      - "6333:6333"
      - "6334:6334"
    volumes:
      - qdrant-storage:/qdrant/storage:z
    networks:
      - gutenberg-network
  add_qdrant_service:
    build: .
    container_name: add_qdrant_service
    depends_on:
      init-topics:
        condition: service_completed_successfully
    environment:
      APP_NAME: "add_qdrant_service"
      BROKERS: "${BROKERS}"
      OUTPUT_TOPIC: "${OUTPUT_TOPIC}"
      GITHUB_TOKEN: "${GITHUB_TOKEN}"
      PROCESSED_TOPIC: "${PROCESSED_TOPIC}"
      CONSUMER_CONFIG: "${CONSUMER_CONFIG}"
      PRODUCER_CONFIG: "${PRODUCER_CONFIG}"
      VECTORDB_TOPIC_NAME: "${VECTORDB_TOPIC_NAME}"
      MODEL_PROVIDER: "${MODEL_PROVIDER}"
      TEMPLATE: "${TEMPLATE}"
      OPENAI_API_KEY: "${OPENAI_API_KEY}"
      LOCAL_LLM_URL: "${LOCAL_LLM_URL}"
    volumes:
      - ./recovery/add_qdrant_service:/bytewax/recovery/add_qdrant_service
      -  ./logs:/bytewax/logs
    networks:
      - gutenberg-network
    command: ["python", "-m", "bytewax.run", "-r", "/bytewax/recovery/add_qdrant_service", "-s", "3", "-b", "0", "-w4", "dataflows/add_qdrant_service"]

  kafka-ui:
    container_name: kafka-ui
    image: provectuslabs/kafka-ui:latest
    depends_on:
      init-topics:
        condition: service_completed_successfully
    ports:
      - 8080:8080
    environment:
      DYNAMIC_CONFIG_ENABLED: 'true'
    volumes:
      - ./kui/config.yml:/etc/kafkaui/dynamic_config.yaml
    networks:
      - gutenberg-network

  postgres:
    image: postgres:16.2
    environment:
      POSTGRES_DB: ${POSTGRES_DB}
      POSTGRES_USER: ${POSTGRES_USER}
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
    ports:
      - "5432:5432"
    command:
      - "postgres"
      - "-c"
      - "wal_level=logical"
    healthcheck:
      test:  [ "CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}" ]
      interval: 10s
      timeout: 5s
      retries: 5
    volumes:
      - postgres-data:/var/lib/postgresql/data
    networks:
      - gutenberg-network
  pgadmin:
    image: dpage/pgadmin4
    environment:
      PGADMIN_DEFAULT_EMAIL: admin@example.com
      PGADMIN_DEFAULT_PASSWORD: adminpassword
      PGADMIN_LISTEN_PORT: 80
    depends_on:
      - postgres
    ports:
      - "5050:80"
    volumes:
      - pgadmin-data:/var/lib/pgadmin
    networks:
      - gutenberg-network
  debezium:
    image: debezium/connect:2.6.2.Final
    hostname: debezium
    ports:
      - "8083:8083"
    depends_on:
      - kafka_b
      - postgres
      - fastapi
    environment:
      BOOTSTRAP_SERVERS: kafka_b:9094
      GROUP_ID: 1
      CONFIG_STORAGE_TOPIC: debezium_connect_configs
      OFFSET_STORAGE_TOPIC: debezium_connect_offsets
      STATUS_STORAGE_TOPIC: debezium_connect_statuses
      KEY_CONVERTER_SCHEMAS_ENABLE: "false"
      VALUE_CONVERTER_SCHEMAS_ENABLE: "false"
      CONNECT_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_REST_ADVERTISED_HOST_NAME: "debezium"
      CONNECT_CONFIG_PROVIDERS: 'env'
      CONNECT_CONFIG_PROVIDERS_ENV_CLASS: 'org.apache.kafka.common.config.provider.EnvVarConfigProvider'
      POSTGRES_HOSTNAME: ${POSTGRES_HOSTNAME}
      POSTGRES_PORT: ${POSTGRES_PORT}
      POSTGRES_USER: ${POSTGRES_USER}
      POSTGRES_DB: ${POSTGRES_DB}
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
    healthcheck:
      test: ["CMD", "curl", "-f", "https://localhost:8083/"]
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 40s
    networks:
      - gutenberg-network
  init-debezium:
    image: curlimages/curl:latest
    depends_on:
      debezium:
        condition: service_healthy
    entrypoint: /kafka/debezium-setup/init-connector-config.sh
    volumes:
      - ./debezium-setup:/kafka/debezium-setup:ro
    networks:
      - gutenberg-network

volumes:
  qdrant-storage:
    driver: local
  kafka_data:
    driver: local
  postgres-data:
  pgadmin-data:
networks:
  gutenberg-network:
    driver: bridge