misc(ingest): Ingest dev Infratructure (getlago#288)

* misc(ingest): Add Ingest dev infra

* git add files

* some fixes

* remove table creation

* fix conflict

docker-compose.dev.yml

@@ -6,6 +6,7 @@ volumes:
  postgres_data:
  redis_data:
  redpanda_data:
+ clickhouse_data:
 
 services:
  traefik:
@@ -213,3 +214,97 @@ services:
  - "traefik.http.routers.mail.entrypoints=websecure"
  - "traefik.http.routers.mail.tls=true"
  - "traefik.http.services.mail.loadbalancer.server.port=8025"
+
+ redpanda:
+ image: docker.redpanda.com/redpandadata/redpanda:v23.2.9
+ container_name: redpanda
+ hostname: redpanda
+ command:
+ - redpanda start
+ - --smp 1
+ - --overprovisioned
+ - --kafka-addr internal:https://0.0.0.0:9092,external:https://0.0.0.0:19092
+ - --advertise-kafka-addr internal:https://redpanda:9092,external:https://localhost:19092
+ volumes:
+ - redpanda_data:/var/lib/redpanda/data
+
+ redpandacreatetopics:
+ image: docker.redpanda.com/redpandadata/redpanda:v23.2.9
+ depends_on:
+ - redpanda
+ entrypoint: >
+ rpk topic create events-raw --brokers redpanda:9092
+
+ redpanda-console:
+ image: docker.redpanda.com/redpandadata/console:v2.3.1
+ entrypoint: /bin/sh
+ command: -c "echo \"$$CONSOLE_CONFIG_FILE\" > /tmp/config.yml; /app/console"
+ environment:
+ CONFIG_FILEPATH: /tmp/config.yml
+ CONSOLE_CONFIG_FILE: |
+ kafka:
+ brokers: ["redpanda:9092"]
+ schemaRegistry:
+ enabled: false
+ urls: ["http:https://redpanda:8081"]
+ redpanda:
+ adminApi:
+ enabled: true
+ urls: ["http:https://redpanda:9644"]
+ depends_on:
+ - redpanda
+ labels:
+ - "traefik.enable=true"
+ - "traefik.http.routers.console.rule=Host(`console.lago.dev`)"
+ - "traefik.http.routers.console.entrypoints=websecure"
+ - "traefik.http.routers.console.tls=true"
+ - "traefik.http.services.console.loadbalancer.server.port=8080"
+
+ clickhouse:
+ image: clickhouse/clickhouse-server
+ container_name: clickhouse
+ hostname: clickhouse
+ user: '101:101'
+ depends_on:
+ - db
+ - minio
+ - miniocreatebuckets
+ - redpanda
+ - redpandacreatetopics
+ volumes:
+ - clickhouse_data:/var/lib/clickhouse
+ - ./extra/clickhouse/config.d/config.xml:/etc/clickhouse-server/config.d/config.xml
+ - ./extra/clickhouse/users.d/users.xml:/etc/clickhouse-server/users.d/users.xml
+ ports:
+ - 9000:9000
+ - 8123:8123
+
+ minio:
+ image: quay.io/minio/minio
+ container_name: minio
+ hostname: minio
+ command: server --address 0.0.0.0:10000 --console-address 0.0.0.0:10001 /data
+ ports:
+ - 10000:10000
+ environment:
+ MINIO_ROOT_USER: lago
+ MINIO_ROOT_PASSWORD: lagopassword
+ labels:
+ - "traefik.enable=true"
+ - "traefik.http.routers.minio.rule=Host(`minio.lago.dev`)"
+ - "traefik.http.routers.minio.entrypoints=websecure"
+ - "traefik.http.routers.minio.tls=true"
+ - "traefik.http.services.minio.loadbalancer.server.port=10001" 
+
+ miniocreatebuckets:
+ image: minio/mc
+ depends_on:
+ - minio
+ entrypoint: >
+ /bin/sh -c "
+ /usr/bin/mc alias set lagominio http:https://minio:10000 lago lagopassword;
+ /usr/bin/mc admin info lagominio;
+ /usr/bin/mc mb lagominio/clickhouse;
+ /usr/bin/mc policy set public lagominio/clickhouse;
+ exit 0;
+ "

extra/clickhouse/config.d/config.xml

@@ -0,0 +1,56 @@
+<clickhouse replace="true">
+ <logger>
+ <level>debug</level>
+ <log>/var/log/clickhouse-server/clickhouse-server.log</log>
+ <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
+ <size>1000M</size>
+ <count>3</count>
+ </logger>
+ <display_name>clickhouse_dev</display_name>
+ <listen_host>0.0.0.0</listen_host>
+ <http_port>8123</http_port>
+ <tcp_port>9000</tcp_port>
+ <user_directories>
+ <users_xml>
+ <path>users.xml</path>
+ </users_xml>
+ <local_directory>
+ <path>/var/lib/clickhouse/access/</path>
+ </local_directory>
+ </user_directories>
+ <storage_configuration>
+ <disks>
+ <default>
+ </default>
+ <s3>
+ <type>s3</type>
+ <endpoint>http:https://minio:10000/clickhouse/</endpoint>
+ <access_key_id>lago</access_key_id>
+ <secret_access_key>lagopassword</secret_access_key>
+ <region></region>
+ <metadata_path>/var/lib/clickhouse/disks/s3/</metadata_path>
+ <cache_enabled>true</cache_enabled>
+ <data_cache_enabled>true</data_cache_enabled>
+ <enable_filesystem_cache>true</enable_filesystem_cache>
+ </s3>
+ <s3_cache>
+ <type>cache</type>
+ <disk>s3</disk>
+ <path>/var/lib/clickhouse/disks/s3_cache/</path>
+ <max_size>10Gi</max_size>
+ </s3_cache>
+ </disks>
+ <policies>
+ <default>
+ <volumes>
+ <default>
+ <disk>default</disk>
+ </default>
+ <cold>
+ <disk>s3</disk>
+ </cold>
+ </volumes>
+ </default>
+ </policies>
+ </storage_configuration>
+</clickhouse>

extra/clickhouse/users.d/users.xml

@@ -0,0 +1,37 @@
+<?xml version="1.0"?>
+<clickhouse replace="true">
+ <profiles>
+ <default>
+ <max_memory_usage>10000000000</max_memory_usage>
+ <use_uncompressed_cache>0</use_uncompressed_cache>
+ <load_balancing>in_order</load_balancing>
+ <log_queries>1</log_queries>
+ </default>
+ </profiles>
+ <users>
+ <default>
+ <access_management>1</access_management>
+ <profile>default</profile>
+ <networks>
+ <ip>::/0</ip>
+ </networks>
+ <quota>default</quota>
+ <access_management>1</access_management>
+ <named_collection_control>1</named_collection_control>
+ <show_named_collections>1</show_named_collections>
+ <show_named_collections_secrets>1</show_named_collections_secrets>
+ </default>
+ </users>
+ <quotas>
+ <default>
+ <interval>
+ <duration>3600</duration>
+ <queries>0</queries>
+ <errors>0</errors>
+ <result_rows>0</result_rows>
+ <read_rows>0</read_rows>
+ <execution_time>0</execution_time>
+ </interval>
+ </default>
+ </quotas>
+</clickhouse>