feat(ci): add vector to handle logs pipeline (#5021)

config/dashboard.toml

@@ -31,7 +31,7 @@ surcharge=false
 dispute_evidence_upload=false
 paypal_automatic_flow=false
 threeds_authenticator=false
-global_search=false
+global_search=true
 dispute_analytics=true
 configure_pmts=false
 branding=false

config/prometheus.yaml

@@ -35,3 +35,11 @@ scrape_configs:
 
  static_configs:
  - targets: ["otel-collector:8888"]
+
+ - job_name: "vector"
+
+ # metrics_path defaults to '/metrics'
+ # scheme defaults to 'http'.
+
+ static_configs:
+ - targets: ["vector:9598"]

config/vector.yaml

@@ -0,0 +1,134 @@
+acknowledgements:
+ enabled: true
+
+api:
+ enabled: true
+ address: 0.0.0.0:8686
+
+sources:
+ kafka_tx_events:
+ type: kafka
+ bootstrap_servers: kafka0:29092
+ group_id: sessionizer
+ topics:
+ - hyperswitch-payment-attempt-events
+ - hyperswitch-payment-intent-events
+ - hyperswitch-refund-events
+ - hyperswitch-dispute-events
+ decoding:
+ codec: json
+
+ app_logs:
+ type: docker_logs
+ include_labels:
+ - "logs=promtail"
+
+ vector_metrics:
+ type: internal_metrics
+
+ node_metrics:
+ type: host_metrics
+
+transforms:
+ plus_1_events:
+ type: filter
+ inputs:
+ - kafka_tx_events
+ condition: ".sign_flag == 1"
+
+ hs_server_logs:
+ type: filter
+ inputs:
+ - app_logs
+ condition: '.labels."com.docker.compose.service" == "hyperswitch-server"'
+
+ parsed_hs_server_logs:
+ type: remap
+ inputs:
+ - app_logs
+ source: |-
+ .message = parse_json!(.message)
+
+ events:
+ type: remap
+ inputs:
+ - plus_1_events
+ source: |-
+ .timestamp = from_unix_timestamp!(.created_at, unit: "seconds")
+
+sinks:
+ opensearch_events:
+ type: elasticsearch
+ inputs:
+ - events
+ endpoints:
+ - "https://opensearch:9200"
+ id_key: message_key
+ api_version: v7
+ tls:
+ verify_certificate: false
+ verify_hostname: false
+ auth:
+ strategy: basic
+ user: admin
+ password: 0penS3arc#
+ encoding:
+ except_fields:
+ - message_key
+ - offset
+ - partition
+ - topic
+ bulk:
+ # Add a date prefixed index for better grouping
+ # index: "vector-{{ .topic }}-%Y-%m-%d"
+ index: "{{ .topic }}"
+
+ opensearch_logs:
+ type: elasticsearch
+ inputs:
+ - parsed_hs_server_logs
+ endpoints:
+ - "https://opensearch:9200"
+ api_version: v7
+ tls:
+ verify_certificate: false
+ verify_hostname: false
+ auth:
+ strategy: basic
+ user: admin
+ password: 0penS3arc#
+ bulk:
+ # Add a date prefixed index for better grouping
+ # index: "vector-{{ .topic }}-%Y-%m-%d"
+ index: "logs-{{ .container_name }}-%Y-%m-%d"
+
+ log_events:
+ type: loki
+ inputs:
+ - kafka_tx_events
+ endpoint: http:https://loki:3100
+ labels:
+ source: vector
+ topic: "{{ .topic }}"
+ job: kafka
+ encoding:
+ codec: json
+
+ log_app_loki:
+ type: loki
+ inputs:
+ - parsed_hs_server_logs
+ endpoint: http:https://loki:3100
+ labels:
+ source: vector
+ job: app_logs
+ container: "{{ .container_name }}"
+ stream: "{{ .stream }}"
+ encoding:
+ codec: json
+
+ metrics:
+ type: prometheus_exporter
+ inputs:
+ - vector_metrics
+ - node_metrics

crates/analytics/src/clickhouse.rs

@@ -262,6 +262,7 @@ impl TryInto<RefundFilterRow> for serde_json::Value {
  ))
  }
 }
+
 impl TryInto<DisputeMetricRow> for serde_json::Value {
  type Error = Report<ParsingError>;
 

docker-compose.yml

@@ -245,18 +245,6 @@ services:
  - ./config/grafana.ini:/etc/grafana/grafana.ini
  - ./config/grafana-datasource.yaml:/etc/grafana/provisioning/datasources/datasource.yml
 
- promtail:
- image: grafana/promtail:latest
- volumes:
- - ./logs:/var/log/router
- - ./config:/etc/promtail
- - /var/run/docker.sock:/var/run/docker.sock
- command: -config.file=/etc/promtail/promtail.yaml
- profiles:
- - monitoring
- networks:
- - router_net
-
  loki:
  image: grafana/loki:latest
  ports:
@@ -387,21 +375,13 @@ services:
  soft: 262144
  hard: 262144
 
- fluentd:
- build: ./docker/fluentd
- volumes:
- - ./docker/fluentd/conf:/fluentd/etc
- networks:
- - router_net
- profiles:
- - olap
-
  opensearch:
- image: public.ecr.aws/opensearchproject/opensearch:1.3.14
+ image: opensearchproject/opensearch:2
  container_name: opensearch
  hostname: opensearch
  environment:
  - "discovery.type=single-node"
+ - OPENSEARCH_INITIAL_ADMIN_PASSWORD=0penS3arc#
  profiles:
  - olap
  ports:
@@ -410,7 +390,7 @@ services:
  - router_net
 
  opensearch-dashboards:
- image: opensearchproject/opensearch-dashboards:1.3.14
+ image: opensearchproject/opensearch-dashboards:2
  ports:
  - 5601:5601
  profiles:
@@ -419,3 +399,18 @@ services:
  OPENSEARCH_HOSTS: '["https://opensearch:9200"]'
  networks:
  - router_net
+
+ vector:
+ image: timberio/vector:latest-debian
+ ports:
+ - "8686"
+ - "9598"
+ profiles:
+ - olap
+ environment:
+ KAFKA_HOST: 'kafka0:29092'
+ networks:
+ - router_net
+ volumes:
+ - ./config/vector.yaml:/etc/vector/vector.yaml
+ - /var/run/docker.sock:/var/run/docker.sock