feat: Add crons task consumers (#3106)

We now process tasks via Kafka consumers instead of celerybeat. This needs to be added to self-hosted as well

docker-compose.yml

@@ -384,6 +384,12 @@ services:
  ingest-monitors:
  <<: *sentry_defaults
  command: run consumer --no-strict-offset-reset ingest-monitors --consumer-group ingest-monitors
+ monitors-clock-tick:
+ <<: *sentry_defaults
+ command: run consumer --no-strict-offset-reset monitors-clock-tick --consumer-group monitors-clock-tick
+ monitors-clock-tasks:
+ <<: *sentry_defaults
+ command: run consumer --no-strict-offset-reset monitors-clock-tasks --consumer-group monitors-clock-tasks
  post-process-forwarder-errors:
  <<: *sentry_defaults
  command: run consumer post-process-forwarder-errors --consumer-group post-process-forwarder --synchronize-commit-log-topic=snuba-commit-log --synchronize-commit-group=snuba-consumers

install/create-kafka-topics.sh

@@ -14,12 +14,19 @@ done
 
 # XXX(BYK): We cannot use auto.create.topics as Confluence and Apache hates it now (and makes it very hard to enable)
 EXISTING_KAFKA_TOPICS=$($dc exec -T kafka kafka-topics --list --bootstrap-server kafka:9092 2>/dev/null)
-NEEDED_KAFKA_TOPICS="ingest-attachments ingest-transactions ingest-events ingest-replay-recordings profiles ingest-occurrences ingest-metrics ingest-performance-metrics ingest-monitors"
+NEEDED_KAFKA_TOPICS="ingest-attachments ingest-transactions ingest-events ingest-replay-recordings profiles ingest-occurrences ingest-metrics ingest-performance-metrics ingest-monitors monitors-clock-tasks"
 for topic in $NEEDED_KAFKA_TOPICS; do
  if ! echo "$EXISTING_KAFKA_TOPICS" | grep -qE "(^| )$topic( |$)"; then
  $dc exec kafka kafka-topics --create --topic $topic --bootstrap-server kafka:9092
  echo ""
  fi
 done
 
+# This topic must have only a single partition for the consumer to work correctly
+# https://github.com/getsentry/ops/blob/7dbc26f39c584ec924c8fef2ad5c532d6a158be3/k8s/clusters/us/_topicctl.yaml#L288-L295
+
+if ! echo "$EXISTING_KAFKA_TOPICS" | grep -qE "(^| )monitors-clock-tick( |$)"; then
+ $dc exec kafka kafka-topics --create --topic monitors-clock-tick --bootstrap-server kafka:9092 --partitions 1
+fi
+
 echo "${_endgroup}"