Integration tests in python (#2892)

* integration tests in python

.github/workflows/test.yml

@@ -74,6 +74,21 @@ jobs:
  - name: Checkout
  uses: actions/checkout@v4
 
+ - name: Setup dev environment
+ run: |
+ pip install -r requirements-dev.txt
+ ### pytest-sentry configuration ###
+ if [ "$GITHUB_REPOSITORY" = "getsentry/self-hosted" ]; then
+ echo "PYTEST_SENTRY_DSN=${{ env.SELF_HOSTED_TESTING_DSN }}" >> $GITHUB_ENV
+ echo "PYTEST_SENTRY_TRACES_SAMPLE_RATE=0" >> $GITHUB_ENV
+
+ # This records failures on master to sentry in order to detect flakey tests, as it's
+ # expected that people have failing tests on their PRs
+ if [ "$GITHUB_REF" = "refs/heads/master" ]; then
+ echo "PYTEST_SENTRY_ALWAYS_REPORT=1" >> $GITHUB_ENV
+ fi
+ fi
+
  - name: Get Compose
  run: |
  # Always remove `docker compose` support as that's the newer version

_integration-test/custom-ca-roots/setup.sh

@@ -1,3 +1,4 @@
+#!/usr/bin/env bash
 set -e
 export COMPOSE_FILE=docker-compose.yml:_integration-test/custom-ca-roots/docker-compose.test.yml
 
@@ -42,4 +43,4 @@ openssl req -x509 -newkey rsa:2048 -nodes -days 1 -keyout $TEST_NGINX_CONF_PATH/
 
 cp _integration-test/custom-ca-roots/test.py sentry/test-custom-ca-roots.py
 
-$dc up -d fixture-custom-ca-roots
+docker compose --ansi never up -d fixture-custom-ca-roots

_integration-test/custom-ca-roots/teardown.sh

@@ -1,3 +1,4 @@
+#!/usr/bin/env bash
 $dc rm -s -f -v fixture-custom-ca-roots
 rm -f certificates/test-custom-ca-roots.crt sentry/test-custom-ca-roots.py
 unset COMPOSE_FILE

_integration-test/run.py

@@ -0,0 +1,209 @@
+import subprocess
+import os
+from functools import lru_cache
+from bs4 import BeautifulSoup
+import httpx
+import pytest
+import sentry_sdk
+import time
+import json
+import re
+from typing import Callable
+
+SENTRY_CONFIG_PY = "sentry/sentry.conf.py"
+SENTRY_TEST_HOST = os.getenv("SENTRY_TEST_HOST", "http:https://localhost:9000")
+TEST_USER = "[email protected]"
+TEST_PASS = "test123TEST"
+TIMEOUT_SECONDS = 60
+
+
+def poll_for_response(
+ request: str, client: httpx.Client, validator: Callable = None
+) -> httpx.Response:
+ for i in range(TIMEOUT_SECONDS):
+ response = client.get(
+ request, follow_redirects=True, headers={"Referer": SENTRY_TEST_HOST}
+ )
+ if response.status_code == 200:
+ if validator is None or validator(response.text):
+ break
+ time.sleep(1)
+ else:
+ raise AssertionError(
+ "timeout waiting for response status code 200 or valid data"
+ )
+ return response
+
+
+@lru_cache
+def get_sentry_dsn(client: httpx.Client) -> str:
+ response = poll_for_response(
+ f"{SENTRY_TEST_HOST}/api/0/projects/sentry/internal/keys/",
+ client,
+ lambda x: len(json.loads(x)[0]["dsn"]["public"]) > 0,
+ )
+ sentry_dsn = json.loads(response.text)[0]["dsn"]["public"]
+ return sentry_dsn
+
+
+@pytest.fixture(scope="session", autouse=True)
+def configure_self_hosted_environment():
+ subprocess.run(["docker", "compose", "--ansi", "never", "up", "-d"], check=True)
+ for i in range(TIMEOUT_SECONDS):
+ try:
+ response = httpx.get(SENTRY_TEST_HOST, follow_redirects=True)
+ except httpx.ConnectionError:
+ time.sleep(1)
+ else:
+ if response.status_code == 200:
+ break
+ else:
+ raise AssertionError("timeout waiting for self-hosted to come up")
+
+ # Create test user
+ subprocess.run(
+ [
+ "docker",
+ "compose",
+ "exec",
+ "web",
+ "sentry",
+ "createuser",
+ "--force-update",
+ "--superuser",
+ "--email",
+ TEST_USER,
+ "--password",
+ TEST_PASS,
+ "--no-input",
+ ],
+ check=True,
+ text=True,
+ )
+
+
+@pytest.fixture()
+def client_login():
+ client = httpx.Client()
+ response = client.get(SENTRY_TEST_HOST, follow_redirects=True)
+ parser = BeautifulSoup(response.text, "html.parser")
+ login_csrf_token = parser.find("input", {"name": "csrfmiddlewaretoken"})["value"]
+ login_response = client.post(
+ f"{SENTRY_TEST_HOST}/auth/login/sentry/",
+ follow_redirects=True,
+ data={
+ "op": "login",
+ "username": TEST_USER,
+ "password": TEST_PASS,
+ "csrfmiddlewaretoken": login_csrf_token,
+ },
+ headers={"Referer": f"{SENTRY_TEST_HOST}/auth/login/sentry/"},
+ )
+ assert login_response.status_code == 200
+ yield (client, login_response)
+
+
+def test_initial_redirect():
+ initial_auth_redirect = httpx.get(SENTRY_TEST_HOST, follow_redirects=True)
+ assert initial_auth_redirect.url == f"{SENTRY_TEST_HOST}/auth/login/sentry/"
+
+
+def test_login(client_login):
+ client, login_response = client_login
+ parser = BeautifulSoup(login_response.text, "html.parser")
+ script_tag = parser.find(
+ "script", string=lambda x: x and "window.__initialData" in x
+ )
+ assert script_tag is not None
+ json_data = json.loads(script_tag.text.split("=", 1)[1].strip().rstrip(";"))
+ assert json_data["isAuthenticated"] is True
+ assert json_data["user"]["username"] == "[email protected]"
+ assert json_data["user"]["isSuperuser"] is True
+ assert login_response.cookies["sc"] is not None
+ # Set up initial/required settings (InstallWizard request)
+ client.headers.update({"X-CSRFToken": login_response.cookies["sc"]})
+ response = client.put(
+ f"{SENTRY_TEST_HOST}/api/0/internal/options/?query=is:required",
+ follow_redirects=True,
+ headers={"Referer": SENTRY_TEST_HOST},
+ data={
+ "mail.use-tls": False,
+ "mail.username": "",
+ "mail.port": 25,
+ "system.admin-email": "[email protected]",
+ "mail.password": "",
+ "system.url-prefix": SENTRY_TEST_HOST,
+ "auth.allow-registration": False,
+ "beacon.anonymous": True,
+ },
+ )
+ assert response.status_code == 200
+
+
+def test_receive_event(client_login):
+ event_id = None
+ client, _ = client_login
+ with sentry_sdk.init(dsn=get_sentry_dsn(client)):
+ event_id = sentry_sdk.capture_exception(Exception("a failure"))
+ assert event_id is not None
+ response = poll_for_response(
+ f"{SENTRY_TEST_HOST}/api/0/projects/sentry/internal/events/{event_id}/", client
+ )
+ response_json = json.loads(response.text)
+ assert response_json["eventID"] == event_id
+ assert response_json["metadata"]["value"] == "a failure"
+
+
+def test_cleanup_crons_running():
+ docker_services = subprocess.check_output(
+ [
+ "docker",
+ "compose",
+ "--ansi",
+ "never",
+ "ps",
+ "-a",
+ ],
+ text=True,
+ )
+ pattern = re.compile(
+ r"(\-cleanup\s+running)|(\-cleanup[_-].+\s+Up\s+)", re.MULTILINE
+ )
+ cleanup_crons = pattern.findall(docker_services)
+ assert len(cleanup_crons) > 0
+
+
+def test_custom_cas():
+ try:
+ subprocess.run(["./_integration-test/custom-ca-roots/setup.sh"], check=True)
+ subprocess.run(
+ ["docker", "compose", "--ansi", "never", "run", "--no-deps", "web", "python3", "/etc/sentry/test-custom-ca-roots.py"], check=True
+ )
+ finally:
+ subprocess.run(["./_integration-test/custom-ca-roots/teardown.sh"], check=True)
+
+
+def test_receive_transaction_events(client_login):
+ client, _ = client_login
+ with sentry_sdk.init(
+ dsn=get_sentry_dsn(client), profiles_sample_rate=1.0, traces_sample_rate=1.0
+ ):
+
+ def placeholder_fn():
+ sum = 0
+ for i in range(5):
+ sum += i
+ time.sleep(0.25)
+
+ with sentry_sdk.start_transaction(op="task", name="Test Transactions"):
+ placeholder_fn()
+ poll_for_response(
+ f"{SENTRY_TEST_HOST}/api/0/organizations/sentry/events/?dataset=profiles&field=profile.id&project=1&statsPeriod=1h",
+ client,
+ lambda x: len(json.loads(x)["data"]) > 0,
+ )
+ poll_for_response(
+ f"{SENTRY_TEST_HOST}/api/0/organizations/sentry/events/?dataset=spansIndexed&field=id&project=1&statsPeriod=1h",
+ client,
+ lambda x: len(json.loads(x)["data"]) > 0,
+ )

integration-test.sh

@@ -15,7 +15,7 @@ export MINIMIZE_DOWNTIME=0
 
 if [[ "$test_option" == "--initial-install" ]]; then
  echo "Testing initial install"
- source _integration-test/run.sh
+ pytest --reruns 5 _integration-test/run.py
  source _integration-test/ensure-customizations-not-present.sh
  source _integration-test/ensure-backup-restore-works.sh
 elif [[ "$test_option" == "--customizations" ]]; then
@@ -34,7 +34,7 @@ EOT
  echo "Testing in-place upgrade and customizations"
  export MINIMIZE_DOWNTIME=1
  ./install.sh
- source _integration-test/run.sh
+ pytest --reruns 5 _integration-test/run.py
  source _integration-test/ensure-customizations-work.sh
  source _integration-test/ensure-backup-restore-works.sh
 fi

requirements-dev.txt

@@ -0,0 +1,6 @@
+sentry-sdk>=1.39.2
+pytest>=8.0.0
+pytest-rerunfailures>=11.0
+pytest-sentry>=0.1.11
+httpx>=0.25.2
+beautifulsoup4>=4.7.1