Write Customization tests in python (#2918)

* port everything integration test related to python

.github/workflows/test.yml

@@ -55,11 +55,11 @@ jobs:
  integration-test:
  if: github.repository_owner == 'getsentry'
  runs-on: ubuntu-20.04
- name: "integration test"
+ name: integration test ${{ matrix.compose_version }} - customizations ${{ matrix.customizations }}
  strategy:
  fail-fast: false
  matrix:
- test_type: ["initial-install", "customizations"]
+ customizations: ["disabled", "enabled"]
  compose_version: ["v2.0.1", "v2.7.0"]
  include:
  - compose_version: "v2.0.1"
@@ -68,18 +68,19 @@ jobs:
  compose_path: "/usr/local/lib/docker/cli-plugins"
  env:
  COMPOSE_PROJECT_NAME: self-hosted-${{ strategy.job-index }}
- SENTRY_DSN: https://[email protected]/6627632
- REPORT_SELF_HOSTED_ISSUES: 1
+ REPORT_SELF_HOSTED_ISSUES: 0
+ SELF_HOSTED_TESTING_DSN: ${{ vars.SELF_HOSTED_TESTING_DSN }}
  steps:
  - name: Checkout
  uses: actions/checkout@v4
 
  - name: Setup dev environment
  run: |
  pip install -r requirements-dev.txt
+ echo "PY_COLORS=1" >> "$GITHUB_ENV"
  ### pytest-sentry configuration ###
  if [ "$GITHUB_REPOSITORY" = "getsentry/self-hosted" ]; then
- echo "PYTEST_SENTRY_DSN=${{ env.SELF_HOSTED_TESTING_DSN }}" >> $GITHUB_ENV
+ echo "PYTEST_SENTRY_DSN=$SELF_HOSTED_TESTING_DSN" >> $GITHUB_ENV
  echo "PYTEST_SENTRY_TRACES_SAMPLE_RATE=0" >> $GITHUB_ENV
 
  # This records failures on master to sentry in order to detect flakey tests, as it's
@@ -102,13 +103,29 @@ jobs:
  sudo chmod +x "${{ matrix.compose_path }}/docker-compose"
 
  - name: Install self-hosted
- run: ./install.sh
+ uses: nick-fields/retry@v3
+ with:
+ timeout_minutes: 10
+ max_attempts: 3
+ command: ./install.sh
 
  - name: Integration Test
- run: ./integration-test.sh --${{ matrix.test_type }}
+ run: pytest --cov --junitxml=junit.xml --reruns 3 _integration-test/ --customizations=${{ matrix.customizations }}
 
  - name: Inspect failure
  if: failure()
  run: |
  docker compose ps
  docker compose logs
+
+ - name: Upload coverage to Codecov
+ uses: codecov/codecov-action@v4
+ with:
+ token: ${{ secrets.CODECOV_TOKEN }}
+ slug: getsentry/self-hosted
+
+ - name: Upload test results to Codecov
+ if: ${{ !cancelled() }}
+ uses: codecov/test-results-action@v1
+ with:
+ token: ${{ secrets.CODECOV_TOKEN }}

_integration-test/conftest.py

@@ -10,27 +10,49 @@
 TEST_PASS = "test123TEST"
 TIMEOUT_SECONDS = 60
 
+def pytest_addoption(parser):
+ parser.addoption("--customizations", default="disabled")
 
 @pytest.fixture(scope="session", autouse=True)
-def configure_self_hosted_environment():
+def configure_self_hosted_environment(request):
  subprocess.run(["docker", "compose", "--ansi", "never", "up", "-d"], check=True)
  for i in range(TIMEOUT_SECONDS):
  try:
  response = httpx.get(SENTRY_TEST_HOST, follow_redirects=True)
- except httpx.ConnectionError:
+ except httpx.NetworkError:
  time.sleep(1)
  else:
  if response.status_code == 200:
  break
  else:
  raise AssertionError("timeout waiting for self-hosted to come up")
 
+ if request.config.getoption("--customizations") == "enabled":
+ os.environ['TEST_CUSTOMIZATIONS'] = "enabled"
+ script_content = '''\
+#!/bin/bash
+touch /created-by-enhance-image
+apt-get update
+apt-get install -y gcc libsasl2-dev python-dev libldap2-dev libssl-dev
+'''
+
+ with open('sentry/enhance-image.sh', 'w') as script_file:
+ script_file.write(script_content)
+ # Set executable permissions for the shell script
+ os.chmod('sentry/enhance-image.sh', 0o755)
+
+ # Write content to the requirements.txt file
+ with open('sentry/requirements.txt', 'w') as req_file:
+ req_file.write('python-ldap\n')
+ os.environ['MINIMIZE_DOWNTIME'] = "1"
+ subprocess.run(["./install.sh"], check=True)
  # Create test user
  subprocess.run(
  [
  "docker",
  "compose",
  "exec",
+ "-T",
  "web",
  "sentry",
  "createuser",
@@ -45,3 +67,8 @@ def configure_self_hosted_environment():
  check=True,
  text=True,
  )
+
+@pytest.fixture()
+def setup_backup_restore_env_variables():
+ os.environ['SENTRY_DOCKER_IO_DIR'] = os.path.join(os.getcwd(), 'sentry')
+ os.environ['SKIP_USER_CREATION'] = "1"

_integration-test/backup.py → _integration-test/test_backup.py

@@ -1,13 +1,8 @@
 import subprocess
 import os
-import pytest
 
-@pytest.fixture()
-def setup_env():
- os.environ['SENTRY_DOCKER_IO_DIR'] = os.path.join(os.getcwd(), 'sentry')
- os.environ['SKIP_USER_CREATION'] = "1"
 
-def test_backup(setup_env):
+def test_backup(setup_backup_restore_env_variables):
  # Docker was giving me permissioning issues when trying to create this file and write to it even after giving read + write access
  # to group and owner. Instead, try creating the empty file and then give everyone write access to the backup file
  file_path = os.path.join(os.getcwd(), 'sentry', 'backup.json')
@@ -18,7 +13,7 @@ def test_backup(setup_env):
  subprocess.run([sentry_admin_sh, "export", "global", "/sentry-admin/backup.json", "--no-prompt"], check=True)
  assert os.path.getsize(file_path) > 0
 
-def test_import(setup_env):
+def test_import(setup_backup_restore_env_variables):
  # Bring postgres down and recreate the docker volume
  subprocess.run(["docker", "compose", "--ansi", "never", "stop", "postgres"], check=True)
  subprocess.run(["docker", "compose", "--ansi", "never", "rm", "-f", "-v", "postgres"], check=True)

_integration-test/run.py → _integration-test/test_run.py

@@ -141,7 +141,18 @@ def test_custom_cas():
  try:
  subprocess.run(["./_integration-test/custom-ca-roots/setup.sh"], check=True)
  subprocess.run(
- ["docker", "compose", "--ansi", "never", "run", "--no-deps", "web", "python3", "/etc/sentry/test-custom-ca-roots.py"], check=True
+ [
+ "docker",
+ "compose",
+ "--ansi",
+ "never",
+ "run",
+ "--no-deps",
+ "web",
+ "python3",
+ "/etc/sentry/test-custom-ca-roots.py",
+ ],
+ check=True,
  )
  finally:
  subprocess.run(["./_integration-test/custom-ca-roots/teardown.sh"], check=True)
@@ -171,3 +182,64 @@ def placeholder_fn():
  client,
  lambda x: len(json.loads(x)["data"]) > 0,
  )
+
+
+def test_customizations():
+ commands = [
+ [
+ "docker",
+ "compose",
+ "--ansi",
+ "never",
+ "run",
+ "--no-deps",
+ "web",
+ "bash",
+ "-c",
+ "if [ ! -e /created-by-enhance-image ]; then exit 1; fi",
+ ],
+ [
+ "docker",
+ "compose",
+ "--ansi",
+ "never",
+ "run",
+ "--no-deps",
+ "--entrypoint=/etc/sentry/entrypoint.sh",
+ "sentry-cleanup",
+ "bash",
+ "-c",
+ "if [ ! -e /created-by-enhance-image ]; then exit 1; fi",
+ ],
+ [
+ "docker",
+ "compose",
+ "--ansi",
+ "never",
+ "run",
+ "--no-deps",
+ "web",
+ "python",
+ "-c",
+ "import ldap",
+ ],
+ [
+ "docker",
+ "compose",
+ "--ansi",
+ "never",
+ "run",
+ "--no-deps",
+ "--entrypoint=/etc/sentry/entrypoint.sh",
+ "sentry-cleanup",
+ "python",
+ "-c",
+ "import ldap",
+ ]
+ ]
+ for command in commands:
+ result = subprocess.run(command, check=False)
+ if os.getenv("TEST_CUSTOMIZATIONS", "disabled") == "enabled":
+ assert result.returncode == 0
+ else:
+ assert result.returncode != 0

requirements-dev.txt

@@ -1,5 +1,7 @@
+codecov-cli>=0.4.8
 sentry-sdk>=1.39.2
 pytest>=8.0.0
+pytest-cov>=4.1.0
 pytest-rerunfailures>=11.0
 pytest-sentry>=0.1.11
 httpx>=0.25.2