test(ingest/minio): Configure delta lake minio tests for arm64 (datah…

…ub-project#8364)

Co-authored-by: Tamas Nemeth <[email protected]>

metadata-ingestion/.gitignore

@@ -6,7 +6,6 @@ pvenv36/
 /venv*/
 bq_credentials.json
 junit.*.xml
-tests/integrations/delta_lake/minio/minio
 /tmp
 *.bak
 

metadata-ingestion/tests/integration/delta_lake/docker-compose.yml

@@ -0,0 +1,12 @@
+version: "3"
+services:
+  minio:
+    image: minio/minio:RELEASE.2023-07-07T07-13-57Z
+    container_name: "minio_test"
+    environment:
+      MINIO_ROOT_USER: "miniouser"
+      MINIO_ROOT_PASSWORD: "miniopassword"
+    ports:
+      - 9000:9000  # S3 API
+      - 9001:9001  # Web UI
+    command: server /data --console-address ":9001"

metadata-ingestion/tests/integration/delta_lake/test_delta_lake_minio.py

@@ -2,47 +2,58 @@
 import subprocess
 
 import boto3
+import freezegun
 import pytest
 
 from datahub.ingestion.run.pipeline import Pipeline
 from tests.test_helpers import mce_helpers
+from tests.test_helpers.docker_helpers import wait_for_port
 
 FROZEN_TIME = "2020-04-14 07:00:00"
+MINIO_PORT = 9000
 
 
-@pytest.fixture(scope="module", autouse=True)
-def minio_startup():
-    cmd = "./tests/integration/delta_lake/minio/setup_minio.sh"
-    ret = subprocess.run(
-        cmd,
-        shell=True,
-    )
-    assert ret.returncode == 0
-    yield
+def is_minio_up(container_name: str) -> bool:
+    """A cheap way to figure out if postgres is responsive on a container"""
 
-    # Shutdown minio server
-    cmd = "./tests/integration/delta_lake/minio/kill_minio.sh"
+    cmd = f"docker logs {container_name} 2>&1 | grep '1 Online'"
     ret = subprocess.run(
         cmd,
         shell=True,
     )
-    assert ret.returncode == 0
+    return ret.returncode == 0
 
 
-@pytest.fixture(scope="module", autouse=True)
-def bucket_name():
-    return "my-test-bucket"
+@pytest.fixture(scope="module")
+def test_resources_dir(pytestconfig):
+    return pytestconfig.rootpath / "tests/integration/delta_lake"
+
+
+@pytest.fixture(scope="module")
+def minio_runner(docker_compose_runner, pytestconfig, test_resources_dir):
+    container_name = "minio_test"
+    with docker_compose_runner(
+        test_resources_dir / "docker-compose.yml", container_name
+    ) as docker_services:
+        wait_for_port(
+            docker_services,
+            container_name,
+            MINIO_PORT,
+            timeout=120,
+            checker=lambda: is_minio_up(container_name),
+        )
+        yield docker_services
 
 
 @pytest.fixture(scope="module", autouse=True)
-def s3_bkt(bucket_name, minio_startup):
+def s3_bkt(minio_runner):
     s3 = boto3.resource(
         "s3",
-        endpoint_url="https://localhost:9000",
-        aws_access_key_id="minioadmin",
-        aws_secret_access_key="minioadmin",
+        endpoint_url=f"https://localhost:{MINIO_PORT}",
+        aws_access_key_id="miniouser",
+        aws_secret_access_key="miniopassword",
     )
-    bkt = s3.Bucket(bucket_name)
+    bkt = s3.Bucket("my-test-bucket")
     bkt.create()
     return bkt
 
@@ -62,9 +73,8 @@ def populate_minio(pytestconfig, s3_bkt):
 
 
 @pytest.mark.slow_integration
-def test_delta_lake_ingest(pytestconfig, tmp_path, mock_time):
-    test_resources_dir = pytestconfig.rootpath / "tests/integration/delta_lake/"
-
+@freezegun.freeze_time("2023-01-01 00:00:00+00:00")
+def test_delta_lake_ingest(pytestconfig, tmp_path, test_resources_dir):
     # Run the metadata ingestion pipeline.
     pipeline = Pipeline.create(
         {
@@ -76,9 +86,9 @@ def test_delta_lake_ingest(pytestconfig, tmp_path, mock_time):
                     "base_path": "s3:https://my-test-bucket/delta_tables/sales",
                     "s3": {
                         "aws_config": {
-                            "aws_access_key_id": "minioadmin",
-                            "aws_secret_access_key": "minioadmin",
-                            "aws_endpoint_url": "https://localhost:9000",
+                            "aws_access_key_id": "miniouser",
+                            "aws_secret_access_key": "miniopassword",
+                            "aws_endpoint_url": f"https://localhost:{MINIO_PORT}",
                             "aws_region": "us-east-1",
                         },
                     },