Update CI to support multi-project setup (MarquezProject#902)

* Add build jobs for core, web, and java client Signed-off-by: wslulciuc <[email protected]> * Override checkout location for java client ci job Signed-off-by: wslulciuc <[email protected]> * Fix path to get-jdk11.sh Signed-off-by: wslulciuc <[email protected]> * Add build-client-java CI job Signed-off-by: wslulciuc <[email protected]> * Add build-client-python CI job Signed-off-by: wslulciuc <[email protected]> * continued: Add build-client-python CI job Signed-off-by: wslulciuc <[email protected]> * Add build-integration-airflow CI job Signed-off-by: wslulciuc <[email protected]> * continued: Add build-integration-airflow CI job Signed-off-by: wslulciuc <[email protected]> * continued: Add build-integration-airflow CI job Signed-off-by: wslulciuc <[email protected]> * continued: Add build-integration-airflow CI job Signed-off-by: wslulciuc <[email protected]> * continued: Add build-integration-airflow CI job Signed-off-by: wslulciuc <[email protected]> * continued: Add build-integration-airflow CI job Signed-off-by: wslulciuc <[email protected]> * continued: Add build-integration-airflow CI job Signed-off-by: wslulciuc <[email protected]> * Fix get_location test in airflow lib Signed-off-by: wslulciuc <[email protected]> * continued: Fix get_location test in airflow lib Signed-off-by: wslulciuc <[email protected]> * Add get-docker-compose.sh Signed-off-by: wslulciuc <[email protected]> * Fix docker/up.sh for airflow lib Signed-off-by: wslulciuc <[email protected]> * continued: Fix docker/up.sh for airflow lib Signed-off-by: wslulciuc <[email protected]> * Add build-integration-spark CI job Signed-off-by: wslulciuc <[email protected]> * continued: Add build-integration-spark CI job Signed-off-by: wslulciuc <[email protected]> * revert: Add build-integration-spark CI job Signed-off-by: wslulciuc <[email protected]> * Add build-integration-spark CI job as WIP Signed-off-by: wslulciuc <[email protected]>
ShakirzyanovArsen · Jan 22, 2021 · 2351c9e · 2351c9e
1 parent d4dac10
commit 2351c9e
Show file tree

Hide file tree

Showing 38 changed files with 152 additions and 1,637 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,26 +1,26 @@
 version: 2.1
 
-defaults: &defaults
+# Define reusable config (see: https://yaml.org/spec/1.2/spec.html#id2765878)
+machine: &machine
  working_directory: ~/marquez
  machine: true
 
-# Define reusable config (see: https://yaml.org/spec/1.2/spec.html#id2765878)
 checkout_project_root: &checkout_project_root
  # Override checkout path to project root (see: https://circleci.com/docs/2.0/configuration-reference/#checkout)
  checkout:
  path: ~/marquez
 
 jobs:
- build:
- <<: *defaults
+ build-api:
+ <<: *machine
  environment:
  TESTCONTAINERS_RYUK_DISABLED: true
  steps:
  - checkout
  - restore_cache:
  keys:
- - v1-jvm-{{ .Branch }}-{{ .Revision }}
- - v1-jvm-{{ .Branch }}
+ - v1-api-{{ .Branch }}-{{ .Revision }}
+ - v1-api-{{ .Branch }}
  - run: ./.circleci/get-jdk11.sh
  - run: echo "checks.disable=true" > ~/.testcontainers.properties
  - run: ./gradlew --no-daemon --stacktrace build
@@ -32,11 +32,11 @@ jobs:
  path: build/reports/tests/test
  destination: test-report
  - save_cache:
- key: v1-jvm-{{ .Branch }}-{{ .Revision }}
+ key: v1-api-{{ .Branch }}-{{ .Revision }}
  paths:
  - ~/.gradle
 
- test-web:
+ build-web:
  working_directory: ~/marquez/web
  docker:
  - image: circleci/node:11.9.0
@@ -46,17 +46,95 @@ jobs:
  - *checkout_project_root
  - restore_cache:
  keys:
- - v1-deps-{{ .Branch }}-{{ .Revision }}
- - v1-deps-{{ .Branch }}
+ - v1-web-{{ .Branch }}-{{ .Revision }}
+ - v1-web-{{ .Branch }}
  - run: npm install
  - run: npm run test
  - save_cache:
  paths:
  - node_modules
- key: v1-deps-{{ .Branch }}-{{ .Revision }}
+ key: v1-web-{{ .Branch }}-{{ .Revision }}
+
+ build-client-java:
+ working_directory: ~/marquez/clients/java
+ machine: true
+ steps:
+ - *checkout_project_root
+ - restore_cache:
+ keys:
+ - v1-client-java-{{ .Branch }}-{{ .Revision }}
+ - v1-client-java-{{ .Branch }}
+ - run: ../../.circleci/get-jdk11.sh
+ - run: ./gradlew --no-daemon --stacktrace build
+ - run: ./gradlew --no-daemon jacocoTestReport
+ - run: bash <(curl -s https://codecov.io/bash)
+ - store_test_results:
+ path: build/test-results/test
+ - store_artifacts:
+ path: build/reports/tests/test
+ destination: test-report
+ - save_cache:
+ key: v1-client-java-{{ .Branch }}-{{ .Revision }}
+ paths:
+ - ~/.gradle
+
+ build-client-python:
+ working_directory: ~/marquez/clients/python
+ docker:
+ - image: circleci/python:3.6
+ steps:
+ - *checkout_project_root
+ - run: pip install --user flake8
+ - run: python -m flake8 --extend-ignore=F401
+ - run: pip install --user -r test-requirements.txt
+ - run: python -m pytest --cov=marquez_python tests/
+ - run: bash <(curl -s https://codecov.io/bash)
+
+# build-integration-spark:
+# working_directory: ~/marquez/experimental/integrations/prototype-spark
+# machine: true
+# steps:
+# - *checkout_project_root
+# - restore_cache:
+# keys:
+# - v1-integration-spark-{{ .Branch }}-{{ .Revision }}
+# - v1-integration-spark-{{ .Branch }}
+# - run: ./gradlew --no-daemon --stacktrace build
+# - run: ./gradlew --no-daemon jacocoTestReport
+# - run: bash <(curl -s https://codecov.io/bash)
+# - store_test_results:
+# path: build/test-results/test
+# - store_artifacts:
+# path: build/reports/tests/test
+# destination: test-report
+# - save_cache:
+# key: v1-integration-spark-{{ .Branch }}-{{ .Revision }}
+# paths:
+# - ~/.gradle
+
+ build-integration-airflow:
+ working_directory: ~/marquez/integrations/airflow
+ docker:
+ - image: circleci/python:3.6
+ steps:
+ - *checkout_project_root
+ - run: pip install --user flake8
+ - run: python -m flake8 --exclude integration
+ - run: pip install --user -r test-requirements.txt --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-1.10.12/constraints-3.7.txt"
+ - run: airflow initdb
+ - run: python -m pytest --cov=marquez_python tests/
+ - run: bash <(curl -s https://codecov.io/bash)
+
+ test-integration-airflow:
+ working_directory: ~/marquez/integrations/airflow/tests/integration
+ machine: true
+ steps:
+ - *checkout_project_root
+ - run: ./../../../../.circleci/get-docker-compose.sh
+ - run: ./docker/up.sh
 
  release:
- <<: *defaults
+ <<: *machine
  steps:
  - checkout
  - run: ./.circleci/get-jdk11.sh
@@ -66,13 +144,18 @@ jobs:
 workflows:
  marquez:
  jobs:
- - build
+ - build-api
+ - build-web
+ - build-client-java
+ - build-client-python
+ # - build-integration-spark
+ - build-integration-airflow
+ - test-integration-airflow:
+ requires:
+ - build-integration-airflow
  - release:
  filters:
  tags:
  only: /^[0-9]+(\.[0-9]+){2}(-rc\.[0-9]+)?$/
  branches:
  ignore: /.*/
- marquez_web:
- jobs:
- - test-web
diff --git a/...s/airflow/.circleci/get-docker-compose.sh → .circleci/get-docker-compose.sh b/...s/airflow/.circleci/get-docker-compose.sh → .circleci/get-docker-compose.sh
@@ -1,4 +1,3 @@
-
 #!/bin/bash
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -20,5 +19,6 @@ set -e
 curl -L https://github.com/docker/compose/releases/download/1.25.3/docker-compose-`uname -s`-`uname -m` > ~/docker-compose
 chmod +x ~/docker-compose
 sudo mv ~/docker-compose /usr/local/bin/docker-compose
+docker-compose --version
 
-echo "DONE!"
+echo "DONE!"
diff --git a/.circleci/get-docker.sh b/.circleci/get-docker.sh
diff --git a/.gitignore b/.gitignore
@@ -1,18 +1,23 @@
+*.egg-info
 *.iml
 *.jar
 *.swp
+./**/*.class
+./out/*
 .classpath
 .editorconfig
 .gradle/
 .idea/
 .project
 .settings
 .vscode/
-./out/*
-out/*
-./**/*.class
+__pycache__
 bin/
 build/
+dist/
+integrations/airflow/tests/integration/integration-requirements.txt
+integrations/airflow/tests/integration/requirements.txt
+out/*
 
 # Marquez configuration
 marquez.yml
diff --git a/clients/java/.circleci/config.yml b/clients/java/.circleci/config.yml
diff --git a/clients/java/.gitignore b/clients/java/.gitignore
diff --git a/clients/java/CHANGELOG.md b/clients/java/CHANGELOG.md