Skip to content

Commit

Permalink
ci: make wheel builds more robust (datahub-project#8815)
Browse files Browse the repository at this point in the history
  • Loading branch information
hsheth2 authored Sep 12, 2023
1 parent f7fee74 commit 449cc9b
Show file tree
Hide file tree
Showing 5 changed files with 22 additions and 15 deletions.
5 changes: 4 additions & 1 deletion docs-website/sphinx/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ $(VENV_SENTINEL): requirements.txt
$(VENV_DIR)/bin/pip install -r requirements.txt
touch $(VENV_SENTINEL)

.PHONY: help html doctest linkcheck clean serve md
.PHONY: help html doctest linkcheck clean clean_all serve md

# Not using Python's http.server because it enables caching headers.
serve:
Expand All @@ -35,3 +35,6 @@ md: html
# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
html doctest linkcheck clean: venv Makefile
@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

clean_all: clean
-rm -rf $(VENV_DIR)
2 changes: 1 addition & 1 deletion docs-website/sphinx/requirements.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
-e ../../metadata-ingestion[datahub-rest]
-e ../../metadata-ingestion[datahub-rest,sql-parsing]
beautifulsoup4==4.11.2
Sphinx==6.1.3
sphinx-click==4.4.0
Expand Down
18 changes: 11 additions & 7 deletions docs-website/yarn.lock
Original file line number Diff line number Diff line change
Expand Up @@ -2986,6 +2986,13 @@
dependencies:
"@types/node" "*"

"@types/websocket@^1.0.3":
version "1.0.6"
resolved "https://registry.yarnpkg.com/@types/websocket/-/websocket-1.0.6.tgz#ec8dce5915741632ac3a4b1f951b6d4156e32d03"
integrity sha512-JXkliwz93B2cMWOI1ukElQBPN88vMg3CruvW4KVSKpflt3NyNCJImnhIuB/f97rG7kakqRJGFiwkA895Kn02Dg==
dependencies:
"@types/node" "*"

"@types/ws@^8.5.5":
version "8.5.5"
resolved "https://registry.yarnpkg.com/@types/ws/-/ws-8.5.5.tgz#af587964aa06682702ee6dcbc7be41a80e4b28eb"
Expand Down Expand Up @@ -7053,7 +7060,6 @@ node-forge@^1:
resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.3.1.tgz#be8da2af243b2417d5f646a770663a92b7e9ded3"
integrity sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==


node-gyp-build@^4.3.0:
version "4.6.1"
resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-4.6.1.tgz#24b6d075e5e391b8d5539d98c7fc5c210cac8a3e"
Expand Down Expand Up @@ -9903,6 +9909,10 @@ use-sidecar@^1.1.2:
detect-node-es "^1.1.0"
tslib "^2.0.0"

use-sync-external-store@^1.2.0:
version "1.2.0"
resolved "https://registry.yarnpkg.com/use-sync-external-store/-/use-sync-external-store-1.2.0.tgz#7dbefd6ef3fe4e767a0cf5d7287aacfb5846928a"
integrity sha512-eEgnFxGQ1Ife9bzYs6VLi8/4X6CObHMw9Qr9tPY43iKwsPw8xE8+EFsf/2cFZ5S3esXgpWgtSCtLNS41F+sKPA==

utf-8-validate@^5.0.2:
version "5.0.10"
Expand All @@ -9911,12 +9921,6 @@ utf-8-validate@^5.0.2:
dependencies:
node-gyp-build "^4.3.0"

use-sync-external-store@^1.2.0:
version "1.2.0"
resolved "https://registry.yarnpkg.com/use-sync-external-store/-/use-sync-external-store-1.2.0.tgz#7dbefd6ef3fe4e767a0cf5d7287aacfb5846928a"
integrity sha512-eEgnFxGQ1Ife9bzYs6VLi8/4X6CObHMw9Qr9tPY43iKwsPw8xE8+EFsf/2cFZ5S3esXgpWgtSCtLNS41F+sKPA==


util-deprecate@^1.0.1, util-deprecate@^1.0.2, util-deprecate@~1.0.1:
version "1.0.2"
resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"
Expand Down
6 changes: 3 additions & 3 deletions metadata-ingestion-modules/airflow-plugin/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -110,14 +110,14 @@ task testFull(type: Exec, dependsOn: [testQuick, installDevTest]) {
commandLine 'bash', '-x', '-c',
"source ${venv_name}/bin/activate && pytest -m 'not slow_integration' -vv --continue-on-collection-errors --junit-xml=junit.full.xml"
}
task buildWheel(type: Exec, dependsOn: [install]) {
commandLine 'bash', '-c', "source ${venv_name}/bin/activate && " + 'pip install build && RELEASE_VERSION="\${RELEASE_VERSION:-0.0.0.dev1}" RELEASE_SKIP_TEST=1 RELEASE_SKIP_UPLOAD=1 ./scripts/release.sh'
}

task cleanPythonCache(type: Exec) {
commandLine 'bash', '-c',
"find src -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete -o -type d -empty -delete"
}
task buildWheel(type: Exec, dependsOn: [install, cleanPythonCache]) {
commandLine 'bash', '-c', "source ${venv_name}/bin/activate && " + 'pip install build && RELEASE_VERSION="\${RELEASE_VERSION:-0.0.0.dev1}" RELEASE_SKIP_TEST=1 RELEASE_SKIP_UPLOAD=1 ./scripts/release.sh'
}

build.dependsOn install
check.dependsOn lint
Expand Down
6 changes: 3 additions & 3 deletions metadata-ingestion/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -185,16 +185,16 @@ task specGen(type: Exec, dependsOn: [codegen, installDevTest]) {
task docGen(type: Exec, dependsOn: [codegen, installDevTest, specGen]) {
commandLine 'bash', '-c', "source ${venv_name}/bin/activate && ./scripts/docgen.sh"
}
task buildWheel(type: Exec, dependsOn: [install, codegen]) {
commandLine 'bash', '-c', "source ${venv_name}/bin/activate && " + 'pip install build && RELEASE_VERSION="\${RELEASE_VERSION:-0.0.0.dev1}" RELEASE_SKIP_TEST=1 RELEASE_SKIP_UPLOAD=1 ./scripts/release.sh'
}



task cleanPythonCache(type: Exec) {
commandLine 'bash', '-c',
"find src tests -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete -o -type d -empty -delete"
}
task buildWheel(type: Exec, dependsOn: [install, codegen, cleanPythonCache]) {
commandLine 'bash', '-c', "source ${venv_name}/bin/activate && " + 'pip install build && RELEASE_VERSION="\${RELEASE_VERSION:-0.0.0.dev1}" RELEASE_SKIP_TEST=1 RELEASE_SKIP_UPLOAD=1 ./scripts/release.sh'
}

build.dependsOn install
check.dependsOn lint
Expand Down

0 comments on commit 449cc9b

Please sign in to comment.