-
Notifications
You must be signed in to change notification settings - Fork 5.5k
/
pipeline.macos.yml
132 lines (120 loc) · 5.04 KB
/
pipeline.macos.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
common: &common
env:
BUILDKITE: "true"
CI: "true"
PYTHON: "3.7"
RAY_USE_RANDOM_PORTS: "1"
RAY_DEFAULT_BUILD: "1"
LC_ALL: en_US.UTF-8
LANG: en_US.UTF-8
BUILD: "1"
DL: "1"
prelude_commands: &prelude_commands |-
rm -rf /tmp/bazel_event_logs
cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
(which bazel && bazel clean) || true;
. ./ci/ci.sh init && source ~/.zshenv
./ci/ci.sh build
./ci/env/install-dependencies.sh
./ci/env/env_info.sh
epilogue_commands: &epilogue_commands |-
# Cleanup runtime environment to save storage
rm -rf /tmp/ray || true
# Cleanup local caches (this shouldn't clean up global disk cache)
bazel clean
steps:
- label: ":mac: :apple: Wheels and Jars"
<<: *common
conditions: ["RAY_CI_MACOS_WHEELS_AFFECTED", "RAY_CI_PYTHON_DEPENDENCIES_AFFECTED"]
commands:
# Cleanup environments
- rm -rf /tmp/bazel_event_logs
- cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
- (which bazel && bazel clean) || true
# TODO(simon): make sure to change both PR and wheel builds
# Special setup for jar builds (will be installed to the machine instead)
# - brew remove --force java & brew uninstall --force java & rm -rf /usr/local/Homebrew/Library/Taps/homebrew/homebrew-cask
# - brew install --cask adoptopenjdk/openjdk/adoptopenjdk8
- diskutil list external physical
- export JAVA_HOME=/Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home
- java -version
# Build wheels
- export UPLOAD_WHEELS_AS_ARTIFACTS=1
- export MAC_WHEELS=1
- export MAC_JARS=1
- export RAY_INSTALL_JAVA=1
- . ./ci/ci.sh init && source ~/.zshenv
- ./ci/ci.sh build
# Test wheels
- ./ci/ci.sh test_wheels
# Build jars
- bash ./java/build-jar-multiplatform.sh darwin
# Upload the wheels and jars
# We don't want to push on PRs, in fact, the copy_files will fail because unauthenticated.
- if [ "$BUILDKITE_PULL_REQUEST" != "false" ]; then exit 0; fi
- pip install -q docker aws_requests_auth boto3
# Upload to branch directory.
- python .buildkite/copy_files.py --destination branch_wheels --path ./.whl
- python .buildkite/copy_files.py --destination branch_jars --path ./.jar/darwin
# Upload to latest directory.
- if [ "$BUILDKITE_BRANCH" = "master" ]; then python .buildkite/copy_files.py --destination wheels --path ./.whl; fi
- if [ "$BUILDKITE_BRANCH" = "master" ]; then python .buildkite/copy_files.py --destination jars --path ./.jar/darwin; fi
- label: ":mac: :apple: Ray Core Unit Tests and Dashboard"
<<: *common
conditions: ["RAY_CI_CORE_CPP_AFFECTED", "RAY_CI_PYTHON_AFFECTED", "RAY_CI_DASHBOARD_AFFECTED"]
commands:
- *prelude_commands
- TORCH_VERSION=1.6 ./ci/env/install-dependencies.sh
# Use --dynamic_mode=off until MacOS CI runs on Big Sur or newer. Otherwise there are problems with running tests
# with dynamic linking.
- bazel test --config=ci --dynamic_mode=off --test_env=CI $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-post_wheel_build --
//:all python/ray/dashboard/... -python/ray/serve/... -rllib/... -core_worker_test
- *epilogue_commands
- label: ":mac: :apple: Ray C++ and Java"
<<: *common
conditions: ["RAY_CI_CPP_AFFECTED", "RAY_CI_JAVA_AFFECTED"]
commands:
- export RAY_INSTALL_JAVA=1
- *prelude_commands
# clang-format is needed by java/test.sh
- pip install clang-format==12.0.1
- ./java/test.sh
- ./ci/ci.sh test_cpp
- *epilogue_commands
- label: ":mac: :apple: Small & Client"
<<: *common
conditions: ["RAY_CI_CORE_CPP_AFFECTED", "RAY_CI_PYTHON_AFFECTED"]
commands:
- *prelude_commands
- bazel test $(./ci/run/bazel_export_options) --config=ci
--test_env=CONDA_EXE --test_env=CONDA_PYTHON_EXE --test_env=CONDA_SHLVL --test_env=CONDA_PREFIX
--test_env=CONDA_DEFAULT_ENV --test_env=CONDA_PROMPT_MODIFIER --test_env=CI
--test_tag_filters=client_tests,small_size_python_tests
-- python/ray/tests/...
- *epilogue_commands
- label: ":mac: :apple: Large"
<<: *common
parallelism: 3
conditions: ["RAY_CI_CORE_CPP_AFFECTED", "RAY_CI_PYTHON_AFFECTED"]
commands:
- *prelude_commands
- ./ci/ci.sh test_large
- *epilogue_commands
- label: ":mac: :apple: Medium A-J"
<<: *common
conditions: ["RAY_CI_CORE_CPP_AFFECTED", "RAY_CI_PYTHON_AFFECTED"]
commands:
- *prelude_commands
- bazel test --config=ci $(./ci/run/bazel_export_options) --test_env=CI
--test_tag_filters=-kubernetes,medium_size_python_tests_a_to_j
python/ray/tests/...
- *epilogue_commands
- label: ":mac: :apple: Medium K-Z"
<<: *common
conditions: ["RAY_CI_CORE_CPP_AFFECTED", "RAY_CI_PYTHON_AFFECTED"]
commands:
- *prelude_commands
- bazel test --config=ci $(./ci/run/bazel_export_options) --test_env=CI
--test_tag_filters=-kubernetes,medium_size_python_tests_k_to_z
python/ray/tests/...
- *epilogue_commands