forked from ray-project/ray
-
Notifications
You must be signed in to change notification settings - Fork 0
/
pipeline.macos.yml
174 lines (161 loc) · 7.14 KB
/
pipeline.macos.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
common: &common
env:
BUILDKITE: "true"
CI: "true"
PYTHON: "3.7"
RAY_USE_RANDOM_PORTS: "1"
RAY_DEFAULT_BUILD: "1"
LC_ALL: en_US.UTF-8
LANG: en_US.UTF-8
BUILD: "1"
DL: "1"
prelude_commands: &prelude_commands |-
rm -rf /tmp/bazel_event_logs
cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
(which bazel && bazel clean) || true;
. ./ci/ci.sh init && source ~/.zshenv
./ci/ci.sh build
./ci/env/install-dependencies.sh
./ci/env/env_info.sh
epilogue_commands: &epilogue_commands |-
# Cleanup runtime environment to save storage
rm -rf /tmp/ray || true
# Cleanup local caches - this should not clean up global disk cache
bazel clean
steps:
- label: ":mac: :apple: Wheels and Jars (intel)"
<<: *common
conditions: ["RAY_CI_MACOS_WHEELS_AFFECTED", "RAY_CI_PYTHON_DEPENDENCIES_AFFECTED"]
commands:
# Cleanup environments
- rm -rf /tmp/bazel_event_logs
- cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
- (which bazel && bazel clean) || true
# TODO(simon): make sure to change both PR and wheel builds
# Special setup for jar builds (will be installed to the machine instead)
# - brew remove --force java & brew uninstall --force java & rm -rf /usr/local/Homebrew/Library/Taps/homebrew/homebrew-cask
# - brew install --cask adoptopenjdk/openjdk/adoptopenjdk8
- diskutil list external physical
- export JAVA_HOME=/Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home
- java -version
# Build wheels
- export UPLOAD_WHEELS_AS_ARTIFACTS=1
- export MAC_WHEELS=1
- export MAC_JARS=1
- export RAY_INSTALL_JAVA=1
- export RAY_ENABLE_WINDOWS_OR_OSX_CLUSTER=1
- . ./ci/ci.sh init && source ~/.zshenv
- ./ci/ci.sh build
# Test wheels
- ./ci/ci.sh test_wheels
# Build jars
- bash ./java/build-jar-multiplatform.sh darwin
# Upload the wheels and jars
# We don't want to push on PRs, in fact, the copy_files will fail because unauthenticated.
- if [ "$BUILDKITE_PULL_REQUEST" != "false" ]; then exit 0; fi
- pip install -q docker aws_requests_auth boto3
# Upload to branch directory.
- python .buildkite/copy_files.py --destination branch_wheels --path ./.whl
- python .buildkite/copy_files.py --destination branch_jars --path ./.jar/darwin
# Upload to latest directory.
- if [ "$BUILDKITE_BRANCH" = "master" ]; then python .buildkite/copy_files.py --destination wheels --path ./.whl; fi
- if [ "$BUILDKITE_BRANCH" = "master" ]; then python .buildkite/copy_files.py --destination jars --path ./.jar/darwin; fi
- label: ":mac: :mechanical_arm: Wheels and Jars (arm64)"
<<: *common
conditions: ["RAY_CI_MACOS_WHEELS_AFFECTED", "RAY_CI_PYTHON_DEPENDENCIES_AFFECTED"]
instance_size: arm64-medium
commands:
# Cleanup environments
- rm -rf /tmp/bazel_event_logs
- cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
- (which bazel && bazel clean) || true
- brew install pkg-config nvm node || true
# TODO(simon): make sure to change both PR and wheel builds
# Special setup for jar builds (will be installed to the machine instead)
# - brew remove --force java & brew uninstall --force java & rm -rf /usr/local/Homebrew/Library/Taps/homebrew/homebrew-cask
# - brew install --cask adoptopenjdk/openjdk/adoptopenjdk8
- diskutil list external physical
- export JAVA_HOME=/Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home
- java -version
# Build wheels
- export UPLOAD_WHEELS_AS_ARTIFACTS=1
- export MAC_WHEELS=1
- export MAC_JARS=1
- export RAY_INSTALL_JAVA=1
- export RAY_ENABLE_WINDOWS_OR_OSX_CLUSTER=1
- export PYTHON=3.8
- . ./ci/ci.sh init && source ~/.zshenv
- ./ci/ci.sh build
# Test wheels
- ./ci/ci.sh test_wheels
# Build jars
- bash ./java/build-jar-multiplatform.sh darwin
# Upload the wheels and jars
# We don't want to push on PRs, in fact, the copy_files will fail because unauthenticated.
- if [ "$BUILDKITE_PULL_REQUEST" != "false" ]; then exit 0; fi
- python -m pip install -q docker aws_requests_auth boto3
# Upload to branch directory.
- python .buildkite/copy_files.py --destination branch_wheels --path ./.whl
- python .buildkite/copy_files.py --destination branch_jars --path ./.jar/darwin
# Upload to latest directory.
- if [ "$BUILDKITE_BRANCH" = "master" ]; then python .buildkite/copy_files.py --destination wheels --path ./.whl; fi
- if [ "$BUILDKITE_BRANCH" = "master" ]; then python .buildkite/copy_files.py --destination jars --path ./.jar/darwin; fi
- label: ":mac: :apple: Ray Core Unit Tests and Dashboard"
<<: *common
conditions: ["RAY_CI_CORE_CPP_AFFECTED", "RAY_CI_PYTHON_AFFECTED", "RAY_CI_DASHBOARD_AFFECTED"]
commands:
- *prelude_commands
- TORCH_VERSION=1.6 ./ci/env/install-dependencies.sh
# Use --dynamic_mode=off until MacOS CI runs on Big Sur or newer. Otherwise there are problems with running tests
# with dynamic linking.
- bazel test --config=ci --dynamic_mode=off --test_env=CI $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-post_wheel_build --
//:all python/ray/dashboard/... -python/ray/serve/... -rllib/... -core_worker_test
- *epilogue_commands
- label: ":mac: :apple: Ray C++ and Java"
<<: *common
conditions: ["RAY_CI_CPP_AFFECTED", "RAY_CI_JAVA_AFFECTED"]
commands:
- export RAY_INSTALL_JAVA=1
- *prelude_commands
# clang-format is needed by java/test.sh
- pip install clang-format==12.0.1
- ./java/test.sh
- ./ci/ci.sh test_cpp
- *epilogue_commands
- label: ":mac: :apple: Small & Client"
<<: *common
conditions: ["RAY_CI_CORE_CPP_AFFECTED", "RAY_CI_PYTHON_AFFECTED"]
commands:
- *prelude_commands
- bazel test $(./ci/run/bazel_export_options) --config=ci
--test_env=CONDA_EXE --test_env=CONDA_PYTHON_EXE --test_env=CONDA_SHLVL --test_env=CONDA_PREFIX
--test_env=CONDA_DEFAULT_ENV --test_env=CONDA_PROMPT_MODIFIER --test_env=CI
--test_tag_filters=client_tests,small_size_python_tests
-- python/ray/tests/...
- *epilogue_commands
- label: ":mac: :apple: Large"
<<: *common
parallelism: 3
conditions: ["RAY_CI_CORE_CPP_AFFECTED", "RAY_CI_PYTHON_AFFECTED"]
commands:
- *prelude_commands
- ./ci/ci.sh test_large
- *epilogue_commands
- label: ":mac: :apple: Medium A-J"
<<: *common
conditions: ["RAY_CI_CORE_CPP_AFFECTED", "RAY_CI_PYTHON_AFFECTED"]
commands:
- *prelude_commands
- bazel test --config=ci $(./ci/run/bazel_export_options) --test_env=CI
--test_tag_filters=-kubernetes,medium_size_python_tests_a_to_j
python/ray/tests/...
- *epilogue_commands
- label: ":mac: :apple: Medium K-Z"
<<: *common
conditions: ["RAY_CI_CORE_CPP_AFFECTED", "RAY_CI_PYTHON_AFFECTED"]
commands:
- *prelude_commands
- bazel test --config=ci $(./ci/run/bazel_export_options) --test_env=CI
--test_tag_filters=-kubernetes,medium_size_python_tests_k_to_z
python/ray/tests/...
- *epilogue_commands