forked from ray-project/ray
-
Notifications
You must be signed in to change notification settings - Fork 0
/
pipeline.gpu_large.yml
97 lines (90 loc) · 5.1 KB
/
pipeline.gpu_large.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
#ci:group=Large GPU tests
- label: ":tv: :steam_locomotive: Train GPU tests "
conditions: ["NO_WHEELS_REQUIRED", "RAY_CI_TRAIN_AFFECTED"]
commands:
- cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
- TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/env/install-dependencies.sh
- pip install -Ur ./python/requirements/ml/dl-gpu-requirements.txt
- ./ci/env/install-horovod.sh
- ./ci/env/env_info.sh
- bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=gpu,gpu_only,-ray_air
python/ray/train/...
- label: ":tv: :database: :steam_locomotive: Datasets Train Integration GPU Tests and Examples (Python 3.7)"
conditions: ["NO_WHEELS_REQUIRED", "RAY_CI_TRAIN_AFFECTED"]
commands:
- cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
- TRAIN_TESTING=1 DATA_PROCESSING_TESTING=1 ./ci/env/install-dependencies.sh
- pip install -Ur ./python/requirements/ml/dl-gpu-requirements.txt
- ./ci/env/env_info.sh
- bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=datasets_train,-doctest
doc/...
- label: ":tv: :brain: RLlib: Multi-GPU Tests"
conditions: ["NO_WHEELS_REQUIRED", "RAY_CI_RLLIB_AFFECTED"]
parallelism: 2
commands:
- cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
- RLLIB_TESTING=1 ./ci/env/install-dependencies.sh
- pip install -Ur ./python/requirements/ml/dl-gpu-requirements.txt
- ./ci/env/env_info.sh
# --jobs 2 is necessary as we only need to have at least 2 gpus on the machine
# and running tests in parallel would cause timeouts as the other scripts would
# wait for the GPU to become available.
- ./ci/run/run_bazel_test_with_sharding.sh --config=ci $(./ci/run/bazel_export_options)
--build_tests_only
--jobs=2
--test_tag_filters=multi_gpu
--test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1
rllib/...
- label: ":tv: :airplane: AIR GPU tests (ray/air)"
conditions: ["NO_WHEELS_REQUIRED", "RAY_CI_ML_AFFECTED"]
commands:
- cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
- DATA_PROCESSING_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/env/install-dependencies.sh
- pip install -Ur ./python/requirements/ml/dl-gpu-requirements.txt
- ./ci/env/install-horovod.sh
- ./ci/env/env_info.sh
- bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=gpu
python/ray/air/... python/ray/train/...
- label: ":tv: :book: Doc GPU tests and examples"
conditions:
["NO_WHEELS_REQUIRED", "RAY_CI_PYTHON_AFFECTED", "RAY_CI_TUNE_AFFECTED", "RAY_CI_DOC_AFFECTED"]
commands:
- cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
- DOC_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/env/install-dependencies.sh
- pip install -Ur ./python/requirements/ml/dl-gpu-requirements.txt
- ./ci/env/env_info.sh
# Test examples with newer version of `transformers`
# TODO(amogkam): Remove when https://github.com/ray-project/ray/issues/36011
# is resolved.
- pip install -U transformers
- bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=gpu,-timeseries_libs,-post_wheel_build,-doctest
doc/...
- label: ":book: Doctest (GPU)"
commands:
- cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
- DOC_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/env/install-dependencies.sh
- pip install -Ur ./python/requirements/ml/dl-gpu-requirements.txt
- ./ci/env/install-horovod.sh
# Test examples with newer version of `transformers`
# TODO(amogkam): Remove when https://github.com/ray-project/ray/issues/36011
# is resolved.
# TODO(scottjlee): Move datasets to train/data-test-requirements.txt
# (see https://github.com/ray-project/ray/pull/38432/)
- pip install transformers==4.30.2 datasets==2.14.0
- ./ci/env/env_info.sh
- bazel test --config=ci $(./scripts/bazel_export_options)
--test_tag_filters=doctest,-cpu
python/ray/... doc/...
- label: ":zap: :python: Lightning 2.0 Train GPU tests"
conditions:
["NO_WHEELS_REQUIRED", "RAY_CI_TRAIN_AFFECTED"]
commands:
- cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
- NO_DASHBOARD=1 ./ci/env/install-minimal.sh 3.8
- PYTHON=3.8 DOC_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/env/install-dependencies.sh
- pip install -Ur ./python/requirements/ml/dl-gpu-requirements.txt
- pip uninstall -y pytorch-lightning
- pip install lightning==2.0.4 pytorch-lightning==2.0.4 # todo move to requirements-test.txt
- ./ci/env/env_info.sh
- bazel test --config=ci $(./scripts/bazel_export_options) --test_tag_filters=ptl_v2
python/ray/train/...