[CI] Parallelize Tune test runners (ray-project#29365)

This PR moves our manual test sharding to use the native bazel system, removes the now unnecessary `tests_dir_[A-Z]` tags from the Tune BUILD file, adjusts some timeouts to match bazel suggestions and makes it so that changes to Tune are considered to affect AIR (as Tune is AIR's execution engine).

Signed-off-by: Antoni Baum <[email protected]>
Signed-off-by: Kai Fricke <[email protected]>
Co-authored-by: Kai Fricke <[email protected]>

.buildkite/pipeline.ml.yml

@@ -169,49 +169,30 @@
  --test_tag_filters=documentation --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1
  rllib/...
 
-- label: ":octopus: Tune tests {A-R; no RLlib}"
+- label: ":octopus: Tune tests and examples (small)"
  conditions: ["NO_WHEELS_REQUIRED", "RAY_CI_TUNE_AFFECTED"]
  instance_size: small
+ parallelism: 3
  commands:
  - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
  - TUNE_TESTING=1 ./ci/env/install-dependencies.sh
  - ./ci/env/env_info.sh
- - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only
- --test_tag_filters=tests_dir_A,tests_dir_B,tests_dir_C,tests_dir_D,tests_dir_E,tests_dir_F,tests_dir_G,tests_dir_H,tests_dir_I,tests_dir_J,tests_dir_K,tests_dir_L,tests_dir_M,tests_dir_N,tests_dir_O,tests_dir_P,tests_dir_Q,tests_dir_R,-example,-py37,-soft_imports,-gpu_only,-rllib
+ - ./ci/run/run_bazel_test_with_sharding.sh
+ --config=ci $(./ci/run/bazel_export_options) --build_tests_only
+ --test_tag_filters=-medium_instance,-py37,-soft_imports,-gpu_only,-rllib,-multinode
  python/ray/tune/...
 
-- label: ":octopus: Tune tests {S-Z; no RLlib}"
+- label: ":octopus: Tune tests and examples (medium)"
  conditions: ["NO_WHEELS_REQUIRED", "RAY_CI_TUNE_AFFECTED"]
  instance_size: medium
  commands:
  - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
  - TUNE_TESTING=1 ./ci/env/install-dependencies.sh
  - ./ci/env/env_info.sh
- - python ./ci/env/setup_credentials.py sigopt
  - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only
- --test_tag_filters=tests_dir_S,tests_dir_T,tests_dir_U,tests_dir_V,tests_dir_W,tests_dir_X,tests_dir_Y,tests_dir_Z,-example,-py37,-soft_imports,-gpu_only,-rllib
+ --test_tag_filters=medium_instance,-py37,-soft_imports,-gpu_only,-rllib,-multinode
  python/ray/tune/...
 
-
-- label: ":octopus: Tune examples {w/o tf/pytorch; no RLlib}"
- conditions: ["NO_WHEELS_REQUIRED", "RAY_CI_TUNE_AFFECTED"]
- instance_size: medium
- commands:
- - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
- - TUNE_TESTING=1 ./ci/env/install-dependencies.sh
- - ./ci/env/env_info.sh
- - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=example,-tf,-pytorch,-py37,-soft_imports,-gpu_only,-rllib python/ray/tune/...
-
-- label: ":octopus: Tune examples {w/ tf/pytorch; no RLlib}"
- conditions: ["NO_WHEELS_REQUIRED", "RAY_CI_TUNE_AFFECTED"]
- instance_size: small
- commands:
- - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
- - TUNE_TESTING=1 ./ci/env/install-dependencies.sh
- - ./ci/env/env_info.sh
- - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=tf,-pytorch,-py37,-soft_imports,-gpu_only,-rllib python/ray/tune/...
- - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-tf,pytorch,-py37,-soft_imports,-gpu_only,-rllib python/ray/tune/...
-
 - label: ":octopus: :brain: Tune tests and examples {using RLlib}"
  conditions: ["NO_WHEELS_REQUIRED", "RAY_CI_TUNE_AFFECTED", "RAY_CI_RLLIB_AFFECTED"]
  instance_size: large

ci/pipeline/determine_tests_to_run.py

@@ -158,6 +158,7 @@ def get_commit_range():
  RAY_CI_LINUX_WHEELS_AFFECTED = 1
  RAY_CI_MACOS_WHEELS_AFFECTED = 1
  elif changed_file.startswith("python/ray/tune"):
+ RAY_CI_ML_AFFECTED = 1
  RAY_CI_DOC_AFFECTED = 1
  RAY_CI_TUNE_AFFECTED = 1
  RAY_CI_RLLIB_AFFECTED = 1