From 0f18ccee1d26652e09162ffc18b37220ca2b0430 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Mon, 26 Jun 2023 10:42:49 +0100
Subject: [PATCH 01/70] Move requirements files

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 .../requirements.txt => air/data-requirements.txt}                | 0
 .../requirements_dataset.txt => air/data-test-requirements.txt}   | 0
 .../{ml/requirements_dl.txt => air/dl-cpu-requirements.txt}       | 0
 .../requirements_ml_docker.txt => air/ml-docker-requirements.txt} | 0
 python/requirements/{ml => air}/requirements_no_deps.txt          | 0
 python/requirements/{ml => air}/requirements_upstream.txt         | 0
 .../{ml/requirements_rllib.txt => air/rllib-requirements.txt}     | 0
 .../{ml/requirements_train.txt => air/train-requirements.txt}     | 0
 .../{ml/requirements_tune.txt => air/tune-requirements.txt}       | 0
 .../lint-requirements.txt}                                        | 0
 .../{requirements_test.txt => requirements/test-requirements.txt} | 0
 11 files changed, 0 insertions(+), 0 deletions(-)
 rename python/requirements/{data_processing/requirements.txt => air/data-requirements.txt} (100%)
 rename python/requirements/{data_processing/requirements_dataset.txt => air/data-test-requirements.txt} (100%)
 rename python/requirements/{ml/requirements_dl.txt => air/dl-cpu-requirements.txt} (100%)
 rename python/requirements/{ml/requirements_ml_docker.txt => air/ml-docker-requirements.txt} (100%)
 rename python/requirements/{ml => air}/requirements_no_deps.txt (100%)
 rename python/requirements/{ml => air}/requirements_upstream.txt (100%)
 rename python/requirements/{ml/requirements_rllib.txt => air/rllib-requirements.txt} (100%)
 rename python/requirements/{ml/requirements_train.txt => air/train-requirements.txt} (100%)
 rename python/requirements/{ml/requirements_tune.txt => air/tune-requirements.txt} (100%)
 rename python/{requirements_linters.txt => requirements/lint-requirements.txt} (100%)
 rename python/{requirements_test.txt => requirements/test-requirements.txt} (100%)

diff --git a/python/requirements/data_processing/requirements.txt b/python/requirements/air/data-requirements.txt
similarity index 100%
rename from python/requirements/data_processing/requirements.txt
rename to python/requirements/air/data-requirements.txt
diff --git a/python/requirements/data_processing/requirements_dataset.txt b/python/requirements/air/data-test-requirements.txt
similarity index 100%
rename from python/requirements/data_processing/requirements_dataset.txt
rename to python/requirements/air/data-test-requirements.txt
diff --git a/python/requirements/ml/requirements_dl.txt b/python/requirements/air/dl-cpu-requirements.txt
similarity index 100%
rename from python/requirements/ml/requirements_dl.txt
rename to python/requirements/air/dl-cpu-requirements.txt
diff --git a/python/requirements/ml/requirements_ml_docker.txt b/python/requirements/air/ml-docker-requirements.txt
similarity index 100%
rename from python/requirements/ml/requirements_ml_docker.txt
rename to python/requirements/air/ml-docker-requirements.txt
diff --git a/python/requirements/ml/requirements_no_deps.txt b/python/requirements/air/requirements_no_deps.txt
similarity index 100%
rename from python/requirements/ml/requirements_no_deps.txt
rename to python/requirements/air/requirements_no_deps.txt
diff --git a/python/requirements/ml/requirements_upstream.txt b/python/requirements/air/requirements_upstream.txt
similarity index 100%
rename from python/requirements/ml/requirements_upstream.txt
rename to python/requirements/air/requirements_upstream.txt
diff --git a/python/requirements/ml/requirements_rllib.txt b/python/requirements/air/rllib-requirements.txt
similarity index 100%
rename from python/requirements/ml/requirements_rllib.txt
rename to python/requirements/air/rllib-requirements.txt
diff --git a/python/requirements/ml/requirements_train.txt b/python/requirements/air/train-requirements.txt
similarity index 100%
rename from python/requirements/ml/requirements_train.txt
rename to python/requirements/air/train-requirements.txt
diff --git a/python/requirements/ml/requirements_tune.txt b/python/requirements/air/tune-requirements.txt
similarity index 100%
rename from python/requirements/ml/requirements_tune.txt
rename to python/requirements/air/tune-requirements.txt
diff --git a/python/requirements_linters.txt b/python/requirements/lint-requirements.txt
similarity index 100%
rename from python/requirements_linters.txt
rename to python/requirements/lint-requirements.txt
diff --git a/python/requirements_test.txt b/python/requirements/test-requirements.txt
similarity index 100%
rename from python/requirements_test.txt
rename to python/requirements/test-requirements.txt

From 99c8e4b89ba84e76e9603bbbb522ae50a2200847 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Mon, 26 Jun 2023 10:47:49 +0100
Subject: [PATCH 02/70] replace

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 .buildkite/pipeline.gpu.yml                   | 12 +++----
 .buildkite/pipeline.gpu_large.yml             | 14 ++++----
 .github/CODEOWNERS                            |  4 +--
 ci/build/build-docker-images.py               | 14 ++++----
 ci/build/upload_build_info.sh                 |  2 +-
 ci/env/install-dependencies.sh                | 34 +++++++++----------
 doc/source/ray-contribute/development.rst     |  4 +--
 doc/source/ray-contribute/docs.ipynb          |  2 +-
 .../ray-contribute/getting-involved.rst       |  6 ++--
 docker/ray-ml/Dockerfile                      | 18 +++++-----
 .../requirements/air/dl-cpu-requirements.txt  |  4 +--
 .../air/ml-docker-requirements.txt            |  2 +-
 .../requirements/air/rllib-requirements.txt   |  4 +--
 .../requirements/air/train-requirements.txt   |  2 +-
 python/requirements/air/tune-requirements.txt |  2 +-
 release/long_running_tests/app_config.yaml    |  2 +-
 .../horovod/driver_requirements.txt           |  2 +-
 .../train/driver_requirements.txt             |  2 +-
 .../tune_rllib/driver_requirements.txt        |  2 +-
 release/rllib_tests/app_config.yaml           |  4 +--
 release/rllib_tests/debug_app_config.yaml     |  4 +--
 21 files changed, 70 insertions(+), 70 deletions(-)

diff --git a/.buildkite/pipeline.gpu.yml b/.buildkite/pipeline.gpu.yml
index e6a4ffb364ec2..34ce078391030 100644
--- a/.buildkite/pipeline.gpu.yml
+++ b/.buildkite/pipeline.gpu.yml
@@ -6,7 +6,7 @@
 #  commands:
 #    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
 #    - TUNE_TESTING=1 ./ci/env/install-dependencies.sh
-#    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+#    - pip install -Ur ./python/requirements/air/ml-docker-requirements.txt
 #    - ./ci/env/env_info.sh
 #    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=gpu python/ray/tune/...
 
@@ -16,7 +16,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - RLLIB_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+    - pip install -Ur ./python/requirements/air/ml-docker-requirements.txt
     - ./ci/env/env_info.sh
     # --jobs 1 is necessary as we only have 1 GPU on the machine and running tests in parallel
     # would cause timeouts as the other scripts would wait for the GPU to become available.
@@ -40,7 +40,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+    - pip install -Ur ./python/requirements/air/ml-docker-requirements.txt
     - bazel test --config=ci $(./ci/run/bazel_export_options) --test_tag_filters=gpu python/ray/serve/...
 
 # Todo: enable once tests pass
@@ -49,7 +49,7 @@
 #  commands:
 #    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
 #    - RLLIB_TESTING=1 ./ci/env/install-dependencies.sh
-#    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+#    - pip install -Ur ./python/requirements/air/ml-docker-requirements.txt
 #    - ./ci/env/env_info.sh
 #    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only
 #      --test_tag_filters=examples_C,examples_D --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 --test_env=RLLIB_NUM_GPUS=1 rllib/...
@@ -60,7 +60,7 @@
 #  commands:
 #    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
 #    - RLLIB_TESTING=1 ./ci/env/install-dependencies.sh
-#    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+#    - pip install -Ur ./python/requirements/air/ml-docker-requirements.txt
 #    - ./ci/env/env_info.sh
 #    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only
 #      --test_tag_filters=examples_E,examples_F,examples_G,examples_H,examples_I,examples_J,examples_K,examples_L,examples_M,examples_N,examples_O,examples_P --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 --test_env=RLLIB_NUM_GPUS=1
@@ -72,7 +72,7 @@
 #  commands:
 #    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
 #    - RLLIB_TESTING=1 ./ci/env/install-dependencies.sh
-#    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+#    - pip install -Ur ./python/requirements/air/ml-docker-requirements.txt
 #    - ./ci/env/env_info.sh
 #    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only
 #      --test_tag_filters=examples_Q,examples_R,examples_S,examples_T,examples_U,examples_V,examples_W,examples_X,examples_Y,examples_Z --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 --test_env=RLLIB_NUM_GPUS=1
diff --git a/.buildkite/pipeline.gpu_large.yml b/.buildkite/pipeline.gpu_large.yml
index c79b7b8bbd2c5..196ced5870b47 100644
--- a/.buildkite/pipeline.gpu_large.yml
+++ b/.buildkite/pipeline.gpu_large.yml
@@ -5,7 +5,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+    - pip install -Ur ./python/requirements/air/ml-docker-requirements.txt
     - ./ci/env/install-horovod.sh
     - ./ci/env/env_info.sh
     - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=gpu,gpu_only,-ray_air python/ray/train/...
@@ -15,7 +15,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - TRAIN_TESTING=1 DATA_PROCESSING_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+    - pip install -Ur ./python/requirements/air/ml-docker-requirements.txt
     - ./ci/env/env_info.sh
     - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=datasets_train,-doctest doc/...
 
@@ -25,7 +25,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - RLLIB_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+    - pip install -Ur ./python/requirements/air/ml-docker-requirements.txt
     - ./ci/env/env_info.sh
     # --jobs 2 is necessary as we only need to have at least 2 gpus on the machine
     # and running tests in parallel would cause timeouts as the other scripts would
@@ -42,7 +42,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - DATA_PROCESSING_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+    - pip install -Ur ./python/requirements/air/ml-docker-requirements.txt
     - ./ci/env/install-horovod.sh
     - ./ci/env/env_info.sh
     - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=gpu python/ray/air/... python/ray/train/...
@@ -53,7 +53,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - DOC_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+    - pip install -Ur ./python/requirements/air/ml-docker-requirements.txt
     - ./ci/env/env_info.sh
     # Test examples with newer version of `transformers`
     # TODO(amogkam): Remove when https://github.com/ray-project/ray/issues/36011
@@ -65,7 +65,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - DOC_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+    - pip install -Ur ./python/requirements/air/ml-docker-requirements.txt
     - ./ci/env/install-horovod.sh
     - ./ci/env/env_info.sh
     # Test examples with newer version of `transformers`
@@ -82,7 +82,7 @@
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - NO_DASHBOARD=1 ./ci/env/install-minimal.sh 3.8
     - PYTHON=3.8 DOC_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
+    - pip install -Ur ./python/requirements/air/ml-docker-requirements.txt
     - pip uninstall -y pytorch-lightning
     - pip install lightning==2.0.0
     - ./ci/env/env_info.sh
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index d206f7ef9df48..d1c7f6e8c80f5 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -101,8 +101,8 @@
 /doc/source/ray-air/ @richardliaw @gjoliver @krfricke @xwjiang2010 @amogkam @matthewdeng @Yard1 @maxpumperla @ray-project/ray-docs
 
 # ML Docker Dependencies
-/python/requirements/ml/requirements_dl.txt @amogkam @krfricke @richardliaw @matthewdeng
-/python/requirements/ml/requirements_ml_docker.txt @amogkam @krfricke @richardliaw @matthewdeng
+/python/requirements/air/dl-cpu-requirements.txt @amogkam @krfricke @richardliaw @matthewdeng
+/python/requirements/air/ml-docker-requirements.txt @amogkam @krfricke @richardliaw @matthewdeng
 
 # Ray symbol export
 /src/ray/ray_version_script.lds @iycheng @ericl @scv119
diff --git a/ci/build/build-docker-images.py b/ci/build/build-docker-images.py
index b79f1ee7fff22..351f2d935ea0b 100644
--- a/ci/build/build-docker-images.py
+++ b/ci/build/build-docker-images.py
@@ -426,13 +426,13 @@ def prep_ray_ml():
         "python/requirements.txt",
     ]
     ml_requirements_files = [
-        "python/requirements/ml/requirements_ml_docker.txt",
-        "python/requirements/ml/requirements_dl.txt",
-        "python/requirements/ml/requirements_tune.txt",
-        "python/requirements/ml/requirements_rllib.txt",
-        "python/requirements/ml/requirements_train.txt",
-        "python/requirements/ml/requirements_upstream.txt",
-        "python/requirements/ml/requirements_no_deps.txt",
+        "python/requirements/air/ml-docker-requirements.txt",
+        "python/requirements/air/dl-cpu-requirements.txt",
+        "python/requirements/air/tune-requirements.txt",
+        "python/requirements/air/rllib-requirements.txt",
+        "python/requirements/air/train-requirements.txt",
+        "python/requirements/air/requirements_upstream.txt",
+        "python/requirements/air/requirements_no_deps.txt",
     ]
     # We don't need these in the ml docker image
     ignore_requirements = [
diff --git a/ci/build/upload_build_info.sh b/ci/build/upload_build_info.sh
index 6876d47cd3c1b..01a1788217227 100755
--- a/ci/build/upload_build_info.sh
+++ b/ci/build/upload_build_info.sh
@@ -35,7 +35,7 @@ if [[ -z "${BUILDKITE-}" ]]; then
     aws s3 cp --recursive /tmp/bazel_event_logs "${DST}"
 else
     # Codepath for Buildkite
-    # Keep cryptography/openssl in sync with `requirements_test.txt`
+    # Keep cryptography/openssl in sync with `requirements/test-requirements.txt`
     pip install -q -c "${RAY_DIR}/python/requirements.txt" docker aws_requests_auth boto3 cryptography==38.0.1 PyOpenSSL==22.1.0
     python .buildkite/copy_files.py --destination logs --path /tmp/bazel_event_logs
 fi
diff --git a/ci/env/install-dependencies.sh b/ci/env/install-dependencies.sh
index 2e855c3c2daa3..4efbb9e50902e 100755
--- a/ci/env/install-dependencies.sh
+++ b/ci/env/install-dependencies.sh
@@ -192,7 +192,7 @@ install_shellcheck() {
 }
 
 install_linters() {
-  pip install -r "${WORKSPACE_DIR}"/python/requirements_linters.txt
+  pip install -r "${WORKSPACE_DIR}"/python/requirements/lint-requirements.txt
 
   install_shellcheck
 }
@@ -328,7 +328,7 @@ install_pip_packages() {
   # files.
   delayed_packages=()
 
-  requirements_files+=("${WORKSPACE_DIR}/python/requirements_test.txt")
+  requirements_files+=("${WORKSPACE_DIR}/python/requirements/test-requirements.txt")
 
   if [ "${LINT-}" = 1 ]; then
     install_linters
@@ -350,8 +350,8 @@ install_pip_packages() {
 
   # Additional RLlib test dependencies.
   if [ "${RLLIB_TESTING-}" = 1 ] || [ "${DOC_TESTING-}" = 1 ]; then
-    requirements_files+=("${WORKSPACE_DIR}/python/requirements/ml/requirements_rllib.txt")
-    #TODO(amogkam): Add this back to requirements_rllib.txt once mlagents no longer pins torch<1.9.0 version.
+    requirements_files+=("${WORKSPACE_DIR}/python/requirements/air/rllib-requirements.txt")
+    #TODO(amogkam): Add this back to rllib-requirements.txt once mlagents no longer pins torch<1.9.0 version.
     pip install --no-dependencies mlagents==0.28.0
   fi
 
@@ -362,18 +362,18 @@ install_pip_packages() {
 
   # Additional Train test dependencies.
   if [ "${TRAIN_TESTING-}" = 1 ] || [ "${DOC_TESTING-}" = 1 ]; then
-    requirements_files+=("${WORKSPACE_DIR}/python/requirements/ml/requirements_train.txt")
+    requirements_files+=("${WORKSPACE_DIR}/python/requirements/air/train-requirements.txt")
     install_ml_no_deps=1
   fi
 
   # Additional Tune/Doc test dependencies.
   if [ "${TUNE_TESTING-}" = 1 ] || [ "${DOC_TESTING-}" = 1 ]; then
-    requirements_files+=("${WORKSPACE_DIR}/python/requirements/ml/requirements_tune.txt")
+    requirements_files+=("${WORKSPACE_DIR}/python/requirements/air/tune-requirements.txt")
   fi
 
   # For Tune, install upstream dependencies.
   if [ "${TUNE_TESTING-}" = 1 ] ||  [ "${DOC_TESTING-}" = 1 ]; then
-    requirements_files+=("${WORKSPACE_DIR}/python/requirements/ml/requirements_upstream.txt")
+    requirements_files+=("${WORKSPACE_DIR}/python/requirements/air/requirements_upstream.txt")
   fi
 
   # Additional dependency for Ludwig.
@@ -386,7 +386,7 @@ install_pip_packages() {
   fi
 
   # Additional dependency for time series libraries.
-  # This cannot be included in requirements_tune.txt as it has conflicting
+  # This cannot be included in tune-requirements.txt as it has conflicting
   # dependencies.
   if [ "${INSTALL_TIMESERIES_LIBS-}" = 1 ]; then
     requirements_packages+=("statsforecast==1.5.0")
@@ -396,10 +396,10 @@ install_pip_packages() {
 
   # Data processing test dependencies.
   if [ "${DATA_PROCESSING_TESTING-}" = 1 ] || [ "${DOC_TESTING-}" = 1 ]; then
-    requirements_files+=("${WORKSPACE_DIR}/python/requirements/data_processing/requirements.txt")
+    requirements_files+=("${WORKSPACE_DIR}/python/requirements/air/data-requirements.txt")
   fi
   if [ "${DATA_PROCESSING_TESTING-}" = 1 ]; then
-    requirements_files+=("${WORKSPACE_DIR}/python/requirements/data_processing/requirements_dataset.txt")
+    requirements_files+=("${WORKSPACE_DIR}/python/requirements/air/data-test-requirements.txt")
     if [ -n "${ARROW_VERSION-}" ]; then
       if [ "${ARROW_VERSION-}" = nightly ]; then
         delayed_packages+=("--extra-index-url")
@@ -419,7 +419,7 @@ install_pip_packages() {
   if [ "${install_ml_no_deps}" = 1 ]; then
     # Install these requirements first. Their dependencies may be overwritten later
     # by the main install.
-    pip install -r "${WORKSPACE_DIR}/python/requirements/ml/requirements_no_deps.txt"
+    pip install -r "${WORKSPACE_DIR}/python/requirements/air/requirements_no_deps.txt"
   fi
 
   retry_pip_install "CC=gcc pip install -Ur ${WORKSPACE_DIR}/python/requirements.txt"
@@ -431,10 +431,10 @@ install_pip_packages() {
         # Install right away, as some dependencies (e.g. torch-spline-conv) need
         # torch to be installed for their own install.
         pip install -U "torch==${TORCH_VERSION-1.9.0}" "torchvision==${TORCHVISION_VERSION-0.10.0}"
-        # We won't add requirements_dl.txt as it would otherwise overwrite our custom
+        # We won't add dl-cpu-requirements.txt as it would otherwise overwrite our custom
         # torch. Thus we have also have to install tensorflow manually.
-        TF_PACKAGE=$(grep "tensorflow==" "${WORKSPACE_DIR}/python/requirements/ml/requirements_dl.txt")
-        TFPROB_PACKAGE=$(grep "tensorflow-probability==" "${WORKSPACE_DIR}/python/requirements/ml/requirements_dl.txt")
+        TF_PACKAGE=$(grep "tensorflow==" "${WORKSPACE_DIR}/python/requirements/air/dl-cpu-requirements.txt")
+        TFPROB_PACKAGE=$(grep "tensorflow-probability==" "${WORKSPACE_DIR}/python/requirements/air/dl-cpu-requirements.txt")
 
         # %%;* deletes everything after ; to get rid of e.g. python version specifiers
         pip install -U "${TF_PACKAGE%%;*}" "${TFPROB_PACKAGE%%;*}"
@@ -442,12 +442,12 @@ install_pip_packages() {
         # Otherwise, use pinned default torch version.
         # Again, install right away, as some dependencies (e.g. torch-spline-conv) need
         # torch to be installed for their own install.
-        TORCH_PACKAGE=$(grep "torch==" "${WORKSPACE_DIR}/python/requirements/ml/requirements_dl.txt")
-        TORCHVISION_PACKAGE=$(grep "torchvision==" "${WORKSPACE_DIR}/python/requirements/ml/requirements_dl.txt")
+        TORCH_PACKAGE=$(grep "torch==" "${WORKSPACE_DIR}/python/requirements/air/dl-cpu-requirements.txt")
+        TORCHVISION_PACKAGE=$(grep "torchvision==" "${WORKSPACE_DIR}/python/requirements/air/dl-cpu-requirements.txt")
 
         # %%;* deletes everything after ; to get rid of e.g. python version specifiers
         pip install "${TORCH_PACKAGE%%;*}" "${TORCHVISION_PACKAGE%%;*}"
-        requirements_files+=("${WORKSPACE_DIR}/python/requirements/ml/requirements_dl.txt")
+        requirements_files+=("${WORKSPACE_DIR}/python/requirements/air/dl-cpu-requirements.txt")
       fi
   fi
 
diff --git a/doc/source/ray-contribute/development.rst b/doc/source/ray-contribute/development.rst
index 736c2906cf4b1..67ba0ce9bc93d 100644
--- a/doc/source/ray-contribute/development.rst
+++ b/doc/source/ray-contribute/development.rst
@@ -360,13 +360,13 @@ Dependencies for the linter (``scripts/format.sh``) can be installed with:
 
 .. code-block:: shell
 
- pip install -r python/requirements_linters.txt
+ pip install -r python/requirements/lint-requirements.txt
 
 Dependencies for running Ray unit tests under ``python/ray/tests`` can be installed with:
 
 .. code-block:: shell
 
- pip install -c python/requirements.txt -r python/requirements_test.txt
+ pip install -c python/requirements.txt -r python/requirements/test-requirements.txt
 
 Requirement files for running Ray Data / ML library tests are under ``python/requirements/``.
 
diff --git a/doc/source/ray-contribute/docs.ipynb b/doc/source/ray-contribute/docs.ipynb
index b5cb3d3c04384..23f89b38d1c3f 100644
--- a/doc/source/ray-contribute/docs.ipynb
+++ b/doc/source/ray-contribute/docs.ipynb
@@ -66,7 +66,7 @@
     "Install the dependencies for our linters to ensure your changes comply with our style guide.\n",
     "\n",
     "```shell\n",
-    "pip install -r ../python/requirements_linters.txt\n",
+    "pip install -r ../python/requirements/lint-requirements.txt\n",
     "```\n",
     "\n",
     "Build the documentation by running the following command:\n",
diff --git a/doc/source/ray-contribute/getting-involved.rst b/doc/source/ray-contribute/getting-involved.rst
index 9bccb133a64d8..9f8dd61e960ad 100644
--- a/doc/source/ray-contribute/getting-involved.rst
+++ b/doc/source/ray-contribute/getting-involved.rst
@@ -96,7 +96,7 @@ If you are running tests for the first time, you can install the required depend
 
 .. code-block:: shell
 
-    pip install -c python/requirements.txt -r python/requirements_test.txt
+    pip install -c python/requirements.txt -r python/requirements/test-requirements.txt
 
 Testing for Python development
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -222,11 +222,11 @@ Lint and Formatting
 
 We also have tests for code formatting and linting that need to pass before merge.
 
-* For Python formatting, install the `required dependencies <https://github.com/ray-project/ray/blob/master/python/requirements_linters.txt>`_ first with:
+* For Python formatting, install the `required dependencies <https://github.com/ray-project/ray/blob/master/python/requirements/lint-requirements.txt>`_ first with:
 
 .. code-block:: shell
 
-  pip install -r python/requirements_linters.txt
+  pip install -r python/requirements/lint-requirements.txt
 
 * If developing for C++, you will need `clang-format <https://www.kernel.org/doc/html/latest/process/clang-format.html>`_ version ``12`` (download this version of Clang from `here <http://releases.llvm.org/download.html>`_)
 
diff --git a/docker/ray-ml/Dockerfile b/docker/ray-ml/Dockerfile
index 72949a1a23741..1905c8a88499b 100644
--- a/docker/ray-ml/Dockerfile
+++ b/docker/ray-ml/Dockerfile
@@ -5,11 +5,11 @@ ARG PYTHON_MINOR_VERSION=7
 # We have to uninstall wrapt this way for Tensorflow compatibility
 COPY requirements.txt ./
 COPY requirements_no_deps.txt ./
-COPY requirements_dl.txt ./
-COPY requirements_ml_docker.txt ./
-COPY requirements_rllib.txt ./
-COPY requirements_tune.txt ./
-COPY requirements_train.txt ./
+COPY dl-cpu-requirements.txt ./
+COPY ml-docker-requirements.txt ./
+COPY rllib-requirements.txt ./
+COPY tune-requirements.txt ./
+COPY train-requirements.txt ./
 COPY requirements_upstream.txt ./
 
 RUN sudo apt-get update \
@@ -31,14 +31,14 @@ RUN sudo apt-get update \
     # Install other requirements. Keep pinned requirements bounds as constraints
     && $HOME/anaconda3/bin/pip --no-cache-dir install -U \
            -c requirements.txt \
-           -r requirements_rllib.txt \
-           -r requirements_train.txt \
-           -r requirements_tune.txt \
+           -r rllib-requirements.txt \
+           -r train-requirements.txt \
+           -r tune-requirements.txt \
            -r requirements_upstream.txt \
     # explicitly install (overwrite) pytorch with CUDA support
     && $HOME/anaconda3/bin/pip --no-cache-dir install -U \
            -c requirements.txt \
-           -r requirements_ml_docker.txt \
+           -r ml-docker-requirements.txt \
     # Remove dataclasses & typing because they are included in Python > 3.6
     && if [ $(python -c 'import sys; print(sys.version_info.minor)') != "6" ]; then \
         $HOME/anaconda3/bin/pip uninstall dataclasses typing -y; fi  \
diff --git a/python/requirements/air/dl-cpu-requirements.txt b/python/requirements/air/dl-cpu-requirements.txt
index 9e395984cb130..4c25ec097b5ad 100644
--- a/python/requirements/air/dl-cpu-requirements.txt
+++ b/python/requirements/air/dl-cpu-requirements.txt
@@ -1,12 +1,12 @@
 # These requirements are used for the CI and CPU-only Docker images so we install CPU only versions of torch.
-# For GPU Docker images, you should install requirements_ml_docker.txt afterwards.
+# For GPU Docker images, you should install ml-docker-requirements.txt afterwards.
 
 tensorflow==2.11.0; sys_platform != 'darwin' or platform_machine != 'arm64'
 tensorflow-macos==2.11.0; sys_platform == 'darwin' and platform_machine == 'arm64'
 tensorflow-probability==0.19.0
 tensorflow-datasets
 
-# If you make changes below this line, please also make the corresponding changes to `requirements_ml_docker.txt`
+# If you make changes below this line, please also make the corresponding changes to `ml-docker-requirements.txt`
 # and to `install-dependencies.sh`!
 
 --extra-index-url https://download.pytorch.org/whl/cpu  # for CPU versions of torch, torchvision
diff --git a/python/requirements/air/ml-docker-requirements.txt b/python/requirements/air/ml-docker-requirements.txt
index 3a027b9b48698..9609f6f40eee5 100644
--- a/python/requirements/air/ml-docker-requirements.txt
+++ b/python/requirements/air/ml-docker-requirements.txt
@@ -6,7 +6,7 @@ ipywidgets>=8
 # Needed for Ray Client error message serialization/deserialization.
 tblib
 
-# If you make changes below this line, please also make the corresponding changes to `requirements_dl.txt`!
+# If you make changes below this line, please also make the corresponding changes to `dl-cpu-requirements.txt`!
 
 --extra-index-url https://download.pytorch.org/whl/cu116  # for GPU versions of torch, torchvision
 --find-links https://data.pyg.org/whl/torch-1.13.0+cu116.html  # for GPU versions of torch-scatter, torch-sparse, torch-cluster, torch-spline-conv
diff --git a/python/requirements/air/rllib-requirements.txt b/python/requirements/air/rllib-requirements.txt
index 5759d04eb2b81..d4d2930e4edfa 100644
--- a/python/requirements/air/rllib-requirements.txt
+++ b/python/requirements/air/rllib-requirements.txt
@@ -1,4 +1,4 @@
--r requirements_dl.txt
+-r dl-cpu-requirements.txt
 
 # Environment adapters.
 # ---------------------
@@ -11,7 +11,7 @@ mujoco-py<2.2,>=2.1
 # Kaggle envs.
 kaggle_environments==1.7.11
 # Unity3D testing
-# TODO(sven): Add this back to requirements_rllib.txt once mlagents no longer pins torch<1.9.0 version.
+# TODO(sven): Add this back to rllib-requirements.txt once mlagents no longer pins torch<1.9.0 version.
 #mlagents==0.28.0
 mlagents_envs==0.28.0
 # For tests on PettingZoo's multi-agent envs.
diff --git a/python/requirements/air/train-requirements.txt b/python/requirements/air/train-requirements.txt
index 9ced832dc787f..57ddb70f882be 100644
--- a/python/requirements/air/train-requirements.txt
+++ b/python/requirements/air/train-requirements.txt
@@ -1,6 +1,6 @@
 # TODO(train-team): Remove this once Ray Train is deprecated.
 
--r requirements_dl.txt
+-r dl-cpu-requirements.txt
 
 mlflow==1.30.0; python_version <= '3.7'
 mlflow==2.4.1; python_version > '3.7'
diff --git a/python/requirements/air/tune-requirements.txt b/python/requirements/air/tune-requirements.txt
index a4be1c4de2ac8..0eb4a82632e7d 100644
--- a/python/requirements/air/tune-requirements.txt
+++ b/python/requirements/air/tune-requirements.txt
@@ -1,4 +1,4 @@
--r requirements_dl.txt
+-r dl-cpu-requirements.txt
 
 aim==3.16.1
 ax-platform[mysql]==0.2.6; python_version < '3.8'
diff --git a/release/long_running_tests/app_config.yaml b/release/long_running_tests/app_config.yaml
index 5d13555ff713a..ae78399b8c85e 100755
--- a/release/long_running_tests/app_config.yaml
+++ b/release/long_running_tests/app_config.yaml
@@ -7,7 +7,7 @@ debian_packages:
 
 python:
   pip_packages:
-    # These dependencies should be handled by requirements_rllib.txt and requirements_ml_docker.txt and removed here
+    # These dependencies should be handled by rllib-requirements.txt and ml-docker-requirements.txt and removed here
     - gym>=0.21.0,<0.24.1
     - ale-py==0.7.5
     - pytest
diff --git a/release/ml_user_tests/horovod/driver_requirements.txt b/release/ml_user_tests/horovod/driver_requirements.txt
index 867abb6f6bd4c..3caff26f694ff 100755
--- a/release/ml_user_tests/horovod/driver_requirements.txt
+++ b/release/ml_user_tests/horovod/driver_requirements.txt
@@ -2,7 +2,7 @@
 # The cluster uses ray-ml Docker image.
 # ray-ml Docker image installs dependencies from ray/python/requirements/ml/ directory.
 # We constrain on these requirements file so that the same versions are installed.
--c ../../../python/requirements/ml/requirements_dl.txt
+-c ../../../python/requirements/air/dl-cpu-requirements.txt
 
 torch
 torchvision
\ No newline at end of file
diff --git a/release/ml_user_tests/train/driver_requirements.txt b/release/ml_user_tests/train/driver_requirements.txt
index eb46d707860e9..30415594267f1 100755
--- a/release/ml_user_tests/train/driver_requirements.txt
+++ b/release/ml_user_tests/train/driver_requirements.txt
@@ -2,7 +2,7 @@
 # The cluster uses ray-ml Docker image.
 # ray-ml Docker image installs dependencies from ray/python/requirements/ml/ directory.
 # We constrain on these requirements file so that the same versions are installed.
--c ../../../python/requirements/ml/requirements_dl.txt
+-c ../../../python/requirements/air/dl-cpu-requirements.txt
 
 torch
 tensorflow
\ No newline at end of file
diff --git a/release/ml_user_tests/tune_rllib/driver_requirements.txt b/release/ml_user_tests/tune_rllib/driver_requirements.txt
index 2ec0944aef35a..584e2d9fe7e6b 100755
--- a/release/ml_user_tests/tune_rllib/driver_requirements.txt
+++ b/release/ml_user_tests/tune_rllib/driver_requirements.txt
@@ -2,7 +2,7 @@
 # The cluster uses ray-ml Docker image.
 # ray-ml Docker image installs dependencies from ray/python/requirements/ml/ directory.
 # We constrain on these requirements file so that the same versions are installed.
--c ../../../python/requirements/ml/requirements_dl.txt
+-c ../../../python/requirements/air/dl-cpu-requirements.txt
 
 tensorflow
 torch
diff --git a/release/rllib_tests/app_config.yaml b/release/rllib_tests/app_config.yaml
index fe802591f9b09..29027b64e53da 100755
--- a/release/rllib_tests/app_config.yaml
+++ b/release/rllib_tests/app_config.yaml
@@ -13,8 +13,8 @@ debian_packages:
 
 python:
   pip_packages:
-  ## These dependencies should be handled by requirements_rllib.txt and
-  ## requirements_ml_docker.txt and removed here
+  ## These dependencies should be handled by rllib-requirements.txt and
+  ## ml-docker-requirements.txt and removed here
   - gymnasium[atari,mujoco]==0.26.3
   - ale-py==0.8.0
   - gym==0.26.2
diff --git a/release/rllib_tests/debug_app_config.yaml b/release/rllib_tests/debug_app_config.yaml
index c51fbcc4d39b0..4230e4f0cc683 100755
--- a/release/rllib_tests/debug_app_config.yaml
+++ b/release/rllib_tests/debug_app_config.yaml
@@ -13,8 +13,8 @@ debian_packages:
 
 python:
   pip_packages:
-  ## These dependencies should be handled by requirements_rllib.txt and
-  ## requirements_ml_docker.txt and removed here
+  ## These dependencies should be handled by rllib-requirements.txt and
+  ## ml-docker-requirements.txt and removed here
   - gymnasium[atari,mujoco]==0.26.3
   - ale-py==0.8.0
   - gym==0.26.2

From 2ed7c756c247a07360e021e8e4461e458aa917e6 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Mon, 26 Jun 2023 11:20:52 +0100
Subject: [PATCH 03/70] split some files

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 ci/build/build-docker-images.py               |  2 -
 ci/env/install-dependencies.sh                | 19 +-------
 docker/ray-ml/Dockerfile                      |  9 +---
 .../accelerate/accelerate_trainer.py          |  1 -
 python/requirements/air/core-requirements.txt | 25 ++++++++++
 python/requirements/air/data-requirements.txt |  3 --
 .../air/data-test-requirements.txt            |  4 +-
 .../requirements/air/dl-cpu-requirements.txt  |  1 +
 ...quirements.txt => dl-gpu-requirements.txt} |  8 ----
 .../requirements/air/requirements_no_deps.txt |  6 ---
 .../air/requirements_upstream.txt             |  9 ----
 .../requirements/air/rllib-requirements.txt   | 36 +-------------
 .../air/rllib-test-requirements.txt           | 35 ++++++++++++++
 .../requirements/air/train-requirements.txt   | 26 +---------
 .../air/train-test-requirements.txt           |  2 +
 python/requirements/air/tune-requirements.txt | 48 ++++---------------
 .../air/tune-test-requirements.txt            | 23 +++++++++
 .../docker/ml-docker-requirements.txt         |  8 ++++
 .../docker/ray-docker-requirements.txt        |  2 +
 python/requirements/test-requirements.txt     |  1 +
 20 files changed, 113 insertions(+), 155 deletions(-)
 create mode 100644 python/requirements/air/core-requirements.txt
 rename python/requirements/air/{ml-docker-requirements.txt => dl-gpu-requirements.txt} (79%)
 delete mode 100644 python/requirements/air/requirements_no_deps.txt
 delete mode 100644 python/requirements/air/requirements_upstream.txt
 create mode 100644 python/requirements/air/rllib-test-requirements.txt
 create mode 100644 python/requirements/air/train-test-requirements.txt
 create mode 100644 python/requirements/air/tune-test-requirements.txt
 create mode 100644 python/requirements/docker/ml-docker-requirements.txt
 create mode 100644 python/requirements/docker/ray-docker-requirements.txt

diff --git a/ci/build/build-docker-images.py b/ci/build/build-docker-images.py
index 351f2d935ea0b..9df3f6c79e530 100644
--- a/ci/build/build-docker-images.py
+++ b/ci/build/build-docker-images.py
@@ -431,8 +431,6 @@ def prep_ray_ml():
         "python/requirements/air/tune-requirements.txt",
         "python/requirements/air/rllib-requirements.txt",
         "python/requirements/air/train-requirements.txt",
-        "python/requirements/air/requirements_upstream.txt",
-        "python/requirements/air/requirements_no_deps.txt",
     ]
     # We don't need these in the ml docker image
     ignore_requirements = [
diff --git a/ci/env/install-dependencies.sh b/ci/env/install-dependencies.sh
index 4efbb9e50902e..64ad41279c2d8 100755
--- a/ci/env/install-dependencies.sh
+++ b/ci/env/install-dependencies.sh
@@ -355,15 +355,9 @@ install_pip_packages() {
     pip install --no-dependencies mlagents==0.28.0
   fi
 
-  # Some Ray Train dependencies have to be installed with --no-deps,
-  # as sub-dependencies conflict. The packages still work for our workflows.
-  # Todo(krfricke): Try to remove once we move to Python 3.8 in CI.
-  local install_ml_no_deps=0
-
   # Additional Train test dependencies.
   if [ "${TRAIN_TESTING-}" = 1 ] || [ "${DOC_TESTING-}" = 1 ]; then
     requirements_files+=("${WORKSPACE_DIR}/python/requirements/air/train-requirements.txt")
-    install_ml_no_deps=1
   fi
 
   # Additional Tune/Doc test dependencies.
@@ -371,13 +365,8 @@ install_pip_packages() {
     requirements_files+=("${WORKSPACE_DIR}/python/requirements/air/tune-requirements.txt")
   fi
 
-  # For Tune, install upstream dependencies.
-  if [ "${TUNE_TESTING-}" = 1 ] ||  [ "${DOC_TESTING-}" = 1 ]; then
-    requirements_files+=("${WORKSPACE_DIR}/python/requirements/air/requirements_upstream.txt")
-  fi
-
   # Additional dependency for Ludwig.
-  # This cannot be included in requirements_upstream.txt as it has conflicting
+  # This cannot be included in requirements files as it has conflicting
   # dependencies with Modin.
   if [ "${INSTALL_LUDWIG-}" = 1 ]; then
     # TODO: eventually pin this to master.
@@ -416,12 +405,6 @@ install_pip_packages() {
     fi
   fi
 
-  if [ "${install_ml_no_deps}" = 1 ]; then
-    # Install these requirements first. Their dependencies may be overwritten later
-    # by the main install.
-    pip install -r "${WORKSPACE_DIR}/python/requirements/air/requirements_no_deps.txt"
-  fi
-
   retry_pip_install "CC=gcc pip install -Ur ${WORKSPACE_DIR}/python/requirements.txt"
 
   # Install deeplearning libraries (Torch + TensorFlow)
diff --git a/docker/ray-ml/Dockerfile b/docker/ray-ml/Dockerfile
index 1905c8a88499b..6623a971b4a08 100644
--- a/docker/ray-ml/Dockerfile
+++ b/docker/ray-ml/Dockerfile
@@ -4,13 +4,11 @@ ARG PYTHON_MINOR_VERSION=7
 
 # We have to uninstall wrapt this way for Tensorflow compatibility
 COPY requirements.txt ./
-COPY requirements_no_deps.txt ./
 COPY dl-cpu-requirements.txt ./
 COPY ml-docker-requirements.txt ./
 COPY rllib-requirements.txt ./
 COPY tune-requirements.txt ./
 COPY train-requirements.txt ./
-COPY requirements_upstream.txt ./
 
 RUN sudo apt-get update \
     && sudo apt-get install -y gcc \
@@ -21,11 +19,7 @@ RUN sudo apt-get update \
         unzip \
         unrar \
     && $HOME/anaconda3/bin/pip --no-cache-dir install -U pip pip-tools \
-    # Install no-deps requirements. Their dependencies may be overwritten
-    # in subsequent steps
-    && $HOME/anaconda3/bin/pip --no-cache-dir install -U \
-           -r requirements_no_deps.txt \
-        # Then, install requirements
+        # Install requirements
     && $HOME/anaconda3/bin/pip --no-cache-dir install -U \
            -r requirements.txt \
     # Install other requirements. Keep pinned requirements bounds as constraints
@@ -34,7 +28,6 @@ RUN sudo apt-get update \
            -r rllib-requirements.txt \
            -r train-requirements.txt \
            -r tune-requirements.txt \
-           -r requirements_upstream.txt \
     # explicitly install (overwrite) pytorch with CUDA support
     && $HOME/anaconda3/bin/pip --no-cache-dir install -U \
            -c requirements.txt \
diff --git a/python/ray/train/huggingface/accelerate/accelerate_trainer.py b/python/ray/train/huggingface/accelerate/accelerate_trainer.py
index b906acd8e2e01..e53e6990110df 100644
--- a/python/ray/train/huggingface/accelerate/accelerate_trainer.py
+++ b/python/ray/train/huggingface/accelerate/accelerate_trainer.py
@@ -107,7 +107,6 @@ def train_loop_per_worker():
     - Type of launcher
 
     This Trainer requires ``accelerate>=0.17.0`` package.
-    It is tested with ``accelerate==0.17.1``.
 
     Example:
         .. code-block:: python
diff --git a/python/requirements/air/core-requirements.txt b/python/requirements/air/core-requirements.txt
new file mode 100644
index 0000000000000..7649a760efaa1
--- /dev/null
+++ b/python/requirements/air/core-requirements.txt
@@ -0,0 +1,25 @@
+# ML tracking integrations
+comet-ml==3.31.9
+mlflow==1.30.0; python_version <= '3.7'
+mlflow==2.4.1; python_version > '3.7'
+wandb==0.13.4
+
+# ML training frameworks
+xgboost==1.6.2; python_version <= '3.7'
+xgboost==1.7.6; python_version > '3.7'
+xgboost_ray==0.1.15
+
+lightgbm==3.3.5
+lightgbm_ray==0.1.8
+
+# Huggingface
+transformers==4.19.1
+accelerate==0.20.3
+
+# DL libraries
+-r dl-cpu-requirements.txt
+
+# Cloud storage tools
+s3fs==2023.1.0; python_version < '3.8'
+s3fs==2023.5.0; python_version >= '3.8'
+``
\ No newline at end of file
diff --git a/python/requirements/air/data-requirements.txt b/python/requirements/air/data-requirements.txt
index c0462a4220322..8c8080a5dd448 100644
--- a/python/requirements/air/data-requirements.txt
+++ b/python/requirements/air/data-requirements.txt
@@ -7,11 +7,8 @@ aioboto3==11.0.1; python_version < '3.8'
 aioboto3==11.2.0; python_version >= '3.8'
 crc32c==2.3
 flask_cors
-s3fs==2023.1.0; python_version < '3.8'
-s3fs==2023.5.0; python_version >= '3.8'
 modin==0.12.1; python_version < '3.8'
 modin==0.22.2; python_version >= '3.8'
-pytest-repeat
 raydp>=0.0.dev0
 responses==0.13.4
 pymars>=0.8.3
diff --git a/python/requirements/air/data-test-requirements.txt b/python/requirements/air/data-test-requirements.txt
index 1b614c54bcb18..894f45e001ed1 100644
--- a/python/requirements/air/data-test-requirements.txt
+++ b/python/requirements/air/data-test-requirements.txt
@@ -4,5 +4,5 @@
 pickle5; python_version < '3.8'
 python-snappy
 tensorflow-datasets
-datasets<=2.10.1; python_version ~= '3.6'
-datasets; python_version >= '3.7'
\ No newline at end of file
+datasets
+pytest-repeat
diff --git a/python/requirements/air/dl-cpu-requirements.txt b/python/requirements/air/dl-cpu-requirements.txt
index 4c25ec097b5ad..ccbc7365f7cab 100644
--- a/python/requirements/air/dl-cpu-requirements.txt
+++ b/python/requirements/air/dl-cpu-requirements.txt
@@ -12,6 +12,7 @@ tensorflow-datasets
 --extra-index-url https://download.pytorch.org/whl/cpu  # for CPU versions of torch, torchvision
 --find-links https://data.pyg.org/whl/torch-1.13.0+cpu.html  # for CPU versions of torch-scatter, torch-sparse, torch-cluster, torch-spline-conv
 torch==1.13.0
+torchmetrics==0.9.3
 torchvision==0.14.0
 torch-scatter==2.1.0
 torch-sparse==0.6.16
diff --git a/python/requirements/air/ml-docker-requirements.txt b/python/requirements/air/dl-gpu-requirements.txt
similarity index 79%
rename from python/requirements/air/ml-docker-requirements.txt
rename to python/requirements/air/dl-gpu-requirements.txt
index 9609f6f40eee5..d989c2ac5bf8c 100644
--- a/python/requirements/air/ml-docker-requirements.txt
+++ b/python/requirements/air/dl-gpu-requirements.txt
@@ -1,11 +1,3 @@
-ipython
-
-# Needed for rich visualization for Ray Train and Ray Data.
-ipywidgets>=8
-
-# Needed for Ray Client error message serialization/deserialization.
-tblib
-
 # If you make changes below this line, please also make the corresponding changes to `dl-cpu-requirements.txt`!
 
 --extra-index-url https://download.pytorch.org/whl/cu116  # for GPU versions of torch, torchvision
diff --git a/python/requirements/air/requirements_no_deps.txt b/python/requirements/air/requirements_no_deps.txt
deleted file mode 100644
index 6d2754211cd22..0000000000000
--- a/python/requirements/air/requirements_no_deps.txt
+++ /dev/null
@@ -1,6 +0,0 @@
-# These requirements have outdated or incompatible downstream dependencies.
-# Thus we install them on a best effort basis before any other packages
-# (without constraints), but their dependencies may be overwritten afterwards.
-
-# mosaicml requires importlib-metadata>5, but flake8 is not compatible with it
-mosaicml==0.12.1
diff --git a/python/requirements/air/requirements_upstream.txt b/python/requirements/air/requirements_upstream.txt
deleted file mode 100644
index 5db407ce322f1..0000000000000
--- a/python/requirements/air/requirements_upstream.txt
+++ /dev/null
@@ -1,9 +0,0 @@
-# Upstream dependencies that depend on Ray.
-# Because they depend on Ray, we can't pin the subdependencies.
-# So we separate its own requirements file.
-
-tune-sklearn==0.4.4
-xgboost_ray==0.1.15
-lightgbm_ray==0.1.8
-modin==0.12.1; python_version < '3.8'
-modin==0.22.2; python_version >= '3.8'
diff --git a/python/requirements/air/rllib-requirements.txt b/python/requirements/air/rllib-requirements.txt
index d4d2930e4edfa..07c0480ddacdf 100644
--- a/python/requirements/air/rllib-requirements.txt
+++ b/python/requirements/air/rllib-requirements.txt
@@ -1,37 +1,3 @@
--r dl-cpu-requirements.txt
-
-# Environment adapters.
-# ---------------------
-# Atari
-# TODO(sven): Still needed for Atari (need to be wrapped by gymnasium as it does NOT support Atari yet)
-gym==0.26.2
-gymnasium[atari,mujoco]==0.26.3
-# For testing MuJoCo envs with gymnasium.
-mujoco-py<2.2,>=2.1
-# Kaggle envs.
-kaggle_environments==1.7.11
-# Unity3D testing
-# TODO(sven): Add this back to rllib-requirements.txt once mlagents no longer pins torch<1.9.0 version.
-#mlagents==0.28.0
-mlagents_envs==0.28.0
-# For tests on PettingZoo's multi-agent envs.
-pettingzoo==1.22.1; python_version >= '3.7'
-# When installing pettingzoo, chess is missing, even though its a dependancy
-# TODO: remove if a future pettingzoo and/or ray version fixes this dependancy issue.
-chess==1.7.0
-pymunk==6.2.1
-supersuit==3.7.0; python_version >= '3.7'
-# For tests on minigrid.
-minigrid==2.1.1
-# For tests on RecSim and Kaggle envs.
-# Explicitly depends on `tensorflow` and doesn't accept `tensorflow-macos`
-recsim==0.2.4; sys_platform != 'darwin' or platform_machine != 'arm64'
-tensorflow_estimator==2.11.0
-# DeepMind's OpenSpiel
-open-spiel==1.2
-
-# Other.
-# ------
 # For MAML on PyTorch.
 higher==0.2.1
 # For auto-generating an env-rendering Window.
@@ -46,4 +12,4 @@ tf2onnx==1.13.0; sys_platform != 'darwin' or platform_machine != 'arm64'
 rich==12.0.1
 # Msgpack checkpoint stuff.
 msgpack
-msgpack_numpy
+msgpack_numpy
\ No newline at end of file
diff --git a/python/requirements/air/rllib-test-requirements.txt b/python/requirements/air/rllib-test-requirements.txt
new file mode 100644
index 0000000000000..a5f2693039adc
--- /dev/null
+++ b/python/requirements/air/rllib-test-requirements.txt
@@ -0,0 +1,35 @@
+-r dl-cpu-requirements.txt
+
+# Environment adapters.
+# ---------------------
+# Atari
+# TODO(sven): Still needed for Atari (need to be wrapped by gymnasium as it does NOT support Atari yet)
+gym==0.26.2
+gymnasium[atari,mujoco]==0.26.3
+# For testing MuJoCo envs with gymnasium.
+mujoco-py<2.2,>=2.1
+# Kaggle envs.
+kaggle_environments==1.7.11
+# Unity3D testing
+# TODO(sven): Add this back to rllib-requirements.txt once mlagents no longer pins torch<1.9.0 version.
+#mlagents==0.28.0
+mlagents_envs==0.28.0
+# For tests on PettingZoo's multi-agent envs.
+pettingzoo==1.22.1; python_version >= '3.7'
+# When installing pettingzoo, chess is missing, even though its a dependancy
+# TODO: remove if a future pettingzoo and/or ray version fixes this dependancy issue.
+chess==1.7.0
+pymunk==6.2.1
+supersuit==3.7.0; python_version >= '3.7'
+# For tests on minigrid.
+minigrid==2.1.1
+# For tests on RecSim and Kaggle envs.
+# Explicitly depends on `tensorflow` and doesn't accept `tensorflow-macos`
+recsim==0.2.4; sys_platform != 'darwin' or platform_machine != 'arm64'
+tensorflow_estimator==2.11.0
+# DeepMind's OpenSpiel
+open-spiel==1.2
+
+# Requires libtorrent which is unavailable for arm64
+autorom[accept-rom-license]; platform_machine != "arm64"
+h5py==3.7.0
diff --git a/python/requirements/air/train-requirements.txt b/python/requirements/air/train-requirements.txt
index 57ddb70f882be..e1c37446f6d00 100644
--- a/python/requirements/air/train-requirements.txt
+++ b/python/requirements/air/train-requirements.txt
@@ -1,26 +1,2 @@
-# TODO(train-team): Remove this once Ray Train is deprecated.
-
--r dl-cpu-requirements.txt
-
-mlflow==1.30.0; python_version <= '3.7'
-mlflow==2.4.1; python_version > '3.7'
-tensorboardX==2.6.0
-
-# Dependencies for Hugging Face examples & tests:
-# `python/ray/train/examples/transformers/transformers_example.py`
-# `python/ray/air/examples/huggingface/huggingface_basic_language_modeling_example.py`
-# `python/ray/train/tests/test_accelerate_trainer_gpu.py`
-# `python/ray/air/tests/test_huggingface_trainer.py`
-
-# We need transformers>=4.19.0 for HuggingFaceTrainer to work (not available on py 3.6)
-# If changing the version here, also change it in HuggingFaceTrainer docstring
-transformers==4.18.0; python_version <= '3.6'
-transformers==4.19.1; python_version > '3.6'
-# We need accelerate>=0.17.0 for AccelerateTrainer to work (not available on py 3.6)
-# If changing the version here, also change it in AccelerateTrainer docstring
-accelerate==0.5.1; python_version <= '3.6'
-accelerate==0.20.3; python_version > '3.6'
-# Tracking issue: https://github.com/ray-project/ray/issues/34399
-deepspeed==0.8.3; python_version > '3.6'
+deepspeed==0.8.3
 datasets==2.0.0
-sentencepiece==0.1.96
diff --git a/python/requirements/air/train-test-requirements.txt b/python/requirements/air/train-test-requirements.txt
new file mode 100644
index 0000000000000..35203bac382ae
--- /dev/null
+++ b/python/requirements/air/train-test-requirements.txt
@@ -0,0 +1,2 @@
+evaluate==0.4.0
+sentencepiece==0.1.96
diff --git a/python/requirements/air/tune-requirements.txt b/python/requirements/air/tune-requirements.txt
index 0eb4a82632e7d..cc1d56d2fe1bd 100644
--- a/python/requirements/air/tune-requirements.txt
+++ b/python/requirements/air/tune-requirements.txt
@@ -1,48 +1,20 @@
--r dl-cpu-requirements.txt
-
-aim==3.16.1
+# Searchers
 ax-platform[mysql]==0.2.6; python_version < '3.8'
 ax-platform[mysql]==0.3.2; python_version >= '3.8'
+
 bayesian-optimization==1.4.3
-comet-ml==3.31.9
+
+# BOHB
 ConfigSpace==0.7.1
+hpbandster==0.7.4
+
 dragonfly-opt==0.1.7
 flaml==1.1.1
-freezegun==1.1.0
-# Requires decord which is unavailable for arm64
-gluoncv==0.10.5.post0; platform_machine != "arm64"
-gpy==1.10.0
-
-# Requires libtorrent which is unavailable for arm64
-autorom[accept-rom-license]; platform_machine != "arm64"
-h5py==3.7.0
-hpbandster==0.7.4
 HEBO==0.3.2
+# pymoo pin for HEBO compatibility
+pymoo==0.5.0
 hyperopt==0.2.7
-jupyterlab==3.6.1
-lightgbm==3.3.5
-matplotlib!=3.4.3
-mlflow==1.30.0; python_version <= '3.7'
-mlflow==2.4.1; python_version > '3.7'
-# Unavailable for arm64 in more recent versions
-mxnet==1.8.0.post0; platform_machine != "arm64"
 nevergrad==0.4.3.post7
 optuna==3.2.0
-# For HEBO compatibility
-pymoo==0.5.0
-pytest-remotedata==0.3.2
-lightning-bolts==0.4.0
-pytorch-lightning==1.6.5
-fairscale==0.4.6
-s3fs==2023.1.0; python_version < '3.8'
-s3fs==2023.5.0; python_version >= '3.8'
-shortuuid==1.0.1
-scikit-optimize==0.9.0
-sigopt==7.5.0
-timm==0.9.2
-transformers==4.18.0; python_version <= '3.6'
-transformers==4.19.1; python_version > '3.6'
-wandb==0.13.4
-xgboost==1.6.2; python_version <= '3.7'
-xgboost==1.7.6; python_version > '3.7'
-zoopt==0.4.1
+
+tune-sklearn==0.4.4
diff --git a/python/requirements/air/tune-test-requirements.txt b/python/requirements/air/tune-test-requirements.txt
new file mode 100644
index 0000000000000..fcb0775e9fe3f
--- /dev/null
+++ b/python/requirements/air/tune-test-requirements.txt
@@ -0,0 +1,23 @@
+-r dl-cpu-requirements.txt
+
+aim==3.16.1
+
+# Requires decord which is unavailable for arm64
+gluoncv==0.10.5.post0; platform_machine != "arm64"
+gpy==1.10.0
+
+jupyterlab==3.6.1
+matplotlib!=3.4.3
+
+# Unavailable for arm64 in more recent versions
+mxnet==1.8.0.post0; platform_machine != "arm64"
+pytest-remotedata==0.3.2
+lightning-bolts==0.4.0
+pytorch-lightning==1.6.5
+fairscale==0.4.6
+shortuuid==1.0.1
+scikit-optimize==0.9.0
+timm==0.9.2
+
+sigopt==7.5.0
+zoopt==0.4.1
diff --git a/python/requirements/docker/ml-docker-requirements.txt b/python/requirements/docker/ml-docker-requirements.txt
new file mode 100644
index 0000000000000..726c7e124359e
--- /dev/null
+++ b/python/requirements/docker/ml-docker-requirements.txt
@@ -0,0 +1,8 @@
+# Requirements that are shipped in the ML docker image.
+ipython
+
+# Needed for rich visualization for Ray Train and Ray Data.
+ipywidgets>=8
+
+# Needed for Ray Client error message serialization/deserialization.
+tblib
diff --git a/python/requirements/docker/ray-docker-requirements.txt b/python/requirements/docker/ray-docker-requirements.txt
new file mode 100644
index 0000000000000..133e5a6877186
--- /dev/null
+++ b/python/requirements/docker/ray-docker-requirements.txt
@@ -0,0 +1,2 @@
+awscli
+gsutil
diff --git a/python/requirements/test-requirements.txt b/python/requirements/test-requirements.txt
index 3306e85f93b4f..c79ac0d55416f 100644
--- a/python/requirements/test-requirements.txt
+++ b/python/requirements/test-requirements.txt
@@ -27,6 +27,7 @@ fastapi==0.75.0
 feather-format==0.4.1
 # Keep compatible with Werkzeug
 flask==2.1.3
+freezegun==1.1.0
 google-api-python-client==2.65.0
 google-cloud-storage==2.5.0
 gradio==3.11; platform_system != "Windows"

From e18ec73471a3e50c5a71dfbdc7f19e320229f88f Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Mon, 26 Jun 2023 11:31:27 +0100
Subject: [PATCH 04/70] ml dockerfile

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 ci/build/build-docker-images.py | 18 +++++++++++---
 ci/env/install-dependencies.sh  |  4 +++
 docker/ray-ml/Dockerfile        | 43 ++++++++++++++++++++++++---------
 3 files changed, 49 insertions(+), 16 deletions(-)

diff --git a/ci/build/build-docker-images.py b/ci/build/build-docker-images.py
index 9df3f6c79e530..7878d519be016 100644
--- a/ci/build/build-docker-images.py
+++ b/ci/build/build-docker-images.py
@@ -426,22 +426,32 @@ def prep_ray_ml():
         "python/requirements.txt",
     ]
     ml_requirements_files = [
-        "python/requirements/air/ml-docker-requirements.txt",
+        "python/requirements/docker/ml-docker-requirements.txt",
+        "python/requirements/docker/ray-docker-requirements.txt",
+        "python/requirements/air/core-requirements.txt",
+        "python/requirements/air/data-requirements.txt",
+        "python/requirements/air/dl-gpu-requirements.txt",
         "python/requirements/air/dl-cpu-requirements.txt",
         "python/requirements/air/tune-requirements.txt",
+        "python/requirements/air/tune-test-requirements.txt",
         "python/requirements/air/rllib-requirements.txt",
+        "python/requirements/air/rllib-test-requirements.txt",
         "python/requirements/air/train-requirements.txt",
+        "python/requirements/air/train-test-requirements.txt",
     ]
-    # We don't need these in the ml docker image
+    # We don't need these in the ml docker image (or they are installed elsewhere)
     ignore_requirements = [
         "python/requirements/compat/requirements_legacy_compat.txt",
+        "python/requirements/air/data-test-requirements.txt",
     ]
 
-    files_on_disk = glob.glob(f"{root_dir}/python/**/requirements*.txt", recursive=True)
+    files_on_disk = glob.glob(
+        f"{root_dir}/python/**/*-requirements.txt", recursive=True
+    )
     for file_on_disk in files_on_disk:
         rel = os.path.relpath(file_on_disk, start=root_dir)
         print(rel)
-        if not rel.startswith("python/requirements/ml"):
+        if not rel.startswith("python/requirements/air"):
             continue
         elif rel not in ml_requirements_files and rel not in ignore_requirements:
             raise RuntimeError(
diff --git a/ci/env/install-dependencies.sh b/ci/env/install-dependencies.sh
index 64ad41279c2d8..264ad1f35a0e8 100755
--- a/ci/env/install-dependencies.sh
+++ b/ci/env/install-dependencies.sh
@@ -329,6 +329,7 @@ install_pip_packages() {
   delayed_packages=()
 
   requirements_files+=("${WORKSPACE_DIR}/python/requirements/test-requirements.txt")
+  requirements_files+=("${WORKSPACE_DIR}/python/requirements/air/core-requirements.txt")
 
   if [ "${LINT-}" = 1 ]; then
     install_linters
@@ -351,6 +352,7 @@ install_pip_packages() {
   # Additional RLlib test dependencies.
   if [ "${RLLIB_TESTING-}" = 1 ] || [ "${DOC_TESTING-}" = 1 ]; then
     requirements_files+=("${WORKSPACE_DIR}/python/requirements/air/rllib-requirements.txt")
+    requirements_files+=("${WORKSPACE_DIR}/python/requirements/air/rllib-test-requirements.txt")
     #TODO(amogkam): Add this back to rllib-requirements.txt once mlagents no longer pins torch<1.9.0 version.
     pip install --no-dependencies mlagents==0.28.0
   fi
@@ -358,11 +360,13 @@ install_pip_packages() {
   # Additional Train test dependencies.
   if [ "${TRAIN_TESTING-}" = 1 ] || [ "${DOC_TESTING-}" = 1 ]; then
     requirements_files+=("${WORKSPACE_DIR}/python/requirements/air/train-requirements.txt")
+    requirements_files+=("${WORKSPACE_DIR}/python/requirements/air/train-test-requirements.txt")
   fi
 
   # Additional Tune/Doc test dependencies.
   if [ "${TUNE_TESTING-}" = 1 ] || [ "${DOC_TESTING-}" = 1 ]; then
     requirements_files+=("${WORKSPACE_DIR}/python/requirements/air/tune-requirements.txt")
+    requirements_files+=("${WORKSPACE_DIR}/python/requirements/air/tune-test-requirements.txt")
   fi
 
   # Additional dependency for Ludwig.
diff --git a/docker/ray-ml/Dockerfile b/docker/ray-ml/Dockerfile
index 6623a971b4a08..a4f807961be52 100644
--- a/docker/ray-ml/Dockerfile
+++ b/docker/ray-ml/Dockerfile
@@ -5,10 +5,17 @@ ARG PYTHON_MINOR_VERSION=7
 # We have to uninstall wrapt this way for Tensorflow compatibility
 COPY requirements.txt ./
 COPY dl-cpu-requirements.txt ./
+COPY dl-gpu-requirements.txt ./
+COPY ray-docker-requirements.txt ./
 COPY ml-docker-requirements.txt ./
+COPY core-requirements.txt ./
+COPY data-requirements.txt ./
 COPY rllib-requirements.txt ./
+COPY rllib-test-requirements.txt ./
 COPY tune-requirements.txt ./
+COPY tune-test-requirements.txt ./
 COPY train-requirements.txt ./
+COPY train-test-requirements.txt ./
 
 RUN sudo apt-get update \
     && sudo apt-get install -y gcc \
@@ -17,26 +24,38 @@ RUN sudo apt-get update \
         zlib1g-dev \
         libgl1-mesa-dev \
         unzip \
-        unrar \
-    && $HOME/anaconda3/bin/pip --no-cache-dir install -U pip pip-tools \
-        # Install requirements
-    && $HOME/anaconda3/bin/pip --no-cache-dir install -U \
-           -r requirements.txt \
+        unrar
+
+RUN $HOME/anaconda3/bin/pip --no-cache-dir install -U pip pip-tools
+
+# Install requirements
+RUN $HOME/anaconda3/bin/pip --no-cache-dir install -U \
+           -r requirements.txt
+ \
     # Install other requirements. Keep pinned requirements bounds as constraints
-    && $HOME/anaconda3/bin/pip --no-cache-dir install -U \
+RUN $HOME/anaconda3/bin/pip --no-cache-dir install -U \
            -c requirements.txt \
+           -r core-requirements.txt \
+           -r data-requirements.txt \
            -r rllib-requirements.txt \
+           -r rllib-test-requirements.txt \
            -r train-requirements.txt \
+           -r train-test-requirements.txt \
            -r tune-requirements.txt \
-    # explicitly install (overwrite) pytorch with CUDA support
-    && $HOME/anaconda3/bin/pip --no-cache-dir install -U \
+           -r tune-test-requirements.txt \
+           -r ray-docker-requirements.txt \
+           -r ml-docker-requirements.txt
+
+# explicitly install (overwrite) pytorch with CUDA support
+RUN $HOME/anaconda3/bin/pip --no-cache-dir install -U \
            -c requirements.txt \
-           -r ml-docker-requirements.txt \
-    # Remove dataclasses & typing because they are included in Python > 3.6
-    && if [ $(python -c 'import sys; print(sys.version_info.minor)') != "6" ]; then \
+           -r dl-gpu-requirements.txt \
+
+# Remove dataclasses & typing because they are included in Python > 3.6
+RUN if [ $(python -c 'import sys; print(sys.version_info.minor)') != "6" ]; then \
         $HOME/anaconda3/bin/pip uninstall dataclasses typing -y; fi  \
     && sudo rm \
-        requirements*.txt \
+        *requirements.txt \
     && sudo apt-get clean
 
 RUN $HOME/anaconda3/bin/pip freeze > /home/ray/pip-freeze.txt

From dab5995b2cab5fcd39773b6bfd6630ed3b1f983b Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Mon, 26 Jun 2023 11:31:59 +0100
Subject: [PATCH 05/70] export [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 docker/ray-ml/Dockerfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docker/ray-ml/Dockerfile b/docker/ray-ml/Dockerfile
index a4f807961be52..0bef762c1c26d 100644
--- a/docker/ray-ml/Dockerfile
+++ b/docker/ray-ml/Dockerfile
@@ -58,6 +58,7 @@ RUN if [ $(python -c 'import sys; print(sys.version_info.minor)') != "6" ]; then
         *requirements.txt \
     && sudo apt-get clean
 
+# Export installed packages
 RUN $HOME/anaconda3/bin/pip freeze > /home/ray/pip-freeze.txt
 
 # Make sure tfp is installed correctly and matches tf version.

From d88f12688db22de694254da1292da958f2968301 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Mon, 26 Jun 2023 13:43:04 +0100
Subject: [PATCH 06/70] fix core-requirements [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 python/requirements/air/core-requirements.txt | 1 -
 1 file changed, 1 deletion(-)

diff --git a/python/requirements/air/core-requirements.txt b/python/requirements/air/core-requirements.txt
index 7649a760efaa1..c3f22cc47b627 100644
--- a/python/requirements/air/core-requirements.txt
+++ b/python/requirements/air/core-requirements.txt
@@ -22,4 +22,3 @@ accelerate==0.20.3
 # Cloud storage tools
 s3fs==2023.1.0; python_version < '3.8'
 s3fs==2023.5.0; python_version >= '3.8'
-``
\ No newline at end of file

From 1397c2ec026c687d489f55ae7f2c3e5654b4793d Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Mon, 26 Jun 2023 14:02:26 +0100
Subject: [PATCH 07/70] move core requirements install [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 ci/env/install-dependencies.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ci/env/install-dependencies.sh b/ci/env/install-dependencies.sh
index 264ad1f35a0e8..505a13f44b7ef 100755
--- a/ci/env/install-dependencies.sh
+++ b/ci/env/install-dependencies.sh
@@ -329,7 +329,6 @@ install_pip_packages() {
   delayed_packages=()
 
   requirements_files+=("${WORKSPACE_DIR}/python/requirements/test-requirements.txt")
-  requirements_files+=("${WORKSPACE_DIR}/python/requirements/air/core-requirements.txt")
 
   if [ "${LINT-}" = 1 ]; then
     install_linters
@@ -436,6 +435,8 @@ install_pip_packages() {
         pip install "${TORCH_PACKAGE%%;*}" "${TORCHVISION_PACKAGE%%;*}"
         requirements_files+=("${WORKSPACE_DIR}/python/requirements/air/dl-cpu-requirements.txt")
       fi
+
+      requirements_files+=("${WORKSPACE_DIR}/python/requirements/air/core-requirements.txt")
   fi
 
   # Inject our own mirror for the CIFAR10 dataset

From d2c5a9048108458fb6af0917fe25a8dc488adff3 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Mon, 26 Jun 2023 15:39:35 +0100
Subject: [PATCH 08/70] Fix pipeline installs [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 .buildkite/pipeline.gpu.yml                        | 12 ++++++------
 .buildkite/pipeline.gpu_large.yml                  | 14 +++++++-------
 .github/CODEOWNERS                                 |  2 +-
 .../docker/ray-docker-requirements.txt             |  3 ++-
 4 files changed, 16 insertions(+), 15 deletions(-)

diff --git a/.buildkite/pipeline.gpu.yml b/.buildkite/pipeline.gpu.yml
index 34ce078391030..ffb5d908759b8 100644
--- a/.buildkite/pipeline.gpu.yml
+++ b/.buildkite/pipeline.gpu.yml
@@ -6,7 +6,7 @@
 #  commands:
 #    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
 #    - TUNE_TESTING=1 ./ci/env/install-dependencies.sh
-#    - pip install -Ur ./python/requirements/air/ml-docker-requirements.txt
+#    - pip install -Ur ./python/requirements/air/dl-gpu-requirements.txt
 #    - ./ci/env/env_info.sh
 #    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=gpu python/ray/tune/...
 
@@ -16,7 +16,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - RLLIB_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/air/ml-docker-requirements.txt
+    - pip install -Ur ./python/requirements/air/dl-gpu-requirements.txt
     - ./ci/env/env_info.sh
     # --jobs 1 is necessary as we only have 1 GPU on the machine and running tests in parallel
     # would cause timeouts as the other scripts would wait for the GPU to become available.
@@ -40,7 +40,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/air/ml-docker-requirements.txt
+    - pip install -Ur ./python/requirements/air/dl-gpu-requirements.txt
     - bazel test --config=ci $(./ci/run/bazel_export_options) --test_tag_filters=gpu python/ray/serve/...
 
 # Todo: enable once tests pass
@@ -49,7 +49,7 @@
 #  commands:
 #    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
 #    - RLLIB_TESTING=1 ./ci/env/install-dependencies.sh
-#    - pip install -Ur ./python/requirements/air/ml-docker-requirements.txt
+#    - pip install -Ur ./python/requirements/air/dl-gpu-requirements.txt
 #    - ./ci/env/env_info.sh
 #    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only
 #      --test_tag_filters=examples_C,examples_D --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 --test_env=RLLIB_NUM_GPUS=1 rllib/...
@@ -60,7 +60,7 @@
 #  commands:
 #    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
 #    - RLLIB_TESTING=1 ./ci/env/install-dependencies.sh
-#    - pip install -Ur ./python/requirements/air/ml-docker-requirements.txt
+#    - pip install -Ur ./python/requirements/air/dl-gpu-requirements.txt
 #    - ./ci/env/env_info.sh
 #    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only
 #      --test_tag_filters=examples_E,examples_F,examples_G,examples_H,examples_I,examples_J,examples_K,examples_L,examples_M,examples_N,examples_O,examples_P --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 --test_env=RLLIB_NUM_GPUS=1
@@ -72,7 +72,7 @@
 #  commands:
 #    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
 #    - RLLIB_TESTING=1 ./ci/env/install-dependencies.sh
-#    - pip install -Ur ./python/requirements/air/ml-docker-requirements.txt
+#    - pip install -Ur ./python/requirements/air/dl-gpu-requirements.txt
 #    - ./ci/env/env_info.sh
 #    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only
 #      --test_tag_filters=examples_Q,examples_R,examples_S,examples_T,examples_U,examples_V,examples_W,examples_X,examples_Y,examples_Z --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 --test_env=RLLIB_NUM_GPUS=1
diff --git a/.buildkite/pipeline.gpu_large.yml b/.buildkite/pipeline.gpu_large.yml
index 196ced5870b47..624790dd67681 100644
--- a/.buildkite/pipeline.gpu_large.yml
+++ b/.buildkite/pipeline.gpu_large.yml
@@ -5,7 +5,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/air/ml-docker-requirements.txt
+    - pip install -Ur ./python/requirements/air/dl-gpu-requirements.txt
     - ./ci/env/install-horovod.sh
     - ./ci/env/env_info.sh
     - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=gpu,gpu_only,-ray_air python/ray/train/...
@@ -15,7 +15,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - TRAIN_TESTING=1 DATA_PROCESSING_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/air/ml-docker-requirements.txt
+    - pip install -Ur ./python/requirements/air/dl-gpu-requirements.txt
     - ./ci/env/env_info.sh
     - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=datasets_train,-doctest doc/...
 
@@ -25,7 +25,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - RLLIB_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/air/ml-docker-requirements.txt
+    - pip install -Ur ./python/requirements/air/dl-gpu-requirements.txt
     - ./ci/env/env_info.sh
     # --jobs 2 is necessary as we only need to have at least 2 gpus on the machine
     # and running tests in parallel would cause timeouts as the other scripts would
@@ -42,7 +42,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - DATA_PROCESSING_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/air/ml-docker-requirements.txt
+    - pip install -Ur ./python/requirements/air/dl-gpu-requirements.txt
     - ./ci/env/install-horovod.sh
     - ./ci/env/env_info.sh
     - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=gpu python/ray/air/... python/ray/train/...
@@ -53,7 +53,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - DOC_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/air/ml-docker-requirements.txt
+    - pip install -Ur ./python/requirements/air/dl-gpu-requirements.txt
     - ./ci/env/env_info.sh
     # Test examples with newer version of `transformers`
     # TODO(amogkam): Remove when https://github.com/ray-project/ray/issues/36011
@@ -65,7 +65,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - DOC_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/air/ml-docker-requirements.txt
+    - pip install -Ur ./python/requirements/air/dl-gpu-requirements.txt
     - ./ci/env/install-horovod.sh
     - ./ci/env/env_info.sh
     # Test examples with newer version of `transformers`
@@ -82,7 +82,7 @@
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - NO_DASHBOARD=1 ./ci/env/install-minimal.sh 3.8
     - PYTHON=3.8 DOC_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/air/ml-docker-requirements.txt
+    - pip install -Ur ./python/requirements/air/dl-gpu-requirements.txt
     - pip uninstall -y pytorch-lightning
     - pip install lightning==2.0.0
     - ./ci/env/env_info.sh
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index d1c7f6e8c80f5..e6109ed43ef32 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -102,7 +102,7 @@
 
 # ML Docker Dependencies
 /python/requirements/air/dl-cpu-requirements.txt @amogkam @krfricke @richardliaw @matthewdeng
-/python/requirements/air/ml-docker-requirements.txt @amogkam @krfricke @richardliaw @matthewdeng
+/python/requirements/air/dl-gpu-requirements.txt @amogkam @krfricke @richardliaw @matthewdeng
 
 # Ray symbol export
 /src/ray/ray_version_script.lds @iycheng @ericl @scv119
diff --git a/python/requirements/docker/ray-docker-requirements.txt b/python/requirements/docker/ray-docker-requirements.txt
index 133e5a6877186..2e593e69e4ac1 100644
--- a/python/requirements/docker/ray-docker-requirements.txt
+++ b/python/requirements/docker/ray-docker-requirements.txt
@@ -1,2 +1,3 @@
-awscli
+# Todo: Fix conflicts with pinned boto3/botocore
+# awscli
 gsutil

From ec48911cf29702caa576a74ffb3b3dd235f663ee Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Mon, 26 Jun 2023 18:17:35 +0100
Subject: [PATCH 09/70] HEBO 0.3.4 [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 docker/ray-ml/Dockerfile                      | 2 +-
 python/requirements/air/tune-requirements.txt | 4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/docker/ray-ml/Dockerfile b/docker/ray-ml/Dockerfile
index 0bef762c1c26d..ac4171afc10b1 100644
--- a/docker/ray-ml/Dockerfile
+++ b/docker/ray-ml/Dockerfile
@@ -49,7 +49,7 @@ RUN $HOME/anaconda3/bin/pip --no-cache-dir install -U \
 # explicitly install (overwrite) pytorch with CUDA support
 RUN $HOME/anaconda3/bin/pip --no-cache-dir install -U \
            -c requirements.txt \
-           -r dl-gpu-requirements.txt \
+           -r dl-gpu-requirements.txt
 
 # Remove dataclasses & typing because they are included in Python > 3.6
 RUN if [ $(python -c 'import sys; print(sys.version_info.minor)') != "6" ]; then \
diff --git a/python/requirements/air/tune-requirements.txt b/python/requirements/air/tune-requirements.txt
index cc1d56d2fe1bd..8ec1c26d0eeb3 100644
--- a/python/requirements/air/tune-requirements.txt
+++ b/python/requirements/air/tune-requirements.txt
@@ -10,7 +10,9 @@ hpbandster==0.7.4
 
 dragonfly-opt==0.1.7
 flaml==1.1.1
-HEBO==0.3.2
+# Todo: Update when released on pypi
+git+https://github.com/huawei-noah/HEBO@v0.3.4#subdirectory=HEBO
+# HEBO==0.3.4
 # pymoo pin for HEBO compatibility
 pymoo==0.5.0
 hyperopt==0.2.7

From 84523511c0d4d995fcec4eddb6c0cf4118fc7a05 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Mon, 26 Jun 2023 21:47:07 +0100
Subject: [PATCH 10/70] mosaicml [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 .buildkite/pipeline.ml.yml                    | 2 ++
 python/requirements/air/tune-requirements.txt | 4 ++--
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/.buildkite/pipeline.ml.yml b/.buildkite/pipeline.ml.yml
index 408c3bb502462..7e371bbdad048 100644
--- a/.buildkite/pipeline.ml.yml
+++ b/.buildkite/pipeline.ml.yml
@@ -35,6 +35,8 @@
   parallelism: 4
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
+    # Todo (krfricke): Move to train-test-requirements.txt
+    - pip install "mosaicml==0.12.1"
     - TRAIN_TESTING=1 DATA_PROCESSING_TESTING=1 INSTALL_HOROVOD=1 ./ci/env/install-dependencies.sh
     - ./ci/env/env_info.sh
     - ./ci/run/run_bazel_test_with_sharding.sh
diff --git a/python/requirements/air/tune-requirements.txt b/python/requirements/air/tune-requirements.txt
index 8ec1c26d0eeb3..5fcdfa5c7790b 100644
--- a/python/requirements/air/tune-requirements.txt
+++ b/python/requirements/air/tune-requirements.txt
@@ -10,8 +10,8 @@ hpbandster==0.7.4
 
 dragonfly-opt==0.1.7
 flaml==1.1.1
-# Todo: Update when released on pypi
-git+https://github.com/huawei-noah/HEBO@v0.3.4#subdirectory=HEBO
+# Todo: Use version pin again after HEBO is updated on pypi
+git+https://github.com/huawei-noah/HEBO@9a2a674c22518eed35a8b98e5134576741a95410#subdirectory=HEBO
 # HEBO==0.3.4
 # pymoo pin for HEBO compatibility
 pymoo==0.5.0

From e858eb6f2a9d7707da9466d408afed132283cf7b Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Tue, 27 Jun 2023 08:39:00 +0100
Subject: [PATCH 11/70] pymoo [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 python/requirements/air/tune-requirements.txt | 2 --
 1 file changed, 2 deletions(-)

diff --git a/python/requirements/air/tune-requirements.txt b/python/requirements/air/tune-requirements.txt
index 5fcdfa5c7790b..fed5be460384f 100644
--- a/python/requirements/air/tune-requirements.txt
+++ b/python/requirements/air/tune-requirements.txt
@@ -13,8 +13,6 @@ flaml==1.1.1
 # Todo: Use version pin again after HEBO is updated on pypi
 git+https://github.com/huawei-noah/HEBO@9a2a674c22518eed35a8b98e5134576741a95410#subdirectory=HEBO
 # HEBO==0.3.4
-# pymoo pin for HEBO compatibility
-pymoo==0.5.0
 hyperopt==0.2.7
 nevergrad==0.4.3.post7
 optuna==3.2.0

From 6ef95153530458770b97623d4557563811af2868 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Tue, 27 Jun 2023 10:28:08 +0100
Subject: [PATCH 12/70] trivial [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 .buildkite/pipeline.ml.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/pipeline.ml.yml b/.buildkite/pipeline.ml.yml
index 7e371bbdad048..e177dd7c0a6c7 100644
--- a/.buildkite/pipeline.ml.yml
+++ b/.buildkite/pipeline.ml.yml
@@ -35,7 +35,7 @@
   parallelism: 4
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
-    # Todo (krfricke): Move to train-test-requirements.txt
+    # Todo (krfricke): Move mosaicml to train-test-requirements.txt
     - pip install "mosaicml==0.12.1"
     - TRAIN_TESTING=1 DATA_PROCESSING_TESTING=1 INSTALL_HOROVOD=1 ./ci/env/install-dependencies.sh
     - ./ci/env/env_info.sh

From fff3935c0301ecf98b84d158ce6da58b10827a97 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Tue, 27 Jun 2023 13:30:44 +0100
Subject: [PATCH 13/70] core [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 .buildkite/pipeline.build.yml  | 2 +-
 ci/env/install-dependencies.sh | 5 ++++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/.buildkite/pipeline.build.yml b/.buildkite/pipeline.build.yml
index b34cf64c59a43..c539a0b57c62c 100644
--- a/.buildkite/pipeline.build.yml
+++ b/.buildkite/pipeline.build.yml
@@ -192,9 +192,9 @@
   instance_size: large
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
-    - ./ci/env/env_info.sh
     - DOC_TESTING=1 ./ci/env/install-dependencies.sh
     - ./ci/env/install-horovod.sh
+    - ./ci/env/env_info.sh
     - bazel test --config=ci $(./scripts/bazel_export_options)
       --test_tag_filters=doctest,-gpu
       python/ray/... doc/...
diff --git a/ci/env/install-dependencies.sh b/ci/env/install-dependencies.sh
index 505a13f44b7ef..5d65134b7d5f5 100755
--- a/ci/env/install-dependencies.sh
+++ b/ci/env/install-dependencies.sh
@@ -435,8 +435,11 @@ install_pip_packages() {
         pip install "${TORCH_PACKAGE%%;*}" "${TORCHVISION_PACKAGE%%;*}"
         requirements_files+=("${WORKSPACE_DIR}/python/requirements/air/dl-cpu-requirements.txt")
       fi
+  fi
 
-      requirements_files+=("${WORKSPACE_DIR}/python/requirements/air/core-requirements.txt")
+  # AIR core dependencies
+  if [ "${RLLIB_TESTING-}" = 1 ] || [ "${TRAIN_TESTING-}" = 1 ] || [ "${TUNE_TESTING-}" = 1 ] || [ "${DOC_TESTING-}" = 1 ]; then
+    requirements_files+=("${WORKSPACE_DIR}/python/requirements/air/core-requirements.txt")
   fi
 
   # Inject our own mirror for the CIFAR10 dataset

From bc2427a456efdafe66381ec5526f9edf7e561f37 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Tue, 27 Jun 2023 14:53:19 +0100
Subject: [PATCH 14/70] doctest mosaic [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 .buildkite/pipeline.build.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.buildkite/pipeline.build.yml b/.buildkite/pipeline.build.yml
index c539a0b57c62c..0bb3614a28797 100644
--- a/.buildkite/pipeline.build.yml
+++ b/.buildkite/pipeline.build.yml
@@ -192,6 +192,8 @@
   instance_size: large
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
+    # Todo (krfricke): Move mosaicml to train-test-requirements.txt
+    - pip install "mosaicml==0.12.1"
     - DOC_TESTING=1 ./ci/env/install-dependencies.sh
     - ./ci/env/install-horovod.sh
     - ./ci/env/env_info.sh

From 1ebd59b8b9c44f65cd52a0a78fc9261b078a12a1 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Wed, 28 Jun 2023 07:54:17 +0100
Subject: [PATCH 15/70] Move back to ml/

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 .buildkite/pipeline.gpu.yml                   | 12 ++++----
 .buildkite/pipeline.gpu_large.yml             | 14 +++++-----
 .github/CODEOWNERS                            |  4 +--
 ci/build/build-docker-images.py               | 24 ++++++++--------
 ci/env/install-dependencies.sh                | 28 +++++++++----------
 .../{ => ml}/air/core-requirements.txt        |  0
 .../{ => ml}/air/data-requirements.txt        |  0
 .../{ => ml}/air/data-test-requirements.txt   |  0
 .../{ => ml}/air/dl-cpu-requirements.txt      |  0
 .../{ => ml}/air/dl-gpu-requirements.txt      |  0
 .../{ => ml}/air/rllib-requirements.txt       |  0
 .../{ => ml}/air/rllib-test-requirements.txt  |  0
 .../{ => ml}/air/train-requirements.txt       |  0
 .../{ => ml}/air/train-test-requirements.txt  |  0
 .../{ => ml}/air/tune-requirements.txt        |  0
 .../{ => ml}/air/tune-test-requirements.txt   |  0
 .../horovod/driver_requirements.txt           |  2 +-
 .../train/driver_requirements.txt             |  2 +-
 .../tune_rllib/driver_requirements.txt        |  2 +-
 19 files changed, 44 insertions(+), 44 deletions(-)
 rename python/requirements/{ => ml}/air/core-requirements.txt (100%)
 rename python/requirements/{ => ml}/air/data-requirements.txt (100%)
 rename python/requirements/{ => ml}/air/data-test-requirements.txt (100%)
 rename python/requirements/{ => ml}/air/dl-cpu-requirements.txt (100%)
 rename python/requirements/{ => ml}/air/dl-gpu-requirements.txt (100%)
 rename python/requirements/{ => ml}/air/rllib-requirements.txt (100%)
 rename python/requirements/{ => ml}/air/rllib-test-requirements.txt (100%)
 rename python/requirements/{ => ml}/air/train-requirements.txt (100%)
 rename python/requirements/{ => ml}/air/train-test-requirements.txt (100%)
 rename python/requirements/{ => ml}/air/tune-requirements.txt (100%)
 rename python/requirements/{ => ml}/air/tune-test-requirements.txt (100%)

diff --git a/.buildkite/pipeline.gpu.yml b/.buildkite/pipeline.gpu.yml
index ffb5d908759b8..f5184a3aa183d 100644
--- a/.buildkite/pipeline.gpu.yml
+++ b/.buildkite/pipeline.gpu.yml
@@ -6,7 +6,7 @@
 #  commands:
 #    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
 #    - TUNE_TESTING=1 ./ci/env/install-dependencies.sh
-#    - pip install -Ur ./python/requirements/air/dl-gpu-requirements.txt
+#    - pip install -Ur ./python/requirements/ml/dl-gpu-requirements.txt
 #    - ./ci/env/env_info.sh
 #    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=gpu python/ray/tune/...
 
@@ -16,7 +16,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - RLLIB_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/air/dl-gpu-requirements.txt
+    - pip install -Ur ./python/requirements/ml/dl-gpu-requirements.txt
     - ./ci/env/env_info.sh
     # --jobs 1 is necessary as we only have 1 GPU on the machine and running tests in parallel
     # would cause timeouts as the other scripts would wait for the GPU to become available.
@@ -40,7 +40,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/air/dl-gpu-requirements.txt
+    - pip install -Ur ./python/requirements/ml/dl-gpu-requirements.txt
     - bazel test --config=ci $(./ci/run/bazel_export_options) --test_tag_filters=gpu python/ray/serve/...
 
 # Todo: enable once tests pass
@@ -49,7 +49,7 @@
 #  commands:
 #    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
 #    - RLLIB_TESTING=1 ./ci/env/install-dependencies.sh
-#    - pip install -Ur ./python/requirements/air/dl-gpu-requirements.txt
+#    - pip install -Ur ./python/requirements/ml/dl-gpu-requirements.txt
 #    - ./ci/env/env_info.sh
 #    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only
 #      --test_tag_filters=examples_C,examples_D --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 --test_env=RLLIB_NUM_GPUS=1 rllib/...
@@ -60,7 +60,7 @@
 #  commands:
 #    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
 #    - RLLIB_TESTING=1 ./ci/env/install-dependencies.sh
-#    - pip install -Ur ./python/requirements/air/dl-gpu-requirements.txt
+#    - pip install -Ur ./python/requirements/ml/dl-gpu-requirements.txt
 #    - ./ci/env/env_info.sh
 #    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only
 #      --test_tag_filters=examples_E,examples_F,examples_G,examples_H,examples_I,examples_J,examples_K,examples_L,examples_M,examples_N,examples_O,examples_P --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 --test_env=RLLIB_NUM_GPUS=1
@@ -72,7 +72,7 @@
 #  commands:
 #    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
 #    - RLLIB_TESTING=1 ./ci/env/install-dependencies.sh
-#    - pip install -Ur ./python/requirements/air/dl-gpu-requirements.txt
+#    - pip install -Ur ./python/requirements/ml/dl-gpu-requirements.txt
 #    - ./ci/env/env_info.sh
 #    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only
 #      --test_tag_filters=examples_Q,examples_R,examples_S,examples_T,examples_U,examples_V,examples_W,examples_X,examples_Y,examples_Z --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 --test_env=RLLIB_NUM_GPUS=1
diff --git a/.buildkite/pipeline.gpu_large.yml b/.buildkite/pipeline.gpu_large.yml
index 624790dd67681..34643c9e0af1a 100644
--- a/.buildkite/pipeline.gpu_large.yml
+++ b/.buildkite/pipeline.gpu_large.yml
@@ -5,7 +5,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/air/dl-gpu-requirements.txt
+    - pip install -Ur ./python/requirements/ml/dl-gpu-requirements.txt
     - ./ci/env/install-horovod.sh
     - ./ci/env/env_info.sh
     - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=gpu,gpu_only,-ray_air python/ray/train/...
@@ -15,7 +15,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - TRAIN_TESTING=1 DATA_PROCESSING_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/air/dl-gpu-requirements.txt
+    - pip install -Ur ./python/requirements/ml/dl-gpu-requirements.txt
     - ./ci/env/env_info.sh
     - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=datasets_train,-doctest doc/...
 
@@ -25,7 +25,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - RLLIB_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/air/dl-gpu-requirements.txt
+    - pip install -Ur ./python/requirements/ml/dl-gpu-requirements.txt
     - ./ci/env/env_info.sh
     # --jobs 2 is necessary as we only need to have at least 2 gpus on the machine
     # and running tests in parallel would cause timeouts as the other scripts would
@@ -42,7 +42,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - DATA_PROCESSING_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/air/dl-gpu-requirements.txt
+    - pip install -Ur ./python/requirements/ml/dl-gpu-requirements.txt
     - ./ci/env/install-horovod.sh
     - ./ci/env/env_info.sh
     - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=gpu python/ray/air/... python/ray/train/...
@@ -53,7 +53,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - DOC_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/air/dl-gpu-requirements.txt
+    - pip install -Ur ./python/requirements/ml/dl-gpu-requirements.txt
     - ./ci/env/env_info.sh
     # Test examples with newer version of `transformers`
     # TODO(amogkam): Remove when https://github.com/ray-project/ray/issues/36011
@@ -65,7 +65,7 @@
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - DOC_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/air/dl-gpu-requirements.txt
+    - pip install -Ur ./python/requirements/ml/dl-gpu-requirements.txt
     - ./ci/env/install-horovod.sh
     - ./ci/env/env_info.sh
     # Test examples with newer version of `transformers`
@@ -82,7 +82,7 @@
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
     - NO_DASHBOARD=1 ./ci/env/install-minimal.sh 3.8
     - PYTHON=3.8 DOC_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/env/install-dependencies.sh
-    - pip install -Ur ./python/requirements/air/dl-gpu-requirements.txt
+    - pip install -Ur ./python/requirements/ml/dl-gpu-requirements.txt
     - pip uninstall -y pytorch-lightning
     - pip install lightning==2.0.0
     - ./ci/env/env_info.sh
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index 99b4213945fb9..a0d41da2195fb 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -101,8 +101,8 @@
 /doc/source/ray-air/ @richardliaw @gjoliver @krfricke @xwjiang2010 @amogkam @matthewdeng @Yard1 @maxpumperla @ray-project/ray-docs
 
 # ML Docker Dependencies
-/python/requirements/air/dl-cpu-requirements.txt @amogkam @krfricke @richardliaw @matthewdeng
-/python/requirements/air/dl-gpu-requirements.txt @amogkam @krfricke @richardliaw @matthewdeng
+/python/requirements/ml/dl-cpu-requirements.txt @amogkam @krfricke @richardliaw @matthewdeng
+/python/requirements/ml/dl-gpu-requirements.txt @amogkam @krfricke @richardliaw @matthewdeng
 
 # Ray symbol export
 /src/ray/ray_version_script.lds @iycheng @ericl @scv119
diff --git a/ci/build/build-docker-images.py b/ci/build/build-docker-images.py
index 7878d519be016..4fb35df47ea80 100644
--- a/ci/build/build-docker-images.py
+++ b/ci/build/build-docker-images.py
@@ -428,21 +428,21 @@ def prep_ray_ml():
     ml_requirements_files = [
         "python/requirements/docker/ml-docker-requirements.txt",
         "python/requirements/docker/ray-docker-requirements.txt",
-        "python/requirements/air/core-requirements.txt",
-        "python/requirements/air/data-requirements.txt",
-        "python/requirements/air/dl-gpu-requirements.txt",
-        "python/requirements/air/dl-cpu-requirements.txt",
-        "python/requirements/air/tune-requirements.txt",
-        "python/requirements/air/tune-test-requirements.txt",
-        "python/requirements/air/rllib-requirements.txt",
-        "python/requirements/air/rllib-test-requirements.txt",
-        "python/requirements/air/train-requirements.txt",
-        "python/requirements/air/train-test-requirements.txt",
+        "python/requirements/ml/core-requirements.txt",
+        "python/requirements/ml/data-requirements.txt",
+        "python/requirements/ml/dl-gpu-requirements.txt",
+        "python/requirements/ml/dl-cpu-requirements.txt",
+        "python/requirements/ml/tune-requirements.txt",
+        "python/requirements/ml/tune-test-requirements.txt",
+        "python/requirements/ml/rllib-requirements.txt",
+        "python/requirements/ml/rllib-test-requirements.txt",
+        "python/requirements/ml/train-requirements.txt",
+        "python/requirements/ml/train-test-requirements.txt",
     ]
     # We don't need these in the ml docker image (or they are installed elsewhere)
     ignore_requirements = [
         "python/requirements/compat/requirements_legacy_compat.txt",
-        "python/requirements/air/data-test-requirements.txt",
+        "python/requirements/ml/data-test-requirements.txt",
     ]
 
     files_on_disk = glob.glob(
@@ -451,7 +451,7 @@ def prep_ray_ml():
     for file_on_disk in files_on_disk:
         rel = os.path.relpath(file_on_disk, start=root_dir)
         print(rel)
-        if not rel.startswith("python/requirements/air"):
+        if not rel.startswith("python/requirements/ml"):
             continue
         elif rel not in ml_requirements_files and rel not in ignore_requirements:
             raise RuntimeError(
diff --git a/ci/env/install-dependencies.sh b/ci/env/install-dependencies.sh
index 5d65134b7d5f5..d6986c07280f5 100755
--- a/ci/env/install-dependencies.sh
+++ b/ci/env/install-dependencies.sh
@@ -350,22 +350,22 @@ install_pip_packages() {
 
   # Additional RLlib test dependencies.
   if [ "${RLLIB_TESTING-}" = 1 ] || [ "${DOC_TESTING-}" = 1 ]; then
-    requirements_files+=("${WORKSPACE_DIR}/python/requirements/air/rllib-requirements.txt")
-    requirements_files+=("${WORKSPACE_DIR}/python/requirements/air/rllib-test-requirements.txt")
+    requirements_files+=("${WORKSPACE_DIR}/python/requirements/ml/rllib-requirements.txt")
+    requirements_files+=("${WORKSPACE_DIR}/python/requirements/ml/rllib-test-requirements.txt")
     #TODO(amogkam): Add this back to rllib-requirements.txt once mlagents no longer pins torch<1.9.0 version.
     pip install --no-dependencies mlagents==0.28.0
   fi
 
   # Additional Train test dependencies.
   if [ "${TRAIN_TESTING-}" = 1 ] || [ "${DOC_TESTING-}" = 1 ]; then
-    requirements_files+=("${WORKSPACE_DIR}/python/requirements/air/train-requirements.txt")
-    requirements_files+=("${WORKSPACE_DIR}/python/requirements/air/train-test-requirements.txt")
+    requirements_files+=("${WORKSPACE_DIR}/python/requirements/ml/train-requirements.txt")
+    requirements_files+=("${WORKSPACE_DIR}/python/requirements/ml/train-test-requirements.txt")
   fi
 
   # Additional Tune/Doc test dependencies.
   if [ "${TUNE_TESTING-}" = 1 ] || [ "${DOC_TESTING-}" = 1 ]; then
-    requirements_files+=("${WORKSPACE_DIR}/python/requirements/air/tune-requirements.txt")
-    requirements_files+=("${WORKSPACE_DIR}/python/requirements/air/tune-test-requirements.txt")
+    requirements_files+=("${WORKSPACE_DIR}/python/requirements/ml/tune-requirements.txt")
+    requirements_files+=("${WORKSPACE_DIR}/python/requirements/ml/tune-test-requirements.txt")
   fi
 
   # Additional dependency for Ludwig.
@@ -388,10 +388,10 @@ install_pip_packages() {
 
   # Data processing test dependencies.
   if [ "${DATA_PROCESSING_TESTING-}" = 1 ] || [ "${DOC_TESTING-}" = 1 ]; then
-    requirements_files+=("${WORKSPACE_DIR}/python/requirements/air/data-requirements.txt")
+    requirements_files+=("${WORKSPACE_DIR}/python/requirements/ml/data-requirements.txt")
   fi
   if [ "${DATA_PROCESSING_TESTING-}" = 1 ]; then
-    requirements_files+=("${WORKSPACE_DIR}/python/requirements/air/data-test-requirements.txt")
+    requirements_files+=("${WORKSPACE_DIR}/python/requirements/ml/data-test-requirements.txt")
     if [ -n "${ARROW_VERSION-}" ]; then
       if [ "${ARROW_VERSION-}" = nightly ]; then
         delayed_packages+=("--extra-index-url")
@@ -419,8 +419,8 @@ install_pip_packages() {
         pip install -U "torch==${TORCH_VERSION-1.9.0}" "torchvision==${TORCHVISION_VERSION-0.10.0}"
         # We won't add dl-cpu-requirements.txt as it would otherwise overwrite our custom
         # torch. Thus we have also have to install tensorflow manually.
-        TF_PACKAGE=$(grep "tensorflow==" "${WORKSPACE_DIR}/python/requirements/air/dl-cpu-requirements.txt")
-        TFPROB_PACKAGE=$(grep "tensorflow-probability==" "${WORKSPACE_DIR}/python/requirements/air/dl-cpu-requirements.txt")
+        TF_PACKAGE=$(grep "tensorflow==" "${WORKSPACE_DIR}/python/requirements/ml/dl-cpu-requirements.txt")
+        TFPROB_PACKAGE=$(grep "tensorflow-probability==" "${WORKSPACE_DIR}/python/requirements/ml/dl-cpu-requirements.txt")
 
         # %%;* deletes everything after ; to get rid of e.g. python version specifiers
         pip install -U "${TF_PACKAGE%%;*}" "${TFPROB_PACKAGE%%;*}"
@@ -428,18 +428,18 @@ install_pip_packages() {
         # Otherwise, use pinned default torch version.
         # Again, install right away, as some dependencies (e.g. torch-spline-conv) need
         # torch to be installed for their own install.
-        TORCH_PACKAGE=$(grep "torch==" "${WORKSPACE_DIR}/python/requirements/air/dl-cpu-requirements.txt")
-        TORCHVISION_PACKAGE=$(grep "torchvision==" "${WORKSPACE_DIR}/python/requirements/air/dl-cpu-requirements.txt")
+        TORCH_PACKAGE=$(grep "torch==" "${WORKSPACE_DIR}/python/requirements/ml/dl-cpu-requirements.txt")
+        TORCHVISION_PACKAGE=$(grep "torchvision==" "${WORKSPACE_DIR}/python/requirements/ml/dl-cpu-requirements.txt")
 
         # %%;* deletes everything after ; to get rid of e.g. python version specifiers
         pip install "${TORCH_PACKAGE%%;*}" "${TORCHVISION_PACKAGE%%;*}"
-        requirements_files+=("${WORKSPACE_DIR}/python/requirements/air/dl-cpu-requirements.txt")
+        requirements_files+=("${WORKSPACE_DIR}/python/requirements/ml/dl-cpu-requirements.txt")
       fi
   fi
 
   # AIR core dependencies
   if [ "${RLLIB_TESTING-}" = 1 ] || [ "${TRAIN_TESTING-}" = 1 ] || [ "${TUNE_TESTING-}" = 1 ] || [ "${DOC_TESTING-}" = 1 ]; then
-    requirements_files+=("${WORKSPACE_DIR}/python/requirements/air/core-requirements.txt")
+    requirements_files+=("${WORKSPACE_DIR}/python/requirements/ml/core-requirements.txt")
   fi
 
   # Inject our own mirror for the CIFAR10 dataset
diff --git a/python/requirements/air/core-requirements.txt b/python/requirements/ml/air/core-requirements.txt
similarity index 100%
rename from python/requirements/air/core-requirements.txt
rename to python/requirements/ml/air/core-requirements.txt
diff --git a/python/requirements/air/data-requirements.txt b/python/requirements/ml/air/data-requirements.txt
similarity index 100%
rename from python/requirements/air/data-requirements.txt
rename to python/requirements/ml/air/data-requirements.txt
diff --git a/python/requirements/air/data-test-requirements.txt b/python/requirements/ml/air/data-test-requirements.txt
similarity index 100%
rename from python/requirements/air/data-test-requirements.txt
rename to python/requirements/ml/air/data-test-requirements.txt
diff --git a/python/requirements/air/dl-cpu-requirements.txt b/python/requirements/ml/air/dl-cpu-requirements.txt
similarity index 100%
rename from python/requirements/air/dl-cpu-requirements.txt
rename to python/requirements/ml/air/dl-cpu-requirements.txt
diff --git a/python/requirements/air/dl-gpu-requirements.txt b/python/requirements/ml/air/dl-gpu-requirements.txt
similarity index 100%
rename from python/requirements/air/dl-gpu-requirements.txt
rename to python/requirements/ml/air/dl-gpu-requirements.txt
diff --git a/python/requirements/air/rllib-requirements.txt b/python/requirements/ml/air/rllib-requirements.txt
similarity index 100%
rename from python/requirements/air/rllib-requirements.txt
rename to python/requirements/ml/air/rllib-requirements.txt
diff --git a/python/requirements/air/rllib-test-requirements.txt b/python/requirements/ml/air/rllib-test-requirements.txt
similarity index 100%
rename from python/requirements/air/rllib-test-requirements.txt
rename to python/requirements/ml/air/rllib-test-requirements.txt
diff --git a/python/requirements/air/train-requirements.txt b/python/requirements/ml/air/train-requirements.txt
similarity index 100%
rename from python/requirements/air/train-requirements.txt
rename to python/requirements/ml/air/train-requirements.txt
diff --git a/python/requirements/air/train-test-requirements.txt b/python/requirements/ml/air/train-test-requirements.txt
similarity index 100%
rename from python/requirements/air/train-test-requirements.txt
rename to python/requirements/ml/air/train-test-requirements.txt
diff --git a/python/requirements/air/tune-requirements.txt b/python/requirements/ml/air/tune-requirements.txt
similarity index 100%
rename from python/requirements/air/tune-requirements.txt
rename to python/requirements/ml/air/tune-requirements.txt
diff --git a/python/requirements/air/tune-test-requirements.txt b/python/requirements/ml/air/tune-test-requirements.txt
similarity index 100%
rename from python/requirements/air/tune-test-requirements.txt
rename to python/requirements/ml/air/tune-test-requirements.txt
diff --git a/release/ml_user_tests/horovod/driver_requirements.txt b/release/ml_user_tests/horovod/driver_requirements.txt
index 3caff26f694ff..6c8cb758d5659 100755
--- a/release/ml_user_tests/horovod/driver_requirements.txt
+++ b/release/ml_user_tests/horovod/driver_requirements.txt
@@ -2,7 +2,7 @@
 # The cluster uses ray-ml Docker image.
 # ray-ml Docker image installs dependencies from ray/python/requirements/ml/ directory.
 # We constrain on these requirements file so that the same versions are installed.
--c ../../../python/requirements/air/dl-cpu-requirements.txt
+-c ../../../python/requirements/ml/dl-cpu-requirements.txt
 
 torch
 torchvision
\ No newline at end of file
diff --git a/release/ml_user_tests/train/driver_requirements.txt b/release/ml_user_tests/train/driver_requirements.txt
index 30415594267f1..df4c7e35f75eb 100755
--- a/release/ml_user_tests/train/driver_requirements.txt
+++ b/release/ml_user_tests/train/driver_requirements.txt
@@ -2,7 +2,7 @@
 # The cluster uses ray-ml Docker image.
 # ray-ml Docker image installs dependencies from ray/python/requirements/ml/ directory.
 # We constrain on these requirements file so that the same versions are installed.
--c ../../../python/requirements/air/dl-cpu-requirements.txt
+-c ../../../python/requirements/ml/dl-cpu-requirements.txt
 
 torch
 tensorflow
\ No newline at end of file
diff --git a/release/ml_user_tests/tune_rllib/driver_requirements.txt b/release/ml_user_tests/tune_rllib/driver_requirements.txt
index 584e2d9fe7e6b..56a9ab752ce29 100755
--- a/release/ml_user_tests/tune_rllib/driver_requirements.txt
+++ b/release/ml_user_tests/tune_rllib/driver_requirements.txt
@@ -2,7 +2,7 @@
 # The cluster uses ray-ml Docker image.
 # ray-ml Docker image installs dependencies from ray/python/requirements/ml/ directory.
 # We constrain on these requirements file so that the same versions are installed.
--c ../../../python/requirements/air/dl-cpu-requirements.txt
+-c ../../../python/requirements/ml/dl-cpu-requirements.txt
 
 tensorflow
 torch

From 25879d8b63a215bcee9992378b614981d01bd52a Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Wed, 28 Jun 2023 08:06:05 +0100
Subject: [PATCH 16/70] Compile requirements [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 ci/build/build-docker-images.py               |  1 -
 ci/env/install-dependencies.sh                | 35 +++++++++++++++++++
 docker/ray-ml/Dockerfile                      |  4 +--
 .../docker/ml-docker-requirements.txt         |  8 -----
 .../docker/ray-docker-requirements.txt        |  9 +++++
 .../ml/air/dl-cpu-requirements.txt            |  4 +--
 release/long_running_tests/app_config.yaml    |  2 +-
 release/rllib_tests/app_config.yaml           |  4 +--
 release/rllib_tests/debug_app_config.yaml     |  4 +--
 9 files changed, 52 insertions(+), 19 deletions(-)
 delete mode 100644 python/requirements/docker/ml-docker-requirements.txt

diff --git a/ci/build/build-docker-images.py b/ci/build/build-docker-images.py
index 4fb35df47ea80..556bd78ae9bc7 100644
--- a/ci/build/build-docker-images.py
+++ b/ci/build/build-docker-images.py
@@ -426,7 +426,6 @@ def prep_ray_ml():
         "python/requirements.txt",
     ]
     ml_requirements_files = [
-        "python/requirements/docker/ml-docker-requirements.txt",
         "python/requirements/docker/ray-docker-requirements.txt",
         "python/requirements/ml/core-requirements.txt",
         "python/requirements/ml/data-requirements.txt",
diff --git a/ci/env/install-dependencies.sh b/ci/env/install-dependencies.sh
index d6986c07280f5..a0d78f30fd404 100755
--- a/ci/env/install-dependencies.sh
+++ b/ci/env/install-dependencies.sh
@@ -313,6 +313,37 @@ retry_pip_install() {
   fi
 }
 
+compile_ray_requirements() {
+  # Compile boundaries
+  # shellcheck disable=SC2262
+  alias pip="python -m pip"
+  pip install pip-tools
+
+  if [ -f "${WORKSPACE_DIR}/python/requirements_compiled.txt" ]; then
+    echo requirements_compiled already exists
+  else
+    pip-compile --resolver=backtracking -q \
+       --pip-args --no-deps --strip-extras --no-annotate --no-header -q -o \
+      "${WORKSPACE_DIR}/python/requirements_compiled.txt" \
+      "${WORKSPACE_DIR}/python/requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/lint-requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/test-requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/docker/ray-docker-requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/ml/core-requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/ml/data-requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/ml/data-test-requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/ml/dl-cpu-requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/ml/rllib-requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/ml/rllib-test-requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/ml/train-requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/ml/train-test-requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/ml/tune-requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/ml/tune-test-requirements.txt"
+  fi
+
+  cat "${WORKSPACE_DIR}/python/requirements_compiled.txt"
+}
+
 install_pip_packages() {
   # Install modules needed in all jobs.
   # shellcheck disable=SC2262
@@ -435,6 +466,10 @@ install_pip_packages() {
         pip install "${TORCH_PACKAGE%%;*}" "${TORCHVISION_PACKAGE%%;*}"
         requirements_files+=("${WORKSPACE_DIR}/python/requirements/ml/dl-cpu-requirements.txt")
       fi
+
+      # Todo: Move this. We currently have it here because some torch-subpackages need
+      # torch to be installed.
+      compile_ray_requirements
   fi
 
   # AIR core dependencies
diff --git a/docker/ray-ml/Dockerfile b/docker/ray-ml/Dockerfile
index ac4171afc10b1..515e4ce72dc95 100644
--- a/docker/ray-ml/Dockerfile
+++ b/docker/ray-ml/Dockerfile
@@ -7,7 +7,6 @@ COPY requirements.txt ./
 COPY dl-cpu-requirements.txt ./
 COPY dl-gpu-requirements.txt ./
 COPY ray-docker-requirements.txt ./
-COPY ml-docker-requirements.txt ./
 COPY core-requirements.txt ./
 COPY data-requirements.txt ./
 COPY rllib-requirements.txt ./
@@ -43,8 +42,7 @@ RUN $HOME/anaconda3/bin/pip --no-cache-dir install -U \
            -r train-test-requirements.txt \
            -r tune-requirements.txt \
            -r tune-test-requirements.txt \
-           -r ray-docker-requirements.txt \
-           -r ml-docker-requirements.txt
+           -r ray-docker-requirements.txt
 
 # explicitly install (overwrite) pytorch with CUDA support
 RUN $HOME/anaconda3/bin/pip --no-cache-dir install -U \
diff --git a/python/requirements/docker/ml-docker-requirements.txt b/python/requirements/docker/ml-docker-requirements.txt
deleted file mode 100644
index 726c7e124359e..0000000000000
--- a/python/requirements/docker/ml-docker-requirements.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-# Requirements that are shipped in the ML docker image.
-ipython
-
-# Needed for rich visualization for Ray Train and Ray Data.
-ipywidgets>=8
-
-# Needed for Ray Client error message serialization/deserialization.
-tblib
diff --git a/python/requirements/docker/ray-docker-requirements.txt b/python/requirements/docker/ray-docker-requirements.txt
index 2e593e69e4ac1..014c3cd5a3092 100644
--- a/python/requirements/docker/ray-docker-requirements.txt
+++ b/python/requirements/docker/ray-docker-requirements.txt
@@ -1,3 +1,12 @@
 # Todo: Fix conflicts with pinned boto3/botocore
 # awscli
 gsutil
+
+# Requirements that are shipped in the ML docker image.
+ipython
+
+# Needed for rich visualization for Ray Train and Ray Data.
+ipywidgets>=8
+
+# Needed for Ray Client error message serialization/deserialization.
+tblib
diff --git a/python/requirements/ml/air/dl-cpu-requirements.txt b/python/requirements/ml/air/dl-cpu-requirements.txt
index ccbc7365f7cab..ae8375e7ccb05 100644
--- a/python/requirements/ml/air/dl-cpu-requirements.txt
+++ b/python/requirements/ml/air/dl-cpu-requirements.txt
@@ -1,12 +1,12 @@
 # These requirements are used for the CI and CPU-only Docker images so we install CPU only versions of torch.
-# For GPU Docker images, you should install ml-docker-requirements.txt afterwards.
+# For GPU Docker images, you should install dl-gpu-requirements.txt afterwards.
 
 tensorflow==2.11.0; sys_platform != 'darwin' or platform_machine != 'arm64'
 tensorflow-macos==2.11.0; sys_platform == 'darwin' and platform_machine == 'arm64'
 tensorflow-probability==0.19.0
 tensorflow-datasets
 
-# If you make changes below this line, please also make the corresponding changes to `ml-docker-requirements.txt`
+# If you make changes below this line, please also make the corresponding changes to `dl-gpu-requirements.txt`
 # and to `install-dependencies.sh`!
 
 --extra-index-url https://download.pytorch.org/whl/cpu  # for CPU versions of torch, torchvision
diff --git a/release/long_running_tests/app_config.yaml b/release/long_running_tests/app_config.yaml
index ae78399b8c85e..04d4d19fb2271 100755
--- a/release/long_running_tests/app_config.yaml
+++ b/release/long_running_tests/app_config.yaml
@@ -7,7 +7,7 @@ debian_packages:
 
 python:
   pip_packages:
-    # These dependencies should be handled by rllib-requirements.txt and ml-docker-requirements.txt and removed here
+    # These dependencies should be handled by rllib-requirements.txt and removed here
     - gym>=0.21.0,<0.24.1
     - ale-py==0.7.5
     - pytest
diff --git a/release/rllib_tests/app_config.yaml b/release/rllib_tests/app_config.yaml
index 29027b64e53da..dc765eab31c3f 100755
--- a/release/rllib_tests/app_config.yaml
+++ b/release/rllib_tests/app_config.yaml
@@ -13,8 +13,8 @@ debian_packages:
 
 python:
   pip_packages:
-  ## These dependencies should be handled by rllib-requirements.txt and
-  ## ml-docker-requirements.txt and removed here
+  ## These dependencies should be handled by rllib-requirements.txt
+  ## and removed here
   - gymnasium[atari,mujoco]==0.26.3
   - ale-py==0.8.0
   - gym==0.26.2
diff --git a/release/rllib_tests/debug_app_config.yaml b/release/rllib_tests/debug_app_config.yaml
index 4230e4f0cc683..7b07f4c2eea75 100755
--- a/release/rllib_tests/debug_app_config.yaml
+++ b/release/rllib_tests/debug_app_config.yaml
@@ -13,8 +13,8 @@ debian_packages:
 
 python:
   pip_packages:
-  ## These dependencies should be handled by rllib-requirements.txt and
-  ## ml-docker-requirements.txt and removed here
+  ## These dependencies should be handled by rllib-requirements.txt
+  ## and removed here
   - gymnasium[atari,mujoco]==0.26.3
   - ale-py==0.8.0
   - gym==0.26.2

From d0d70c4e23a580d3ad275c216e79f067f967a566 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Wed, 28 Jun 2023 08:47:23 +0100
Subject: [PATCH 17/70] move [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 python/requirements/ml/{air => }/core-requirements.txt       | 0
 python/requirements/ml/{air => }/data-requirements.txt       | 0
 python/requirements/ml/{air => }/data-test-requirements.txt  | 0
 python/requirements/ml/{air => }/dl-cpu-requirements.txt     | 0
 python/requirements/ml/{air => }/dl-gpu-requirements.txt     | 0
 python/requirements/ml/{air => }/rllib-requirements.txt      | 0
 python/requirements/ml/{air => }/rllib-test-requirements.txt | 0
 python/requirements/ml/{air => }/train-requirements.txt      | 0
 python/requirements/ml/{air => }/train-test-requirements.txt | 0
 python/requirements/ml/{air => }/tune-requirements.txt       | 0
 python/requirements/ml/{air => }/tune-test-requirements.txt  | 0
 11 files changed, 0 insertions(+), 0 deletions(-)
 rename python/requirements/ml/{air => }/core-requirements.txt (100%)
 rename python/requirements/ml/{air => }/data-requirements.txt (100%)
 rename python/requirements/ml/{air => }/data-test-requirements.txt (100%)
 rename python/requirements/ml/{air => }/dl-cpu-requirements.txt (100%)
 rename python/requirements/ml/{air => }/dl-gpu-requirements.txt (100%)
 rename python/requirements/ml/{air => }/rllib-requirements.txt (100%)
 rename python/requirements/ml/{air => }/rllib-test-requirements.txt (100%)
 rename python/requirements/ml/{air => }/train-requirements.txt (100%)
 rename python/requirements/ml/{air => }/train-test-requirements.txt (100%)
 rename python/requirements/ml/{air => }/tune-requirements.txt (100%)
 rename python/requirements/ml/{air => }/tune-test-requirements.txt (100%)

diff --git a/python/requirements/ml/air/core-requirements.txt b/python/requirements/ml/core-requirements.txt
similarity index 100%
rename from python/requirements/ml/air/core-requirements.txt
rename to python/requirements/ml/core-requirements.txt
diff --git a/python/requirements/ml/air/data-requirements.txt b/python/requirements/ml/data-requirements.txt
similarity index 100%
rename from python/requirements/ml/air/data-requirements.txt
rename to python/requirements/ml/data-requirements.txt
diff --git a/python/requirements/ml/air/data-test-requirements.txt b/python/requirements/ml/data-test-requirements.txt
similarity index 100%
rename from python/requirements/ml/air/data-test-requirements.txt
rename to python/requirements/ml/data-test-requirements.txt
diff --git a/python/requirements/ml/air/dl-cpu-requirements.txt b/python/requirements/ml/dl-cpu-requirements.txt
similarity index 100%
rename from python/requirements/ml/air/dl-cpu-requirements.txt
rename to python/requirements/ml/dl-cpu-requirements.txt
diff --git a/python/requirements/ml/air/dl-gpu-requirements.txt b/python/requirements/ml/dl-gpu-requirements.txt
similarity index 100%
rename from python/requirements/ml/air/dl-gpu-requirements.txt
rename to python/requirements/ml/dl-gpu-requirements.txt
diff --git a/python/requirements/ml/air/rllib-requirements.txt b/python/requirements/ml/rllib-requirements.txt
similarity index 100%
rename from python/requirements/ml/air/rllib-requirements.txt
rename to python/requirements/ml/rllib-requirements.txt
diff --git a/python/requirements/ml/air/rllib-test-requirements.txt b/python/requirements/ml/rllib-test-requirements.txt
similarity index 100%
rename from python/requirements/ml/air/rllib-test-requirements.txt
rename to python/requirements/ml/rllib-test-requirements.txt
diff --git a/python/requirements/ml/air/train-requirements.txt b/python/requirements/ml/train-requirements.txt
similarity index 100%
rename from python/requirements/ml/air/train-requirements.txt
rename to python/requirements/ml/train-requirements.txt
diff --git a/python/requirements/ml/air/train-test-requirements.txt b/python/requirements/ml/train-test-requirements.txt
similarity index 100%
rename from python/requirements/ml/air/train-test-requirements.txt
rename to python/requirements/ml/train-test-requirements.txt
diff --git a/python/requirements/ml/air/tune-requirements.txt b/python/requirements/ml/tune-requirements.txt
similarity index 100%
rename from python/requirements/ml/air/tune-requirements.txt
rename to python/requirements/ml/tune-requirements.txt
diff --git a/python/requirements/ml/air/tune-test-requirements.txt b/python/requirements/ml/tune-test-requirements.txt
similarity index 100%
rename from python/requirements/ml/air/tune-test-requirements.txt
rename to python/requirements/ml/tune-test-requirements.txt

From 79e0457b3608336e4ba3d7bc3abeed8e9dda5f4b Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Wed, 28 Jun 2023 09:17:53 +0100
Subject: [PATCH 18/70] myst

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 doc/requirements-doc.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/requirements-doc.txt b/doc/requirements-doc.txt
index 613798164c2a5..3253da3f1be50 100644
--- a/doc/requirements-doc.txt
+++ b/doc/requirements-doc.txt
@@ -65,8 +65,8 @@ autodoc_pydantic==1.6.1
 sphinx_design==0.4.1
 
 # MyST
-myst-parser==0.15.2
-myst-nb==0.13.1
+myst-parser==0.19.2
+myst-nb==0.17.2
 
 # Jupyter conversion
 jupytext==1.13.6

From 4cd074b03f03486d4f6b5d8bd2eeb4fc099c200b Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Wed, 28 Jun 2023 10:07:46 +0100
Subject: [PATCH 19/70] myst [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 python/requirements/test-requirements.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/python/requirements/test-requirements.txt b/python/requirements/test-requirements.txt
index c79ac0d55416f..2055dda8e19cc 100644
--- a/python/requirements/test-requirements.txt
+++ b/python/requirements/test-requirements.txt
@@ -80,8 +80,8 @@ xlrd==2.0.1
 memray; platform_system != "Windows" and sys_platform != "darwin"
 
 # For doc tests
-myst-parser==0.15.2
-myst-nb==0.13.1
+myst-parser==0.19.2
+myst-nb==0.17.2
 sphinx==4.3.2
 jupytext==1.13.6
 jinja2==3.0.3

From cd76a02a26ee69d4e10f3ae41fc3a353d1bd487a Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Wed, 28 Jun 2023 10:20:24 +0100
Subject: [PATCH 20/70] myst-parser [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 doc/requirements-doc.txt                  | 2 +-
 python/requirements/test-requirements.txt | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/requirements-doc.txt b/doc/requirements-doc.txt
index 3253da3f1be50..a82551337c524 100644
--- a/doc/requirements-doc.txt
+++ b/doc/requirements-doc.txt
@@ -65,7 +65,7 @@ autodoc_pydantic==1.6.1
 sphinx_design==0.4.1
 
 # MyST
-myst-parser==0.19.2
+myst-parser==0.15.2
 myst-nb==0.17.2
 
 # Jupyter conversion
diff --git a/python/requirements/test-requirements.txt b/python/requirements/test-requirements.txt
index 2055dda8e19cc..c5056a6956b15 100644
--- a/python/requirements/test-requirements.txt
+++ b/python/requirements/test-requirements.txt
@@ -80,7 +80,7 @@ xlrd==2.0.1
 memray; platform_system != "Windows" and sys_platform != "darwin"
 
 # For doc tests
-myst-parser==0.19.2
+myst-parser==0.15.2
 myst-nb==0.17.2
 sphinx==4.3.2
 jupytext==1.13.6

From 936c07f5470d5e66a1aa9123f260e639409343f2 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Wed, 28 Jun 2023 11:44:56 +0100
Subject: [PATCH 21/70] myst-parser [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 doc/requirements-doc.txt                  | 2 +-
 python/requirements/test-requirements.txt | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/requirements-doc.txt b/doc/requirements-doc.txt
index a82551337c524..ef6abb18064c8 100644
--- a/doc/requirements-doc.txt
+++ b/doc/requirements-doc.txt
@@ -65,7 +65,7 @@ autodoc_pydantic==1.6.1
 sphinx_design==0.4.1
 
 # MyST
-myst-parser==0.15.2
+myst-parser==0.18.1
 myst-nb==0.17.2
 
 # Jupyter conversion
diff --git a/python/requirements/test-requirements.txt b/python/requirements/test-requirements.txt
index c5056a6956b15..73c434c337860 100644
--- a/python/requirements/test-requirements.txt
+++ b/python/requirements/test-requirements.txt
@@ -80,7 +80,7 @@ xlrd==2.0.1
 memray; platform_system != "Windows" and sys_platform != "darwin"
 
 # For doc tests
-myst-parser==0.15.2
+myst-parser==0.18.1
 myst-nb==0.17.2
 sphinx==4.3.2
 jupytext==1.13.6

From e05e1cb9ef85531727fbcda315855d5bcc4016e8 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Wed, 28 Jun 2023 13:26:23 +0100
Subject: [PATCH 22/70] move to test dockerfile [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 ci/ci.sh                       | 35 ++++++++++++++++++++++++++++++++++
 ci/docker/base.test.Dockerfile |  1 +
 ci/env/install-dependencies.sh | 35 ----------------------------------
 3 files changed, 36 insertions(+), 35 deletions(-)

diff --git a/ci/ci.sh b/ci/ci.sh
index a6952f72093b5..9791c95e97394 100755
--- a/ci/ci.sh
+++ b/ci/ci.sh
@@ -115,6 +115,41 @@ upload_wheels() {
   )
 }
 
+
+compile_pip_dependencies() {
+    # Compile boundaries
+  # shellcheck disable=SC2262
+  alias pip="python -m pip"
+  pip install pip-tools
+
+  # Required packages to lookup e.g. dragonfly-opt
+  pip install numpy
+
+  if [ -f "${WORKSPACE_DIR}/python/requirements_compiled.txt" ]; then
+    echo requirements_compiled already exists
+  else
+    pip-compile --resolver=backtracking -q \
+       --pip-args --no-deps --strip-extras --no-annotate --no-header -q -o \
+      "${WORKSPACE_DIR}/python/requirements_compiled.txt" \
+      "${WORKSPACE_DIR}/python/requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/lint-requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/test-requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/docker/ray-docker-requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/ml/core-requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/ml/data-requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/ml/data-test-requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/ml/dl-cpu-requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/ml/rllib-requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/ml/rllib-test-requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/ml/train-requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/ml/train-test-requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/ml/tune-requirements.txt" \
+      "${WORKSPACE_DIR}/python/requirements/ml/tune-test-requirements.txt"
+  fi
+
+  cat "${WORKSPACE_DIR}/python/requirements_compiled.txt"
+}
+
 test_core() {
   local args=(
     "//:*"
diff --git a/ci/docker/base.test.Dockerfile b/ci/docker/base.test.Dockerfile
index 012ceb6ccc54c..524e64b72e29d 100644
--- a/ci/docker/base.test.Dockerfile
+++ b/ci/docker/base.test.Dockerfile
@@ -64,4 +64,5 @@ WORKDIR /ray
 # Below should be re-run each time
 COPY . .
 
+RUN ./ci/ci.sh compile_pip_dependencies && pip uninstall -y torch
 RUN ./ci/env/install-dependencies.sh init
diff --git a/ci/env/install-dependencies.sh b/ci/env/install-dependencies.sh
index a0d78f30fd404..d6986c07280f5 100755
--- a/ci/env/install-dependencies.sh
+++ b/ci/env/install-dependencies.sh
@@ -313,37 +313,6 @@ retry_pip_install() {
   fi
 }
 
-compile_ray_requirements() {
-  # Compile boundaries
-  # shellcheck disable=SC2262
-  alias pip="python -m pip"
-  pip install pip-tools
-
-  if [ -f "${WORKSPACE_DIR}/python/requirements_compiled.txt" ]; then
-    echo requirements_compiled already exists
-  else
-    pip-compile --resolver=backtracking -q \
-       --pip-args --no-deps --strip-extras --no-annotate --no-header -q -o \
-      "${WORKSPACE_DIR}/python/requirements_compiled.txt" \
-      "${WORKSPACE_DIR}/python/requirements.txt" \
-      "${WORKSPACE_DIR}/python/requirements/lint-requirements.txt" \
-      "${WORKSPACE_DIR}/python/requirements/test-requirements.txt" \
-      "${WORKSPACE_DIR}/python/requirements/docker/ray-docker-requirements.txt" \
-      "${WORKSPACE_DIR}/python/requirements/ml/core-requirements.txt" \
-      "${WORKSPACE_DIR}/python/requirements/ml/data-requirements.txt" \
-      "${WORKSPACE_DIR}/python/requirements/ml/data-test-requirements.txt" \
-      "${WORKSPACE_DIR}/python/requirements/ml/dl-cpu-requirements.txt" \
-      "${WORKSPACE_DIR}/python/requirements/ml/rllib-requirements.txt" \
-      "${WORKSPACE_DIR}/python/requirements/ml/rllib-test-requirements.txt" \
-      "${WORKSPACE_DIR}/python/requirements/ml/train-requirements.txt" \
-      "${WORKSPACE_DIR}/python/requirements/ml/train-test-requirements.txt" \
-      "${WORKSPACE_DIR}/python/requirements/ml/tune-requirements.txt" \
-      "${WORKSPACE_DIR}/python/requirements/ml/tune-test-requirements.txt"
-  fi
-
-  cat "${WORKSPACE_DIR}/python/requirements_compiled.txt"
-}
-
 install_pip_packages() {
   # Install modules needed in all jobs.
   # shellcheck disable=SC2262
@@ -466,10 +435,6 @@ install_pip_packages() {
         pip install "${TORCH_PACKAGE%%;*}" "${TORCHVISION_PACKAGE%%;*}"
         requirements_files+=("${WORKSPACE_DIR}/python/requirements/ml/dl-cpu-requirements.txt")
       fi
-
-      # Todo: Move this. We currently have it here because some torch-subpackages need
-      # torch to be installed.
-      compile_ray_requirements
   fi
 
   # AIR core dependencies

From bc1eb80fd759f806e11e8dc1ab129ad8b50a8fb9 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Wed, 28 Jun 2023 14:02:25 +0100
Subject: [PATCH 23/70] pip [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 ci/docker/base.test.Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/docker/base.test.Dockerfile b/ci/docker/base.test.Dockerfile
index 524e64b72e29d..cc2bbbdbbd3c5 100644
--- a/ci/docker/base.test.Dockerfile
+++ b/ci/docker/base.test.Dockerfile
@@ -64,5 +64,5 @@ WORKDIR /ray
 # Below should be re-run each time
 COPY . .
 
-RUN ./ci/ci.sh compile_pip_dependencies && pip uninstall -y torch
 RUN ./ci/env/install-dependencies.sh init
+RUN bash --login -i ./ci/ci.sh compile_pip_dependencies && python -m pip uninstall -y torch

From 698b1b079b12232aa32dbe39ace14fa7038cc4a1 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Wed, 28 Jun 2023 14:30:02 +0100
Subject: [PATCH 24/70] bash [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 ci/docker/base.test.Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/docker/base.test.Dockerfile b/ci/docker/base.test.Dockerfile
index cc2bbbdbbd3c5..981d3c0c9b091 100644
--- a/ci/docker/base.test.Dockerfile
+++ b/ci/docker/base.test.Dockerfile
@@ -65,4 +65,4 @@ WORKDIR /ray
 COPY . .
 
 RUN ./ci/env/install-dependencies.sh init
-RUN bash --login -i ./ci/ci.sh compile_pip_dependencies && python -m pip uninstall -y torch
+RUN bash --login -i ./ci/ci.sh compile_pip_dependencies && bash --login -i python -m pip uninstall -y torch

From 9ac12e1811f34f7aa497f04e521588d6b3821948 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Wed, 28 Jun 2023 14:47:36 +0100
Subject: [PATCH 25/70] no uninstall [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 ci/docker/base.test.Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/docker/base.test.Dockerfile b/ci/docker/base.test.Dockerfile
index 981d3c0c9b091..26a9b0628ea45 100644
--- a/ci/docker/base.test.Dockerfile
+++ b/ci/docker/base.test.Dockerfile
@@ -65,4 +65,4 @@ WORKDIR /ray
 COPY . .
 
 RUN ./ci/env/install-dependencies.sh init
-RUN bash --login -i ./ci/ci.sh compile_pip_dependencies && bash --login -i python -m pip uninstall -y torch
+RUN bash --login -i ./ci/ci.sh compile_pip_dependencies

From ab5c6e4c201666f1dad2c029cd0a9bb1c01d501b Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Wed, 28 Jun 2023 15:14:57 +0100
Subject: [PATCH 26/70] install/uninstall torch [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 ci/ci.sh | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/ci/ci.sh b/ci/ci.sh
index 9791c95e97394..8dc1066a67894 100755
--- a/ci/ci.sh
+++ b/ci/ci.sh
@@ -123,7 +123,9 @@ compile_pip_dependencies() {
   pip install pip-tools
 
   # Required packages to lookup e.g. dragonfly-opt
-  pip install numpy
+  HAS_TORCH=0
+  python -c "import torch" 2>/dev/null && HAS_TORCH=1
+  pip install numpy torch
 
   if [ -f "${WORKSPACE_DIR}/python/requirements_compiled.txt" ]; then
     echo requirements_compiled already exists
@@ -148,6 +150,10 @@ compile_pip_dependencies() {
   fi
 
   cat "${WORKSPACE_DIR}/python/requirements_compiled.txt"
+
+  if [ "$HAS_TORCH" -eq 0 ]; then
+    pip uninstall -y torch
+  fi
 }
 
 test_core() {

From 1831ca286790076bae732b57e2d431fe0ade3828 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Wed, 28 Jun 2023 15:43:21 +0100
Subject: [PATCH 27/70] not quiet [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 ci/ci.sh | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/ci/ci.sh b/ci/ci.sh
index 8dc1066a67894..ed11f74e8d09f 100755
--- a/ci/ci.sh
+++ b/ci/ci.sh
@@ -123,15 +123,15 @@ compile_pip_dependencies() {
   pip install pip-tools
 
   # Required packages to lookup e.g. dragonfly-opt
-  HAS_TORCH=0
-  python -c "import torch" 2>/dev/null && HAS_TORCH=1
-  pip install numpy torch
+  #  HAS_TORCH=0
+  #  python -c "import torch" 2>/dev/null && HAS_TORCH=1
+  pip install --no-cache-dir numpy # torch
 
   if [ -f "${WORKSPACE_DIR}/python/requirements_compiled.txt" ]; then
     echo requirements_compiled already exists
   else
     pip-compile --resolver=backtracking -q \
-       --pip-args --no-deps --strip-extras --no-annotate --no-header -q -o \
+       --pip-args --no-deps --strip-extras --no-annotate --no-header -o \
       "${WORKSPACE_DIR}/python/requirements_compiled.txt" \
       "${WORKSPACE_DIR}/python/requirements.txt" \
       "${WORKSPACE_DIR}/python/requirements/lint-requirements.txt" \
@@ -151,9 +151,9 @@ compile_pip_dependencies() {
 
   cat "${WORKSPACE_DIR}/python/requirements_compiled.txt"
 
-  if [ "$HAS_TORCH" -eq 0 ]; then
-    pip uninstall -y torch
-  fi
+#  if [ "$HAS_TORCH" -eq 0 ]; then
+#    pip uninstall -y torch
+#  fi
 }
 
 test_core() {

From 24ededff172a150c8d0cd021b15066c5cf28713c Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Wed, 28 Jun 2023 16:00:54 +0100
Subject: [PATCH 28/70] torch again [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 ci/ci.sh | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/ci/ci.sh b/ci/ci.sh
index ed11f74e8d09f..6bedb3dd6cac1 100755
--- a/ci/ci.sh
+++ b/ci/ci.sh
@@ -123,9 +123,9 @@ compile_pip_dependencies() {
   pip install pip-tools
 
   # Required packages to lookup e.g. dragonfly-opt
-  #  HAS_TORCH=0
-  #  python -c "import torch" 2>/dev/null && HAS_TORCH=1
-  pip install --no-cache-dir numpy # torch
+  HAS_TORCH=0
+  python -c "import torch" 2>/dev/null && HAS_TORCH=1
+  pip install --no-cache-dir numpy torch
 
   if [ -f "${WORKSPACE_DIR}/python/requirements_compiled.txt" ]; then
     echo requirements_compiled already exists
@@ -151,9 +151,9 @@ compile_pip_dependencies() {
 
   cat "${WORKSPACE_DIR}/python/requirements_compiled.txt"
 
-#  if [ "$HAS_TORCH" -eq 0 ]; then
-#    pip uninstall -y torch
-#  fi
+  if [ "$HAS_TORCH" -eq 0 ]; then
+    pip uninstall -y torch
+  fi
 }
 
 test_core() {

From fab40efc0d98bf5af82e505b06489b93fc6bcd0b Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Wed, 28 Jun 2023 16:30:53 +0100
Subject: [PATCH 29/70] aim [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 python/requirements/ml/tune-test-requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/requirements/ml/tune-test-requirements.txt b/python/requirements/ml/tune-test-requirements.txt
index fcb0775e9fe3f..e58057cda4cc0 100644
--- a/python/requirements/ml/tune-test-requirements.txt
+++ b/python/requirements/ml/tune-test-requirements.txt
@@ -1,6 +1,6 @@
 -r dl-cpu-requirements.txt
 
-aim==3.16.1
+aim==3.17.5
 
 # Requires decord which is unavailable for arm64
 gluoncv==0.10.5.post0; platform_machine != "arm64"

From c33d28b3ba6feeb274a4d6c32820e18214f02e29 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Wed, 28 Jun 2023 16:51:22 +0100
Subject: [PATCH 30/70] mxnet [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 doc/requirements-doc.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/requirements-doc.txt b/doc/requirements-doc.txt
index ef6abb18064c8..6a6c5ab720d60 100644
--- a/doc/requirements-doc.txt
+++ b/doc/requirements-doc.txt
@@ -38,7 +38,7 @@ torchvision
 transformers
 
 # Train
-mxnet==1.8.0.post0
+mxnet==1.8.0.post0; platform_machine != "arm64"
 
 # Ray libraries
 git+https://github.com/ray-project/tune-sklearn@master#tune-sklearn

From bc52cbc55aa9fba6e47e93894987702a6194e119 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Wed, 28 Jun 2023 17:16:11 +0100
Subject: [PATCH 31/70] mxnet [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 doc/requirements-doc.txt                          | 2 +-
 python/requirements/ml/tune-test-requirements.txt | 3 +--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/doc/requirements-doc.txt b/doc/requirements-doc.txt
index 6a6c5ab720d60..19da45e94d058 100644
--- a/doc/requirements-doc.txt
+++ b/doc/requirements-doc.txt
@@ -38,7 +38,7 @@ torchvision
 transformers
 
 # Train
-mxnet==1.8.0.post0; platform_machine != "arm64"
+mxnet==1.9.1
 
 # Ray libraries
 git+https://github.com/ray-project/tune-sklearn@master#tune-sklearn
diff --git a/python/requirements/ml/tune-test-requirements.txt b/python/requirements/ml/tune-test-requirements.txt
index e58057cda4cc0..e5e4a2b9c37f6 100644
--- a/python/requirements/ml/tune-test-requirements.txt
+++ b/python/requirements/ml/tune-test-requirements.txt
@@ -9,8 +9,7 @@ gpy==1.10.0
 jupyterlab==3.6.1
 matplotlib!=3.4.3
 
-# Unavailable for arm64 in more recent versions
-mxnet==1.8.0.post0; platform_machine != "arm64"
+mxnet==1.9.1
 pytest-remotedata==0.3.2
 lightning-bolts==0.4.0
 pytorch-lightning==1.6.5

From b67a7245da439ebaf449f5e042e33f63411d6c63 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Wed, 28 Jun 2023 19:13:42 +0100
Subject: [PATCH 32/70] skip on arm64 [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 ci/ci.sh | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/ci/ci.sh b/ci/ci.sh
index 6bedb3dd6cac1..c053ddc39b6bd 100755
--- a/ci/ci.sh
+++ b/ci/ci.sh
@@ -117,7 +117,13 @@ upload_wheels() {
 
 
 compile_pip_dependencies() {
-    # Compile boundaries
+  # Compile boundaries
+
+  if [[ "${HOSTTYPE}" == "aarch64" || "${HOSTTYPE}" = "arm64" ]]; then
+    echo "Skipping for arch64"
+    return 0
+  fi
+
   # shellcheck disable=SC2262
   alias pip="python -m pip"
   pip install pip-tools

From 2fa59174ca810a269cf44e7dc0d7004496f48b5d Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Wed, 28 Jun 2023 22:19:27 +0100
Subject: [PATCH 33/70] trivial [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 ci/ci.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/ci.sh b/ci/ci.sh
index c053ddc39b6bd..be2496c95d92d 100755
--- a/ci/ci.sh
+++ b/ci/ci.sh
@@ -120,7 +120,7 @@ compile_pip_dependencies() {
   # Compile boundaries
 
   if [[ "${HOSTTYPE}" == "aarch64" || "${HOSTTYPE}" = "arm64" ]]; then
-    echo "Skipping for arch64"
+    echo "Skipping for aarch64"
     return 0
   fi
 

From 39d21db1551efd6472ac3cc1d1d76350b3e8a5b9 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Thu, 29 Jun 2023 08:34:32 +0100
Subject: [PATCH 34/70] comments [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 ci/ci.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/ci/ci.sh b/ci/ci.sh
index be2496c95d92d..af9f04f72fd56 100755
--- a/ci/ci.sh
+++ b/ci/ci.sh
@@ -120,6 +120,9 @@ compile_pip_dependencies() {
   # Compile boundaries
 
   if [[ "${HOSTTYPE}" == "aarch64" || "${HOSTTYPE}" = "arm64" ]]; then
+    # Resolution currently does not work on aarch64 as some pinned packages
+    # are not available. Once they are reasonably upgraded we should be able
+    # to enable this here.
     echo "Skipping for aarch64"
     return 0
   fi

From c252bcb5191c2ea1075a2a16beb73c8ae47cf448 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Thu, 29 Jun 2023 11:22:29 +0100
Subject: [PATCH 35/70] Doctests [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 ci/ci.sh                                   | 2 +-
 doc/source/conf.py                         | 1 +
 doc/source/data/examples/ocr_example.ipynb | 2 +-
 3 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/ci/ci.sh b/ci/ci.sh
index af9f04f72fd56..778d36b7ea810 100755
--- a/ci/ci.sh
+++ b/ci/ci.sh
@@ -122,7 +122,7 @@ compile_pip_dependencies() {
   if [[ "${HOSTTYPE}" == "aarch64" || "${HOSTTYPE}" = "arm64" ]]; then
     # Resolution currently does not work on aarch64 as some pinned packages
     # are not available. Once they are reasonably upgraded we should be able
-    # to enable this here.
+    # to enable this here.p
     echo "Skipping for aarch64"
     return 0
   fi
diff --git a/doc/source/conf.py b/doc/source/conf.py
index e5531b0f441e5..d6c4e53426882 100644
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -101,6 +101,7 @@
     "smartquotes",
     "replacements",
 ]
+myst_heading_anchors = 3
 
 
 # Cache notebook outputs in _build/.jupyter_cache
diff --git a/doc/source/data/examples/ocr_example.ipynb b/doc/source/data/examples/ocr_example.ipynb
index 3a287ff1710fe..1b081d2d184e8 100644
--- a/doc/source/data/examples/ocr_example.ipynb
+++ b/doc/source/data/examples/ocr_example.ipynb
@@ -83,7 +83,7 @@
     "\n",
     "````{note}\n",
     "If you want to load the data from a private bucket, you have to run\n",
-    "```python\n",
+    "```\n",
     "import pyarrow.fs\n",
     "\n",
     "ds = ray.data.read_binary_files(\"s3://<bucket>/<folder>\",\n",

From 404cff1feecec09518774c171b0e985bfca4c35a Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Fri, 30 Jun 2023 09:20:12 +0100
Subject: [PATCH 36/70] merge, split install into file [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 docker/ray-ml/Dockerfile                      | 36 +---------------
 .../ray-ml/install-ml-docker-requirements.sh  | 41 +++++++++++++++++++
 2 files changed, 43 insertions(+), 34 deletions(-)
 create mode 100644 docker/ray-ml/install-ml-docker-requirements.sh

diff --git a/docker/ray-ml/Dockerfile b/docker/ray-ml/Dockerfile
index 02d2e652874da..4fec62a2523e9 100644
--- a/docker/ray-ml/Dockerfile
+++ b/docker/ray-ml/Dockerfile
@@ -15,41 +15,9 @@ COPY tune-test-requirements.txt ./
 COPY train-requirements.txt ./
 COPY train-test-requirements.txt ./
 
-RUN sudo apt-get update \
-    && sudo apt-get install -y gcc \
-        cmake \
-        libgtk2.0-dev \
-        zlib1g-dev \
-        libgl1-mesa-dev \
-        unzip \
-        unrar
+COPY install-ml-docker-requirements.sh ./
 
-RUN $HOME/anaconda3/bin/pip --no-cache-dir install -U pip pip-tools
-
-# Install requirements
-RUN $HOME/anaconda3/bin/pip --no-cache-dir install -U \
-           -r requirements.txt
- \
-    # Install other requirements. Keep pinned requirements bounds as constraints
-RUN $HOME/anaconda3/bin/pip --no-cache-dir install -U \
-           -c requirements.txt \
-           -r core-requirements.txt \
-           -r data-requirements.txt \
-           -r rllib-requirements.txt \
-           -r rllib-test-requirements.txt \
-           -r train-requirements.txt \
-           -r train-test-requirements.txt \
-           -r tune-requirements.txt \
-           -r tune-test-requirements.txt \
-           -r ray-docker-requirements.txt
-
-# explicitly install (overwrite) pytorch with CUDA support
-RUN $HOME/anaconda3/bin/pip --no-cache-dir install -U \
-           -c requirements.txt \
-           -r dl-gpu-requirements.txt
-
-RUN sudo rm requirements.txt \
-    && sudo apt-get clean
+RUN chmod +x install-ml-docker-requirements.sh && ./install-ml-docker-requirements.sh
 
 # Export installed packages
 RUN $HOME/anaconda3/bin/pip freeze > /home/ray/pip-freeze.txt
diff --git a/docker/ray-ml/install-ml-docker-requirements.sh b/docker/ray-ml/install-ml-docker-requirements.sh
new file mode 100644
index 0000000000000..1f1b589a3c91b
--- /dev/null
+++ b/docker/ray-ml/install-ml-docker-requirements.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+
+# shellcheck disable=SC2139
+alias pip="$HOME/anaconda3/bin/pip"
+
+sudo apt-get update \
+    && sudo apt-get install -y gcc \
+        cmake \
+        libgtk2.0-dev \
+        zlib1g-dev \
+        libgl1-mesa-dev \
+        unzip \
+        unrar
+
+pip --no-cache-dir install -U pip pip-tools
+
+# Install requirements
+pip --no-cache-dir install -U \
+           -r requirements.txt
+ \
+    # Install other requirements. Keep pinned requirements bounds as constraints
+pip --no-cache-dir install -U \
+           -c requirements.txt \
+           -r core-requirements.txt \
+           -r data-requirements.txt \
+           -r rllib-requirements.txt \
+           -r rllib-test-requirements.txt \
+           -r train-requirements.txt \
+           -r train-test-requirements.txt \
+           -r tune-requirements.txt \
+           -r tune-test-requirements.txt \
+           -r ray-docker-requirements.txt
+
+# explicitly install (overwrite) pytorch with CUDA support
+pip --no-cache-dir install -U \
+           -c requirements.txt \
+           -r dl-gpu-requirements.txt
+
+sudo apt-get clean
+
+sudo rm requirements*.txt

From 88b742d5ae8c158ceb1c1a8bf022b145a8f1c849 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Fri, 30 Jun 2023 09:22:12 +0100
Subject: [PATCH 37/70] myst try [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 doc/source/conf.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/doc/source/conf.py b/doc/source/conf.py
index d6c4e53426882..80eb292ca1103 100644
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -101,7 +101,8 @@
     "smartquotes",
     "replacements",
 ]
-myst_heading_anchors = 3
+
+myst_heading_anchors = 7
 
 
 # Cache notebook outputs in _build/.jupyter_cache

From 3a6a9edc72e7bb88eb7600454fbdbfa96714257a Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Fri, 30 Jun 2023 10:57:55 +0100
Subject: [PATCH 38/70] revert some changes [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 doc/requirements-doc.txt                               | 4 ++--
 doc/source/data/examples/ocr_example.ipynb             | 2 +-
 docker/ray-ml/install-ml-docker-requirements.sh        | 5 +++--
 python/requirements/docker/ray-docker-requirements.txt | 4 +++-
 python/requirements/ml/tune-test-requirements.txt      | 2 +-
 python/requirements/test-requirements.txt              | 2 +-
 6 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/doc/requirements-doc.txt b/doc/requirements-doc.txt
index 19da45e94d058..d8849672aba93 100644
--- a/doc/requirements-doc.txt
+++ b/doc/requirements-doc.txt
@@ -38,7 +38,7 @@ torchvision
 transformers
 
 # Train
-mxnet==1.9.1
+mxnet==1.9.1; sys_platform != "darwin"
 
 # Ray libraries
 git+https://github.com/ray-project/tune-sklearn@master#tune-sklearn
@@ -65,7 +65,7 @@ autodoc_pydantic==1.6.1
 sphinx_design==0.4.1
 
 # MyST
-myst-parser==0.18.1
+myst-parser==0.15.2
 myst-nb==0.17.2
 
 # Jupyter conversion
diff --git a/doc/source/data/examples/ocr_example.ipynb b/doc/source/data/examples/ocr_example.ipynb
index 1b081d2d184e8..3a287ff1710fe 100644
--- a/doc/source/data/examples/ocr_example.ipynb
+++ b/doc/source/data/examples/ocr_example.ipynb
@@ -83,7 +83,7 @@
     "\n",
     "````{note}\n",
     "If you want to load the data from a private bucket, you have to run\n",
-    "```\n",
+    "```python\n",
     "import pyarrow.fs\n",
     "\n",
     "ds = ray.data.read_binary_files(\"s3://<bucket>/<folder>\",\n",
diff --git a/docker/ray-ml/install-ml-docker-requirements.sh b/docker/ray-ml/install-ml-docker-requirements.sh
index 1f1b589a3c91b..80569241fafc9 100644
--- a/docker/ray-ml/install-ml-docker-requirements.sh
+++ b/docker/ray-ml/install-ml-docker-requirements.sh
@@ -17,8 +17,9 @@ pip --no-cache-dir install -U pip pip-tools
 # Install requirements
 pip --no-cache-dir install -U \
            -r requirements.txt
- \
-    # Install other requirements. Keep pinned requirements bounds as constraints
+
+
+# Install other requirements. Keep pinned requirements bounds as constraints
 pip --no-cache-dir install -U \
            -c requirements.txt \
            -r core-requirements.txt \
diff --git a/python/requirements/docker/ray-docker-requirements.txt b/python/requirements/docker/ray-docker-requirements.txt
index 014c3cd5a3092..3023dd86d01eb 100644
--- a/python/requirements/docker/ray-docker-requirements.txt
+++ b/python/requirements/docker/ray-docker-requirements.txt
@@ -6,7 +6,9 @@ gsutil
 ipython
 
 # Needed for rich visualization for Ray Train and Ray Data.
-ipywidgets>=8
+# Todo: Pin to >=8 when myst-parser is upgraded
+# ipywidgets>=8
+ipywidgets
 
 # Needed for Ray Client error message serialization/deserialization.
 tblib
diff --git a/python/requirements/ml/tune-test-requirements.txt b/python/requirements/ml/tune-test-requirements.txt
index e5e4a2b9c37f6..49cab3fc3cb06 100644
--- a/python/requirements/ml/tune-test-requirements.txt
+++ b/python/requirements/ml/tune-test-requirements.txt
@@ -9,7 +9,7 @@ gpy==1.10.0
 jupyterlab==3.6.1
 matplotlib!=3.4.3
 
-mxnet==1.9.1
+mxnet==1.9.1; sys_platform != "darwin"
 pytest-remotedata==0.3.2
 lightning-bolts==0.4.0
 pytorch-lightning==1.6.5
diff --git a/python/requirements/test-requirements.txt b/python/requirements/test-requirements.txt
index adaa269710d46..3faf4888a5f7c 100644
--- a/python/requirements/test-requirements.txt
+++ b/python/requirements/test-requirements.txt
@@ -81,7 +81,7 @@ xlrd==2.0.1
 memray; platform_system != "Windows" and sys_platform != "darwin"
 
 # For doc tests
-myst-parser==0.18.1
+myst-parser==0.15.2
 myst-nb==0.17.2
 sphinx==4.3.2
 jupytext==1.13.6

From 1a17fa79941c0068932a5f8d1552794a1fb7f2c5 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Fri, 30 Jun 2023 12:42:54 +0100
Subject: [PATCH 39/70] myst-nb revert [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 doc/requirements-doc.txt                  | 2 +-
 python/requirements/test-requirements.txt | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/requirements-doc.txt b/doc/requirements-doc.txt
index d8849672aba93..59a593414d32d 100644
--- a/doc/requirements-doc.txt
+++ b/doc/requirements-doc.txt
@@ -66,7 +66,7 @@ sphinx_design==0.4.1
 
 # MyST
 myst-parser==0.15.2
-myst-nb==0.17.2
+myst-nb==0.13.1
 
 # Jupyter conversion
 jupytext==1.13.6
diff --git a/python/requirements/test-requirements.txt b/python/requirements/test-requirements.txt
index 3faf4888a5f7c..79484a15d7b72 100644
--- a/python/requirements/test-requirements.txt
+++ b/python/requirements/test-requirements.txt
@@ -82,7 +82,7 @@ memray; platform_system != "Windows" and sys_platform != "darwin"
 
 # For doc tests
 myst-parser==0.15.2
-myst-nb==0.17.2
+myst-nb==0.13.1
 sphinx==4.3.2
 jupytext==1.13.6
 jinja2==3.0.3

From 864be292ffca136396005cd8ac3a0a1754328c11 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Fri, 30 Jun 2023 14:18:38 +0100
Subject: [PATCH 40/70] chmod [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 doc/source/conf.py                              | 2 --
 docker/ray-ml/Dockerfile                        | 2 +-
 docker/ray-ml/install-ml-docker-requirements.sh | 3 +--
 3 files changed, 2 insertions(+), 5 deletions(-)
 mode change 100644 => 100755 docker/ray-ml/install-ml-docker-requirements.sh

diff --git a/doc/source/conf.py b/doc/source/conf.py
index 80eb292ca1103..e5531b0f441e5 100644
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -102,8 +102,6 @@
     "replacements",
 ]
 
-myst_heading_anchors = 7
-
 
 # Cache notebook outputs in _build/.jupyter_cache
 # To prevent notebook execution, set this to "off". To force re-execution, set this to "force".
diff --git a/docker/ray-ml/Dockerfile b/docker/ray-ml/Dockerfile
index 4fec62a2523e9..c93bd09f92c27 100644
--- a/docker/ray-ml/Dockerfile
+++ b/docker/ray-ml/Dockerfile
@@ -17,7 +17,7 @@ COPY train-test-requirements.txt ./
 
 COPY install-ml-docker-requirements.sh ./
 
-RUN chmod +x install-ml-docker-requirements.sh && ./install-ml-docker-requirements.sh
+RUN sudo chmod +x install-ml-docker-requirements.sh && ./install-ml-docker-requirements.sh
 
 # Export installed packages
 RUN $HOME/anaconda3/bin/pip freeze > /home/ray/pip-freeze.txt
diff --git a/docker/ray-ml/install-ml-docker-requirements.sh b/docker/ray-ml/install-ml-docker-requirements.sh
old mode 100644
new mode 100755
index 80569241fafc9..6da9ae18d1fca
--- a/docker/ray-ml/install-ml-docker-requirements.sh
+++ b/docker/ray-ml/install-ml-docker-requirements.sh
@@ -15,8 +15,7 @@ sudo apt-get update \
 pip --no-cache-dir install -U pip pip-tools
 
 # Install requirements
-pip --no-cache-dir install -U \
-           -r requirements.txt
+pip --no-cache-dir install -U -r requirements.txt
 
 
 # Install other requirements. Keep pinned requirements bounds as constraints

From e66581e597758e83ce4b1b39c8bddc349f54a953 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Fri, 30 Jun 2023 12:47:51 +0100
Subject: [PATCH 41/70] [requirements] Use compiled requirements in CI and
 docker install [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 ci/ci.sh                                        | 2 +-
 ci/env/install-dependencies.sh                  | 5 +++++
 docker/ray-ml/install-ml-docker-requirements.sh | 6 ++++++
 3 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/ci/ci.sh b/ci/ci.sh
index db26d774b6e66..cb50805a9d3f7 100755
--- a/ci/ci.sh
+++ b/ci/ci.sh
@@ -131,7 +131,7 @@ compile_pip_dependencies() {
   alias pip="python -m pip"
   pip install pip-tools
 
-  # Required packages to lookup e.g. dragonfly-opt
+  # Required packages to lookup, e.g. dragonfly-opt
   HAS_TORCH=0
   python -c "import torch" 2>/dev/null && HAS_TORCH=1
   pip install --no-cache-dir numpy torch
diff --git a/ci/env/install-dependencies.sh b/ci/env/install-dependencies.sh
index d6986c07280f5..5e5d1a2aa4b6d 100755
--- a/ci/env/install-dependencies.sh
+++ b/ci/env/install-dependencies.sh
@@ -457,6 +457,11 @@ install_pip_packages() {
 
   # Generate the pip command with collected requirements files
   pip_cmd="pip install -U -c ${WORKSPACE_DIR}/python/requirements.txt"
+
+  if [ -f "${WORKSPACE_DIR}/python/requirements_compiled.txt" ]; then
+    pip_cmd+=" -c ${WORKSPACE_DIR}/python/requirements_compiled.txt"
+  fi
+
   for file in "${requirements_files[@]}"; do
      pip_cmd+=" -r ${file}"
   done
diff --git a/docker/ray-ml/install-ml-docker-requirements.sh b/docker/ray-ml/install-ml-docker-requirements.sh
index 6da9ae18d1fca..bb6f6b3fea835 100755
--- a/docker/ray-ml/install-ml-docker-requirements.sh
+++ b/docker/ray-ml/install-ml-docker-requirements.sh
@@ -17,10 +17,16 @@ pip --no-cache-dir install -U pip pip-tools
 # Install requirements
 pip --no-cache-dir install -U -r requirements.txt
 
+if [ -f "${WORKSPACE_DIR}/python/requirements_compiled.txt" ]; then
+  CONSTRAINT="-c ${WORKSPACE_DIR}/python/requirements_compiled.txt"
+else
+  CONSTRAINT=""
+fi
 
 # Install other requirements. Keep pinned requirements bounds as constraints
 pip --no-cache-dir install -U \
            -c requirements.txt \
+           "$CONSTRAINT" \
            -r core-requirements.txt \
            -r data-requirements.txt \
            -r rllib-requirements.txt \

From 415d67986b93546754faaac957f504d2e70fd3ff Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Mon, 3 Jul 2023 16:37:27 -0700
Subject: [PATCH 42/70] dl-gpu tensorflow

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 python/requirements/ml/dl-gpu-requirements.txt | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/python/requirements/ml/dl-gpu-requirements.txt b/python/requirements/ml/dl-gpu-requirements.txt
index d989c2ac5bf8c..0debcfaefb4f3 100644
--- a/python/requirements/ml/dl-gpu-requirements.txt
+++ b/python/requirements/ml/dl-gpu-requirements.txt
@@ -1,3 +1,8 @@
+tensorflow==2.11.0; sys_platform != 'darwin' or platform_machine != 'arm64'
+tensorflow-macos==2.11.0; sys_platform == 'darwin' and platform_machine == 'arm64'
+tensorflow-probability==0.19.0
+tensorflow-datasets
+
 # If you make changes below this line, please also make the corresponding changes to `dl-cpu-requirements.txt`!
 
 --extra-index-url https://download.pytorch.org/whl/cu116  # for GPU versions of torch, torchvision

From b31cbab33c61e9c1565c5bd685ab971f5a42ce4f Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Tue, 4 Jul 2023 08:44:43 -0700
Subject: [PATCH 43/70] trivial [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 python/requirements/ml/dl-gpu-requirements.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/python/requirements/ml/dl-gpu-requirements.txt b/python/requirements/ml/dl-gpu-requirements.txt
index 0debcfaefb4f3..49c8b7e8fa16b 100644
--- a/python/requirements/ml/dl-gpu-requirements.txt
+++ b/python/requirements/ml/dl-gpu-requirements.txt
@@ -1,10 +1,10 @@
+# If you make changes below this line, please also make the corresponding changes to `dl-cpu-requirements.txt`!
+
 tensorflow==2.11.0; sys_platform != 'darwin' or platform_machine != 'arm64'
 tensorflow-macos==2.11.0; sys_platform == 'darwin' and platform_machine == 'arm64'
 tensorflow-probability==0.19.0
 tensorflow-datasets
 
-# If you make changes below this line, please also make the corresponding changes to `dl-cpu-requirements.txt`!
-
 --extra-index-url https://download.pytorch.org/whl/cu116  # for GPU versions of torch, torchvision
 --find-links https://data.pyg.org/whl/torch-1.13.0+cu116.html  # for GPU versions of torch-scatter, torch-sparse, torch-cluster, torch-spline-conv
 # specifying explicit plus-notation below so pip overwrites the existing cpu verisons

From 1f167b3b72186e94e0071e620901832766b0c686 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Wed, 5 Jul 2023 08:41:01 -0700
Subject: [PATCH 44/70] Preserve requirements_compiled.txt [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 ci/docker/base.gpu.Dockerfile |  1 +
 ci/docker/base.ml.Dockerfile  | 11 -----------
 ci/docker/build.Dockerfile    |  4 ++++
 ci/docker/gpu.Dockerfile      |  3 +++
 ci/docker/ml.Dockerfile       |  3 +++
 ci/docker/test.Dockerfile     |  3 +++
 6 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/ci/docker/base.gpu.Dockerfile b/ci/docker/base.gpu.Dockerfile
index 2dbbb96a0915f..bba013b61d2c0 100644
--- a/ci/docker/base.gpu.Dockerfile
+++ b/ci/docker/base.gpu.Dockerfile
@@ -63,4 +63,5 @@ WORKDIR /ray
 COPY . .
 
 RUN ./ci/env/install-dependencies.sh init
+RUN bash --login -i ./ci/ci.sh compile_pip_dependencies
 RUN RLLIB_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 bash --login -i ./ci/env/install-dependencies.sh
diff --git a/ci/docker/base.ml.Dockerfile b/ci/docker/base.ml.Dockerfile
index 405ec1df0ca4c..8448e1b257257 100644
--- a/ci/docker/base.ml.Dockerfile
+++ b/ci/docker/base.ml.Dockerfile
@@ -1,15 +1,4 @@
 ARG DOCKER_IMAGE_BASE_TEST
 FROM $DOCKER_IMAGE_BASE_TEST
 
-# Move out of working dir /ray
-# Delete stale data
-WORKDIR /
-RUN rm -rf /ray
-
-RUN mkdir /ray
-WORKDIR /ray
-
-# Copy new ray files
-COPY . .
-
 RUN RLLIB_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/env/install-dependencies.sh
diff --git a/ci/docker/build.Dockerfile b/ci/docker/build.Dockerfile
index c05e10b07db2d..965dccbcd88f9 100644
--- a/ci/docker/build.Dockerfile
+++ b/ci/docker/build.Dockerfile
@@ -16,10 +16,14 @@ ENV CXX=clang++-12
 # Move out of working dir /ray
 # Delete stale data
 WORKDIR /
+# Preserve requirements_compiled.txt
+RUN mv /ray/python/requirements_compiled.txt /tmp/requirements_compiled.txt || true
 RUN rm -rf /ray
 
 RUN mkdir /ray
 WORKDIR /ray
+RUN mv /tmp/requirements_compiled.txt /ray/python/requirements_compiled.txt || true
+
 
 # Below should be re-run each time
 COPY . .
diff --git a/ci/docker/gpu.Dockerfile b/ci/docker/gpu.Dockerfile
index 02440fa953176..ad6e2192e0bf5 100644
--- a/ci/docker/gpu.Dockerfile
+++ b/ci/docker/gpu.Dockerfile
@@ -13,10 +13,13 @@ ENV TRAVIS_COMMIT=${BUILDKITE_COMMIT}
 # Move out of working dir /ray
 # Delete stale data
 WORKDIR /
+# Preserve requirements_compiled.txt
+RUN mv /ray/python/requirements_compiled.txt /tmp/requirements_compiled.txt || true
 RUN rm -rf /ray
 
 RUN mkdir /ray
 WORKDIR /ray
+RUN mv /tmp/requirements_compiled.txt /ray/python/requirements_compiled.txt || true
 
 # Copy new ray files
 COPY . .
diff --git a/ci/docker/ml.Dockerfile b/ci/docker/ml.Dockerfile
index 44ce9b320a257..57f25a35d2e99 100644
--- a/ci/docker/ml.Dockerfile
+++ b/ci/docker/ml.Dockerfile
@@ -4,10 +4,13 @@ FROM $DOCKER_IMAGE_BASE_ML
 # Move out of working dir /ray
 # Delete stale data
 WORKDIR /
+# Preserve requirements_compiled.txt
+RUN mv /ray/python/requirements_compiled.txt /tmp/requirements_compiled.txt || true
 RUN rm -rf /ray
 
 RUN mkdir /ray
 WORKDIR /ray
+RUN mv /tmp/requirements_compiled.txt /ray/python/requirements_compiled.txt || true
 
 # Copy new ray files
 COPY . .
diff --git a/ci/docker/test.Dockerfile b/ci/docker/test.Dockerfile
index e0fdcb2c73141..7d7189347cc9c 100644
--- a/ci/docker/test.Dockerfile
+++ b/ci/docker/test.Dockerfile
@@ -4,10 +4,13 @@ FROM $DOCKER_IMAGE_BASE_TEST
 # Move out of working dir /ray
 # Delete stale data
 WORKDIR /
+# Preserve requirements_compiled.txt
+RUN mv /ray/python/requirements_compiled.txt /tmp/requirements_compiled.txt || true
 RUN rm -rf /ray
 
 RUN mkdir /ray
 WORKDIR /ray
+RUN mv /tmp/requirements_compiled.txt /ray/python/requirements_compiled.txt || true
 
 # Copy new ray files.
 COPY . .

From b186fcf5e40d2f07dc632c74ebb633cc1d4dcb57 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Wed, 5 Jul 2023 09:26:31 -0700
Subject: [PATCH 45/70] move [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 ci/docker/build.Dockerfile | 4 ++--
 ci/docker/gpu.Dockerfile   | 3 ++-
 ci/docker/ml.Dockerfile    | 3 ++-
 ci/docker/test.Dockerfile  | 3 ++-
 4 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/ci/docker/build.Dockerfile b/ci/docker/build.Dockerfile
index 965dccbcd88f9..d0e6f9692c51e 100644
--- a/ci/docker/build.Dockerfile
+++ b/ci/docker/build.Dockerfile
@@ -22,12 +22,12 @@ RUN rm -rf /ray
 
 RUN mkdir /ray
 WORKDIR /ray
-RUN mv /tmp/requirements_compiled.txt /ray/python/requirements_compiled.txt || true
-
 
 # Below should be re-run each time
 COPY . .
 
+RUN mv /tmp/requirements_compiled.txt /ray/python/requirements_compiled.txt || true
+
 RUN env
 
 # init also calls install-dependencies.sh
diff --git a/ci/docker/gpu.Dockerfile b/ci/docker/gpu.Dockerfile
index ad6e2192e0bf5..0c0c58613029c 100644
--- a/ci/docker/gpu.Dockerfile
+++ b/ci/docker/gpu.Dockerfile
@@ -19,11 +19,12 @@ RUN rm -rf /ray
 
 RUN mkdir /ray
 WORKDIR /ray
-RUN mv /tmp/requirements_compiled.txt /ray/python/requirements_compiled.txt || true
 
 # Copy new ray files
 COPY . .
 
+RUN mv /tmp/requirements_compiled.txt /ray/python/requirements_compiled.txt || true
+
 RUN env
 
 RUN RLLIB_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 bash --login -i ./ci/env/install-dependencies.sh
diff --git a/ci/docker/ml.Dockerfile b/ci/docker/ml.Dockerfile
index 57f25a35d2e99..37bf6327c79c6 100644
--- a/ci/docker/ml.Dockerfile
+++ b/ci/docker/ml.Dockerfile
@@ -10,11 +10,12 @@ RUN rm -rf /ray
 
 RUN mkdir /ray
 WORKDIR /ray
-RUN mv /tmp/requirements_compiled.txt /ray/python/requirements_compiled.txt || true
 
 # Copy new ray files
 COPY . .
 
+RUN mv /tmp/requirements_compiled.txt /ray/python/requirements_compiled.txt || true
+
 RUN RLLIB_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 bash --login -i ./ci/env/install-dependencies.sh
 
 # Install Ray
diff --git a/ci/docker/test.Dockerfile b/ci/docker/test.Dockerfile
index 7d7189347cc9c..4c6256e4b30ea 100644
--- a/ci/docker/test.Dockerfile
+++ b/ci/docker/test.Dockerfile
@@ -10,11 +10,12 @@ RUN rm -rf /ray
 
 RUN mkdir /ray
 WORKDIR /ray
-RUN mv /tmp/requirements_compiled.txt /ray/python/requirements_compiled.txt || true
 
 # Copy new ray files.
 COPY . .
 
+RUN mv /tmp/requirements_compiled.txt /ray/python/requirements_compiled.txt || true
+
 RUN bash --login -i ./ci/env/install-dependencies.sh
 
 # Install Ray

From f1d46a48c0e6d012b2ec7f505d538df22f298378 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Wed, 5 Jul 2023 10:44:21 -0700
Subject: [PATCH 46/70] remove ray== pin [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 ci/ci.sh                 | 2 ++
 doc/requirements-doc.txt | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/ci/ci.sh b/ci/ci.sh
index db26d774b6e66..29b97b6b338fb 100755
--- a/ci/ci.sh
+++ b/ci/ci.sh
@@ -158,6 +158,8 @@ compile_pip_dependencies() {
       "${WORKSPACE_DIR}/python/requirements/ml/tune-test-requirements.txt"
   fi
 
+  # Remove ray== pin (this can come up from upstream dependencies)
+  sed -i "/^ray==/d" "${WORKSPACE_DIR}/python/requirements_compiled.txt"
   cat "${WORKSPACE_DIR}/python/requirements_compiled.txt"
 
   if [ "$HAS_TORCH" -eq 0 ]; then
diff --git a/doc/requirements-doc.txt b/doc/requirements-doc.txt
index 59a593414d32d..787efc883b60a 100644
--- a/doc/requirements-doc.txt
+++ b/doc/requirements-doc.txt
@@ -27,7 +27,7 @@ pytorch-lightning==1.6.5
 scikit-optimize
 redis
 starlette
-uvicorn==0.16.0
+uvicorn==0.22.0
 werkzeug
 wandb
 tensorflow; sys_platform != 'darwin' or platform_machine != 'arm64'

From b675fbccf239df1d34dc6d59aaccbaebe26b7a49 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Wed, 5 Jul 2023 11:51:43 -0700
Subject: [PATCH 47/70] remove more pins [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 ci/ci.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/ci/ci.sh b/ci/ci.sh
index 29b97b6b338fb..f7bda1d838d1e 100755
--- a/ci/ci.sh
+++ b/ci/ci.sh
@@ -158,8 +158,9 @@ compile_pip_dependencies() {
       "${WORKSPACE_DIR}/python/requirements/ml/tune-test-requirements.txt"
   fi
 
-  # Remove ray== pin (this can come up from upstream dependencies)
-  sed -i "/^ray==/d" "${WORKSPACE_DIR}/python/requirements_compiled.txt"
+  # Remove some pins from upstream dependencies:
+  # ray, xgboost-ray, lightgbm-ray, tune-sklearn
+  sed -i "/^ray==/d;/^xgboost-ray==/d;/^lightgbm-ray==/d;/^tune-sklearn==/d" "${WORKSPACE_DIR}/python/requirements_compiled.txt"
   cat "${WORKSPACE_DIR}/python/requirements_compiled.txt"
 
   if [ "$HAS_TORCH" -eq 0 ]; then

From 334eff092927e250220339adf1ff66c08d5f4bfe Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Wed, 5 Jul 2023 16:51:59 -0700
Subject: [PATCH 48/70] require on buildkite [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 ci/env/install-dependencies.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/ci/env/install-dependencies.sh b/ci/env/install-dependencies.sh
index 5e5d1a2aa4b6d..51c2dd2685040 100755
--- a/ci/env/install-dependencies.sh
+++ b/ci/env/install-dependencies.sh
@@ -458,7 +458,11 @@ install_pip_packages() {
   # Generate the pip command with collected requirements files
   pip_cmd="pip install -U -c ${WORKSPACE_DIR}/python/requirements.txt"
 
-  if [ -f "${WORKSPACE_DIR}/python/requirements_compiled.txt" ]; then
+  if { [ -f "${WORKSPACE_DIR}/python/requirements_compiled.txt" ] || [ -n "${BUILDKITE-}" ]; } && [ "${PYTHON-}" != "3.7" ]; then
+    # On Buildkite, we always require the compiled constraints file.
+    # On Python 3.7, we don't, as the dependencies are compiled for 3.8+
+    # and we don't build ray-ml images. This means we don't have to keep
+    # consistency between CI and docker images.
     pip_cmd+=" -c ${WORKSPACE_DIR}/python/requirements_compiled.txt"
   fi
 

From 48be89fcc6749a41992f80cd39a6a4c619f3fc42 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Wed, 5 Jul 2023 17:31:07 -0700
Subject: [PATCH 49/70] relax install condition [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 ci/env/install-dependencies.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/ci/env/install-dependencies.sh b/ci/env/install-dependencies.sh
index 51c2dd2685040..8dd6b02ef2ccc 100755
--- a/ci/env/install-dependencies.sh
+++ b/ci/env/install-dependencies.sh
@@ -458,8 +458,7 @@ install_pip_packages() {
   # Generate the pip command with collected requirements files
   pip_cmd="pip install -U -c ${WORKSPACE_DIR}/python/requirements.txt"
 
-  if { [ -f "${WORKSPACE_DIR}/python/requirements_compiled.txt" ] || [ -n "${BUILDKITE-}" ]; } && [ "${PYTHON-}" != "3.7" ]; then
-    # On Buildkite, we always require the compiled constraints file.
+  if [ -f "${WORKSPACE_DIR}/python/requirements_compiled.txt" ] && [ "${PYTHON-}" != "3.7" ]; then
     # On Python 3.7, we don't, as the dependencies are compiled for 3.8+
     # and we don't build ray-ml images. This means we don't have to keep
     # consistency between CI and docker images.

From f963e91b0d3256915405fb26878d3e4b7615a735 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Thu, 6 Jul 2023 09:35:04 -0700
Subject: [PATCH 50/70] bash [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 docker/ray-ml/Dockerfile                        | 2 +-
 docker/ray-ml/install-ml-docker-requirements.sh | 4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/docker/ray-ml/Dockerfile b/docker/ray-ml/Dockerfile
index c93bd09f92c27..492346591e633 100644
--- a/docker/ray-ml/Dockerfile
+++ b/docker/ray-ml/Dockerfile
@@ -17,7 +17,7 @@ COPY train-test-requirements.txt ./
 
 COPY install-ml-docker-requirements.sh ./
 
-RUN sudo chmod +x install-ml-docker-requirements.sh && ./install-ml-docker-requirements.sh
+RUN sudo chmod +x install-ml-docker-requirements.sh && bash --login -i ./install-ml-docker-requirements.sh
 
 # Export installed packages
 RUN $HOME/anaconda3/bin/pip freeze > /home/ray/pip-freeze.txt
diff --git a/docker/ray-ml/install-ml-docker-requirements.sh b/docker/ray-ml/install-ml-docker-requirements.sh
index bb6f6b3fea835..338dcea1978a5 100755
--- a/docker/ray-ml/install-ml-docker-requirements.sh
+++ b/docker/ray-ml/install-ml-docker-requirements.sh
@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -xe
+
 # shellcheck disable=SC2139
 alias pip="$HOME/anaconda3/bin/pip"
 
@@ -44,4 +46,4 @@ pip --no-cache-dir install -U \
 
 sudo apt-get clean
 
-sudo rm requirements*.txt
+sudo rm ./*requirements.txt

From 38665296a1ef5d6be9fa55721b183f0ecaf014da Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Thu, 6 Jul 2023 12:54:27 -0700
Subject: [PATCH 51/70] constraints [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 docker/ray-ml/install-ml-docker-requirements.sh | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/docker/ray-ml/install-ml-docker-requirements.sh b/docker/ray-ml/install-ml-docker-requirements.sh
index 338dcea1978a5..6cb97fd0d010b 100755
--- a/docker/ray-ml/install-ml-docker-requirements.sh
+++ b/docker/ray-ml/install-ml-docker-requirements.sh
@@ -19,16 +19,16 @@ pip --no-cache-dir install -U pip pip-tools
 # Install requirements
 pip --no-cache-dir install -U -r requirements.txt
 
-if [ -f "${WORKSPACE_DIR}/python/requirements_compiled.txt" ]; then
-  CONSTRAINT="-c ${WORKSPACE_DIR}/python/requirements_compiled.txt"
+if [ -f "requirements_compiled.txt" ]; then
+  CONSTRAINT="-c requirements.txt -c requirements_compiled.txt"
 else
-  CONSTRAINT=""
+  CONSTRAINT="-c requirements.txt"
 fi
 
 # Install other requirements. Keep pinned requirements bounds as constraints
+# shellcheck disable=SC2086
 pip --no-cache-dir install -U \
-           -c requirements.txt \
-           "$CONSTRAINT" \
+           $CONSTRAINT \
            -r core-requirements.txt \
            -r data-requirements.txt \
            -r rllib-requirements.txt \

From 2ffbadb70c68e23217cb273bf456745fc88653b1 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Thu, 6 Jul 2023 13:23:32 -0700
Subject: [PATCH 52/70] rllib

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 rllib/core/learner/tests/test_learner.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/rllib/core/learner/tests/test_learner.py b/rllib/core/learner/tests/test_learner.py
index da3e2102b7ae6..6708a3e30e8ff 100644
--- a/rllib/core/learner/tests/test_learner.py
+++ b/rllib/core/learner/tests/test_learner.py
@@ -49,10 +49,10 @@ def test_end_to_end_update(self):
             min_loss = min(loss, min_loss)
             print(f"[iter = {iter_i}] Loss: {loss:.3f}, Min Loss: {min_loss:.3f}")
             # The loss is initially around 0.69 (ln2). When it gets to around
-            # 0.57 the return of the policy gets to around 100.
-            if min_loss < 0.57:
+            # 0.58 the return of the policy gets to around 100.
+            if min_loss < 0.58:
                 break
-        self.assertLess(min_loss, 0.57)
+        self.assertLess(min_loss, 0.58)
 
     def test_compute_gradients(self):
         """Tests the compute_gradients correctness.

From e62f18e94b9e848ddb0a2ab9d295b126170e4610 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Thu, 6 Jul 2023 13:24:58 -0700
Subject: [PATCH 53/70] Revert "rllib" [build_base]

This reverts commit 2ffbadb70c68e23217cb273bf456745fc88653b1.
---
 rllib/core/learner/tests/test_learner.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/rllib/core/learner/tests/test_learner.py b/rllib/core/learner/tests/test_learner.py
index 6708a3e30e8ff..da3e2102b7ae6 100644
--- a/rllib/core/learner/tests/test_learner.py
+++ b/rllib/core/learner/tests/test_learner.py
@@ -49,10 +49,10 @@ def test_end_to_end_update(self):
             min_loss = min(loss, min_loss)
             print(f"[iter = {iter_i}] Loss: {loss:.3f}, Min Loss: {min_loss:.3f}")
             # The loss is initially around 0.69 (ln2). When it gets to around
-            # 0.58 the return of the policy gets to around 100.
-            if min_loss < 0.58:
+            # 0.57 the return of the policy gets to around 100.
+            if min_loss < 0.57:
                 break
-        self.assertLess(min_loss, 0.58)
+        self.assertLess(min_loss, 0.57)
 
     def test_compute_gradients(self):
         """Tests the compute_gradients correctness.

From 8a382cfd2fc44a1e732fd5b0ef316af204703574 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Thu, 6 Jul 2023 16:58:07 -0700
Subject: [PATCH 54/70] Don't remove constraints in wheel build [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 python/build-wheel-manylinux2014.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/python/build-wheel-manylinux2014.sh b/python/build-wheel-manylinux2014.sh
index e07ac26405eda..4c2540d3f3386 100755
--- a/python/build-wheel-manylinux2014.sh
+++ b/python/build-wheel-manylinux2014.sh
@@ -81,8 +81,9 @@ for PYTHON_NUMPY in "${PYTHON_NUMPYS[@]}" ; do
   # The -f flag is passed twice to also run git clean in the arrow subdirectory.
   # The -d flag removes directories. The -x flag ignores the .gitignore file,
   # and the -e flag ensures that we don't remove the .whl directory, the
-  # dashboard directory and jars directory.
-  git clean -f -f -x -d -e .whl -e python/ray/dashboard/client -e dashboard/client -e python/ray/jars
+  # dashboard directory and jars directory, as well as the compiled
+  # dependency constraints.
+  git clean -f -f -x -d -e .whl -e python/ray/dashboard/client -e dashboard/client -e python/ray/jars -e python/requirements_compiled.txt
 
   (
     cd python

From 1f919d145df24e6d6443fd1e56ff549e11b033a4 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Fri, 7 Jul 2023 08:21:28 -0700
Subject: [PATCH 55/70] one more constraint [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 docker/ray-ml/install-ml-docker-requirements.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/ray-ml/install-ml-docker-requirements.sh b/docker/ray-ml/install-ml-docker-requirements.sh
index 6cb97fd0d010b..397ea7533f89e 100755
--- a/docker/ray-ml/install-ml-docker-requirements.sh
+++ b/docker/ray-ml/install-ml-docker-requirements.sh
@@ -41,7 +41,7 @@ pip --no-cache-dir install -U \
 
 # explicitly install (overwrite) pytorch with CUDA support
 pip --no-cache-dir install -U \
-           -c requirements.txt \
+           $CONSTRAINT \
            -r dl-gpu-requirements.txt
 
 sudo apt-get clean

From 4a5417e52d8bbd947d1806140f3c7d3dc4124eef Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Fri, 7 Jul 2023 09:23:25 -0700
Subject: [PATCH 56/70] shellcheck [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 docker/ray-ml/install-ml-docker-requirements.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docker/ray-ml/install-ml-docker-requirements.sh b/docker/ray-ml/install-ml-docker-requirements.sh
index 397ea7533f89e..c82561175c1d4 100755
--- a/docker/ray-ml/install-ml-docker-requirements.sh
+++ b/docker/ray-ml/install-ml-docker-requirements.sh
@@ -40,6 +40,7 @@ pip --no-cache-dir install -U \
            -r ray-docker-requirements.txt
 
 # explicitly install (overwrite) pytorch with CUDA support
+# shellcheck disable=SC2086
 pip --no-cache-dir install -U \
            $CONSTRAINT \
            -r dl-gpu-requirements.txt

From f0f879ca54369ccf18dfc7fb767f0800c142e740 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Fri, 7 Jul 2023 12:42:09 -0700
Subject: [PATCH 57/70] one more exception [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 python/build-wheel-manylinux2014.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/build-wheel-manylinux2014.sh b/python/build-wheel-manylinux2014.sh
index 4c2540d3f3386..6b70c2d8c9615 100755
--- a/python/build-wheel-manylinux2014.sh
+++ b/python/build-wheel-manylinux2014.sh
@@ -123,7 +123,7 @@ for path in .whl/*.whl; do
 done
 
 # Clean the build output so later operations is on a clean directory.
-git clean -f -f -x -d -e .whl -e python/ray/dashboard/client
+git clean -f -f -x -d -e .whl -e python/ray/dashboard/client -e python/requirements_compiled.txt
 
 echo "--- Build JAR"
 if [ "${BUILD_JAR-}" == "1" ]; then

From a77fdfb52dd3113cbe7d41f50504235b1076dc73 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Fri, 7 Jul 2023 14:28:52 -0700
Subject: [PATCH 58/70] trivial [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 release/ray_release/scripts/build_pipeline.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/release/ray_release/scripts/build_pipeline.py b/release/ray_release/scripts/build_pipeline.py
index 3068346a3f2f1..123df18f9d138 100644
--- a/release/ray_release/scripts/build_pipeline.py
+++ b/release/ray_release/scripts/build_pipeline.py
@@ -94,7 +94,7 @@ def main(
     if repo and not no_clone_repo:
         # If the Ray test repo is set, we clone that repo to fetch
         # the test configuration file. Otherwise, we might be missing newly
-        # added test.
+        # added tests.
 
         tmpdir = tempfile.mktemp()
 

From c7530397824d33521586372bf63ee5e0a0ac22ce Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Fri, 7 Jul 2023 15:03:01 -0700
Subject: [PATCH 59/70] bdi [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 ci/build/build-docker-images.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ci/build/build-docker-images.py b/ci/build/build-docker-images.py
index 556bd78ae9bc7..f7f5a6ee28137 100644
--- a/ci/build/build-docker-images.py
+++ b/ci/build/build-docker-images.py
@@ -424,6 +424,7 @@ def prep_ray_ml():
 
     requirements_files = [
         "python/requirements.txt",
+        "python/requirements_compiled.txt",
     ]
     ml_requirements_files = [
         "python/requirements/docker/ray-docker-requirements.txt",

From 0e7c60f2996effae414e9493de79230d298d2398 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Fri, 7 Jul 2023 15:20:06 -0700
Subject: [PATCH 60/70] always require constraints in ray-ml [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 docker/ray-ml/Dockerfile                        |  1 +
 docker/ray-ml/install-ml-docker-requirements.sh | 14 ++++----------
 2 files changed, 5 insertions(+), 10 deletions(-)

diff --git a/docker/ray-ml/Dockerfile b/docker/ray-ml/Dockerfile
index 492346591e633..783c16d0b64a2 100644
--- a/docker/ray-ml/Dockerfile
+++ b/docker/ray-ml/Dockerfile
@@ -3,6 +3,7 @@ FROM rayproject/ray:nightly"$BASE_IMAGE"
 
 # We have to uninstall wrapt this way for Tensorflow compatibility
 COPY requirements.txt ./
+COPY requirements_compiled.txt ./
 COPY dl-cpu-requirements.txt ./
 COPY dl-gpu-requirements.txt ./
 COPY ray-docker-requirements.txt ./
diff --git a/docker/ray-ml/install-ml-docker-requirements.sh b/docker/ray-ml/install-ml-docker-requirements.sh
index c82561175c1d4..0a90c24d7fa12 100755
--- a/docker/ray-ml/install-ml-docker-requirements.sh
+++ b/docker/ray-ml/install-ml-docker-requirements.sh
@@ -19,16 +19,10 @@ pip --no-cache-dir install -U pip pip-tools
 # Install requirements
 pip --no-cache-dir install -U -r requirements.txt
 
-if [ -f "requirements_compiled.txt" ]; then
-  CONSTRAINT="-c requirements.txt -c requirements_compiled.txt"
-else
-  CONSTRAINT="-c requirements.txt"
-fi
-
 # Install other requirements. Keep pinned requirements bounds as constraints
-# shellcheck disable=SC2086
 pip --no-cache-dir install -U \
-           $CONSTRAINT \
+           -c requirements.txt \
+           -c requirements_compiled.txt \
            -r core-requirements.txt \
            -r data-requirements.txt \
            -r rllib-requirements.txt \
@@ -40,9 +34,9 @@ pip --no-cache-dir install -U \
            -r ray-docker-requirements.txt
 
 # explicitly install (overwrite) pytorch with CUDA support
-# shellcheck disable=SC2086
 pip --no-cache-dir install -U \
-           $CONSTRAINT \
+           -c requirements.txt \
+           -c requirements_compiled.txt \
            -r dl-gpu-requirements.txt
 
 sudo apt-get clean

From 0b301ffe4d3f25358fd5a599803235c67f20ef2b Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Fri, 7 Jul 2023 15:47:56 -0700
Subject: [PATCH 61/70] no constraints for gpu [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 docker/ray-ml/install-ml-docker-requirements.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/docker/ray-ml/install-ml-docker-requirements.sh b/docker/ray-ml/install-ml-docker-requirements.sh
index 0a90c24d7fa12..c1cf3f5663f1e 100755
--- a/docker/ray-ml/install-ml-docker-requirements.sh
+++ b/docker/ray-ml/install-ml-docker-requirements.sh
@@ -34,9 +34,8 @@ pip --no-cache-dir install -U \
            -r ray-docker-requirements.txt
 
 # explicitly install (overwrite) pytorch with CUDA support
-pip --no-cache-dir install -U \
+pip --no-cache-dir install \
            -c requirements.txt \
-           -c requirements_compiled.txt \
            -r dl-gpu-requirements.txt
 
 sudo apt-get clean

From 64cfa7e18e2ce90cc3a0fbc968c7bca7c35ac88f Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Sat, 8 Jul 2023 18:22:38 -0700
Subject: [PATCH 62/70] comment [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 docker/ray-ml/install-ml-docker-requirements.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docker/ray-ml/install-ml-docker-requirements.sh b/docker/ray-ml/install-ml-docker-requirements.sh
index c1cf3f5663f1e..dad56710bdd2a 100755
--- a/docker/ray-ml/install-ml-docker-requirements.sh
+++ b/docker/ray-ml/install-ml-docker-requirements.sh
@@ -34,6 +34,9 @@ pip --no-cache-dir install -U \
            -r ray-docker-requirements.txt
 
 # explicitly install (overwrite) pytorch with CUDA support
+# do not use compiled requirements as constraint here as some
+# device-specific constraints can't be met (e.g. torch+cpu).
+# Instead, we don't use `-U` so other dependencies don't get updated.
 pip --no-cache-dir install \
            -c requirements.txt \
            -r dl-gpu-requirements.txt

From a3abd5a9ed896f6625dcf5d5ccf5db25e51fba6a Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Tue, 11 Jul 2023 11:14:18 -0700
Subject: [PATCH 63/70] Check in compiled requirements [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 ci/docker/base.gpu.Dockerfile                 |   1 -
 ci/docker/base.test.Dockerfile                |   1 -
 ci/docker/build.Dockerfile                    |   4 -
 ci/docker/gpu.Dockerfile                      |   4 -
 ci/docker/ml.Dockerfile                       |   4 -
 ci/docker/test.Dockerfile                     |   4 -
 ci/env/install-dependencies.sh                |   2 +-
 docker/ray-ml/Dockerfile                      |   2 +-
 .../ray-ml/install-ml-docker-requirements.sh  |   2 +-
 python/requirements_compiled.txt              | 549 ++++++++++++++++++
 10 files changed, 552 insertions(+), 21 deletions(-)
 create mode 100644 python/requirements_compiled.txt

diff --git a/ci/docker/base.gpu.Dockerfile b/ci/docker/base.gpu.Dockerfile
index bba013b61d2c0..2dbbb96a0915f 100644
--- a/ci/docker/base.gpu.Dockerfile
+++ b/ci/docker/base.gpu.Dockerfile
@@ -63,5 +63,4 @@ WORKDIR /ray
 COPY . .
 
 RUN ./ci/env/install-dependencies.sh init
-RUN bash --login -i ./ci/ci.sh compile_pip_dependencies
 RUN RLLIB_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 bash --login -i ./ci/env/install-dependencies.sh
diff --git a/ci/docker/base.test.Dockerfile b/ci/docker/base.test.Dockerfile
index 26a9b0628ea45..012ceb6ccc54c 100644
--- a/ci/docker/base.test.Dockerfile
+++ b/ci/docker/base.test.Dockerfile
@@ -65,4 +65,3 @@ WORKDIR /ray
 COPY . .
 
 RUN ./ci/env/install-dependencies.sh init
-RUN bash --login -i ./ci/ci.sh compile_pip_dependencies
diff --git a/ci/docker/build.Dockerfile b/ci/docker/build.Dockerfile
index d0e6f9692c51e..c05e10b07db2d 100644
--- a/ci/docker/build.Dockerfile
+++ b/ci/docker/build.Dockerfile
@@ -16,8 +16,6 @@ ENV CXX=clang++-12
 # Move out of working dir /ray
 # Delete stale data
 WORKDIR /
-# Preserve requirements_compiled.txt
-RUN mv /ray/python/requirements_compiled.txt /tmp/requirements_compiled.txt || true
 RUN rm -rf /ray
 
 RUN mkdir /ray
@@ -26,8 +24,6 @@ WORKDIR /ray
 # Below should be re-run each time
 COPY . .
 
-RUN mv /tmp/requirements_compiled.txt /ray/python/requirements_compiled.txt || true
-
 RUN env
 
 # init also calls install-dependencies.sh
diff --git a/ci/docker/gpu.Dockerfile b/ci/docker/gpu.Dockerfile
index 0c0c58613029c..02440fa953176 100644
--- a/ci/docker/gpu.Dockerfile
+++ b/ci/docker/gpu.Dockerfile
@@ -13,8 +13,6 @@ ENV TRAVIS_COMMIT=${BUILDKITE_COMMIT}
 # Move out of working dir /ray
 # Delete stale data
 WORKDIR /
-# Preserve requirements_compiled.txt
-RUN mv /ray/python/requirements_compiled.txt /tmp/requirements_compiled.txt || true
 RUN rm -rf /ray
 
 RUN mkdir /ray
@@ -23,8 +21,6 @@ WORKDIR /ray
 # Copy new ray files
 COPY . .
 
-RUN mv /tmp/requirements_compiled.txt /ray/python/requirements_compiled.txt || true
-
 RUN env
 
 RUN RLLIB_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 bash --login -i ./ci/env/install-dependencies.sh
diff --git a/ci/docker/ml.Dockerfile b/ci/docker/ml.Dockerfile
index 37bf6327c79c6..44ce9b320a257 100644
--- a/ci/docker/ml.Dockerfile
+++ b/ci/docker/ml.Dockerfile
@@ -4,8 +4,6 @@ FROM $DOCKER_IMAGE_BASE_ML
 # Move out of working dir /ray
 # Delete stale data
 WORKDIR /
-# Preserve requirements_compiled.txt
-RUN mv /ray/python/requirements_compiled.txt /tmp/requirements_compiled.txt || true
 RUN rm -rf /ray
 
 RUN mkdir /ray
@@ -14,8 +12,6 @@ WORKDIR /ray
 # Copy new ray files
 COPY . .
 
-RUN mv /tmp/requirements_compiled.txt /ray/python/requirements_compiled.txt || true
-
 RUN RLLIB_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 bash --login -i ./ci/env/install-dependencies.sh
 
 # Install Ray
diff --git a/ci/docker/test.Dockerfile b/ci/docker/test.Dockerfile
index 4c6256e4b30ea..e0fdcb2c73141 100644
--- a/ci/docker/test.Dockerfile
+++ b/ci/docker/test.Dockerfile
@@ -4,8 +4,6 @@ FROM $DOCKER_IMAGE_BASE_TEST
 # Move out of working dir /ray
 # Delete stale data
 WORKDIR /
-# Preserve requirements_compiled.txt
-RUN mv /ray/python/requirements_compiled.txt /tmp/requirements_compiled.txt || true
 RUN rm -rf /ray
 
 RUN mkdir /ray
@@ -14,8 +12,6 @@ WORKDIR /ray
 # Copy new ray files.
 COPY . .
 
-RUN mv /tmp/requirements_compiled.txt /ray/python/requirements_compiled.txt || true
-
 RUN bash --login -i ./ci/env/install-dependencies.sh
 
 # Install Ray
diff --git a/ci/env/install-dependencies.sh b/ci/env/install-dependencies.sh
index 4b095843893df..7f70b38f4bf49 100755
--- a/ci/env/install-dependencies.sh
+++ b/ci/env/install-dependencies.sh
@@ -458,7 +458,7 @@ install_pip_packages() {
   # Generate the pip command with collected requirements files
   pip_cmd="pip install -U -c ${WORKSPACE_DIR}/python/requirements.txt"
 
-  if [ -f "${WORKSPACE_DIR}/python/requirements_compiled.txt" ] && [ "${PYTHON-}" != "3.7" ]; then
+  if [[ -f "${WORKSPACE_DIR}/python/requirements_compiled.txt"  &&  "${PYTHON-}" != "3.7" ]]; then
     # On Python 3.7, we don't, as the dependencies are compiled for 3.8+
     # and we don't build ray-ml images. This means we don't have to keep
     # consistency between CI and docker images.
diff --git a/docker/ray-ml/Dockerfile b/docker/ray-ml/Dockerfile
index 783c16d0b64a2..44ce0a61e14f7 100644
--- a/docker/ray-ml/Dockerfile
+++ b/docker/ray-ml/Dockerfile
@@ -18,7 +18,7 @@ COPY train-test-requirements.txt ./
 
 COPY install-ml-docker-requirements.sh ./
 
-RUN sudo chmod +x install-ml-docker-requirements.sh && bash --login -i ./install-ml-docker-requirements.sh
+RUN sudo chmod +x install-ml-docker-requirements.sh && ./install-ml-docker-requirements.sh
 
 # Export installed packages
 RUN $HOME/anaconda3/bin/pip freeze > /home/ray/pip-freeze.txt
diff --git a/docker/ray-ml/install-ml-docker-requirements.sh b/docker/ray-ml/install-ml-docker-requirements.sh
index dad56710bdd2a..a17aa55568e3d 100755
--- a/docker/ray-ml/install-ml-docker-requirements.sh
+++ b/docker/ray-ml/install-ml-docker-requirements.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-set -xe
+set -e
 
 # shellcheck disable=SC2139
 alias pip="$HOME/anaconda3/bin/pip"
diff --git a/python/requirements_compiled.txt b/python/requirements_compiled.txt
new file mode 100644
index 0000000000000..f4da8c8028b9a
--- /dev/null
+++ b/python/requirements_compiled.txt
@@ -0,0 +1,549 @@
+# ./ci.sh compile_pip_dependencies
+about-time==4.2.1
+absl-py==1.4.0
+accelerate==0.20.3
+adal==1.2.7
+aim==3.17.5
+aim-ui==3.17.5
+aimrecords==0.0.7
+aimrocks==0.4.0
+aioboto3==11.2.0 ; python_version >= "3.8"
+aiobotocore==2.5.0
+aiofiles==22.1.0
+aiohttp==3.8.4
+aiohttp-cors==0.7.0
+aioitertools==0.11.0
+aiorwlock==1.3.0
+aiosignal==1.3.1
+aiosqlite==0.19.0
+alabaster==0.7.13
+ale-py==0.8.1
+alembic==1.11.1
+alive-progress==3.1.4
+anyio==3.7.1
+applicationinsights==0.11.10
+argcomplete==1.12.3
+argon2-cffi==21.3.0
+argon2-cffi-bindings==21.2.0
+array-record==0.4.0
+arrow==1.2.3
+asttokens==2.2.1
+astunparse==1.6.3
+async-exit-stack==1.0.1
+async-generator==1.10
+async-timeout==4.0.2
+asyncmock==0.4.2
+attrs==21.4.0
+autograd==1.6.2
+autorom==0.6.1 ; platform_machine != "arm64"
+autorom-accept-rom-license==0.6.1
+aws-sam-translator==1.71.0
+aws-xray-sdk==2.12.0
+ax-platform==0.3.2 ; python_version >= "3.8"
+azure-cli-core==2.40.0
+azure-cli-telemetry==1.0.8
+azure-common==1.1.28
+azure-core==1.27.1
+azure-identity==1.10.0
+azure-mgmt-compute==23.1.0
+azure-mgmt-core==1.4.0
+azure-mgmt-network==19.0.0
+azure-mgmt-resource==20.0.0
+babel==2.12.1
+backcall==0.2.0
+backoff==1.10.0
+base58==2.0.1
+bayesian-optimization==1.4.3
+bcrypt==4.0.1
+beautifulsoup4==4.11.1
+black==22.10.0
+bleach==6.0.0
+blessed==1.20.0
+bokeh==2.4.3
+boto==2.49.0
+boto3==1.26.76 ; python_version >= "3.8"
+botocore==1.29.76
+botorch==0.8.5
+cached-property==1.5.2
+cachetools==5.3.1
+catboost==1.2
+certifi==2023.5.7
+cffi==1.15.1
+cfn-lint==0.77.10
+charset-normalizer==3.2.0
+chess==1.7.0
+chex==0.1.7
+click==8.1.4
+cloudpickle==2.2.0
+cma==3.2.2
+cmaes==0.9.1
+colorama==0.4.6
+coloredlogs==15.0.1
+colorful==0.5.5
+colorlog==6.7.0
+comet-ml==3.31.9
+comm==0.1.3
+commonmark==0.9.1
+configobj==5.0.8
+configspace==0.7.1
+contextlib2==21.6.0
+contourpy==1.1.0
+crc32c==2.3
+crcmod==1.7
+cryptography==38.0.1
+cycler==0.11.0
+cython==0.29.32
+dask==2022.10.1 ; python_version >= "3.8"
+databricks-cli==0.17.7
+datasets==2.0.0
+debugpy==1.6.7
+decorator==5.1.1
+deepspeed==0.8.3
+defusedxml==0.7.1
+deprecated==1.2.14
+dill==0.3.6
+distlib==0.3.6
+distributed==2022.10.1
+dm-tree==0.1.8
+dnspython==2.3.0
+docker==6.1.3
+docker-pycreds==0.4.0
+docutils==0.17.1
+dopamine-rl==4.0.5
+dragonfly-opt==0.1.7
+dulwich==0.21.5
+ecdsa==0.18.0
+entrypoints==0.4
+et-xmlfile==1.1.0
+etils==1.3.0
+evaluate==0.4.0
+everett==3.2.0
+exceptiongroup==1.1.2
+execnet==2.0.0
+executing==1.2.0
+fairscale==0.4.6
+fastapi==0.75.0
+fasteners==0.18
+fastjsonschema==2.17.1
+feather-format==0.4.1
+ffmpy==0.3.0
+filelock==3.12.2
+flake8==3.9.1
+flake8-bugbear==21.9.2
+flake8-comprehensions==3.10.1
+flake8-quotes==2.0.0
+flaml==1.1.1
+flask==2.1.3
+flask-cors==4.0.0
+flatbuffers==2.0.7
+flax==0.6.11
+fonttools==4.40.0
+fqdn==1.5.1
+freezegun==1.1.0
+frozenlist==1.3.3
+fsspec==2023.5.0
+future==0.18.3
+gast==0.4.0
+gcs-oauth2-boto-plugin==3.0
+gin-config==0.5.0
+gitdb==4.0.10
+gitpython==3.1.31
+glfw==2.6.2
+google-api-core==2.11.1
+google-api-python-client==2.65.0
+google-apitools==0.5.32
+google-auth==2.21.0
+google-auth-httplib2==0.1.0
+google-auth-oauthlib==0.4.6
+google-cloud-core==2.3.3
+google-cloud-storage==2.5.0
+google-crc32c==1.5.0
+google-pasta==0.2.0
+google-reauth==0.1.1
+google-resumable-media==2.5.0
+googleapis-common-protos==1.59.1
+gpustat==1.1
+gpy==1.10.0
+gpytorch==1.10
+gradio==3.11.0 ; platform_system != "Windows"
+grapheme==0.6.0
+graphql-core==3.2.3
+graphviz==0.20.1
+greenlet==2.0.2
+grpcio==1.50.0 ; python_version < "3.10"
+gsutil==5.25
+gunicorn==20.1.0
+gym==0.26.2
+gym-notices==0.0.8
+gymnasium==0.26.3
+gymnasium-notices==0.0.1
+h11==0.12.0
+h5py==3.7.0
+hebo @ git+https://github.com/huawei-noah/HEBO@9a2a674c22518eed35a8b98e5134576741a95410#subdirectory=HEBO
+higher==0.2.1
+hjson==3.1.0
+hpbandster==0.7.4
+httpcore==0.15.0
+httplib2==0.20.4
+httpx==0.24.1
+huggingface-hub==0.16.4
+humanfriendly==10.0
+hyperopt==0.2.7
+idna==3.4
+imageio==2.31.1
+imageio-ffmpeg==0.4.5
+imagesize==1.4.1
+importlib-metadata==4.10.0
+importlib-resources==5.13.0
+iniconfig==2.0.0
+ipykernel==6.24.0
+ipython==8.12.2
+ipython-genutils==0.2.0
+ipywidgets==7.7.5
+isodate==0.6.1
+isoduration==20.11.0
+isort==5.10.1
+itsdangerous==2.1.2
+jax==0.4.13
+jaxlib==0.4.13
+jedi==0.18.2
+jinja2==3.0.3
+jmespath==1.0.1
+joblib==1.2.0
+jschema-to-python==1.2.3
+json5==0.9.14
+jsondiff==2.0.0
+jsonpatch==1.32
+jsonpickle==3.0.1
+jsonpointer==2.4
+jsonschema==4.17.3
+jsonschema-spec==0.1.6
+junit-xml==1.9
+jupyter-cache==0.4.3
+jupyter-client==7.3.4
+jupyter-core==5.3.1
+jupyter-events==0.6.3
+jupyter-server==1.24.0
+jupyter-server-fileid==0.9.0
+jupyter-server-mathjax==0.2.6
+jupyter-server-ydoc==0.6.1
+jupyter-sphinx==0.3.2
+jupyter-ydoc==0.2.4
+jupyterlab==3.6.1
+jupyterlab-pygments==0.2.2
+jupyterlab-server==2.23.0
+jupyterlab-widgets==1.1.4
+jupytext==1.13.6
+kaggle-environments==1.7.11
+keras==2.11.0
+kiwisolver==1.4.4
+knack==0.10.1
+kubernetes==24.2.0
+lazy-object-proxy==1.9.0
+libclang==16.0.0
+lightgbm==3.3.5
+linear-operator==0.4.0
+linkify-it-py==1.0.3
+llvmlite==0.39.1 ; python_version < "3.11"
+locket==1.0.0
+lxml==4.9.1
+lz4==4.3.2
+mako==1.2.4
+markdown==3.4.3
+markdown-it-py==1.1.0
+markupsafe==2.1.3
+matplotlib==3.7.2
+matplotlib-inline==0.1.6
+mccabe==0.6.1
+mdit-py-plugins==0.2.8
+memray==1.8.1 ; platform_system != "Windows" and sys_platform != "darwin"
+minigrid==2.1.1
+mistune==0.8.4
+ml-dtypes==0.2.0
+mlagents-envs==0.28.0
+mlflow==2.4.1 ; python_version > "3.7"
+mock==5.0.2
+modin==0.22.2 ; python_version >= "3.8"
+monotonic==1.6
+more-itertools==9.1.0
+moto==4.0.7
+mpmath==1.3.0
+msal==1.18.0b1
+msal-extensions==1.0.0
+msgpack==1.0.5
+msgpack-numpy==0.4.8
+msrest==0.7.1
+msrestazure==0.6.4
+mujoco==2.2.0
+mujoco-py==2.1.2.14
+multidict==6.0.4
+multipledispatch==1.0.0
+multiprocess==0.70.14
+mypy==0.982
+mypy-extensions==1.0.0
+myst-nb==0.13.1
+myst-parser==0.15.2
+nbclassic==1.0.0
+nbclient==0.5.13
+nbconvert==6.5.4
+nbdime==3.2.1
+nbformat==5.9.0
+nest-asyncio==1.5.6
+netifaces==0.11.0
+networkx==2.6.3
+nevergrad==0.4.3.post7
+ninja==1.11.1
+notebook==6.5.4
+notebook-shim==0.2.3
+numba==0.56.4
+numexpr==2.8.4
+numpy==1.23.5 ; python_version < "3.9"
+nvidia-ml-py==12.535.77
+oauth2client==4.1.3
+oauthlib==3.2.2
+onnx==1.12.0 ; sys_platform != "darwin" or platform_machine != "arm64"
+onnxruntime==1.14.1 ; sys_platform != "darwin" or platform_machine != "arm64"
+open-spiel==1.2
+openapi-schema-validator==0.4.4
+openapi-spec-validator==0.5.7
+opencensus==0.11.2
+opencensus-context==0.1.3
+opencensus-proto==0.1.0
+opencv-python==4.8.0.74
+openpyxl==3.0.10
+opentelemetry-api==1.1.0
+opentelemetry-exporter-opencensus==0.20b0
+opentelemetry-exporter-otlp==1.1.0
+opentelemetry-exporter-otlp-proto-grpc==1.1.0
+opentelemetry-proto==1.1.0
+opentelemetry-sdk==1.1.0
+opentelemetry-semantic-conventions==0.20b0
+opt-einsum==3.3.0
+optax==0.1.5
+optuna==3.2.0
+orbax-checkpoint==0.2.3
+orjson==3.9.2
+packaging==21.3
+pandas==1.5.3
+pandocfilters==1.5.0
+paramiko==2.12.0
+paramz==0.9.5
+parso==0.8.3
+partd==1.4.0
+path==16.7.1
+path-py==12.5.0
+pathable==0.4.3
+pathspec==0.11.1
+pathtools==0.1.2
+patsy==0.5.3
+pbr==5.11.1
+pettingzoo==1.22.1 ; python_version >= "3.7"
+pexpect==4.8.0
+pickleshare==0.7.5
+pillow==9.2.0 ; platform_system != "Windows"
+pkginfo==1.9.6
+pkgutil-resolve-name==1.3.10
+platformdirs==3.8.1
+plotly==5.15.0
+pluggy==1.2.0
+polars==0.14.21
+portalocker==2.7.0
+prometheus-client==0.17.0
+promise==2.3
+prompt-toolkit==3.0.39
+protobuf==3.19.6
+proxy-py==2.4.3
+psutil==5.9.5
+ptyprocess==0.7.0
+pure-eval==0.2.2
+py==1.11.0
+py-cpuinfo==9.0.0
+py-spy==0.3.14
+py3nvml==0.2.7
+py4j==0.10.9.5
+pyaml==23.7.0
+pyarrow==6.0.1 ; python_version < "3.11"
+pyasn1==0.5.0
+pyasn1-modules==0.3.0
+pycodestyle==2.7.0
+pycparser==2.21
+pycryptodome==3.18.0
+pydantic==1.9.2 ; python_version < "3.11"
+pydeprecate==0.3.2
+pydub==0.25.1
+pyflakes==2.3.1
+pygame==2.1.2 ; python_version < "3.11"
+pyglet==1.5.15
+pygments==2.13.0
+pyjwt==2.7.0
+pymars==0.10.0
+pymongo==4.3.2
+pymoo==0.6.0.1
+pymunk==6.2.1
+pynacl==1.5.0
+pyopengl==3.1.7
+pyopenssl==22.1.0
+pyparsing==3.0.9
+pypng==0.20220715.0
+pyro-api==0.1.2
+pyro-ppl==1.8.5
+pyro4==4.82
+pyrsistent==0.19.3
+pysocks==1.7.1
+pyspark==3.3.1
+pytest==7.0.1
+pytest-asyncio==0.16.0
+pytest-docker-tools==3.1.3
+pytest-fixture-config==1.7.0
+pytest-forked==1.4.0
+pytest-httpserver==1.0.6
+pytest-lazy-fixture==0.6.3
+pytest-remotedata==0.3.2
+pytest-repeat==0.9.1
+pytest-rerunfailures==11.1.2
+pytest-shutil==1.7.0
+pytest-sphinx @ git+https://github.com/ray-project/pytest-sphinx
+pytest-sugar==0.9.5
+pytest-timeout==2.1.0
+pytest-virtualenv==1.7.0
+python-dateutil==2.8.2
+python-jose==3.3.0
+python-json-logger==2.0.7
+python-multipart==0.0.6
+python-snappy==0.6.1
+pytorch-lightning==1.6.5
+pytz==2022.7.1
+pyu2f==0.1.5
+pywavelets==1.4.1
+pyyaml==6.0
+pyzmq==25.1.0
+querystring-parser==1.2.4
+raydp==1.6.0b20230707.dev0
+recsim==0.2.4 ; sys_platform != "darwin" or platform_machine != "arm64"
+redis==4.4.2
+regex==2023.6.3
+requests==2.31.0
+requests-oauthlib==1.3.1
+requests-toolbelt==1.0.0
+responses==0.13.4
+restrictedpython==6.0
+retry-decorator==1.1.1
+rfc3339-validator==0.1.4
+rfc3986-validator==0.1.1
+rich==12.0.1
+rsa==4.7.2
+s3fs==2023.5.0 ; python_version >= "3.8"
+s3transfer==0.6.1
+safetensors==0.3.1
+sarif-om==1.0.4
+scikit-image==0.19.3
+scikit-learn==1.0.2 ; python_version < "3.11"
+scikit-optimize==0.9.0
+scipy==1.10.1
+segment-analytics-python==2.2.0
+semantic-version==2.10.0
+send2trash==1.8.2
+sentencepiece==0.1.96
+sentry-sdk==1.27.1
+serpent==1.41
+setproctitle==1.3.2
+shortuuid==1.0.1
+sigopt==7.5.0
+six==1.16.0
+smart-open==6.2.0
+smmap==5.0.0
+sniffio==1.3.0
+snowballstemmer==2.2.0
+sortedcontainers==2.4.0
+soupsieve==2.4.1
+sphinx==4.3.2
+sphinx-togglebutton==0.2.3
+sphinxcontrib-applehelp==1.0.4
+sphinxcontrib-devhelp==1.0.2
+sphinxcontrib-htmlhelp==2.0.1
+sphinxcontrib-jsmath==1.0.1
+sphinxcontrib-qthelp==1.0.3
+sphinxcontrib-serializinghtml==1.1.5
+sqlalchemy==1.4.17
+sqlparse==0.4.4
+sshpubkeys==3.3.1
+stack-data==0.6.2
+starlette==0.17.1
+statsmodels==0.14.0
+supersuit==3.7.0 ; python_version >= "3.7"
+sympy==1.12
+tabulate==0.9.0
+tblib==2.0.0
+tenacity==8.2.2
+tensorboard==2.11.2
+tensorboard-data-server==0.6.1
+tensorboard-plugin-wit==1.8.1
+tensorboardx==2.6
+tensorflow==2.11.0 ; sys_platform != "darwin" or platform_machine != "arm64"
+tensorflow-datasets==4.9.0
+tensorflow-estimator==2.11.0
+tensorflow-io-gcs-filesystem==0.32.0
+tensorflow-metadata==1.13.0
+tensorflow-probability==0.19.0
+tensorstore==0.1.40
+termcolor==2.3.0
+terminado==0.17.1
+testfixtures==7.0.0
+tf-slim==1.1.0
+tf2onnx==1.13.0 ; sys_platform != "darwin" or platform_machine != "arm64"
+threadpoolctl==3.1.0
+tifffile==2023.7.4
+timm==0.9.2
+tinycss2==1.2.1
+tinyscaler==1.2.6
+tokenizers==0.12.1
+toml==0.10.2
+tomli==2.0.1
+toolz==0.12.0
+torch==2.0.1+cpu ; python_version > "3.7"
+torch-cluster==1.6.1+pt20cpu
+torch-geometric==2.3.1
+torch-scatter==2.1.1+pt20cpu
+torch-sparse==0.6.17+pt20cpu
+torch-spline-conv==1.2.2+pt20cpu
+torchdata==0.6.1
+torchmetrics==0.10.3 ; python_version > "3.7"
+torchtext==0.15.2+cpu ; python_version > "3.7"
+torchvision==0.15.2+cpu ; python_version > "3.7"
+tornado==6.1
+tqdm==4.64.1
+traitlets==5.9.0
+transformers==4.19.1
+trustme==0.9.0
+typeguard==2.13.3
+typer==0.9.0
+types-pyyaml==6.0.12.2
+typing-extensions==4.5.0
+uc-micro-py==1.0.2
+uri-template==1.3.0
+uritemplate==4.1.1
+urllib3==1.26.16
+uvicorn==0.22.0
+uvloop==0.17.0
+virtualenv==20.21.0
+wandb==0.13.4
+wcwidth==0.2.6
+webcolors==1.13
+webencodings==0.5.1
+websocket-client==1.6.1
+websockets==11.0.3
+werkzeug==2.1.2
+wheel==0.40.0
+widgetsnbextension==3.6.4
+wrapt==1.15.0
+wurlitzer==3.0.3
+xgboost==1.7.6 ; python_version > "3.7"
+xlrd==2.0.1
+xmltodict==0.13.0
+xxhash==3.2.0
+y-py==0.5.9
+yarl==1.9.2
+ypy-websocket==0.8.2
+zict==3.0.0
+zipp==3.15.0
+zoopt==0.4.1

From eadc1fe58a09f641b119513445f65920c324dfdb Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Tue, 11 Jul 2023 11:25:04 -0700
Subject: [PATCH 64/70] device-specific constraints [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 docker/ray-ml/install-ml-docker-requirements.sh | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/docker/ray-ml/install-ml-docker-requirements.sh b/docker/ray-ml/install-ml-docker-requirements.sh
index a17aa55568e3d..53db300e8e829 100755
--- a/docker/ray-ml/install-ml-docker-requirements.sh
+++ b/docker/ray-ml/install-ml-docker-requirements.sh
@@ -33,14 +33,18 @@ pip --no-cache-dir install -U \
            -r tune-test-requirements.txt \
            -r ray-docker-requirements.txt
 
+
+# Remove any device-specific constraints from requirements_compiled.txt.
+# E.g.: torch-scatter==2.1.1+pt20cpu or torchvision==0.15.2+cpu
+# These are replaced with gpu-specific requirements in dl-gpu-requirements.txt.
+sed "/[0-9]\+cpu/d;/[0-9]\+pt/d" "requirements_compiled.txt" > requirements_compiled_gpu.txt
+
 # explicitly install (overwrite) pytorch with CUDA support
-# do not use compiled requirements as constraint here as some
-# device-specific constraints can't be met (e.g. torch+cpu).
-# Instead, we don't use `-U` so other dependencies don't get updated.
 pip --no-cache-dir install \
            -c requirements.txt \
+           -c requirements_compiled_gpu.txt \
            -r dl-gpu-requirements.txt
 
 sudo apt-get clean
 
-sudo rm ./*requirements.txt
+sudo rm ./*requirements*.txt

From 9a66aedb8192f75cd702d4da54eb6217b22bf2cc Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Tue, 11 Jul 2023 14:22:04 -0700
Subject: [PATCH 65/70] device-specific [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 python/requirements_compiled.txt | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/python/requirements_compiled.txt b/python/requirements_compiled.txt
index f4da8c8028b9a..5b62fa96e35f8 100644
--- a/python/requirements_compiled.txt
+++ b/python/requirements_compiled.txt
@@ -500,16 +500,16 @@ tokenizers==0.12.1
 toml==0.10.2
 tomli==2.0.1
 toolz==0.12.0
-torch==2.0.1+cpu ; python_version > "3.7"
-torch-cluster==1.6.1+pt20cpu
+torch==2.0.1 ; python_version > "3.7"
+torch-cluster==1.6.1
 torch-geometric==2.3.1
-torch-scatter==2.1.1+pt20cpu
-torch-sparse==0.6.17+pt20cpu
-torch-spline-conv==1.2.2+pt20cpu
+torch-scatter==2.1.1
+torch-sparse==0.6.17
+torch-spline-conv==1.2.2
 torchdata==0.6.1
 torchmetrics==0.10.3 ; python_version > "3.7"
-torchtext==0.15.2+cpu ; python_version > "3.7"
-torchvision==0.15.2+cpu ; python_version > "3.7"
+torchtext==0.15.2 ; python_version > "3.7"
+torchvision==0.15.2 ; python_version > "3.7"
 tornado==6.1
 tqdm==4.64.1
 traitlets==5.9.0

From f6ef8bf557dd725bebb045e76e40d77ab6253be6 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Tue, 11 Jul 2023 16:55:21 -0700
Subject: [PATCH 66/70] skip constraints on windows [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 ci/ci.sh                       | 1 +
 ci/env/install-dependencies.sh | 4 +++-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/ci/ci.sh b/ci/ci.sh
index 9fd1c2eb1ff48..cb8f22068c5f7 100755
--- a/ci/ci.sh
+++ b/ci/ci.sh
@@ -134,6 +134,7 @@ compile_pip_dependencies() {
   # Remove some pins from upstream dependencies:
   # ray, xgboost-ray, lightgbm-ray, tune-sklearn
   sed -i "/^ray==/d;/^xgboost-ray==/d;/^lightgbm-ray==/d;/^tune-sklearn==/d" "${WORKSPACE_DIR}/python/requirements_compiled.txt"
+
   cat "${WORKSPACE_DIR}/python/requirements_compiled.txt"
 
   if [ "$HAS_TORCH" -eq 0 ]; then
diff --git a/ci/env/install-dependencies.sh b/ci/env/install-dependencies.sh
index 7f70b38f4bf49..85c22a0a22f41 100755
--- a/ci/env/install-dependencies.sh
+++ b/ci/env/install-dependencies.sh
@@ -458,10 +458,12 @@ install_pip_packages() {
   # Generate the pip command with collected requirements files
   pip_cmd="pip install -U -c ${WORKSPACE_DIR}/python/requirements.txt"
 
-  if [[ -f "${WORKSPACE_DIR}/python/requirements_compiled.txt"  &&  "${PYTHON-}" != "3.7" ]]; then
+  if [[ -f "${WORKSPACE_DIR}/python/requirements_compiled.txt"  &&  "${PYTHON-}" != "3.7" && "${OSTYPE}" != msys ]]; then
     # On Python 3.7, we don't, as the dependencies are compiled for 3.8+
     # and we don't build ray-ml images. This means we don't have to keep
     # consistency between CI and docker images.
+    # On Windows, some pinned dependencies are not built for win, so we
+    # skip this until we have a good wy to resolve cross-platform dependencies.
     pip_cmd+=" -c ${WORKSPACE_DIR}/python/requirements_compiled.txt"
   fi
 

From 83e3fd8f62435f42ba50698ea96bdaa91074fed8 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Wed, 12 Jul 2023 09:43:01 -0700
Subject: [PATCH 67/70] trivial [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 docker/ray-ml/install-ml-docker-requirements.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/ray-ml/install-ml-docker-requirements.sh b/docker/ray-ml/install-ml-docker-requirements.sh
index 53db300e8e829..57744ad8b3ef0 100755
--- a/docker/ray-ml/install-ml-docker-requirements.sh
+++ b/docker/ray-ml/install-ml-docker-requirements.sh
@@ -19,7 +19,7 @@ pip --no-cache-dir install -U pip pip-tools
 # Install requirements
 pip --no-cache-dir install -U -r requirements.txt
 
-# Install other requirements. Keep pinned requirements bounds as constraints
+# Install other requirements. Keep pinned requirements bounds as constraints.
 pip --no-cache-dir install -U \
            -c requirements.txt \
            -c requirements_compiled.txt \

From 2d1922d56c61138dc1e192fdd613ea803824d7b1 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Thu, 13 Jul 2023 07:47:30 -0700
Subject: [PATCH 68/70] trivial [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 docker/ray-ml/install-ml-docker-requirements.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/ray-ml/install-ml-docker-requirements.sh b/docker/ray-ml/install-ml-docker-requirements.sh
index 57744ad8b3ef0..53db300e8e829 100755
--- a/docker/ray-ml/install-ml-docker-requirements.sh
+++ b/docker/ray-ml/install-ml-docker-requirements.sh
@@ -19,7 +19,7 @@ pip --no-cache-dir install -U pip pip-tools
 # Install requirements
 pip --no-cache-dir install -U -r requirements.txt
 
-# Install other requirements. Keep pinned requirements bounds as constraints.
+# Install other requirements. Keep pinned requirements bounds as constraints
 pip --no-cache-dir install -U \
            -c requirements.txt \
            -c requirements_compiled.txt \

From 3324f69e2575f3433f05c980d5340abef91c03c4 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Thu, 13 Jul 2023 09:27:24 -0700
Subject: [PATCH 69/70] fastapi/starlette [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 python/requirements_compiled.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/python/requirements_compiled.txt b/python/requirements_compiled.txt
index 5b62fa96e35f8..d06cf3014920b 100644
--- a/python/requirements_compiled.txt
+++ b/python/requirements_compiled.txt
@@ -122,7 +122,7 @@ exceptiongroup==1.1.2
 execnet==2.0.0
 executing==1.2.0
 fairscale==0.4.6
-fastapi==0.75.0
+fastapi==0.99.1
 fasteners==0.18
 fastjsonschema==2.17.1
 feather-format==0.4.1
@@ -468,7 +468,7 @@ sqlalchemy==1.4.17
 sqlparse==0.4.4
 sshpubkeys==3.3.1
 stack-data==0.6.2
-starlette==0.17.1
+starlette==0.27.0
 statsmodels==0.14.0
 supersuit==3.7.0 ; python_version >= "3.7"
 sympy==1.12

From 8d9957ae422efb8326b730fcebc3e3a228ef0157 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Fri, 14 Jul 2023 10:41:30 -0700
Subject: [PATCH 70/70] upgrade grpcio [build_base]

Signed-off-by: Kai Fricke <kai@anyscale.com>
---
 python/requirements_compiled.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/requirements_compiled.txt b/python/requirements_compiled.txt
index d06cf3014920b..bd2db12da3a72 100644
--- a/python/requirements_compiled.txt
+++ b/python/requirements_compiled.txt
@@ -170,7 +170,7 @@ grapheme==0.6.0
 graphql-core==3.2.3
 graphviz==0.20.1
 greenlet==2.0.2
-grpcio==1.50.0 ; python_version < "3.10"
+grpcio==1.54.2
 gsutil==5.25
 gunicorn==20.1.0
 gym==0.26.2