Revert "Revert "[Core] Unpin grpcio and make Ray run on mac M1 out of…

… the box"" (ray-project#36182)

Reverts ray-project#36172

The Ray Client doesn't support more recent gRPC versions, so we had to keep the upper bound there.

.buildkite/pipeline.build.yml

@@ -372,9 +372,7 @@
  - DL=1 ./ci/env/install-dependencies.sh
  - bash ./ci/ci.sh prepare_docker
  - ./ci/env/env_info.sh
- # This is needed or else the Ray Client tests run into a gRPC forking problem
- # similar to https://github.com/grpc/grpc/issues/31885
- - pip install pip install grpcio==1.50.0
+ - pip install ray[client]
  - bazel test --config=ci $(./ci/run/bazel_export_options)
  --test_tag_filters=client_tests,small_size_python_tests
  -- python/ray/tests/...
@@ -425,9 +423,7 @@
  - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
  - DL=1 ./ci/env/install-dependencies.sh
  - ./ci/env/env_info.sh
- # This is needed or else the Ray Client tests run into a gRPC forking problem
- # similar to https://github.com/grpc/grpc/issues/31885
- - pip install pip install grpcio==1.50.0
+ - pip install ray[client]
  - bazel test --config=ci $(./scripts/bazel_export_options)
  --test_tag_filters=client_tests,small_size_python_tests
  --test_env=TEST_EXTERNAL_REDIS=1
@@ -484,7 +480,6 @@
  commands:
  - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
  - RLLIB_TESTING=1 ./ci/env/install-dependencies.sh
- - pip install "grpcio >= 1.28.1, <= 1.43.0"
  - ./ci/env/env_info.sh
  - bazel test --config=ci --config=asan $(./ci/run/bazel_export_options)
  --config=asan-buildkite

ci/build/test-wheels.sh

@@ -148,14 +148,6 @@ elif [[ "$platform" == "macosx" ]]; then
  # Update pip
  "$PIP_CMD" install -U pip
 
- # Install grpcio
- if [ "$(uname -m)" = "arm64" ]; then
- "$PIP_CMD" uninstall -y grpcio || true
- conda install -y --force-reinstall grpcio || true
- else
- "$PIP_CMD" install -q grpcio
- fi
-
  # Install the wheel.
  "$PIP_CMD" uninstall -y ray
  "$PIP_CMD" install -q "$PYTHON_WHEEL"

python/ray/__init__.py

@@ -57,21 +57,6 @@ def _configure_system():
  )
  sys.path.insert(0, pickle5_path)
 
- # Check that grpc can actually be imported on Apple Silicon. Some package
- # managers (such as `pip`) can't properly install the grpcio library yet,
- # so provide a proactive error message if that's the case.
- if platform.system() == "Darwin" and platform.machine() == "arm64":
- try:
- import grpc # noqa: F401
- except ImportError:
- raise ImportError(
- "Failed to import grpc on Apple Silicon. On Apple"
- " Silicon machines, try `pip uninstall grpcio; conda "
- "install grpcio`. Check out "
- "https://docs.ray.io/en/master/ray-overview/installation.html"
- "#m1-mac-apple-silicon-support for more details."
- )
-
  # Importing psutil & setproctitle. Must be before ray._raylet is
  # initialized.
  thirdparty_files = os.path.join(

python/ray/tune/tests/test_client.py

@@ -86,6 +86,14 @@ def test_cifar10_pytorch(legacy_progress_reporter, start_client_server_2_cpus):
  assert ray.util.client.ray.is_connected()
  from ray.tune.examples.cifar10_pytorch import main
 
+ # This is a workaround that is needed because of
+ # https://github.com/grpc/grpc/issues/32758 -- this happens only
+ # if the DataLoader is used on the Ray Client side, which we should
+ # not encourage anyways (heavy processing should be done on the cluster).
+ import torch
+
+ torch.multiprocessing.set_start_method("spawn")
+
  main(num_samples=1, max_num_epochs=1, gpus_per_trial=0)
 
 

python/requirements.txt

@@ -22,12 +22,8 @@ packaging
 
 # Python version-specific requirements
 dataclasses; python_version < '3.7'
-# Tracking issue: https://github.com/ray-project/ray/issues/30984
-grpcio >= 1.32.0, <= 1.49.1; python_version < '3.10' and sys_platform == 'darwin'
-grpcio >= 1.42.0, <= 1.49.1; python_version >= '3.10' and sys_platform == 'darwin'
-# Original issue: https://github.com/ray-project/ray/issues/33833
-grpcio >= 1.32.0, <= 1.51.3; python_version < '3.10' and sys_platform != 'darwin'
-grpcio >= 1.42.0, <= 1.51.3; python_version >= '3.10' and sys_platform != 'darwin'
+grpcio >= 1.32.0; python_version < '3.10'
+grpcio >= 1.42.0; python_version >= '3.10'
 numpy>=1.16; python_version < '3.9'
 numpy>=1.19.3; python_version >= '3.9'
 typing_extensions; python_version < '3.8'

python/setup.py

@@ -257,6 +257,11 @@ def get_packages(self):
  "smart_open",
  "virtualenv >=20.0.24, < 20.21.1", # For pip runtime env.
  ],
+ "client": [
+ # The Ray client needs a specific range of gRPC to work:
+ # Tracking issue: https://github.com/grpc/grpc/issues/31885
+ "grpcio >= 1.42.0, <= 1.50.0",
+ ],
  "serve": ["uvicorn", "requests", "starlette", "fastapi", "aiorwlock"],
  "tune": ["pandas", "tensorboardX>=1.9", "requests", pyarrow_dep],
  "k8s": ["kubernetes", "urllib3"],
@@ -315,12 +320,8 @@ def get_packages(self):
  "click >= 7.0",
  "dataclasses; python_version < '3.7'",
  "filelock",
- # Tracking issue: https://github.com/ray-project/ray/issues/30984
- "grpcio >= 1.32.0, <= 1.49.1; python_version < '3.10' and sys_platform == 'darwin'", # noqa:E501
- "grpcio >= 1.42.0, <= 1.49.1; python_version >= '3.10' and sys_platform == 'darwin'", # noqa:E501
- # Original issue: https://github.com/ray-project/ray/issues/33833
- "grpcio >= 1.32.0, <= 1.51.3; python_version < '3.10' and sys_platform != 'darwin'", # noqa:E501
- "grpcio >= 1.42.0, <= 1.51.3; python_version >= '3.10' and sys_platform != 'darwin'", # noqa:E501
+ "grpcio >= 1.32.0; python_version < '3.10'", # noqa:E501
+ "grpcio >= 1.42.0; python_version >= '3.10'", # noqa:E501
  "jsonschema",
  "msgpack >= 1.0.0, < 2.0.0",
  "numpy >= 1.16; python_version < '3.9'",