[Core] Auto generate protobuf and grpc code (ray-project#40663)

Revert back to the old way (before ray-project#39210) of auto-generating protobuf and grpc code so we don't need to manually generate and check them in. Also decouple the protobuf version we use in c++ and the protobuf version we use to auto generate python and java code (the protobuf version to auto generate python code affects the python protobuf library version that Ray can support) so that they can be upgraded independently.

Signed-off-by: Jiajun Yao <[email protected]>

.buildkite/lint.rayci.yml

@@ -72,9 +72,3 @@ steps:
  commands:
  - ./ci/lint/check-documentation-style.sh
  depends_on: forge
-
- - label: ":lint-roller: lint: python proto"
- commands:
- - pip install -r ci/compile_py_proto/requirements_compile_py_proto.txt
- - ./ci/lint/check-python-proto.sh
- depends_on: forge

.gitattributes

@@ -1,3 +0,0 @@
-/python/ray/core/generated/**/* linguist-generated=true
-/python/ray/serve/generated/**/* linguist-generated=true
-

.gitignore

@@ -1,7 +1,7 @@
 # The build output should clearly not be checked in
 *test-output.xml
 /bazel-*
-/python/ray/core/**/*
+/python/ray/core
 /python/ray/thirdparty_files/
 /python/ray/_private/runtime_env/agent/thirdparty_files/
 /python/ray/pyarrow_files/
@@ -97,10 +97,6 @@ python/ray/_raylet.pyd
 *_pb2.py
 *.pb.h
 *.pb.cc
-!/python/ray/core/generated/
-!/python/ray/core/generated/*.py
-!/python/ray/serve/generated/
-!/python/ray/serve/generated/*.py
 
 # Ray cluster configuration
 scripts/nodes.txt

BUILD.bazel

@@ -2550,11 +2550,44 @@ copy_to_workspace(
  dstdir = "python/ray/core/",
 )
 
+genrule(
+ name = "install_py_proto",
+ srcs = [
+ ":cp_all_py_proto",
+ ":cp_serve_py_proto",
+ ],
+ outs = ["install_py_proto.out"],
+ cmd = """
+ # NOTE(hchen): Protobuf doesn't allow specifying Python package name. So we use this `sed`
+ # command to change the import path in the generated file.
+ # shellcheck disable=SC2006
+ files=(
+ `ls python/ray/core/generated/*_pb2*.py` \
+ `ls python/ray/serve/generated/*_pb2*.py` \
+ )
+ sed -i -E 's/from src.ray.protobuf/from ./' "$${files[@]}"
+ # Help the generated serve files to have the correct module
+ serve_files=(`ls python/ray/serve/generated/*_pb2*.py`)
+ sed -i -E 's/'"'"'src.ray.protobuf./'"'"'ray.serve.generated./' "$${serve_files[@]}"
+ # TODO(sang): Build our own proto instead of creating a new proto for opencensus separately.
+ # https://github.com/ray-project/ray/issues/31358
+ sed -i -E 's/from opencensus.proto.metrics.v1 import/from . import/' "$${files[@]}"
+ sed -i -E 's/from opencensus.proto.resource.v1 import/from . import/' "$${files[@]}"
+ # TODO(rickyx): Remove this experimental path rewrite once we move autoscaler proto out
+ # of experimental.
+ autoscale_files=(`ls python/ray/core/generated/instance_manager_pb2*.py`)
+ sed -i -E 's/from ..experimental/from ./' "$${autoscale_files[@]}"
+ echo "$${PWD}" > $@
+ """,
+ local = 1,
+)
+
 genrule(
  name = "ray_pkg",
  srcs = [
  ":cp_raylet_so",
  ":python_sources",
+ ":install_py_proto",
  ":cp_redis",
  ":cp_raylet",
  ":cp_gcs_server",

WORKSPACE

@@ -64,22 +64,13 @@ pip_parse(
  requirements_lock = "//ci/ray_ci:requirements.txt",
 )
 
-pip_parse(
- name = "py_deps_compile_py_proto",
- python_interpreter_target = python39,
- requirements_lock = "//ci/compile_py_proto:requirements_compile_py_proto.txt",
-)
-
 load("@py_deps_buildkite//:requirements.bzl", install_py_deps_buildkite = "install_deps")
 load("@py_deps_ray_ci//:requirements.bzl", install_py_deps_ray_ci = "install_deps")
-load("@py_deps_compile_py_proto//:requirements.bzl", install_py_deps_compile_py_proto = "install_deps")
 
 install_py_deps_buildkite()
 
 install_py_deps_ray_ci()
 
-install_py_deps_compile_py_proto()
-
 register_toolchains("//:python_toolchain")
 
 register_execution_platforms(

bazel/ray_deps_setup.bzl

@@ -294,10 +294,25 @@ def ray_deps_setup():
  ],
  )
 
+ # The protobuf version we use to auto generate python and java code.
+ # This can be different from the protobuf version that Ray core uses internally.
+ # Generally this should be a lower version since protobuf guarantees that
+ # code generated by protoc of version X can be used with protobuf library of version >= X.
+ # So the version here effectively determines the lower bound of python/java
+ # protobuf library that Ray supports.
+ http_archive(
+ name = "com_google_protobuf_rules_proto_grpc",
+ strip_prefix = "protobuf-3.19.4",
+ urls = ["https://github.com/protocolbuffers/protobuf/archive/v3.19.4.tar.gz"],
+ sha256 = "3bd7828aa5af4b13b99c191e8b1e884ebfa9ad371b0ce264605d347f135d2568",
+ )
  auto_http_archive(
  name = "rules_proto_grpc",
  url = "https://github.com/rules-proto-grpc/rules_proto_grpc/archive/a74fef39c5fe636580083545f76d1eab74f6450d.tar.gz",
  sha256 = "2f6606151ec042e23396f07de9e7dcf6ca9a5db1d2b09f0cc93a7fc7f4008d1b",
+ repo_mapping = {
+ "@com_google_protobuf": "@com_google_protobuf_rules_proto_grpc",
+ },
  )
 
  auto_http_archive(