[Streaming]Farewell : remove all of streaming related from ray repo. (r…

…ay-project#21770) New repo url is https://github.com/ray-project/mobius Co-authored-by: 林濯 <[email protected]>
krfricke · Jan 23, 2022 · ec62d7f · ec62d7f
1 parent 2da2ac5
commit ec62d7f
Show file tree

Hide file tree

Showing 412 changed files with 13,509 additions and 36,499 deletions.
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -3,7 +3,6 @@
  [
  "RAY_CI_LINUX_WHEELS_AFFECTED",
  "RAY_CI_JAVA_AFFECTED",
- "RAY_CI_STREAMING_JAVA_AFFECTED",
  ]
  commands:
  # Build the wheels and jars
@@ -41,7 +40,6 @@
  [
  "RAY_CI_LINUX_WHEELS_AFFECTED",
  "RAY_CI_JAVA_AFFECTED",
- "RAY_CI_STREAMING_JAVA_AFFECTED",
  ]
  commands:
  # Build the debug wheels
@@ -61,7 +59,6 @@
 # [
 # "RAY_CI_LINUX_WHEELS_AFFECTED",
 # "RAY_CI_JAVA_AFFECTED",
-# "RAY_CI_STREAMING_JAVA_AFFECTED",
 # ]
 # commands:
 # # Build the asan wheels
@@ -156,24 +153,6 @@
  commands:
  - RAY_bootstrap_with_gcs=1 RAY_gcs_grpc_based_pubsub=1 RAY_gcs_storage=memory ./java/test.sh
 
-- label: ":java: Streaming"
- conditions:
- ["RAY_CI_STREAMING_PYTHON_AFFECTED", "RAY_CI_STREAMING_JAVA_AFFECTED", "RAY_CI_PYTHON_AFFECTED"]
- commands:
- - bazel test --config=ci $(./scripts/bazel_export_options)
- //streaming:all
- - bash streaming/src/test/run_streaming_queue_test.sh
-
-- label: ":java: :redis: Streaming"
- conditions:
- ["RAY_CI_STREAMING_PYTHON_AFFECTED", "RAY_CI_STREAMING_JAVA_AFFECTED", "RAY_CI_PYTHON_AFFECTED"]
- commands:
- - bazel test --config=ci
- --test_env=RAY_gcs_grpc_based_pubsub=1
- --test_env=RAY_bootstrap_with_gcs=1
- --test_env=RAY_gcs_storage=memory $(./scripts/bazel_export_options) //streaming:all
- - RAY_bootstrap_with_gcs=1 RAY_gcs_grpc_based_pubsub=1 RAY_gcs_storage=memory bash streaming/src/test/run_streaming_queue_test.sh
-
 - label: ":cpp: Ray CPP Worker"
  conditions: [ "RAY_CI_CPP_AFFECTED" ]
  commands:

diff --git a/.flake8 b/.flake8
@@ -1,7 +1,6 @@
 [flake8]
 exclude =
  python/ray/core/generated/
- streaming/python/generated
  doc/source/conf.py
  python/ray/cloudpickle/
  python/ray/thirdparty_files/

diff --git a/.gitignore b/.gitignore
@@ -169,20 +169,10 @@ dependency-reduced-pom.xml
 # Cpp
 cpp/example/thirdparty/
 
-# streaming/python
-streaming/python/generated/
-streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/generated/
-streaming/build/java
 .clwb
-streaming/**/.settings
-streaming/java/**/target
-streaming/java/**/.classpath
-streaming/java/**/.project
-streaming/java/**/*.log
 
 # pom.xml files generated from pom_template.xml
 java/**/pom.xml
-streaming/java/**/pom.xml
 
 # python virtual env
 venv

diff --git a/BUILD.bazel b/BUILD.bazel
@@ -450,18 +450,18 @@ cc_library(
  ],
 )
 
-# This header is used to warp some streaming code so we can reduce suspicious
+# This header is used to warp some internal code so we can reduce suspicious
 # symbols export.
 cc_library(
- name = "exported_streaming_internal",
+ name = "exported_internal",
  srcs = glob(
  [
- "src/ray/streaming/streaming.cc",
+ "src/ray/internal/internal.cc",
  ],
  ),
  hdrs = glob(
  [
- "src/ray/streaming/streaming.h",
+ "src/ray/internal/internal.h",
  ],
  ),
  copts = COPTS,
@@ -674,7 +674,7 @@ cc_library(
  ],
  }),
  strip_include_prefix = "src",
- visibility = ["//streaming:__subpackages__"],
+ visibility = ["//visibility:public"],
  deps = [
  ":agent_manager_rpc",
  ":gcs",
@@ -722,7 +722,7 @@ cc_library(
  ],
  }),
  strip_include_prefix = "src",
- visibility = ["//streaming:__subpackages__"],
+ visibility = ["//visibility:public"],
  deps = [
  ":agent_manager_rpc",
  ":node_manager_fbs",
@@ -2337,7 +2337,7 @@ pyx_library(
  ),
  deps = [
  "//:core_worker_lib",
- "//:exported_streaming_internal",
+ "//:exported_internal",
  "//:global_state_accessor_lib",
  "//:ray_util",
  "//:raylet_lib",
@@ -2348,33 +2348,6 @@ pyx_library(
  ],
 )
 
-pyx_library(
- name = "_streaming",
- srcs = glob([
- "python/ray/streaming/_streaming.pyx",
- "python/ray/__init__.py",
- "python/ray/_raylet.pxd",
- "python/ray/includes/*.pxd",
- "python/ray/includes/*.pxi",
- "python/ray/streaming/__init__.pxd",
- "python/ray/streaming/includes/*.pxd",
- "python/ray/streaming/includes/*.pxi",
- ]),
- cc_kwargs = dict(
- srcs = PYX_SRCS,
- copts = COPTS + PYX_COPTS,
- ),
- deps = [
- "//streaming:streaming_lib",
- "@com_github_grpc_grpc//:grpc++",
- "@com_google_absl//absl/container:flat_hash_map",
- "@com_google_absl//absl/container:flat_hash_set",
- "@com_google_absl//absl/memory",
- "@com_google_absl//absl/synchronization",
- "@com_google_absl//absl/time",
- ],
-)
-
 cc_binary(
  name = "libcore_worker_library_java.so",
  srcs = glob([
@@ -2398,7 +2371,7 @@ cc_binary(
  visibility = ["//java:__subpackages__"],
  deps = [
  "//:core_worker_lib",
- "//:exported_streaming_internal",
+ "//:exported_internal",
  "//:global_state_accessor_lib",
  "//:src/ray/ray_exported_symbols.lds",
  "//:src/ray/ray_version_script.lds",
@@ -2491,12 +2464,6 @@ copy_to_workspace(
  dstdir = "python/ray",
 )
 
-copy_to_workspace(
- name = "cp_streaming",
- srcs = ["python/ray/streaming/_streaming.so"],
- dstdir = "python/ray/streaming",
-)
-
 copy_to_workspace(
  name = "cp_all_py_proto",
  srcs = [":all_py_proto"],
@@ -2535,7 +2502,6 @@ genrule(
  srcs = [
  ":cp_all_py_proto",
  ":cp_serve_py_proto",
- "//streaming:copy_streaming_py_proto",
  ],
  outs = ["install_py_proto.out"],
  cmd = """
@@ -2555,7 +2521,6 @@ genrule(
  name = "ray_pkg",
  srcs = [
  ":cp_raylet_so",
- ":cp_streaming",
  ":python_sources",
  ":install_py_proto",
  ":cp_redis",

diff --git a/bazel/ray_deps_build_all.bzl b/bazel/ray_deps_build_all.bzl
@@ -1,6 +1,5 @@
 load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace")
 load("@com_github_ray_project_ray//java:dependencies.bzl", "gen_java_deps")
-load("@com_github_ray_project_ray//streaming/java:dependencies.bzl", "gen_streaming_java_deps")
 load("@com_github_nelhage_rules_boost//:boost/boost.bzl", "boost_deps")
 load("@com_github_jupp0r_prometheus_cpp//bazel:repositories.bzl", "prometheus_cpp_repositories")
 load("@com_github_grpc_grpc//third_party/py:python_configure.bzl", "python_configure")
@@ -12,7 +11,6 @@ load("@com_github_johnynek_bazel_jar_jar//:jar_jar.bzl", "jar_jar_repositories")
 def ray_deps_build_all():
  bazel_skylib_workspace()
  gen_java_deps()
- gen_streaming_java_deps()
  boost_deps()
  prometheus_cpp_repositories()
  python_configure(name = "local_config_python")

diff --git a/ci/travis/bazel-format.sh b/ci/travis/bazel-format.sh
@@ -45,6 +45,6 @@ done
 
 pushd "$ROOT_DIR"/../..
 BAZEL_FILES=(bazel/BUILD bazel/ray.bzl BUILD.bazel java/BUILD.bazel \
- cpp/BUILD.bazel cpp/example/BUILD.bazel streaming/BUILD.bazel streaming/java/BUILD.bazel WORKSPACE)
+ cpp/BUILD.bazel cpp/example/BUILD.bazel WORKSPACE)
 buildifier -mode=$RUN_TYPE -diff_command="diff -u" "${BAZEL_FILES[@]}"
 popd
diff --git a/ci/travis/determine_tests_to_run.py b/ci/travis/determine_tests_to_run.py
@@ -88,9 +88,6 @@ def get_commit_range():
  RAY_CI_PYTHON_AFFECTED = 0
  RAY_CI_LINUX_WHEELS_AFFECTED = 0
  RAY_CI_MACOS_WHEELS_AFFECTED = 0
- RAY_CI_STREAMING_CPP_AFFECTED = 0
- RAY_CI_STREAMING_PYTHON_AFFECTED = 0
- RAY_CI_STREAMING_JAVA_AFFECTED = 0
  RAY_CI_DASHBOARD_AFFECTED = 0
  RAY_CI_DOCKER_AFFECTED = 0
  RAY_CI_DOC_AFFECTED = 0
@@ -179,7 +176,6 @@ def get_commit_range():
  RAY_CI_DASHBOARD_AFFECTED = 1
  RAY_CI_LINUX_WHEELS_AFFECTED = 1
  RAY_CI_MACOS_WHEELS_AFFECTED = 1
- RAY_CI_STREAMING_PYTHON_AFFECTED = 1
  RAY_CI_DOC_AFFECTED = 1
  # Python changes might impact cross language stack in Java.
  # Java also depends on Python CLI to manage processes.
@@ -189,7 +185,6 @@ def get_commit_range():
  RAY_CI_PYTHON_DEPENDENCIES_AFFECTED = 1
  elif changed_file.startswith("java/"):
  RAY_CI_JAVA_AFFECTED = 1
- RAY_CI_STREAMING_JAVA_AFFECTED = 1
  elif changed_file.startswith("cpp/"):
  RAY_CI_CPP_AFFECTED = 1
  elif changed_file.startswith("docker/"):
@@ -218,19 +213,8 @@ def get_commit_range():
  RAY_CI_PYTHON_AFFECTED = 1
  RAY_CI_LINUX_WHEELS_AFFECTED = 1
  RAY_CI_MACOS_WHEELS_AFFECTED = 1
- RAY_CI_STREAMING_CPP_AFFECTED = 1
- RAY_CI_STREAMING_PYTHON_AFFECTED = 1
- RAY_CI_STREAMING_JAVA_AFFECTED = 1
  RAY_CI_DASHBOARD_AFFECTED = 1
  RAY_CI_DOC_AFFECTED = 1
- elif changed_file.startswith("streaming/src"):
- RAY_CI_STREAMING_CPP_AFFECTED = 1
- RAY_CI_STREAMING_PYTHON_AFFECTED = 1
- RAY_CI_STREAMING_JAVA_AFFECTED = 1
- elif changed_file.startswith("streaming/python"):
- RAY_CI_STREAMING_PYTHON_AFFECTED = 1
- elif changed_file.startswith("streaming/java"):
- RAY_CI_STREAMING_JAVA_AFFECTED = 1
  else:
  RAY_CI_TUNE_AFFECTED = 1
  RAY_CI_SGD_AFFECTED = 1
@@ -244,9 +228,6 @@ def get_commit_range():
  RAY_CI_DOC_AFFECTED = 1
  RAY_CI_LINUX_WHEELS_AFFECTED = 1
  RAY_CI_MACOS_WHEELS_AFFECTED = 1
- RAY_CI_STREAMING_CPP_AFFECTED = 1
- RAY_CI_STREAMING_PYTHON_AFFECTED = 1
- RAY_CI_STREAMING_JAVA_AFFECTED = 1
  RAY_CI_DASHBOARD_AFFECTED = 1
  else:
  RAY_CI_TUNE_AFFECTED = 1
@@ -262,9 +243,6 @@ def get_commit_range():
  RAY_CI_DOC_AFFECTED = 1
  RAY_CI_LINUX_WHEELS_AFFECTED = 1
  RAY_CI_MACOS_WHEELS_AFFECTED = 1
- RAY_CI_STREAMING_CPP_AFFECTED = 1
- RAY_CI_STREAMING_PYTHON_AFFECTED = 1
- RAY_CI_STREAMING_JAVA_AFFECTED = 1
  RAY_CI_DASHBOARD_AFFECTED = 1
 
  # Log the modified environment variables visible in console.
@@ -284,12 +262,6 @@ def get_commit_range():
  "RAY_CI_PYTHON_AFFECTED={}".format(RAY_CI_PYTHON_AFFECTED),
  "RAY_CI_LINUX_WHEELS_AFFECTED={}".format(RAY_CI_LINUX_WHEELS_AFFECTED),
  "RAY_CI_MACOS_WHEELS_AFFECTED={}".format(RAY_CI_MACOS_WHEELS_AFFECTED),
- "RAY_CI_STREAMING_CPP_AFFECTED={}".format(
- RAY_CI_STREAMING_CPP_AFFECTED),
- "RAY_CI_STREAMING_PYTHON_AFFECTED={}".format(
- RAY_CI_STREAMING_PYTHON_AFFECTED),
- "RAY_CI_STREAMING_JAVA_AFFECTED={}".format(
- RAY_CI_STREAMING_JAVA_AFFECTED),
  "RAY_CI_DOCKER_AFFECTED={}".format(RAY_CI_DOCKER_AFFECTED),
  "RAY_CI_PYTHON_DEPENDENCIES_AFFECTED={}".format(
  RAY_CI_PYTHON_DEPENDENCIES_AFFECTED),

diff --git a/java/BUILD.bazel b/java/BUILD.bazel
@@ -312,8 +312,6 @@ java_binary(
  "//java:io_ray_ray_api",
  "//java:io_ray_ray_runtime",
  "//java:io_ray_ray_serve",
- "//streaming/java:io_ray_ray_streaming-api",
- "//streaming/java:io_ray_ray_streaming-runtime",
  ],
 )
 
@@ -335,7 +333,6 @@ genrule(
  srcs = [
  "//java:ray_dist_shaded.jar",
  "//java:gen_maven_deps",
- "//streaming/java:gen_maven_deps",
  ],
  outs = ["ray_java_pkg.out"],
  cmd = """

diff --git a/python/ray/streaming b/python/ray/streaming
diff --git a/python/ray/streaming/__init__.py b/python/ray/streaming/__init__.py
@@ -0,0 +1,26 @@
+# flake8: noqa
+# Ray should be imported before streaming
+import ray
+
+
+# Redirect ray.streaming.* to raystreaming, so user can keep
+# their original habit.
+def _update_modules():
+ try:
+ import raystreaming
+ import raystreaming.context
+ import sys
+ ray_streaming_module_name = raystreaming.__name__
+ ray_streaming_modules = {}
+ for mod_name, module in sys.modules.items():
+ if mod_name.startswith(ray_streaming_module_name):
+ ray_streaming_modules[mod_name.replace(
+ "raystreaming", "ray.streaming")] = module
+ sys.modules.update(ray_streaming_modules)
+ except Exception as e:
+ print("import raystreaming error: ", e)
+
+
+_update_modules()
+
+__all__ = ["StreamingContext"]
diff --git a/src/ray/streaming/streaming.cc → src/ray/internal/internal.cc b/src/ray/streaming/streaming.cc → src/ray/internal/internal.cc
@@ -12,12 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "ray/streaming/streaming.h"
+#include "ray/internal/internal.h"
 
 #include "ray/core_worker/core_worker.h"
 
 namespace ray {
-namespace streaming {
+namespace internal {
 
 using ray::core::CoreWorkerProcess;
 using ray::core::TaskOptions;
@@ -52,5 +52,5 @@ std::vector<rpc::ObjectReference> SendInternal(const ActorID &peer_actor_id,
  }
  return result.value();
 }
-} // namespace streaming
+} // namespace internal
 } // namespace ray
diff --git a/src/ray/streaming/streaming.h → src/ray/internal/internal.h b/src/ray/streaming/streaming.h → src/ray/internal/internal.h
@@ -17,10 +17,10 @@
 #include "ray/common/id.h"
 #include "ray/core_worker/common.h"
 
-// This header is used to warp some streaming code so we can reduce suspicious
+// This header is used to warp some internal code so we can reduce suspicious
 // symbols export.
 namespace ray {
-namespace streaming {
+namespace internal {
 
 using ray::core::RayFunction;
 
@@ -33,5 +33,5 @@ std::vector<rpc::ObjectReference> SendInternal(const ActorID &peer_actor_id,
  std::shared_ptr<LocalMemoryBuffer> buffer,
  RayFunction &function, int return_num);
 
-} // namespace streaming
+} // namespace internal
 } // namespace ray