Revert "Package and upload ray cross-platform jar (ray-project#9540)" (…

…ray-project#9730)

This reverts commit 8810325.

.gitignore

@@ -10,7 +10,6 @@
 /python/python-driver-*
 /thirdparty/pkg/
 /build/java
-.jar
 
 # Files generated by flatc should be ignored
 /src/ray/gcs/format/*_generated.h

.travis.yml

@@ -99,37 +99,33 @@ matrix:
  script:
  - true # we still need this block to exist, otherwise it will fall back to the global one
 
- # Build Linux wheels and multi-platform jars.
+ # Build Linux wheels.
  - os: linux
  env:
- - LINUX_WHEELS=1 LINUX_JARS=1
+ - LINUX_WHEELS=1
  - PYTHONWARNINGS=ignore
  - RAY_INSTALL_JAVA=1
  install:
- - . ./ci/travis/ci.sh init RAY_CI_LINUX_WHEELS_AFFECTED,RAY_CI_JAVA_AFFECTED,RAY_CI_STREAMING_JAVA_AFFECTED
+ - . ./ci/travis/ci.sh init RAY_CI_LINUX_WHEELS_AFFECTED
  before_script:
  - . ./ci/travis/ci.sh build
  script:
  - . ./ci/travis/ci.sh test_wheels
- - bash ./java/build-jar-multiplatform.sh linux
  cache: false
 
- # Build MacOS wheels and MacOS jars
+ # Build MacOS wheels.
  - os: osx
  osx_image: xcode7
  env:
- - MAC_WHEELS=1 MULTIPLATFORM_JARS=1
+ - MAC_WHEELS=1
  - PYTHONWARNINGS=ignore
  - RAY_INSTALL_JAVA=1
  install:
- - . ./ci/travis/ci.sh init RAY_CI_MACOS_WHEELS_AFFECTED,RAY_CI_JAVA_AFFECTED,RAY_CI_STREAMING_JAVA_AFFECTED
+ - . ./ci/travis/ci.sh init RAY_CI_MACOS_WHEELS_AFFECTED
  before_script:
  - . ./ci/travis/ci.sh build
  script:
  - . ./ci/travis/ci.sh test_wheels
- - bash ./java/build-jar-multiplatform.sh darwin
- - bash ./java/build-jar-multiplatform.sh multiplatform
- #- bash ./java/build-jar-multiplatform.sh deploy
 
  # RLlib: Learning tests (from rllib/tuned_examples/*.yaml).
  - os: linux
@@ -396,39 +392,6 @@ deploy:
  repo: ray-project/ray
  condition: $LINUX_WHEELS = 1 || $MAC_WHEELS = 1
 
- # Upload jars so that we can debug locally for every commit
- - provider: s3
- edge: true # This supposedly opts in to deploy v2.
- access_key_id: AKIAU6DMUCJUFL3EX3SM
- secret_access_key:
- secure: J1sX71fKFPQhgWzColllxfzcF877ScBZ1cIl71krZ6SO0LKnwsCScpQck5eZOyQo/Iverwye0iKtE87qNsiRi3+V2D9iulSr18T09j7+FjPKfxAmXmjfrNafoMXTDQroSJblCri5vl+DysISPqImJkWTNaYhGJ9QakoSd5djnAopLNWj6PCR3S50baS49+nB5nSIY3jMhtUzlaBdniFPFC81Cxyuafr4pv6McGRfR/dK+ZnPhdGtMnVeIJXB+ooZKQ26mDJKBPka4jm3u1Oa72b/Atu2RO3MwxTg79LTrMxXKh2OcCqhtD2Z3lz1OltvNSunCuwY8AejCJsfSLbM9mGDoz+xhNUWmYNy48YFf+61OY8PXi8S/9Q817yb3GpLbb2l/P+KMgq9eSEiELIOwuYsDxPX5TuAg6dx0wCNgDEBJoThSQjYl6MgJrLrs7p+JBxp3giedHiy0TLa5hCVKTj3euONAXDArYnnT+DvUIOkaeTk5DClRZbZ0sUXhLy//HuT5WJvjFBJJZ0u0f4RLVb5D7DI4uMZr7+yJPDR2AXCyW9YMaBEbmEYbPaKi283jlEyn7R33+AZlnXv0THHwZ4xvjKKG3/fBSXsOUmv5wmUveEqVGDj1mKPGj9NF8iA5qMm2AaZuJpEEBVBZtSlTZt6ZG7rzAJZGNL52t7xuMo=
- bucket: ray-wheels
- acl: public_read
- region: us-west-2
- local_dir: .jar
- upload-dir: "jars/$TRAVIS_BRANCH/$TRAVIS_COMMIT"
- skip_cleanup: true
- on:
- repo: ray-project/ray
- all_branches: true
- condition: $MULTIPLATFORM_JARS = 1 || $LINUX_JARS=1
-
- - provider: s3
- edge: true # This supposedly opts in to deploy v2.
- access_key_id: AKIAU6DMUCJUFL3EX3SM
- secret_access_key:
- secure: J1sX71fKFPQhgWzColllxfzcF877ScBZ1cIl71krZ6SO0LKnwsCScpQck5eZOyQo/Iverwye0iKtE87qNsiRi3+V2D9iulSr18T09j7+FjPKfxAmXmjfrNafoMXTDQroSJblCri5vl+DysISPqImJkWTNaYhGJ9QakoSd5djnAopLNWj6PCR3S50baS49+nB5nSIY3jMhtUzlaBdniFPFC81Cxyuafr4pv6McGRfR/dK+ZnPhdGtMnVeIJXB+ooZKQ26mDJKBPka4jm3u1Oa72b/Atu2RO3MwxTg79LTrMxXKh2OcCqhtD2Z3lz1OltvNSunCuwY8AejCJsfSLbM9mGDoz+xhNUWmYNy48YFf+61OY8PXi8S/9Q817yb3GpLbb2l/P+KMgq9eSEiELIOwuYsDxPX5TuAg6dx0wCNgDEBJoThSQjYl6MgJrLrs7p+JBxp3giedHiy0TLa5hCVKTj3euONAXDArYnnT+DvUIOkaeTk5DClRZbZ0sUXhLy//HuT5WJvjFBJJZ0u0f4RLVb5D7DI4uMZr7+yJPDR2AXCyW9YMaBEbmEYbPaKi283jlEyn7R33+AZlnXv0THHwZ4xvjKKG3/fBSXsOUmv5wmUveEqVGDj1mKPGj9NF8iA5qMm2AaZuJpEEBVBZtSlTZt6ZG7rzAJZGNL52t7xuMo=
- bucket: ray-wheels
- acl: public_read
- region: us-west-2
- local_dir: .jar
- upload-dir: "jars/latest"
- skip_cleanup: true
- on:
- repo: ray-project/ray
- branch: master
- condition: $MULTIPLATFORM_JARS = 1 || $LINUX_JARS=1
-
  - provider: script
  edge: true # This supposedly opts in to deploy v2.
  script: bash $TRAVIS_BUILD_DIR/ci/travis/build-autoscaler-images.sh || true

BUILD.bazel

@@ -1815,7 +1815,6 @@ cc_binary(
  }),
  linkshared = 1,
  linkstatic = 1,
- visibility = ["//java:__subpackages__"],
  deps = [
  "//:core_worker_lib",
  "//:global_state_accessor_lib",
@@ -1826,6 +1825,23 @@ cc_binary(
  ],
 )
 
+genrule(
+ name = "core_worker-jni-darwin-compat",
+ srcs = [":libcore_worker_library_java.so"],
+ outs = ["libcore_worker_library_java.dylib"],
+ cmd = "cp $< $@",
+ output_to_bindir = 1,
+)
+
+filegroup(
+ name = "core_worker_library_java",
+ srcs = select({
+ "@bazel_tools//src/conditions:darwin": [":libcore_worker_library_java.dylib"],
+ "//conditions:default": [":libcore_worker_library_java.so"],
+ }),
+ visibility = ["//java:__subpackages__"],
+)
+
 filegroup(
  name = "python_sources",
  srcs = glob([

bazel/ray.bzl

@@ -1,6 +1,5 @@
 load("@com_github_google_flatbuffers//:build_defs.bzl", "flatbuffer_library_public")
 load("@com_github_checkstyle_java//checkstyle:checkstyle.bzl", "checkstyle_test")
-load("@bazel_skylib//rules:copy_file.bzl", "copy_file")
 load("@bazel_common//tools/maven:pom_file.bzl", "pom_file")
 
 COPTS = ["-DRAY_USE_GLOG"] + select({
@@ -126,7 +125,7 @@ def copy_to_workspace(name, srcs, dstdir = ""):
  dstdir = "." + ("/" + dstdir.replace("\\", "/")).rstrip("/") + "/",
  ),
  # Keep this batch script equivalent to the Bash script above (or take out the batch script)
- cmd_bat = """
+ cmd_bat = r"""
  (
  if not exist {dstdir} mkdir {dstdir}
  ) && (
@@ -141,57 +140,3 @@ def copy_to_workspace(name, srcs, dstdir = ""):
  ),
  local = 1,
  )
-
-def native_java_binary(module_name, name, native_binary_name):
- """Copy native binary file to different path based on operating systems"""
- copy_file(
- name = name + "_darwin",
- src = native_binary_name,
- out = module_name + "/src/main/resources/native/darwin/" + name,
- )
-
- copy_file(
- name = name + "_linux",
- src = native_binary_name,
- out = module_name + "/src/main/resources/native/linux/" + name,
- )
-
- copy_file(
- name = name + "_windows",
- src = native_binary_name,
- out = module_name + "/src/main/resources/native/windows/" + name,
- )
-
- native.filegroup(
- name = name,
- srcs = select({
- "@bazel_tools//src/conditions:darwin": [name + "_darwin"],
- "@bazel_tools//src/conditions:windows": [name + "_windows"],
- "//conditions:default": [name + "_linux"],
- }),
- visibility = ["//visibility:public"],
- )
-
-def native_java_library(module_name, name, native_library_name):
- """Copy native library file to different path based on operating systems"""
- copy_file(
- name = name + "_darwin",
- src = native_library_name,
- out = module_name + "/src/main/resources/native/darwin/lib{}.dylib".format(name),
- )
-
- copy_file(
- name = name + "_linux",
- src = native_library_name,
- out = module_name + "/src/main/resources/native/linux/lib{}.so".format(name),
- )
-
- native.filegroup(
- name = name,
- srcs = select({
- "@bazel_tools//src/conditions:darwin": [name + "_darwin"],
- "@bazel_tools//src/conditions:windows": [],
- "//conditions:default": [name + "_linux"],
- }),
- visibility = ["//visibility:public"],
- )

java/BUILD.bazel

@@ -1,6 +1,5 @@
 load("//bazel:ray.bzl", "define_java_module")
-load("//bazel:ray.bzl", "native_java_binary")
-load("//bazel:ray.bzl", "native_java_library")
+load("@bazel_skylib//rules:copy_file.bzl", "copy_file")
 load("@rules_proto_grpc//java:defs.bzl", "java_proto_compile")
 
 exports_files([
@@ -144,69 +143,52 @@ filegroup(
  ],
 )
 
-native_java_binary("runtime", "raylet", "//:raylet")
-
-native_java_binary("runtime", "plasma_store_server", "//:plasma_store_server")
-
-native_java_binary("runtime", "redis-server", "//:redis-server")
-
-native_java_binary("runtime", "gcs_server", "//:gcs_server")
-
-native_java_binary("runtime", "libray_redis_module.so", "//:libray_redis_module.so")
+# plasma_store_server binary should be located in jar root path
+# and keep in sync with mvn resource
+copy_file(
+ name = "cp_plasma_store_server",
+ src = "//:plasma_store_server",
+ out = "plasma_store_server",
+)
 
-native_java_library("runtime", "core_worker_library_java", "//:libcore_worker_library_java.so")
+# `//:redis-server`'s full path is `external/com_github_antirez_redis/redis-server`,
+# This rule removes the prefix, and only keeps `redis-server`.
+copy_file(
+ name = "redis-server-without-prefix",
+ src = "//:redis-server",
+ out = "redis-server",
+)
 
 filegroup(
  name = "java_native_deps",
  srcs = [
- ":core_worker_library_java",
- ":gcs_server",
- ":libray_redis_module.so",
- ":plasma_store_server",
- ":raylet",
- ":redis-server",
+ ":cp_plasma_store_server",
+ ":redis-server-without-prefix",
+ "//:core_worker_library_java",
+ "//:gcs_server",
+ "//:libray_redis_module.so",
+ "//:raylet",
  ],
 )
 
 # Generates the dependencies needed by maven.
 genrule(
- name = "cp_java_generated",
+ name = "gen_maven_deps",
  srcs = [
  ":all_java_proto",
+ ":java_native_deps",
  ":copy_pom_file",
  ],
- outs = ["cp_java_generated.out"],
+ outs = ["gen_maven_deps.out"],
  cmd = """
  WORK_DIR="$$(pwd)"
  # Copy protobuf-generated files.
  rm -rf "$$WORK_DIR/java/runtime/src/main/java/io/ray/runtime/generated"
  for f in $(locations //java:all_java_proto); do
  unzip "$$f" -x META-INF/MANIFEST.MF -d "$$WORK_DIR/java/runtime/src/main/java"
  done
- date > $@
- """,
- local = 1,
- tags = ["no-cache"],
-)
-
-# Generates the dependencies needed by maven.
-genrule(
- name = "gen_maven_deps",
- srcs = [
- ":cp_java_generated",
- ":java_native_deps",
- ],
- outs = ["gen_maven_deps.out"],
- cmd = """
- WORK_DIR="$${PWD}"
- # Copy native dependencies.
- OS_NAME=""
- case "$${OSTYPE}" in
- linux*) OS_NAME="linux";;
- darwin*) OS_NAME="darwin";;
- *) echo "$${OSTYPE} is not supported currently"; exit 1;;
- esac
- NATIVE_DEPS_DIR="$$WORK_DIR/java/runtime/native_dependencies/native/$$OS_NAME"
+ # Copy native dependecies.
+ NATIVE_DEPS_DIR="$$WORK_DIR/java/runtime/native_dependencies/"
  rm -rf "$$NATIVE_DEPS_DIR"
  mkdir -p "$$NATIVE_DEPS_DIR"
  for f in $(locations //java:java_native_deps); do