[Dependencies] Remove kubernetes requirement (ray-project#36333)

The kubernetes package is installed with ray["all"] but it's currently only used in two places:

A helper function for converting units
The GCS FT on K8s release test:
ray/release/k8s_tests/run_gcs_ft_on_k8s.py

Line 4 in 4070624

 from kubernetes import client, config, watch 
This PR removes kubernetes from Ray's dependencies and removes it from the Ray docker image. This change is needed to reduce the overall package size of ray["all"].

For the helper function, we vendor it from the upstream.

For the release test, kubernetes is installed separately:

ray/release/k8s_tests/app_config.yaml

Line 13 in 4070624

 - kubernetes 
. I've also added it to release/requirements.txt to be safe.
Related issue number
Closes ray-project#36331

---------

Signed-off-by: Archit Kulkarni <[email protected]>

LICENSE

@@ -430,3 +430,21 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
+
+--------------------------------------------------------------------------------
+
+Code in python/ray/autoscaler/_private/kuberay/utils.py is adapted from https://github.com/kubernetes-client/python/blob/master/kubernetes/utils/quantity.py
+
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

docker/ray-deps/Dockerfile

@@ -19,7 +19,6 @@ RUN $HOME/anaconda3/bin/pip --no-cache-dir install --find-links $FIND_LINKS_PATH
  "cryptography==38.0.1" \
  "google-api-python-client==1.7.8" \
  "google-oauth" \
- "kubernetes" \
  "azure-cli-core==2.40.0" \
  "azure-identity==1.10.0" \
  "azure-mgmt-compute==23.1.0" \

python/ray/autoscaler/_private/kuberay/autoscaling_config.py

@@ -4,7 +4,6 @@
 import time
 from typing import Any, Dict, Optional
 
-import kubernetes
 import requests
 
 from ray.autoscaler._private.constants import (
@@ -14,7 +13,7 @@
  WORKER_LIVENESS_CHECK_KEY,
  WORKER_RPC_DRAIN_KEY,
 )
-from ray.autoscaler._private.kuberay import node_provider
+from ray.autoscaler._private.kuberay import node_provider, utils
 from ray.autoscaler._private.util import validate_config
 
 logger = logging.getLogger(__name__)
@@ -343,9 +342,7 @@ def _round_up_k8s_quantity(quantity: str) -> int:
  Returns:
  The quantity, rounded up, as an integer.
  """
- resource_decimal: decimal.Decimal = kubernetes.utils.quantity.parse_quantity(
- quantity
- )
+ resource_decimal: decimal.Decimal = utils.parse_quantity(quantity)
  rounded = resource_decimal.to_integral_value(rounding=decimal.ROUND_UP)
  return int(rounded)
 

python/ray/autoscaler/_private/kuberay/utils.py

@@ -0,0 +1,75 @@
+# Source:
+# https://github.com/kubernetes-client/python/blob/master/kubernetes/utils/quantity.py
+from decimal import Decimal, InvalidOperation
+
+
+def parse_quantity(quantity):
+ """
+ Parse kubernetes canonical form quantity like 200Mi to a decimal number.
+ Supported SI suffixes:
+ base1024: Ki | Mi | Gi | Ti | Pi | Ei
+ base1000: n | u | m | "" | k | M | G | T | P | E
+
+ See
+ https://github.com/kubernetes/apimachinery/blob/master/pkg/api/resource/quantity.go
+
+ Input:
+ quantity: string. kubernetes canonical form quantity
+
+ Returns:
+ Decimal
+
+ Raises:
+ ValueError on invalid or unknown input
+ """
+ if isinstance(quantity, (int, float, Decimal)):
+ return Decimal(quantity)
+
+ exponents = {
+ "n": -3,
+ "u": -2,
+ "m": -1,
+ "K": 1,
+ "k": 1,
+ "M": 2,
+ "G": 3,
+ "T": 4,
+ "P": 5,
+ "E": 6,
+ }
+
+ quantity = str(quantity)
+ number = quantity
+ suffix = None
+ if len(quantity) >= 2 and quantity[-1] == "i":
+ if quantity[-2] in exponents:
+ number = quantity[:-2]
+ suffix = quantity[-2:]
+ elif len(quantity) >= 1 and quantity[-1] in exponents:
+ number = quantity[:-1]
+ suffix = quantity[-1:]
+
+ try:
+ number = Decimal(number)
+ except InvalidOperation:
+ raise ValueError("Invalid number format: {}".format(number))
+
+ if suffix is None:
+ return number
+
+ if suffix.endswith("i"):
+ base = 1024
+ elif len(suffix) == 1:
+ base = 1000
+ else:
+ raise ValueError("{} has unknown suffix".format(quantity))
+
+ # handle SI inconsistency
+ if suffix == "ki":
+ raise ValueError("{} has unknown suffix".format(quantity))
+
+ if suffix[0] not in exponents:
+ raise ValueError("{} has unknown suffix".format(quantity))
+
+ exponent = Decimal(exponents[suffix[0]])
+ return number * (base**exponent)

python/requirements.txt

@@ -44,7 +44,6 @@ matplotlib!=3.4.3; platform_system != "Windows"
 tensorboardX>=1.9
 smart_open
 requests
-kubernetes
 colorful
 lz4
 virtualenv>=20.0.24, < 20.21.1

python/setup.py

@@ -264,7 +264,7 @@ def get_packages(self):
  ],
  "serve": ["uvicorn", "requests", "starlette", "fastapi", "aiorwlock"],
  "tune": ["pandas", "tensorboardX>=1.9", "requests", pyarrow_dep],
- "k8s": ["kubernetes", "urllib3"],
+ "k8s": ["urllib3"],
  "observability": [
  "opentelemetry-api",
  "opentelemetry-sdk",

release/requirements.txt

@@ -20,4 +20,5 @@ protobuf >= 3.15.3, != 3.19.5
 pytz
 retry
 git+https://github.com/ray-project/xgboost_ray.git#egg=xgboost_ray
-git+https://github.com/ray-project/lightgbm_ray.git#lightgbm_ray
+git+https://github.com/ray-project/lightgbm_ray.git#lightgbm_ray
+kubernetes