[Serve][Nightly] Environment for Nightly K8s Tests (ray-project#27126)

Signed-off-by: Stefan van der Kleij <[email protected]>

release/k8s_tests/app_config.yaml

@@ -0,0 +1,22 @@
+base_image: {{ env["RAY_IMAGE_NIGHTLY_CPU"] | default("anyscale/ray:nightly-py37") }}
+env_vars: {}
+debian_packages:
+ - curl
+ - unzip
+ - jq
+ - apt-transport-https
+ - ca-certificates
+ - gnupg
+
+python:
+ pip_packages:
+ - kubernetes
+ conda_packages: []
+
+post_build_cmds:
+ # Install gcloud tools
+ - sudo apt-get remove -y google-cloud-sdk
+ - echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
+ - curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key --keyring /usr/share/keyrings/cloud.google.gpg add -
+ - sudo apt-get update && sudo apt-get install google-cloud-cli
+ - sudo apt-get install google-cloud-sdk-gke-gcloud-auth-plugin

release/k8s_tests/compute_tpl.yaml

@@ -0,0 +1,29 @@
+cloud_id: {{env["ANYSCALE_CLOUD_ID"]}}
+region: us-west-2
+
+max_workers: 0
+
+head_node_type:
+ name: head_node
+ instance_type: m5.2xlarge
+
+worker_node_types:
+ - name: worker_node
+ instance_type: m5.2xlarge
+ min_workers: 0
+ max_workers: 0
+ use_spot: false
+
+aws:
+ TagSpecifications:
+ - ResourceType: "instance"
+ Tags:
+ - Key: anyscale-user
+ Value: '{{env["ANYSCALE_USER"]}}'
+ - Key: anyscale-expiration
+ Value: '{{env["EXPIRATION_2D"]}}'
+
+ BlockDeviceMappings:
+ - DeviceName: /dev/sda1
+ Ebs:
+ VolumeSize: 200

release/k8s_tests/prepare.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+set -ex
+
+aws secretsmanager get-secret-value \
+ --secret-id oss-nightly-ci-gke-cluster-service-account \
+ --region us-west-2 |
+ jq -r .SecretString |
+ jq -r .json_text \
+ >gke_service_account.json
+
+gcloud auth activate-service-account \
+ [email protected] \
+ --key-file=gke_service_account.json
+
+USE_GKE_GCLOUD_AUTH_PLUGIN=True \
+ gcloud container clusters get-credentials \
+ serve-ha-nightly \
+ --zone us-central1-c \
+ --project oss-nightly-test
+
+kubectl get nodes

release/k8s_tests/run_gcs_ft_on_k8s.py

@@ -0,0 +1,10 @@
+from kubernetes import client, config
+
+# Configs can be set in Configuration class directly or using helper utility
+config.load_kube_config()
+
+v1 = client.CoreV1Api()
+print("Listing pods with their IPs:")
+ret = v1.list_pod_for_all_namespaces(watch=False)
+for i in ret.items:
+ print("%s\t%s\t%s" % (i.status.pod_ip, i.metadata.namespace, i.metadata.name))

release/release_tests.yaml

@@ -4581,3 +4581,25 @@
  num_nodes: 20
  type: sdk_command
  file_manager: sdk
+
+- name: k8s_serve_ha_test
+ group: k8s-test
+ working_dir: k8s_tests
+ legacy:
+ test_name: k8s_serve_ha_test
+ test_suite: k8s_tests
+
+ stable: false
+
+ frequency: nightly
+ team: serve
+ cluster:
+ cluster_env: app_config.yaml
+ cluster_compute: compute_tpl.yaml
+
+ run:
+ timeout: 1800
+ prepare: bash prepare.sh
+ script: python run_gcs_ft_on_k8s.py
+ type: sdk_command
+ file_manager: sdk