[Serve] Fix test_cli flakiness (ray-project#26471)

dashboard/modules/serve/tests/test_serve_agent.py

@@ -1,5 +1,4 @@
 import copy
-import subprocess
 import sys
 from typing import Dict
 
@@ -10,24 +9,12 @@
 from ray import serve
 from ray._private.test_utils import wait_for_condition
 import ray._private.ray_constants as ray_constants
+from ray.serve.tests.conftest import * # noqa: F401 F403
 
 GET_OR_PUT_URL = "http:https://localhost:52365/api/serve/deployments/"
 STATUS_URL = "http:https://localhost:52365/api/serve/deployments/status"
 
 
-@pytest.fixture
-def ray_start_stop():
- subprocess.check_output(["ray", "stop", "--force"])
- subprocess.check_output(["ray", "start", "--head"])
- wait_for_condition(
- lambda: requests.get("http:https://localhost:52365/api/ray/version").status_code
- == 200,
- timeout=15,
- )
- yield
- subprocess.check_output(["ray", "stop", "--force"])
-
-
 def deploy_and_check_config(config: Dict):
  put_response = requests.put(GET_OR_PUT_URL, json=config, timeout=30)
  assert put_response.status_code == 200

python/ray/serve/tests/conftest.py

@@ -5,9 +5,11 @@
 import subprocess
 import random
 
+import requests
 import ray
 from ray import serve
 
+from ray._private.test_utils import wait_for_condition
 from ray.tests.conftest import pytest_runtest_makereport # noqa
 
 # https://tools.ietf.org/html/rfc6335#section-6
@@ -80,3 +82,32 @@ def serve_instance(_shared_serve_instance):
  _shared_serve_instance.delete_deployments(serve.list_deployments().keys())
  # Clear the ServeHandle cache between tests to avoid them piling up.
  _shared_serve_instance.handle_cache.clear()
+
+
+def check_ray_stop():
+ try:
+ requests.get("http:https://localhost:52365/api/ray/version")
+ return False
+ except Exception:
+ return True
+
+
+@pytest.fixture(scope="function")
+def ray_start_stop():
+ subprocess.check_output(["ray", "stop", "--force"])
+ wait_for_condition(
+ check_ray_stop,
+ timeout=15,
+ )
+ subprocess.check_output(["ray", "start", "--head"])
+ wait_for_condition(
+ lambda: requests.get("http:https://localhost:52365/api/ray/version").status_code
+ == 200,
+ timeout=15,
+ )
+ yield
+ subprocess.check_output(["ray", "stop", "--force"])
+ wait_for_condition(
+ check_ray_stop,
+ timeout=15,
+ )

python/ray/serve/tests/test_cli.py

@@ -44,19 +44,6 @@ def assert_deployments_live(names: List[str]):
  assert all_deployments_live, f'"{nonliving_deployment}" deployment is not live.'
 
 
-@pytest.fixture
-def ray_start_stop():
- subprocess.check_output(["ray", "stop", "--force"])
- subprocess.check_output(["ray", "start", "--head"])
- wait_for_condition(
- lambda: requests.get("http:https://localhost:52365/api/ray/version").status_code
- == 200,
- timeout=15,
- )
- yield
- subprocess.check_output(["ray", "stop", "--force"])
-
-
 def test_start_shutdown(ray_start_stop):
  subprocess.check_output(["serve", "start"])
  subprocess.check_output(["serve", "shutdown", "-y"])
@@ -65,7 +52,6 @@ def test_start_shutdown(ray_start_stop):
 @pytest.mark.skipif(sys.platform == "win32", reason="File path incorrect on Windows.")
 def test_deploy(ray_start_stop):
  """Deploys some valid config files and checks that the deployments work."""
- ray.shutdown()
  # Initialize serve in test to enable calling serve.list_deployments()
  ray.init(address="auto", namespace=SERVE_NAMESPACE)
 
@@ -426,8 +412,6 @@ def test_build(ray_start_stop, node):
 @pytest.mark.parametrize("use_command", [True, False])
 def test_idempotence_after_controller_death(ray_start_stop, use_command: bool):
  """Check that CLI is idempotent even if controller dies."""
- ray.shutdown()
-
  config_file_name = os.path.join(
  os.path.dirname(__file__), "test_config_files", "basic_graph.yaml"
  )