[core][dashboard] [no_early_kickoff] Make dashboard address connectab…

…le from remote nodes when not set to 127.0.0.1 (localhost) (ray-project#35027)

Why are these changes needed?
This PR changes how dashboard host ip in the cluster is made available through internal kv. This PR sets the dashboard url to node's ip address if it's not using localhost. This is needed such that state api works across clusters.
Signed-off-by: rickyyx <[email protected]>

dashboard/head.py

@@ -304,12 +304,21 @@ async def _async_notify():
  logger.info("Initialize the http server.")
  self.http_server = await self._configure_http_server(modules)
  http_host, http_port = self.http_server.get_address()
+ logger.info(f"http server initialized at {http_host}:{http_port}")
  else:
  logger.info("http server disabled.")
+
+ # We need to expose dashboard's node's ip for other worker nodes
+ # if it's listening to all interfaces.
+ dashboard_http_host = (
+ self.ip
+ if self.http_host != ray_constants.DEFAULT_DASHBOARD_IP
+ else http_host
+ )
  await asyncio.gather(
  self.gcs_aio_client.internal_kv_put(
  ray_constants.DASHBOARD_ADDRESS.encode(),
- f"{http_host}:{http_port}".encode(),
+ f"{dashboard_http_host}:{http_port}".encode(),
  True,
  namespace=ray_constants.KV_NAMESPACE_DASHBOARD,
  ),

dashboard/modules/job/tests/test_http_job_server.py

@@ -610,7 +610,6 @@ def test_version_endpoint(job_sdk_client):
 
 def test_request_headers(job_sdk_client):
  client = job_sdk_client
-
  with patch("requests.request") as mock_request:
  _ = client._do_request(
  "POST",

dashboard/tests/test_dashboard.py

@@ -277,22 +277,38 @@ def test_agent_report_unexpected_raylet_death_large_file(shutdown_only):
  "ray_start_with_dashboard",
  [
  {"dashboard_host": "127.0.0.1"},
+ {"dashboard_host": "localhost"},
+ ],
+ indirect=True,
+)
+def test_dashboard_address_local(ray_start_with_dashboard):
+ webui_url = ray_start_with_dashboard["webui_url"]
+ if os.environ.get("RAY_MINIMAL") == "1":
+ # In the minimal installation, webui url shouldn't be configured.
+ assert webui_url == ""
+ else:
+ webui_ip = webui_url.split(":")[0]
+ assert not ipaddress.ip_address(webui_ip).is_unspecified
+ assert webui_ip == "127.0.0.1"
+
+
+@pytest.mark.parametrize(
+ "ray_start_with_dashboard",
+ [
  {"dashboard_host": "0.0.0.0"},
  {"dashboard_host": "::"},
  ],
  indirect=True,
 )
-def test_dashboard_address(ray_start_with_dashboard):
+def test_dashboard_address_global(ray_start_with_dashboard):
  webui_url = ray_start_with_dashboard["webui_url"]
  if os.environ.get("RAY_MINIMAL") == "1":
  # In the minimal installation, webui url shouldn't be configured.
  assert webui_url == ""
  else:
  webui_ip = webui_url.split(":")[0]
- print(ipaddress.ip_address(webui_ip))
- print(webui_ip)
  assert not ipaddress.ip_address(webui_ip).is_unspecified
- assert webui_ip in ["127.0.0.1", ray_start_with_dashboard["node_ip_address"]]
+ assert webui_ip == ray_start_with_dashboard["node_ip_address"]
 
 
 @pytest.mark.skipif(

python/ray/_private/services.py

@@ -1084,7 +1084,9 @@ def start_api_server(
  no redirection should happen, then this should be None.
 
  Returns:
- ProcessInfo for the process that was started.
+ A tuple of :
+ - Dashboard URL if dashboard enabled and started.
+ - ProcessInfo for the process that was started.
  """
  try:
  # Make sure port is available.

python/ray/scripts/scripts.py

@@ -404,10 +404,10 @@ def debug(address):
 @click.option(
  "--dashboard-host",
  required=False,
- default="localhost",
+ default=ray_constants.DEFAULT_DASHBOARD_IP,
  help="the host to bind the dashboard server to, either localhost "
  "(127.0.0.1) or 0.0.0.0 (available from all interfaces). By default, this "
- "is localhost.",
+ "is 127.0.0.1",
 )
 @click.option(
  "--dashboard-port",