[runtime env] Improve runtime env timeout log (#44344)

When a runtime env hits a timeout, the current error message doesn't show the provided timeout, and is confusing (it seems to imply the default timeout was used, even if it wasn't.) This PR clarifies the error message. --------- Signed-off-by: Archit Kulkarni <[email protected]>
ray-project · Apr 4, 2024 · e8dd54e · e8dd54e
1 parent c54dc5c
commit e8dd54e
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 5 deletions.
diff --git a/dashboard/modules/job/tests/test_http_job_server.py b/dashboard/modules/job/tests/test_http_job_server.py
@@ -377,7 +377,7 @@ def test_timeout(job_sdk_client):
  data = client.get_job_info(job_id)
  assert "Failed to set up runtime environment" in data.message
  assert "Timeout" in data.message
- assert "consider increasing `setup_timeout_seconds`" in data.message
+ assert "setup_timeout_seconds" in data.message
 
 
 def test_per_task_runtime_env(job_sdk_client: JobSubmissionClient):

diff --git a/dashboard/modules/job/tests/test_job_agent.py b/dashboard/modules/job/tests/test_job_agent.py
@@ -308,7 +308,7 @@ async def test_timeout(job_sdk_client):
  data = head_client.get_job_info(job_id)
  assert "Failed to set up runtime environment" in data.message
  assert "Timeout" in data.message
- assert "consider increasing `setup_timeout_seconds`" in data.message
+ assert "setup_timeout_seconds" in data.message
 
 
 @pytest.mark.asyncio

diff --git a/python/ray/_private/runtime_env/agent/runtime_env_agent.py b/python/ray/_private/runtime_env/agent/runtime_env_agent.py
@@ -399,12 +399,14 @@ async def _create_runtime_env_with_retry(
  )
  if isinstance(e, asyncio.TimeoutError):
  hint = (
- f"Failed due to timeout; check runtime_env setup logs"
- " and consider increasing `setup_timeout_seconds` beyond "
- f"the default of {DEFAULT_RUNTIME_ENV_TIMEOUT_SECONDS}."
+ f"Failed to install runtime_env within the "
+ f"timeout of {setup_timeout_seconds} seconds. Consider "
+ "increasing the timeout in the runtime_env config. "
  "For example: \n"
  ' runtime_env={"config": {"setup_timeout_seconds":'
  " 1800}, ...}\n"
+ "If not provided, the default timeout is "
+ f"{DEFAULT_RUNTIME_ENV_TIMEOUT_SECONDS} seconds. "
  )
  error_message = hint + error_message
  await asyncio.sleep(