Fail gracefully when cluster status is not available yet. (#45620)

ray-project · May 29, 2024 · 0bb2600 · 0bb2600
1 parent 3518518
commit 0bb2600
Show file tree

Hide file tree

Showing 4 changed files with 40 additions and 29 deletions.
diff --git a/dashboard/client/src/components/AutoscalerStatusCards.tsx b/dashboard/client/src/components/AutoscalerStatusCards.tsx
@@ -2,38 +2,53 @@ import { Box, Typography } from "@mui/material";
 import React from "react";
 import { RayStatusResp } from "../service/status";
 
-const formatNodeStatus = (cluster_status: string) => {
+const formatNodeStatus = (clusterStatus?: string) => {
  // ==== auto scaling status
  // Node status
  // ....
  // Resources
  // ....
- const sections = cluster_status.split("Resources");
- return formatClusterStatus(
- "Node Status",
- sections[0].split("Node status")[1],
- );
+ if (!clusterStatus) {
+ return "No cluster status.";
+ }
+ try {
+ // Try to parse the node status.
+ const sections = clusterStatus.split("Resources");
+ return formatClusterStatus(
+ "Node Status",
+ sections[0].split("Node status")[1],
+ );
+ } catch (e) {
+ return "No cluster status.";
+ }
 };
 
-const formatResourcesStatus = (cluster_status: string) => {
+const formatResourcesStatus = (clusterStatus?: string) => {
  // ==== auto scaling status
  // Node status
  // ....
  // Resources
  // ....
- const sections = cluster_status.split("Resources");
- return formatClusterStatus("Resource Status", sections[1]);
+ if (!clusterStatus) {
+ return "No cluster status.";
+ }
+ try {
+ const sections = clusterStatus.split("Resources");
+ return formatClusterStatus("Resource Status", sections[1]);
+ } catch (e) {
+ return "No cluster status.";
+ }
 };
 
-const formatClusterStatus = (title: string, cluster_status: string) => {
- const cluster_status_rows = cluster_status.split("\n");
+const formatClusterStatus = (title: string, clusterStatus: string) => {
+ const clusterStatusRows = clusterStatus.split("\n");
 
  return (
  <div>
  <Box marginBottom={2}>
  <Typography variant="h3">{title}</Typography>
  </Box>
- {cluster_status_rows.map((i, key) => {
+ {clusterStatusRows.map((i, key) => {
  // Format the output.
  // See format_info_string in util.py
  if (i.startsWith("-----") || i.startsWith("=====") || i === "") {
@@ -54,35 +69,31 @@ const formatClusterStatus = (title: string, cluster_status: string) => {
 };
 
 type StatusCardProps = {
- cluster_status: RayStatusResp | undefined;
+ clusterStatus: RayStatusResp | undefined;
 };
 
-export const NodeStatusCard = ({ cluster_status }: StatusCardProps) => {
+export const NodeStatusCard = ({ clusterStatus }: StatusCardProps) => {
  return (
  <Box
  style={{
  overflow: "hidden",
  overflowY: "scroll",
  }}
  >
- {cluster_status?.data && cluster_status?.data?.clusterStatus
- ? formatNodeStatus(cluster_status?.data.clusterStatus)
- : "No cluster status."}
+ {formatNodeStatus(clusterStatus?.data.clusterStatus)}
  </Box>
  );
 };
 
-export const ResourceStatusCard = ({ cluster_status }: StatusCardProps) => {
+export const ResourceStatusCard = ({ clusterStatus }: StatusCardProps) => {
  return (
  <Box
  style={{
  overflow: "hidden",
  overflowY: "scroll",
  }}
  >
- {cluster_status?.data && cluster_status?.data?.clusterStatus
- ? formatResourcesStatus(cluster_status?.data.clusterStatus)
- : "No cluster status."}
+ {formatResourcesStatus(clusterStatus?.data.clusterStatus)}
  </Box>
  );
 };
diff --git a/dashboard/client/src/pages/job/JobDetail.tsx b/dashboard/client/src/pages/job/JobDetail.tsx
@@ -55,7 +55,7 @@ export const JobDetailChartsPage = () => {
  const [actorListFilter, setActorListFilter] = useState<string>();
  const [actorTableExpanded, setActorTableExpanded] = useState(false);
  const actorTableRef = useRef<HTMLDivElement>(null);
- const { cluster_status } = useRayStatus();
+ const { clusterStatus } = useRayStatus();
 
  const { data } = useSWR(
  job?.job_id ? ["useDataDatasets", job.job_id] : null,
@@ -182,10 +182,10 @@ export const JobDetailChartsPage = () => {
  >
  <NodeCountCard className={classes.nodeCountCard} />
  <Section flex="1 1 500px">
- <NodeStatusCard cluster_status={cluster_status} />
+ <NodeStatusCard clusterStatus={clusterStatus} />
  </Section>
  <Section flex="1 1 500px">
- <ResourceStatusCard cluster_status={cluster_status} />
+ <ResourceStatusCard clusterStatus={clusterStatus} />
  </Section>
  </Box>
  </CollapsibleSection>

diff --git a/dashboard/client/src/pages/job/hook/useClusterStatus.ts b/dashboard/client/src/pages/job/hook/useClusterStatus.ts
@@ -3,7 +3,7 @@ import { API_REFRESH_INTERVAL_MS } from "../../../common/constants";
 import { getRayStatus } from "../../../service/status";
 
 export const useRayStatus = () => {
- const { data: cluster_status } = useSWR(
+ const { data: clusterStatus } = useSWR(
  "useClusterStatus",
  async () => {
  try {
@@ -19,6 +19,6 @@ export const useRayStatus = () => {
  );
 
  return {
- cluster_status,
+ clusterStatus,
  };
 };
diff --git a/dashboard/client/src/pages/overview/OverviewPage.tsx b/dashboard/client/src/pages/overview/OverviewPage.tsx
@@ -52,7 +52,7 @@ const useStyles = makeStyles((theme) =>
 export const OverviewPage = () => {
  const classes = useStyles();
 
- const { cluster_status } = useRayStatus();
+ const { clusterStatus } = useRayStatus();
 
  return (
  <div className={classes.root}>
@@ -80,7 +80,7 @@ export const OverviewPage = () => {
  classes.autoscalerCard,
  )}
  >
- <NodeStatusCard cluster_status={cluster_status} />
+ <NodeStatusCard clusterStatus={clusterStatus} />
  </OverviewCard>
  <OverviewCard
  className={classNames(
@@ -89,7 +89,7 @@ export const OverviewPage = () => {
  classes.autoscalerCard,
  )}
  >
- <ResourceStatusCard cluster_status={cluster_status} />
+ <ResourceStatusCard clusterStatus={clusterStatus} />
  </OverviewCard>
  </div>
  }