more retrying (#1143)

intelligent-machine-learning · May 28, 2024 · 93df4af · 93df4af
1 parent aff7c11
commit 93df4af
Showing 1 changed file with 3 additions and 2 deletions.
diff --git a/dlrover/python/master/scaler/pod_scaler.py b/dlrover/python/master/scaler/pod_scaler.py
@@ -174,13 +174,14 @@ def _retry_to_get_master_pod(self):
  Notice: Labels might be different in different environments for now.
  """
 
- RETRY_WAIT_SECONDS = 3
+ RETRY_TIMES = 20
+ RETRY_WAIT_SECONDS = 5
 
  master_labels_selector = k8s_util.gen_k8s_label_selector_from_dict(
  self._get_master_pod_labels()
  )
  logger.info(f"Get master pod by labels : {master_labels_selector}.")
- for _ in range(3):
+ for _ in range(RETRY_TIMES):
  pods = self._k8s_client.list_namespaced_pod(master_labels_selector)
  if pods and len(pods.items) > 0:
  if (