Remove duplicate deepspeed config arg and allow users to force multin…

…ode exec

megatron/neox_arguments/deepspeed_args.py

@@ -351,6 +351,11 @@ class NeoXArgsDeepspeedRunner(NeoXArgsTemplate):
  If true, overrides the default check where DeepSpeed confirms that the headnode is accessible via ssh.
  """
 
+ force_multi: bool = False
+ """
+ If true, Force multi-node launcher mode, helps in cases where user wants to launch on single remote node.
+ """
+
  comment: str = None
  """
  Adds a `--comment` to the DeepSpeed launch command. In DeeperSpeed this is passed on to the SlurmLauncher as well. Sometime necessary for cluster rules, or so I've heard.

megatron/training.py

@@ -624,7 +624,8 @@ def setup_model_and_optimizer(neox_args, use_cache=False, iteration=None):
  lr_scheduler=_lr_scheduler,
  dist_init_required=False,
  model_parameters=_model_params,
- config_params=neox_args.deepspeed_config,
+ # Need to remove the below so that it doesn't conflict with --deepspeed_config required by autotuning
+ #config_params=neox_args.deepspeed_config,
  mpu=mpu if not neox_args.is_pipe_parallel else None,
  )
  model.total_params = get_total_params(model.module)