Fix final value of exponential decay lr (#1040)

EleutherAI · Sep 23, 2023 · e431ff5 · e431ff5
1 parent 390d37c
commit e431ff5
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 2 deletions.
diff --git a/configs/neox_arguments.md b/configs/neox_arguments.md
@@ -111,7 +111,7 @@ Logging Arguments
 
 - **git_hash**: str
 
- Default = aa6c176
+ Default = 1f832c1
 
  current git hash of repository
 

diff --git a/megatron/learning_rates.py b/megatron/learning_rates.py
@@ -82,7 +82,8 @@ def get_lr(self):
  )
  elif self.decay_style == "exponential":
  # exp(-0.693) = 1/2
- lr = self.start_lr * math.exp(-0.693 * num_iters_ / self.end_iter)
+ end_iter = self.end_iter - self.warmup_iter
+ lr = self.start_lr * math.exp(-0.693 * num_iters_ / end_iter)
  else:
  lr = self.start_lr
  return max(lr, self.min_lr)