Merge pull request EleutherAI#722 from EleutherAI/opt-embeddings-fix

Fix learned and sinusoidal embeddings after OPT regression
jamestiotio · Jan 18, 2023 · 01f75eb · 01f75eb
2 parents 375de3f + c72ca1c
commit 01f75eb
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 7 deletions.
diff --git a/configs/neox_arguments.md b/configs/neox_arguments.md
@@ -111,7 +111,7 @@ Logging Arguments
 
 - **git_hash**: str
 
- Default = 075a525
+ Default = 62e9440
 
  current git hash of repository
 

diff --git a/megatron/model/word_embeddings.py b/megatron/model/word_embeddings.py
@@ -132,13 +132,11 @@ def forward(self, input_ids, position_ids, tokentype_ids=None):
  # Embeddings.
  words_embeddings = self.word_embeddings(input_ids)
  if self.use_pos_emb and self.embedding_type in ["learned", "sinusoidal"]:
- if self.layer_past is not None:
- position_ids = position_ids + self.layer_past + 1
-
- self.layer_past = position_ids[:, -1]
-
- # OPT always adds 2 for some reason, according to the HF implementation
  if self.opt_pos_emb_offset:
+ if self.layer_past is not None:
+ position_ids = position_ids + self.layer_past + 1
+ self.layer_past = position_ids[:, -1]
+ # OPT always adds 2 for some reason, according to the HF implementation
  position_ids = position_ids + self.opt_pos_emb_offset
  position_embeddings = self.position_embeddings(position_ids)
  embeddings = words_embeddings + position_embeddings