Update transformer.py

bug fixed. if not, the hidden size can not divided.
LeonTing1010 · Apr 2, 2023 · 273e8a5 · 273e8a5
1 parent 9244b3e
commit 273e8a5
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/megatron/model/transformer.py b/megatron/model/transformer.py
@@ -87,7 +87,7 @@ def __init__(
  self.bias_gelu_fusion = neox_args.bias_gelu_fusion
 
  # auto scale so geglu has equal parameters
- ff_mult = 4 * 2 / 3 if self.activation_type == "geglu" else 4
+ ff_mult = int(4 * 2 / 3) if self.activation_type == "geglu" else 4
  ff_dim = (
  int(ff_mult * neox_args.hidden_size) * 2
  if self.activation_type == "geglu"