batch_size with auto defaults to 1 if `No executable batch size f…

…ound` (EleutherAI#1405) Fixes EleutherAI#1323
wx-zhang · Mar 13, 2024 · 14a28ab · 14a28ab
1 parent a041c68
commit 14a28ab
Showing 1 changed file with 7 additions and 1 deletion.
diff --git a/lm_eval/models/huggingface.py b/lm_eval/models/huggingface.py
@@ -617,7 +617,13 @@ def forward_batch(batch_size):
 
  return batch_size
 
- batch_size = forward_batch()
+ try:
+ batch_size = forward_batch()
+ except RuntimeError as e:
+ if "No executable batch size found" in str(e):
+ batch_size = 1
+ else:
+ raise
 
  if self.world_size > 1:
  # if multi-GPU, always take minimum over all selected batch sizes