bugfix to run eval separately on each dataset

michaelbyun · Apr 13, 2023 · 7af1a1b · 7af1a1b
1 parent ba090b7
commit 7af1a1b
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/elk/evaluation/evaluate.py b/elk/evaluation/evaluate.py
@@ -48,6 +48,7 @@ def execute(self):
  transfer_dir = elk_reporter_dir() / self.source / "transfer_eval"
 
  for dataset in datasets:
+ self.data.prompts.datasets = [dataset]
  run = Evaluate(cfg=self, out_dir=transfer_dir / dataset)
  run.evaluate()