save eval runs to separate subfolders by target dataset (EleutherAI#174)

* save eval runs to separate subfolders by target dataset

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* eval multiple datasets

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

elk/evaluation/evaluate.py

@@ -45,10 +45,13 @@ class Eval(Serializable):
  concatenated_layer_offset: int = 0
 
  def execute(self):
- transfer_eval = elk_reporter_dir() / self.source / "transfer_eval"
+ datasets = self.data.prompts.datasets
 
- run = Evaluate(cfg=self, out_dir=transfer_eval)
- run.evaluate()
+ transfer_dir = elk_reporter_dir() / self.source / "transfer_eval"
+
+ for dataset in datasets:
+ run = Evaluate(cfg=self, out_dir=transfer_dir / dataset)
+ run.evaluate()
 
 
 @dataclass