fixed issues related to printing alias of group and updated yaml

lm_eval/evaluator_utils.py

@@ -164,7 +164,10 @@ def get_sample_size(task, limit: Optional[int]) -> Union[int, None]:
 
 
 def prepare_print_tasks(
- task_hierarchy: dict, results: dict, tab=0
+ task_hierarchy: dict,
+ results: dict,
+ tab=0,
+ group_tab=0,
 ) -> Tuple[dict, dict]:
  """
  @param task_hierarchy: Dictionary representing the group hierarchy of tasks. Each key is a group name and its
@@ -197,17 +200,20 @@ def prepare_print_tasks(
  results_agg[group_name]["alias"] = tab_string + group_name
 
  if len(task_list) > 0:
- groups_agg[group_name] = results[group_name].copy()
- # groups_agg[group_name]["tab"] = tab
- if "samples" in groups_agg[group_name]:
- groups_agg[group_name].pop("samples")
-
- if "alias" in groups_agg[group_name]:
- groups_agg[group_name]["alias"] = (
- tab_string + groups_agg[group_name]["alias"]
- )
- else:
- groups_agg[group_name]["alias"] = tab_string + group_name
+ if " " not in results[group_name]:
+ group_tab_string = " " * group_tab + "- " if group_tab > 0 else ""
+ groups_agg[group_name] = results[group_name].copy()
+ group_tab += 1
+
+ if "samples" in groups_agg[group_name]:
+ groups_agg[group_name].pop("samples")
+
+ if "alias" in groups_agg[group_name]:
+ groups_agg[group_name]["alias"] = (
+ group_tab_string + groups_agg[group_name]["alias"]
+ )
+ else:
+ groups_agg[group_name]["alias"] = group_tab_string + group_name
 
  for task_name in task_list:
  if task_name in task_hierarchy:
@@ -222,7 +228,7 @@ def prepare_print_tasks(
  }
 
  _results_agg, _groups_agg = prepare_print_tasks(
- _task_hierarchy, results, tab + 1
+ _task_hierarchy, results, tab + 1, group_tab
  )
  results_agg = {**results_agg, **_results_agg}
  groups_agg = {**groups_agg, **_groups_agg}

lm_eval/tasks/__init__.py

@@ -178,7 +178,6 @@ def load_task(config, task, group=None, yaml_path=None):
  yaml_path = self._get_yaml_path(group_name)
 
  if (update_config is not None) and ("group_alias" in update_config):
- group_name = update_config["group_alias"]
  update_config.pop("group_alias")
 
  if isinstance(name_or_config, dict):
@@ -240,8 +239,9 @@ def load_task(config, task, group=None, yaml_path=None):
 
  all_subtasks = {}
  if parent_name is not None:
- # all_subtasks = {group_name: (parent_name, None)}
  parent_group_config = self._get_config(parent_name)
+ if "group_alias" in parent_group_config:
+ parent_name = parent_group_config["group_alias"]
  all_subtasks = {group_name: (parent_name, parent_group_config)}
 
  fn = partial(

lm_eval/tasks/benchmarks/flan/flan_held_in.yaml

@@ -4,6 +4,8 @@ task:
  # ANLI R1
  - group: anli_r1_flan
  group_alias: ANLI R1
+ group_config:
+ aggregate_metric: True
  task:
  - task: anli_r1
  task_alias: prompt-0
@@ -53,6 +55,8 @@ task:
  # ANLI R2
  - group: anli_r2_flan
  group_alias: ANLI R2
+ group_config:
+ aggregate_metric: True
  task:
  - task: anli_r2
  task_alias: prompt-0
@@ -102,6 +106,8 @@ task:
  # ANLI R3
  - group: anli_r3_flan
  group_alias: ANLI R3
+ group_config:
+ aggregate_metric: True
  task:
  - task: anli_r3
  task_alias: prompt-0
@@ -151,6 +157,8 @@ task:
  # Arc Easy
  - group: arc_easy_flan
  group_alias: Arc Easy
+ group_config:
+ aggregate_metric: True
  task:
  - task: arc_easy
  task_alias: prompt-0
@@ -190,6 +198,8 @@ task:
  # Arc Challenge
  - group: arc_challenge_flan
  group_alias: Arc Challenge
+ group_config:
+ aggregate_metric: True
  task:
  - task: arc_challenge
  task_alias: prompt-0
@@ -229,6 +239,8 @@ task:
  # BoolQ
  - group: boolq_flan
  group_alias: BoolQ
+ group_config:
+ aggregate_metric: True
  task:
  - task: boolq
  task_alias: prompt-0
@@ -283,6 +295,8 @@ task:
  # RTE
  - group: rte_flan
  group_alias: RTE
+ group_config:
+ aggregate_metric: True
  task:
  - task: rte
  task_alias: prompt-0