use HFEvaluateAdaptor for hf metrics

lm_eval/api/metrics.py

@@ -159,15 +159,29 @@ def acc_mutual_info_fn(items):
  return mean(items)
 
 
-exact_match = evaluate.load("exact_match")
+class HFEvaluateAdaptor:
+ def __init__(self, *metric_args, **kwargs):
+
+ metric_object = evaluate.load(*metric_args)
+ self.hf_evaluate_fn = partial(metric_object, **kwargs)
+
+ def __call__(self, items):
+ refs = list(zip(*items))[0]
+ preds = list(zip(*items))[1]
 
+ return self.hf_evaluate_fn(
+ references=refs,
+ predictions=preds
+ )
+
+exact_match = evaluate.load("exact_match")
 
 @register_metric(
  metric="exact_match",
  higher_is_better=True,
  output_type="generate_until",
 )
-def exact_match_fn(**kwargs):
+def hf_evaluate_fn(**kwargs):
  return exact_match.compute(**kwargs)
 
 

lm_eval/api/registry.py

@@ -1,7 +1,7 @@
 import os
 import evaluate
 from lm_eval.api.model import LM
-
+from lm_eval.api.metrics import HFEvaluateAdaptor
 import logging
 
 eval_logger = logging.getLogger("lm-eval")
@@ -115,7 +115,7 @@ def decorate(fn):
  return decorate
 
 
-def get_metric(name, hf_evaluate_metric=False):
+def get_metric(name, hf_evaluate_metric=False, **kwargs):
 
  if not hf_evaluate_metric:
  if name in METRIC_FUNCTION_REGISTRY:
@@ -126,8 +126,8 @@ def get_metric(name, hf_evaluate_metric=False):
  )
 
  try:
- metric_object = evaluate.load(name)
- return metric_object.compute
+ from lm_eval.metrics import HFEvaluateAdaptor
+ return HFEvaluateAdaptor(name, **kwargs)
  except Exception:
  eval_logger.error(
  f"{name} not found in the evaluate library! Please check https://huggingface.co/evaluate-metric",

lm_eval/api/task.py

@@ -17,7 +17,6 @@
 
 from typing import Union, List, Any, Tuple, Literal
 from collections.abc import Callable
-from functools import partial
 
 from lm_eval import utils
 from lm_eval.api import samplers
@@ -588,11 +587,11 @@ def __init__(
  metric_name = metric_name.__name__
  else:
  metric_fn = get_metric(
- metric_name, hf_evaluate_metric
+ metric_name, hf_evaluate_metric, **kwargs
  )
 
  self._metric_fn_kwargs[metric_name] = kwargs
- self._metric_fn_list[metric_name] = partial(metric_fn, **kwargs) if kwargs != {} else metric_fn
+ self._metric_fn_list[metric_name] = metric_fn
 
  self.download(self.config.dataset_kwargs)
  self._training_docs = None
@@ -1106,6 +1105,8 @@ def process_results(self, doc, results):
  gold = type(result)(gold)
 
  for metric in self._metric_fn_list.keys():
+ result_dict[metric] = (gold, result)
+ continue
  if self.multiple_target:
  # in the case where we have multiple targets,
  # return true if any are true
@@ -1141,7 +1142,6 @@ def process_results(self, doc, results):
  result_score = self._metric_fn_list[metric](
  references=[gold],
  predictions=[result],
- **self._metric_fn_kwargs[metric],
  )
  except TypeError: # needed for now in order to use a different interface between our own metrics and HF Evaluate metrics
  result_score = self._metric_fn_list[metric]([gold, result])