refactor!: Use Secret for API key in UpTrainEvaluator (deepset-ai…

…#347)

* refactor!: Use `Secret` for API key in `UpTrainEvaluator`

Some other misc build fixes

* Disambiguate module names

* `mypy` fixes

integrations/uptrain/example/example.py

@@ -2,6 +2,7 @@
 
 from haystack import Pipeline
 from haystack_integrations.components.evaluators import UpTrainEvaluator, UpTrainMetric
+from haystack.utils import Secret
 
 QUESTIONS = [
  "Which is the most popular global sport?",
@@ -24,13 +25,13 @@
 evaluator = UpTrainEvaluator(
  metric=UpTrainMetric.FACTUAL_ACCURACY,
  api="openai",
- api_key_env_var="OPENAI_API_KEY",
+ api_key=Secret.from_env_var("OPENAI_API_KEY"),
 )
 pipeline.add_component("evaluator", evaluator)
 
 # Each metric expects a specific set of parameters as input. Refer to the
 # UpTrainMetric class' documentation for more details.
-output = pipeline.run({"evaluator": {"questions": QUESTIONS, "contexts": CONTEXTS, "responses": RESPONSES}})
+results = pipeline.run({"evaluator": {"questions": QUESTIONS, "contexts": CONTEXTS, "responses": RESPONSES}})
 
-for output in output["evaluator"]["results"]:
+for output in results["evaluator"]["results"]:
  print(output)

integrations/uptrain/pyproject.toml

@@ -7,22 +7,21 @@ name = "uptrain-haystack"
 dynamic = ["version"]
 description = 'An integration of UpTrain LLM evaluation framework with Haystack'
 readme = "README.md"
-requires-python = ">=3.7"
+requires-python = ">=3.8"
 license = "Apache-2.0"
 keywords = []
 authors = [{ name = "deepset GmbH", email = "[email protected]" }]
 classifiers = [
  "Development Status :: 4 - Beta",
  "Programming Language :: Python",
- "Programming Language :: Python :: 3.7",
  "Programming Language :: Python :: 3.8",
  "Programming Language :: Python :: 3.9",
  "Programming Language :: Python :: 3.10",
  "Programming Language :: Python :: 3.11",
  "Programming Language :: Python :: Implementation :: CPython",
  "Programming Language :: Python :: Implementation :: PyPy",
 ]
-dependencies = ["haystack-ai", "uptrain>=0.5"]
+dependencies = ["haystack-ai>=2.0.0b6", "uptrain>=0.5"]
 
 [project.urls]
 Source = "https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/uptrain"
@@ -49,7 +48,7 @@ cov-report = ["- coverage combine", "coverage report"]
 cov = ["test-cov", "cov-report"]
 
 [[tool.hatch.envs.all.matrix]]
-python = ["3.7", "3.8", "3.9", "3.10", "3.11"]
+python = ["3.8", "3.9", "3.10", "3.11"]
 
 [tool.hatch.envs.lint]
 detached = true
@@ -113,6 +112,7 @@ ignore = [
  # Misc
  "S101",
  "TID252",
+ "B008",
 ]
 unfixable = [
  # Don't touch unused imports

integrations/uptrain/src/haystack_integrations/components/evaluators/__init__.py

@@ -1,5 +1,5 @@
-from .evaluator import UpTrainEvaluator
-from .metrics import UpTrainMetric
+from .uptrain_evaluator import UpTrainEvaluator
+from .uptrain_metrics import UpTrainMetric
 
 __all__ = (
  "UpTrainEvaluator",

integrations/uptrain/src/haystack_integrations/components/evaluators/evaluator.py → integrations/uptrain/src/haystack_integrations/components/evaluators/uptrain_evaluator.py

@@ -1,9 +1,9 @@
 import json
-import os
 from typing import Any, Dict, List, Optional, Union
 
 from haystack import DeserializationError, component, default_from_dict, default_to_dict
-from haystack_integrations.components.evaluators.metrics import (
+from haystack.utils import Secret, deserialize_secrets_inplace
+from haystack_integrations.components.evaluators.uptrain_metrics import (
  METRIC_DESCRIPTORS,
  InputConverters,
  OutputConverters,
@@ -32,7 +32,7 @@ def __init__(
  metric_params: Optional[Dict[str, Any]] = None,
  *,
  api: str = "openai",
- api_key_env_var: Optional[str] = "OPENAI_API_KEY",
+ api_key: Secret = Secret.from_env_var("OPENAI_API_KEY"),
  api_params: Optional[Dict[str, Any]] = None,
  ):
  """
@@ -46,16 +46,16 @@ def __init__(
  The API to use for evaluation.
 
  Supported APIs: "openai", "uptrain".
- :param api_key_env_var:
- The name of the environment variable containing the API key.
+ :param api_key:
+ The API key to use.
  :param api_params:
  Additional parameters to pass to the API client.
  """
  self.metric = metric if isinstance(metric, UpTrainMetric) else UpTrainMetric.from_str(metric)
  self.metric_params = metric_params
  self.descriptor = METRIC_DESCRIPTORS[self.metric]
  self.api = api
- self.api_key_env_var = api_key_env_var
+ self.api_key = api_key
  self.api_params = api_params
 
  self._init_backend()
@@ -73,7 +73,7 @@ def run(self, **inputs) -> Dict[str, Any]:
  evaluator = UpTrainEvaluator(
  metric=UpTrainMetric.FACTUAL_ACCURACY,
  api="openai",
- api_key_env_var="OPENAI_API_KEY",
+ api_key=Secret.from_env_var("OPENAI_API_KEY"),
  )
  pipeline.add_component("evaluator", evaluator)
 
@@ -140,7 +140,7 @@ def check_serializable(obj: Any):
  metric=self.metric,
  metric_params=self.metric_params,
  api=self.api,
- api_key_env_var=self.api_key_env_var,
+ api_key=self.api_key.to_dict(),
  api_params=self.api_params,
  )
 
@@ -152,6 +152,7 @@ def from_dict(cls, data: Dict[str, Any]) -> "UpTrainEvaluator":
  :param data:
  The dictionary to deserialize from.
  """
+ deserialize_secrets_inplace(data["init_parameters"], ["api_key"])
  return default_from_dict(cls, data)
 
  def _init_backend(self):
@@ -185,11 +186,8 @@ def _init_backend(self):
  msg = f"Unsupported API '{self.api}' for UpTrain evaluator. Supported APIs: {supported_apis}"
  raise ValueError(msg)
 
- api_key = os.environ.get(self.api_key_env_var)
- if api_key is None:
- msg = f"Missing API key environment variable '{self.api_key_env_var}' for UpTrain evaluator"
- raise ValueError(msg)
-
+ api_key = self.api_key.resolve_value()
+ assert api_key is not None
  if self.api == "openai":
  backend_client = EvalLLM(openai_api_key=api_key)
  elif self.api == "uptrain":

integrations/uptrain/src/haystack_integrations/components/evaluators/metrics.py → integrations/uptrain/src/haystack_integrations/components/evaluators/uptrain_metrics.py

@@ -257,7 +257,18 @@ def validate_outputs(outputs: List[Dict[str, Any]]):
  msg = "UpTrain evaluator expects outputs to be a list of `dict`s"
  elif not all(isinstance(y, str) for x in outputs for y in x.keys()):
  msg = "UpTrain evaluator expects keys in the output dicts to be `str`"
- elif not all(isinstance(y, (float, str)) for x in outputs for y in x.values()):
+ elif not all(
+ y is None
+ or isinstance(
+ y,
+ (
+ float,
+ str,
+ ),
+ )
+ for x in outputs
+ for y in x.values()
+ ):
  msg = "UpTrain evaluator expects values in the output dicts to be either `str` or `float`"
 
  if msg is not None:

integrations/uptrain/tests/test_evaluator.py

@@ -8,6 +8,7 @@
 from haystack import DeserializationError
 
 from haystack_integrations.components.evaluators import UpTrainEvaluator, UpTrainMetric
+from haystack.utils import Secret
 
 DEFAULT_QUESTIONS = [
  "Which is the most popular global sport?",
@@ -102,43 +103,47 @@ def log_and_evaluate(self, data, checks, **kwargs):
  return data
 
 
-@patch("os.environ.get")
-def test_evaluator_api(os_environ_get):
- api_key_var = "test-api-key"
- os_environ_get.return_value = api_key_var
+def test_evaluator_api(monkeypatch):
+ monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
+ monkeypatch.setenv("UPTRAIN_API_KEY", "test-api-key")
 
  eval = UpTrainEvaluator(UpTrainMetric.RESPONSE_COMPLETENESS)
  assert eval.api == "openai"
- assert eval.api_key_env_var == "OPENAI_API_KEY"
+ assert eval.api_key == Secret.from_env_var("OPENAI_API_KEY")
 
- eval = UpTrainEvaluator(UpTrainMetric.RESPONSE_COMPLETENESS, api="uptrain", api_key_env_var="UPTRAIN_API_KEY")
+ eval = UpTrainEvaluator(
+ UpTrainMetric.RESPONSE_COMPLETENESS, api="uptrain", api_key=Secret.from_env_var("UPTRAIN_API_KEY")
+ )
  assert eval.api == "uptrain"
- assert eval.api_key_env_var == "UPTRAIN_API_KEY"
+ assert eval.api_key == Secret.from_env_var("UPTRAIN_API_KEY")
 
  with pytest.raises(ValueError, match="Unsupported API"):
  UpTrainEvaluator(UpTrainMetric.CONTEXT_RELEVANCE, api="cohere")
 
- os_environ_get.return_value = None
- with pytest.raises(ValueError, match="Missing API key"):
- UpTrainEvaluator(UpTrainMetric.CONTEXT_RELEVANCE, api="uptrain")
-
+ with pytest.raises(ValueError, match="None of the following authentication environment variables are set"):
+ UpTrainEvaluator(UpTrainMetric.CONTEXT_RELEVANCE, api="uptrain", api_key=Secret.from_env_var("asd39920qqq"))
 
-@patch("os.environ.get")
-def test_evaluator_metric_init_params(os_environ_get):
- api_key = "test-api-key"
- os_environ_get.return_value = api_key
 
- eval = UpTrainEvaluator(UpTrainMetric.CRITIQUE_TONE, metric_params={"llm_persona": "village idiot"})
+def test_evaluator_metric_init_params():
+ eval = UpTrainEvaluator(
+ UpTrainMetric.CRITIQUE_TONE,
+ metric_params={"llm_persona": "village idiot"},
+ api_key=Secret.from_token("Aaa"),
+ )
  assert eval._backend_metric.llm_persona == "village idiot"
 
  with pytest.raises(ValueError, match="Invalid init parameters"):
- UpTrainEvaluator(UpTrainMetric.CRITIQUE_TONE, metric_params={"role": "village idiot"})
+ UpTrainEvaluator(
+ UpTrainMetric.CRITIQUE_TONE, metric_params={"role": "village idiot"}, api_key=Secret.from_token("Aaa")
+ )
 
  with pytest.raises(ValueError, match="unexpected init parameters"):
- UpTrainEvaluator(UpTrainMetric.FACTUAL_ACCURACY, metric_params={"check_numbers": True})
+ UpTrainEvaluator(
+ UpTrainMetric.FACTUAL_ACCURACY, metric_params={"check_numbers": True}, api_key=Secret.from_token("Aaa")
+ )
 
  with pytest.raises(ValueError, match="expected init parameters"):
- UpTrainEvaluator(UpTrainMetric.RESPONSE_MATCHING)
+ UpTrainEvaluator(UpTrainMetric.RESPONSE_MATCHING, api_key=Secret.from_token("Aaa"))
 
 
 @patch("os.environ.get")
@@ -149,7 +154,7 @@ def test_evaluator_serde(os_environ_get):
  "metric": UpTrainMetric.RESPONSE_MATCHING,
  "metric_params": {"method": "rouge"},
  "api": "uptrain",
- "api_key_env_var": "abacab",
+ "api_key": Secret.from_env_var("ENV_VAR", strict=False),
  "api_params": {"eval_name": "test"},
  }
  eval = UpTrainEvaluator(**init_params)
@@ -158,7 +163,7 @@ def test_evaluator_serde(os_environ_get):
 
  assert eval.metric == new_eval.metric
  assert eval.api == new_eval.api
- assert eval.api_key_env_var == new_eval.api_key_env_var
+ assert eval.api_key == new_eval.api_key
  assert eval.metric_params == new_eval.metric_params
  assert eval.api_params == new_eval.api_params
  assert type(new_eval._backend_client) == type(eval._backend_client)
@@ -191,14 +196,12 @@ def test_evaluator_serde(os_environ_get):
  (UpTrainMetric.RESPONSE_MATCHING, {"ground_truths": [], "responses": []}, {"method": "llm"}),
  ],
 )
-@patch("os.environ.get")
-def test_evaluator_valid_inputs(os_environ_get, metric, inputs, params):
- os_environ_get.return_value = "abacab"
+def test_evaluator_valid_inputs(metric, inputs, params):
  init_params = {
  "metric": metric,
  "metric_params": params,
  "api": "uptrain",
- "api_key_env_var": "abacab",
+ "api_key": Secret.from_token("Aaa"),
  "api_params": None,
  }
  eval = UpTrainEvaluator(**init_params)
@@ -220,15 +223,13 @@ def test_evaluator_valid_inputs(os_environ_get, metric, inputs, params):
  (UpTrainMetric.RESPONSE_RELEVANCE, {"responses": []}, "expected input parameter ", None),
  ],
 )
-@patch("os.environ.get")
-def test_evaluator_invalid_inputs(os_environ_get, metric, inputs, error_string, params):
- os_environ_get.return_value = "abacab"
+def test_evaluator_invalid_inputs(metric, inputs, error_string, params):
  with pytest.raises(ValueError, match=error_string):
  init_params = {
  "metric": metric,
  "metric_params": params,
  "api": "uptrain",
- "api_key_env_var": "abacab",
+ "api_key": Secret.from_token("Aaa"),
  "api_params": None,
  }
  eval = UpTrainEvaluator(**init_params)
@@ -299,14 +300,12 @@ def test_evaluator_invalid_inputs(os_environ_get, metric, inputs, error_string,
  ),
  ],
 )
-@patch("os.environ.get")
-def test_evaluator_outputs(os_environ_get, metric, inputs, expected_outputs, metric_params):
- os_environ_get.return_value = "abacab"
+def test_evaluator_outputs(metric, inputs, expected_outputs, metric_params):
  init_params = {
  "metric": metric,
  "metric_params": metric_params,
  "api": "uptrain",
- "api_key_env_var": "abacab",
+ "api_key": Secret.from_token("Aaa"),
  "api_params": None,
  }
  eval = UpTrainEvaluator(**init_params)