Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: Implement DeepEvalEvaluator #346

Merged
merged 10 commits into from
Feb 9, 2024
Merged

feat: Implement DeepEvalEvaluator #346

Changes from 1 commit
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
248dfb0
feat: Implement `DeepEvalEvaluator` and `DeepEvalMetrics`
shadeMe Feb 5, 2024
053b71f
Mock OpenAI API keys
shadeMe Feb 6, 2024
60e5922
Disambiguate module names
shadeMe Feb 6, 2024
4243ee3
Update labeler workflow
shadeMe Feb 6, 2024
696d08f
Revert accidental changes to uptrain tests
shadeMe Feb 6, 2024
84780c4
Fix typo
shadeMe Feb 7, 2024
196603c
Standardize namespacing and add API doc generation to CI
shadeMe Feb 9, 2024
5b3b694
Lint
shadeMe Feb 9, 2024
d7362bd
`mypy` fixes
shadeMe Feb 9, 2024
0a4b1fb
Docs build fix
shadeMe Feb 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Revert accidental changes to uptrain tests
  • Loading branch information
@shadeMe
shadeMe committed Feb 6, 2024
commit 696d08f40a2a1a4455f01ddb0ff3a97cdff0b6d8
29 changes: 16 additions & 13 deletions integrations/uptrain/tests/test_evaluator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,8 +118,10 @@ def test_evaluator_api(os_environ_get):
UpTrainEvaluator(UpTrainMetric.CONTEXT_RELEVANCE, api="uptrain")


def test_evaluator_metric_init_params(monkeypatch):
monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
@patch("os.environ.get")
def test_evaluator_metric_init_params(os_environ_get):
api_key = "test-api-key"
os_environ_get.return_value = api_key

eval = UpTrainEvaluator(UpTrainMetric.CRITIQUE_TONE, metric_params={"llm_persona": "village idiot"})
assert eval._backend_metric.llm_persona == "village idiot"
Expand All @@ -134,8 +136,9 @@ def test_evaluator_metric_init_params(monkeypatch):
UpTrainEvaluator(UpTrainMetric.RESPONSE_MATCHING)


def test_evaluator_serde(monkeypatch):
monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
@patch("os.environ.get")
def test_evaluator_serde(os_environ_get):
os_environ_get.return_value = "abacab"

init_params = {
"metric": UpTrainMetric.RESPONSE_MATCHING,
Expand Down Expand Up @@ -183,9 +186,9 @@ def test_evaluator_serde(monkeypatch):
(UpTrainMetric.RESPONSE_MATCHING, {"ground_truths": [], "responses": []}, {"method": "llm"}),
],
)
def test_evaluator_valid_inputs(metric, inputs, params, monkeypatch):
monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")

@patch("os.environ.get")
def test_evaluator_valid_inputs(os_environ_get, metric, inputs, params):
os_environ_get.return_value = "abacab"
init_params = {
"metric": metric,
"metric_params": params,
Expand All @@ -212,9 +215,9 @@ def test_evaluator_valid_inputs(metric, inputs, params, monkeypatch):
(UpTrainMetric.RESPONSE_RELEVANCE, {"responses": []}, "expected input parameter ", None),
],
)
def test_evaluator_invalid_inputs(metric, inputs, error_string, params, monkeypatch):
monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")

@patch("os.environ.get")
def test_evaluator_invalid_inputs(os_environ_get, metric, inputs, error_string, params):
os_environ_get.return_value = "abacab"
with pytest.raises(ValueError, match=error_string):
init_params = {
"metric": metric,
Expand Down Expand Up @@ -291,9 +294,9 @@ def test_evaluator_invalid_inputs(metric, inputs, error_string, params, monkeypa
),
],
)
def test_evaluator_outputs(metric, inputs, expected_outputs, metric_params, monkeypatch):
monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")

@patch("os.environ.get")
def test_evaluator_outputs(os_environ_get, metric, inputs, expected_outputs, metric_params):
os_environ_get.return_value = "abacab"
init_params = {
"metric": metric,
"metric_params": metric_params,
Expand Down