Add test for FuzzyMatch (openai#802)

.github/workflows/run_tests.yaml

@@ -0,0 +1,33 @@
+name: Run unit tests
+
+on:
+ pull_request:
+ branches:
+ - main
+
+jobs:
+ check_files:
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v2
+ with:
+ fetch-depth: 0
+ lfs: true
+
+ - name: Set up Python
+ uses: actions/setup-python@v2
+ with:
+ python-version: 3.9
+
+ - name: Install dependencies
+ run: |
+ python -m pip install --upgrade pip
+ pip install pyyaml
+ pip install pytest
+ pip install -e .
+
+ - name: Run unit tests
+ run: |
+ pytest

evals/elsuite/basic/fuzzy_match_test.py

@@ -0,0 +1,32 @@
+from mock import patch
+from evals.api import DummyCompletionFn
+from evals.elsuite.basic.fuzzy_match import FuzzyMatch
+from evals.record import DummyRecorder
+
+
+def test_eval_sample_str():
+ eval = FuzzyMatch(
+ completion_fns=[DummyCompletionFn()],
+ samples_jsonl="",
+ )
+
+ recorder = DummyRecorder(None)
+ with recorder.as_default_recorder("x"), patch.object(
+ recorder, "record_metrics", wraps=recorder.record_metrics
+ ) as record_metrics:
+ eval.eval_sample(dict(input="Hello", ideal="world"), None)
+ record_metrics.assert_called_once_with(accuracy=0.0, f1_score=0.0)
+
+
+def test_eval_sample_list():
+ eval = FuzzyMatch(
+ completion_fns=[DummyCompletionFn()],
+ samples_jsonl="",
+ )
+
+ recorder = DummyRecorder(None)
+ with recorder.as_default_recorder("x"), patch.object(
+ recorder, "record_metrics", wraps=recorder.record_metrics
+ ) as record_metrics:
+ eval.eval_sample(dict(input="Hello", ideal=["world", "dummy"]), None)
+ record_metrics.assert_called_once_with(accuracy=1.0, f1_score=0.4)