Merge pull request EleutherAI#8 from dirkgr/PytestUpdate

Pytest update
LZY-the-boys · Apr 27, 2022 · 75435b6 · 75435b6
2 parents e5a55c7 + f4d7e69
commit 75435b6
Show file tree

Hide file tree

Showing 25 changed files with 16 additions and 18 deletions.
diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml
@@ -32,7 +32,7 @@ jobs:
  run: |
  python -m pip install --upgrade pip
  pip install flake8 pytest pytest-cov
- pip install -e .
+ pip install -e .[dev]
  if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
  - name: Lint with flake8
  run: |

diff --git a/docs/task_guide.md b/docs/task_guide.md
@@ -11,7 +11,7 @@ If you haven't already, go ahead and fork the main repo, clone it, create a bran
 git clone https://github.com/<YOUR-USERNAME>/lm-evaluation-harness.git
 cd lm-evaluation-harness
 git checkout -b <task-name>
-pip install -r requirements.txt
+pip install -e ".[dev]"
 ```
 
 ## Creating Your Task File

diff --git a/lm_eval/datasets/arithmetic/__init__.py b/lm_eval/datasets/arithmetic/__init__.py
diff --git a/lm_eval/datasets/asdiv/__init__.py b/lm_eval/datasets/asdiv/__init__.py
diff --git a/lm_eval/datasets/coqa/__init__.py b/lm_eval/datasets/coqa/__init__.py
diff --git a/lm_eval/datasets/drop/__init__.py b/lm_eval/datasets/drop/__init__.py
diff --git a/lm_eval/datasets/gsm8k/__init__.py b/lm_eval/datasets/gsm8k/__init__.py
diff --git a/lm_eval/datasets/headqa/__init__.py b/lm_eval/datasets/headqa/__init__.py
diff --git a/lm_eval/datasets/hendrycks_ethics/__init__.py b/lm_eval/datasets/hendrycks_ethics/__init__.py
diff --git a/lm_eval/datasets/hendrycks_math/__init__.py b/lm_eval/datasets/hendrycks_math/__init__.py
diff --git a/lm_eval/datasets/lambada/__init__.py b/lm_eval/datasets/lambada/__init__.py
diff --git a/lm_eval/datasets/logiqa/__init__.py b/lm_eval/datasets/logiqa/__init__.py
diff --git a/lm_eval/datasets/mutual/__init__.py b/lm_eval/datasets/mutual/__init__.py
diff --git a/lm_eval/datasets/pile/__init__.py b/lm_eval/datasets/pile/__init__.py
diff --git a/lm_eval/datasets/quac/__init__.py b/lm_eval/datasets/quac/__init__.py
diff --git a/lm_eval/datasets/sat_analogies/__init__.py b/lm_eval/datasets/sat_analogies/__init__.py
diff --git a/lm_eval/datasets/triviaqa/__init__.py b/lm_eval/datasets/triviaqa/__init__.py
diff --git a/lm_eval/datasets/truthfulqa/__init__.py b/lm_eval/datasets/truthfulqa/__init__.py
diff --git a/lm_eval/datasets/unscramble/__init__.py b/lm_eval/datasets/unscramble/__init__.py
diff --git a/lm_eval/datasets/wikitext/__init__.py b/lm_eval/datasets/wikitext/__init__.py
diff --git a/lm_eval/tasks/hendrycks_ethics.py b/lm_eval/tasks/hendrycks_ethics.py
@@ -277,20 +277,18 @@ class EthicsUtilitarianism(Ethics):
  DATASET_NAME = "utilitarianism"
 
  def training_docs(self):
- rnd = random.Random()
  for doc in self.dataset["train"]:
- yield self._process_doc(doc, rnd)
+ yield self._process_doc(doc)
 
  def validation_docs(self):
  raise NotImplementedError
 
  def test_docs(self):
- rnd = random.Random()
  for doc in self.dataset["test"]:
- yield self._process_doc(doc, rnd)
+ yield self._process_doc(doc)
 
- def _process_doc(self, doc, rnd):
- rnd.seed(doc["activity"])
+ def _process_doc(self, doc):
+ rnd = random.Random(doc["activity"])
  scenarios = [doc["activity"], doc["baseline"]]
  ordering = [0, 1]
  rnd.shuffle(ordering)

diff --git a/lm_eval/tasks/hendrycks_math.py b/lm_eval/tasks/hendrycks_math.py
@@ -38,15 +38,15 @@ def has_test_docs(self):
  return True
 
  def training_docs(self):
- return map(self._load_doc, self.dataset["train"])
+ return map(self._process_doc, self.dataset["train"])
 
  def validation_docs(self):
  return NotImplemented
 
  def test_docs(self):
- return map(self._load_doc, self.dataset["test"])
+ return map(self._process_doc, self.dataset["test"])
 
- def _load_doc(self, doc):
+ def _process_doc(self, doc):
  doc["answer"] = self.remove_boxed(
  self.last_boxed_only_string(doc["solution"]))
  return doc

diff --git a/lm_eval/tasks/wikitext.py b/lm_eval/tasks/wikitext.py
@@ -76,15 +76,15 @@ def has_test_docs(self):
  return True
 
  def training_docs(self):
- return map(self._load_doc, self.dataset["train"])
+ return map(self._process_doc, self.dataset["train"])
 
  def validation_docs(self):
- return map(self._load_doc, self.dataset["validation"])
+ return map(self._process_doc, self.dataset["validation"])
 
  def test_docs(self):
- return map(self._load_doc, self.dataset["test"])
+ return map(self._process_doc, self.dataset["test"])
 
- def _load_doc(self, doc):
+ def _process_doc(self, doc):
  return doc["page"]
 
  def doc_to_target(self, doc):

diff --git a/lm_eval/tasks/wsc273.py b/lm_eval/tasks/wsc273.py
@@ -53,9 +53,9 @@ def has_test_docs(self):
  return True
 
  def test_docs(self):
- return map(self._load_doc, self.dataset["test"])
+ return map(self._process_doc, self.dataset["test"])
 
- def _load_doc(self, doc):
+ def _process_doc(self, doc):
  # The HF implementation of `wsc273` is not `partial evaluation` friendly.
  doc["text"] = doc["text"].replace(" ", " ")
  doc["options"][0] = self.__normalize_option(doc, doc["options"][0])

diff --git a/setup.py b/setup.py
@@ -37,7 +37,6 @@
  "pycountry==20.7.3",
  "numexpr==2.7.2",
  "lm_dataformat==0.0.20",
- "pytest==6.2.3",
  "pybind11==2.6.2",
  "tqdm-multiprocess==0.0.11",
  "zstandard==0.15.2",
@@ -51,4 +50,5 @@
  dependency_links=[
  "https://github.com/google-research/bleurt/archive/b610120347ef22b494b6d69b4316e303f5932516.zip#egg=bleurt",
  ],
+ extras_require={'dev': [ 'pytest', 'black' ]}
 )