From 3c68794d93cb7c7e0fe54f0a7484e6ce21f639d5 Mon Sep 17 00:00:00 2001 From: "scitools-ci[bot]" <107775138+scitools-ci[bot]@users.noreply.github.com> Date: Fri, 31 May 2024 14:17:21 +0000 Subject: [PATCH 1/3] =?UTF-8?q?=F0=9F=91=81=20TEMPLATE=20DEVIATIONS:=20syn?= =?UTF-8?q?ced=20local=20'.pre-commit-config.yaml'=20with=20remote=20'temp?= =?UTF-8?q?lates/.pre-commit-config.yaml'?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .pre-commit-config.yaml | 105 ++++++++++++++++++++++++++++++++++------ 1 file changed, 89 insertions(+), 16 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 40da54c..15a6640 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,31 +1,104 @@ # See https://pre-commit.com for more information # See https://pre-commit.com/hooks.html for more hooks +# See https://pre-commit.ci/#configuration +# See https://github.com/scientific-python/cookie#sp-repo-review + +ci: + autofix_prs: false + autoupdate_commit_msg: "chore: update pre-commit hooks" + + +# Alphabetised, for lack of a better order. +files: | + (?x)( + benchmarks\/.+\.py| + docs\/.+\.py| + lib\/.+\.py| + noxfile\.py| + pyproject\.toml| + setup\.py| + src\/.+\.py + ) +minimum_pre_commit_version: 1.21.0 + repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: 'v4.6.0' + rev: v4.6.0 hooks: # Prevent giant files from being committed. - id: check-added-large-files # Check whether files parse as valid Python. - id: check-ast - # Check for file name conflicts on case-insensitive filesytems. + # Check for file name conflicts on case-insensitive filesystems. - id: check-case-conflict # Check for files that contain merge conflict strings. - id: check-merge-conflict # Check for debugger imports and py37+ `breakpoint()` calls in Python source. - id: debug-statements - # Don't commit to master branch. + # Check TOML file syntax. + - id: check-toml + # Check YAML file syntax. + - id: check-yaml + # Makes sure files end in a newline and only a newline. + # Duplicates Ruff W292 but also works on non-Python files. + - id: end-of-file-fixer + # Replaces or checks mixed line ending. + - id: mixed-line-ending + # Don't commit to main branch. - id: no-commit-to-branch -- repo: https://github.com/psf/black - rev: '24.4.2' - hooks: - - id: black - # Force black to run on whole repo, using settings from pyproject.toml - pass_filenames: false - args: [--config=./pyproject.toml, .] -- repo: https://github.com/PyCQA/flake8 - rev: '7.0.0' - hooks: - # Run flake8. - - id: flake8 - args: [--config=./.flake8] + # Trims trailing whitespace. + # Duplicates Ruff W291 but also works on non-Python files. + - id: trailing-whitespace + +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: "v0.4.4" + hooks: + - id: ruff + types: [file, python] + args: [--fix, --show-fixes] + - id: ruff-format + types: [file, python] + +- repo: https://github.com/codespell-project/codespell + rev: "v2.2.6" + hooks: + - id: codespell + types_or: [asciidoc, python, markdown, rst] + additional_dependencies: [tomli] + +- repo: https://github.com/adamchainz/blacken-docs + rev: 1.16.0 + hooks: + - id: blacken-docs + types: [file, rst] + +- repo: https://github.com/aio-libs/sort-all + rev: v1.2.0 + hooks: + - id: sort-all + types: [file, python] + +- repo: https://github.com/pre-commit/mirrors-mypy + rev: 'v1.10.0' + hooks: + - id: mypy + exclude: 'noxfile\.py|docs/conf\.py' + +- repo: https://github.com/abravalheri/validate-pyproject + # More exhaustive than Ruff RUF200. + rev: "v0.18" + hooks: + - id: validate-pyproject + +- repo: https://github.com/scientific-python/cookie + rev: 2024.04.23 + hooks: + - id: sp-repo-review + additional_dependencies: ["repo-review[cli]"] + args: ["--show=errskip"] + +- repo: https://github.com/numpy/numpydoc + rev: v1.7.0 + hooks: + - id: numpydoc-validation + types: [file, python] \ No newline at end of file From 9cb106d1eb37bee9ee1e4298b1ab8d7614e6316d Mon Sep 17 00:00:00 2001 From: Martin Yeo <40734014+trexfeathers@users.noreply.github.com> Date: Fri, 31 May 2024 15:26:54 +0100 Subject: [PATCH 2/3] Remove the file filter for test-iris-imagehash --- .pre-commit-config.yaml | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 15a6640..cb3c3a6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,17 +8,6 @@ ci: autoupdate_commit_msg: "chore: update pre-commit hooks" -# Alphabetised, for lack of a better order. -files: | - (?x)( - benchmarks\/.+\.py| - docs\/.+\.py| - lib\/.+\.py| - noxfile\.py| - pyproject\.toml| - setup\.py| - src\/.+\.py - ) minimum_pre_commit_version: 1.21.0 repos: @@ -101,4 +90,4 @@ repos: rev: v1.7.0 hooks: - id: numpydoc-validation - types: [file, python] \ No newline at end of file + types: [file, python] From e93fc9517ddaa240d4a1ee3c6df29fe357872705 Mon Sep 17 00:00:00 2001 From: Martin Yeo <40734014+trexfeathers@users.noreply.github.com> Date: Fri, 28 Jun 2024 12:03:13 +0100 Subject: [PATCH 3/3] Pre-commit fixes for SciTools/test-iris-imagehash#87 (#88) * Pre-commit auto-fixes. * Remove references to Black. * Copy Iris config for numpydoc. * Ruff compliant. * Apply repo-review exceptions from Iris-grib. * Repo-review compliant. * More numpydoc compliance. * Better repo-review compliance. * Remove comment. --- .cirrus.yml | 2 +- .pre-commit-config.yaml | 3 +- CONTRIBUTING.md | 2 +- README.rst | 6 +- etc/readme.txt | 4 +- pyproject.toml | 156 ++++++++++++++++++++++++++++++----- recreate_v4_files_listing.py | 59 +++++++++---- requirements.yml | 3 +- run_test.py | 39 ++++----- 9 files changed, 202 insertions(+), 72 deletions(-) mode change 100644 => 100755 recreate_v4_files_listing.py diff --git a/.cirrus.yml b/.cirrus.yml index 3f00cf2..cd0e686 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -65,4 +65,4 @@ linux_task: - conda info --envs test_script: - source activate ${ENV_NAME} - - python run_test.py + - pytest run_test.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cb3c3a6..31454e7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -71,7 +71,7 @@ repos: rev: 'v1.10.0' hooks: - id: mypy - exclude: 'noxfile\.py|docs/conf\.py' + exclude: 'run_test\.py' - repo: https://github.com/abravalheri/validate-pyproject # More exhaustive than Ruff RUF200. @@ -91,3 +91,4 @@ repos: hooks: - id: numpydoc-validation types: [file, python] + exclude: 'run_test\.py' diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 054951d..5523bc4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,6 +2,6 @@ Feel free to raise an **Issue** or a **Pull Request**. -Note that all authors on pull requests will automatically be asked to sign the +Note that all authors on pull requests will automatically be asked to sign the [SciTools Contributor Licence Agreement](https://cla-assistant.io/SciTools/) (CLA), if they have not already done so. diff --git a/README.rst b/README.rst index 323203f..e82b511 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,7 @@ Test Iris ImageHash =================== -|CirrusCI|_ |PreCommit|_ |Black|_ +|CirrusCI|_ |PreCommit|_ |Ruff|_ \(C) British Crown Copyright 2015 - 2024, Met Office @@ -15,5 +15,5 @@ All files in the images folder shall be named with the perceptual image hash of .. _CirrusCI: https://cirrus-ci.com/github/SciTools/iris .. |PreCommit| image:: https://results.pre-commit.ci/badge/github/SciTools/test-iris-imagehash/gh-pages.svg .. _PreCommit: https://results.pre-commit.ci/latest/github/SciTools/test-iris-imagehash/gh-pages -.. |Black| image:: https://img.shields.io/badge/code%20style-black-000000.svg -.. _Black: https://github.com/psf/black +.. |Ruff| image:: https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json +.. _Ruff: https://github.com/astral-sh/ruff diff --git a/etc/readme.txt b/etc/readme.txt index 2b7ccfa..06ab244 100644 --- a/etc/readme.txt +++ b/etc/readme.txt @@ -23,11 +23,11 @@ The following packages will be downloaded: The imagerepo.json file (and associated phash files) were created with the imagehash.phash function, using a hash_size=16, a tolerance=2, and the above -software stack. +software stack. A phash image is considered not similar to the list of registered expected phash images for an individual test case iff it has a phash hamming distance greater -than the specified tolerance for *all* of the registered expected phash images. +than the specified tolerance for *all* of the registered expected phash images. Note that, the calculation of a phash for an image *may* be sensitive to the version of pillow. The use of pillow 3.3.1 and at least 3.4.0 showed a minor diff --git a/pyproject.toml b/pyproject.toml index abb4363..ce9b7fc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,22 +1,134 @@ -[tool.black] -line-length = 79 -target-version = ['py38'] -include = '\.pyi?$' -exclude = ''' - -( - /( - \.eggs # exclude a few common directories in the - | \.git # root of the project - | \.hg - | \.mypy_cache - | \.tox - | \.venv - | _build - | buck-out - | build - | dist - | etc - )/ -) -''' +[tool.ruff] +line-length = 88 +target-version = "py310" + +[tool.ruff.format] +preview = false + +[tool.ruff.lint] +ignore = [ + # NOTE: Non-permanent exclusions should be added to the ".ruff.toml" file. + + # flake8-commas (COM) + # https://docs.astral.sh/ruff/rules/#flake8-commas-com + "COM812", # Trailing comma missing. + "COM819", # Trailing comma prohibited. + + # flake8-implicit-str-concat (ISC) + # https://docs.astral.sh/ruff/rules/single-line-implicit-string-concatenation/ + # NOTE: This rule may cause conflicts when used with "ruff format". + "ISC001", # Implicitly concatenate string literals on one line. + ] + preview = false + select = [ + "ALL", + # list specific rules to include that is skipped using numpy convention. + "D212", # Multi-line docstring summary should start at the first line + ] + +[tool.ruff.lint.isort] +force-sort-within-sections = true +known-first-party = ["iris"] + +[tool.ruff.lint.per-file-ignores] +# Test script. + +"run_test.py" = [ + # https://docs.astral.sh/ruff/rules/undocumented-public-module/ + "D100", # Missing docstring in public module + "D101", # Missing docstring in public class + "D102", # Missing docstring in public method + "D205", # 1 blank line required between summary line and description + "D401", # 1 First line of docstring should be in imperative mood + + "ANN", # https://docs.astral.sh/ruff/rules/#flake8-annotations-ann + "S101", # Use of assert detected +] + +[tool.ruff.lint.pydocstyle] +convention = "numpy" + +[tool.numpydoc_validation] +checks = [ + "all", # Enable all numpydoc validation rules, apart from the following: + + # -> Docstring text (summary) should start in the line immediately + # after the opening quotes (not in the same line, or leaving a + # blank line in between) + "GL01", # Permit summary line on same line as docstring opening quotes. + + # -> Closing quotes should be placed in the line after the last text + # in the docstring (do not close the quotes in the same line as + # the text, or leave a blank line between the last text and the + # quotes) + "GL02", # Permit a blank line before docstring closing quotes. + + # -> Double line break found; please use only one blank line to + # separate sections or paragraphs, and do not leave blank lines + # at the end of docstrings + "GL03", # Ignoring. + + # -> See Also section not found + "SA01", # Not all docstrings require a "See Also" section. + + # -> No extended summary found + "ES01", # Not all docstrings require an "Extended Summary" section. + + # -> No examples section found + "EX01", # Not all docstrings require an "Examples" section. + + # -> No Yields section found + "YD01", # Not all docstrings require a "Yields" section. +] +exclude = [ + '\.__eq__$', + '\.__ne__$', + '\.__repr__$', +] + +[tool.mypy] +# See https://mypy.readthedocs.io/en/stable/config_file.html +ignore_missing_imports = true +warn_unused_configs = true +warn_unreachable = true +enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"] +exclude = [ + "^run_test\\.py$", +] +strict = true + +[tool.pytest.ini_options] +# configure settings as recommended by repo-review: +addopts = ["-ra", "--strict-config", "--strict-markers"] +filterwarnings = ["error"] +log_cli = "True" +log_cli_level = "INFO" +minversion = "6.0" +testpaths = "./run_test.py" +xfail_strict = "True" + +[tool.repo-review] +ignore = [ + # https://learn.scientific-python.org/development/guides/style/#PC170 + "PC170", # Uses PyGrep hooks + # https://learn.scientific-python.org/development/guides/style/#PC180 + "PC180", # Uses prettier + + # https://learn.scientific-python.org/development/guides/gha-basic/ + "GH", # TODO: switch to GitHub Actions + + # test-iris-imagehash is very simple, and not a package, which exempts the + # following rules. + # https://learn.scientific-python.org/development/guides/style/#PP002 + "PP002", # Build-system + # https://learn.scientific-python.org/development/guides/style/#PP003 + "PP003", # Wheel dependency + # https://learn.scientific-python.org/development/guides/style/#PY004 + "PY004", # Docs folder + # https://learn.scientific-python.org/development/guides/style/#PY007 + "PY007", # Nox/tox + # https://learn.scientific-python.org/development/guides/style/#RF003 + "RF003", # Ruff src directory + # https://learn.scientific-python.org/development/guides/docs/#readthedocsyaml + "RTD", # ReadTheDocs +] diff --git a/recreate_v4_files_listing.py b/recreate_v4_files_listing.py old mode 100644 new mode 100755 index e0cb115..3073061 --- a/recreate_v4_files_listing.py +++ b/recreate_v4_files_listing.py @@ -1,30 +1,59 @@ #!/usr/bin/env python -import os -import os.path -from glob import glob +"""Regenerate v4_files_listing.txt .""" +from __future__ import annotations -REPO_MAIN_DIRPATH = os.path.dirname(os.path.abspath(__file__)) -V4_DIR = os.sep.join([REPO_MAIN_DIRPATH, "images", "v4"]) +from pathlib import Path + +REPO_MAIN_DIRPATH = Path(__file__).resolve().parent +V4_DIR = REPO_MAIN_DIRPATH / "images" / "v4" V4_LISTFILE_NAME = "v4_files_listing.txt" -V4_LISTFILE_PATH = os.sep.join([REPO_MAIN_DIRPATH, V4_LISTFILE_NAME]) +V4_LISTFILE_PATH = REPO_MAIN_DIRPATH / V4_LISTFILE_NAME + + +def get_v4_imagefile_names( + search_dirpath: Path = V4_DIR, + filespec: str = "*.png", +) -> list[str]: + """Return a list of the current image files in the v4 subdirectory. + Parameters + ---------- + search_dirpath : Path + The directory to search for files. + filespec : str + The glob string to search for. -def get_v4_imagefile_names(search_dirpath=V4_DIR, filespec="*.png"): - """Return a list of the current image files in the v4 subdirectory.""" - files_spec = os.path.join(search_dirpath, filespec) - file_paths = glob(files_spec) - return [os.path.basename(file_path) for file_path in file_paths] + Returns + ------- + list[str] + The names of the files found in the search directory. + """ + file_paths = search_dirpath.glob(filespec) + return [file_path.name for file_path in file_paths] def create_v4_images_listfile( - search_dirpath=V4_DIR, filespec="*.png", output_filepath=V4_LISTFILE_PATH -): + search_dirpath: Path = V4_DIR, + filespec: str = "*.png", + output_filepath: Path = V4_LISTFILE_PATH, +) -> None: + """Create a listfile of the current image files in the v4 subdirectory. + + Parameters + ---------- + search_dirpath : Path + The directory to search for files. + filespec : str + The glob string to search for. + output_filepath : Path + The file to write the list of file names to. + """ file_names = get_v4_imagefile_names(search_dirpath, filespec) - with open(output_filepath, "w") as f_out: + with output_filepath.open("w") as f_out: for file_name in sorted(file_names): - f_out.write("{}\n".format(file_name)) + f_out.write(f"{file_name}\n") if __name__ == "__main__": diff --git a/requirements.yml b/requirements.yml index a539322..afbe2da 100644 --- a/requirements.yml +++ b/requirements.yml @@ -4,8 +4,7 @@ channels: - conda-forge dependencies: - - black=20.8b1 - - flake8 - imagehash>=4.0 - pillow<7 - pre-commit + - pytest diff --git a/run_test.py b/run_test.py index f4f7909..c70a4a2 100755 --- a/run_test.py +++ b/run_test.py @@ -1,41 +1,36 @@ #!/usr/bin/env python -import glob -import os -import unittest +from pathlib import Path -from PIL import Image import imagehash - +from PIL import Image _HASH_SIZE = 16 -class TestHash(unittest.TestCase): +class TestHash: def test(self): self.maxDiff = None exceptions = [] - for fname in glob.glob("images/v4/*.png"): + for fname in Path("images/v4").glob("*.png"): phash = imagehash.phash(Image.open(fname), hash_size=_HASH_SIZE) - fname_base = os.path.basename(fname) - fname_hash = os.path.splitext(fname_base)[0] + fname_base = fname.name + fname_hash = fname.stem if str(phash) != fname_hash: msg = "Calculated phash {} does not match filename {!r}." - exceptions.append( - ValueError(msg.format(str(phash), fname_base)) - ) - self.assertEqual([], exceptions) + exceptions.append(ValueError(msg.format(str(phash), fname_base))) + assert exceptions == [], "\n".join(str(e) for e in exceptions) -class TestListing(unittest.TestCase): +class TestListing: def test(self): # Check that the image listing file contents are up to date. import recreate_v4_files_listing as v4list file_names = set(v4list.get_v4_imagefile_names()) - listing_filepath = v4list.V4_LISTFILE_NAME - self.assertTrue(os.path.exists(listing_filepath)) - with open(listing_filepath) as listing_file: + listing_filepath = Path(v4list.V4_LISTFILE_NAME) + assert listing_filepath.exists() + with listing_filepath.open() as listing_file: listed_names = [line.strip() for line in listing_file.readlines()] files = set(file_names) @@ -54,11 +49,5 @@ def test(self): if missing: msg += "\n Names in the listing, but not in the directory:" msg += "".join(["\n " + name for name in missing]) - msg += '\n\n*** Please run "{}.py" to correct. ***'.format( - os.path.basename(v4list.__name__) - ) - self.assertEqual(listed, files, msg) - - -if __name__ == "__main__": - unittest.main() + msg += f'\n\n*** Please run "{Path(v4list.__file__).name}" to correct. ***' + assert listed == files, msg