Merge remote-tracking branch 'upstream/main' into dark_theme

* upstream/main: (61 commits)
  Updated environment lockfiles (SciTools#5270)
  Drop python3.8 support (SciTools#5269)
  build wheel from sdist, not src (SciTools#5266)
  Lazy netcdf saves (SciTools#5191)
  move setup.cfg to pyproject.toml (SciTools#5262)
  Support Python 3.11 (SciTools#5226)
  Remove Resolve test workaround (SciTools#5267)
  add missing whatsnew entry (SciTools#5265)
  make help (SciTools#5258)
  automate pypi manifest checking (SciTools#5259)
  drop sphinxcontrib-napoleon (SciTools#5263)
  add missing test result data (SciTools#5260)
  fix indentation and remove ref to ssstack (SciTools#5256)
  review actions
  update .git-blame-ignore-revs
  adopt codespell
  Adopt sphinx design (SciTools#5127)
  Bump scitools/workflows from 2023.04.2 to 2023.04.3 (SciTools#5253)
  refresh manual pypi publish instructions (SciTools#5252)
  Updated environment lockfiles (SciTools#5250)
  ...

.flake8

@@ -0,0 +1,49 @@
+[flake8]
+# References:
+#   https://flake8.readthedocs.io/en/latest/user/configuration.html
+#   https://flake8.readthedocs.io/en/latest/user/error-codes.html
+#   https://pycodestyle.readthedocs.io/en/latest/intro.html#error-codes
+
+max-line-length = 80
+max-complexity = 50
+select = C,E,F,W,B,B950
+ignore =
+    # E203: whitespace before ':'
+    E203,
+    # E226: missing whitespace around arithmetic operator
+    E226,
+    # E231: missing whitespace after ',', ';', or ':'
+    E231,
+    # E402: module level imports on one line
+    E402,
+    # E501: line too long
+    E501,
+    # E731: do not assign a lambda expression, use a def
+    E731,
+    # W503: line break before binary operator
+    W503,
+    # W504: line break after binary operator
+    W504,
+exclude =
+    #
+    # ignore the following directories
+    #
+    .eggs,
+    build,
+    docs/src/sphinxext/*,
+    tools/*,
+    benchmarks/*,
+    #
+    # ignore auto-generated files
+    #
+    _ff_cross_refrences.py,
+    std_names.py,
+    um_cf_map.py,
+    #
+    # ignore third-party files
+    #
+    gitwash_dumper.py,
+    #
+    # convenience imports
+    #
+    lib/iris/common/__init__.py

.git-blame-ignore-revs

@@ -0,0 +1,20 @@
+# Format: numpy array format (#5235)
+c18dcd8dafef0cc7bbbf80dfce66f76a46ce59c5
+
+# style: flake8 (#3755)
+7c86bc0168684345dc475457b1a77dadc77ce9bb
+
+# style: black (#3518)
+ffcfad475e0593e1e40895453cf1df154e5f6f2c
+
+# style: isort (#4174)
+15bbcc5ac3d539cb6e820148b66e7cf55d91c5d2
+
+# style: blacken-docs (#4205)
+1572e180243e492d8ff76fa8cdefb82ef6f90415
+
+# style: sort-all (#4353)
+64705dbc40881233aae45f051d96049150369e53
+
+# style: codespell (#5186)
+417aa6bbd9b10d25cad7def54d47ef4d718bc38d

.github/workflows/benchmark.yml

@@ -21,21 +21,20 @@ jobs:
     env:
       IRIS_TEST_DATA_LOC_PATH: benchmarks
       IRIS_TEST_DATA_PATH: benchmarks/iris-test-data
-      IRIS_TEST_DATA_VERSION: "2.18"
+      IRIS_TEST_DATA_VERSION: "2.19"
       # Lets us manually bump the cache to rebuild
       ENV_CACHE_BUILD: "0"
       TEST_DATA_CACHE_BUILD: "2"
-      PY_VER: 3.8
 
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
-      - name: Install Nox
+      - name: Install ASV & Nox
         run: |
-          pip install nox
+          pip install asv nox
 
       - name: Cache environment directories
         id: cache-env-dir
@@ -78,7 +77,7 @@ jobs:
           
           if [ "$first_commit" != "" ]
           then
-            nox --session="benchmarks(overnight)" -- $first_commit
+            python benchmarks/bm_runner.py overnight $first_commit
           fi
 
       - name: Create issues for performance shifts

.github/workflows/ci-manifest.yml

@@ -0,0 +1,26 @@
+# Reference
+#    - https://github.com/actions/checkout
+
+name: ci-manifest
+
+on:
+  pull_request:
+    branches:
+      - "*"
+
+  push:
+    branches-ignore:
+      - "auto-update-lockfiles"
+      - "pre-commit-ci-update-config"
+      - "dependabot/*"
+
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  manifest:
+    name: "check-manifest"
+    uses: scitools/workflows/.github/workflows/[email protected]

.github/workflows/ci-tests.yml

@@ -35,22 +35,22 @@ jobs:
       fail-fast: false
       matrix:
         os: ["ubuntu-latest"]
-        python-version: ["3.10"]
+        python-version: ["3.11"]
         session: ["doctest", "gallery", "linkcheck"]
         include:
           - os: "ubuntu-latest"
-            python-version: "3.10"
+            python-version: "3.11"
             session: "tests"
             coverage: "--coverage"
           - os: "ubuntu-latest"
-            python-version: "3.9"
+            python-version: "3.10"
             session: "tests"
           - os: "ubuntu-latest"
-            python-version: "3.8"
+            python-version: "3.9"
             session: "tests"
 
     env:
-      IRIS_TEST_DATA_VERSION: "2.18"
+      IRIS_TEST_DATA_VERSION: "2.19"
       ENV_NAME: "ci-tests"
 
     steps:
@@ -63,7 +63,7 @@ jobs:
           CACHE_WEEKS: 2
         run: |
           echo "CACHE_PERIOD=$(date +%Y).$(expr $(date +%U) / ${CACHE_WEEKS})" >> ${GITHUB_ENV}
-          echo "LOCK_FILE=requirements/ci/nox.lock/py$(echo ${{ matrix.python-version }} | tr -d '.')-linux-64.lock" >> ${GITHUB_ENV}
+          echo "LOCK_FILE=requirements/locks/py$(echo ${{ matrix.python-version }} | tr -d '.')-linux-64.lock" >> ${GITHUB_ENV}
 
       - name: "data cache"
         uses: ./.github/workflows/composite/iris-data-cache
@@ -111,7 +111,7 @@ jobs:
       - name: "nox cache"
         uses: ./.github/workflows/composite/nox-cache
         with:
-          cache_build: 0
+          cache_build: 2
           env_name: ${{ env.ENV_NAME }}
           lock_file: ${{ env.LOCK_FILE }}
 

.github/workflows/ci-wheels.yml

@@ -35,9 +35,7 @@ jobs:
     - name: "building"
       shell: bash
       run: |
-        # require build with explicit --sdist and --wheel in order to
-        # get correct version associated with sdist and bdist artifacts
-        pipx run build --sdist --wheel
+        pipx run build
 
     - uses: actions/upload-artifact@v3
       with:
@@ -54,7 +52,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.9", "3.10", "3.11"]
         session: ["wheel"]
     env:
       ENV_NAME: "ci-wheels"
@@ -74,7 +72,7 @@ jobs:
           CACHE_WEEKS: 2
       run: |
         echo "CACHE_PERIOD=$(date +%Y).$(expr $(date +%U) / ${CACHE_WEEKS})" >> ${GITHUB_ENV}
-        echo "LOCK_FILE=requirements/ci/nox.lock/py$(echo ${{ matrix.python-version }} | tr -d '.')-linux-64.lock" >> ${GITHUB_ENV}
+        echo "LOCK_FILE=requirements/locks/py$(echo ${{ matrix.python-version }} | tr -d '.')-linux-64.lock" >> ${GITHUB_ENV}
 
     - name: "conda package cache"
       uses: ./.github/workflows/composite/conda-pkg-cache
@@ -103,7 +101,7 @@ jobs:
     - name: "nox cache"
       uses: ./.github/workflows/composite/nox-cache
       with:
-        cache_build: 0
+        cache_build: 1
         env_name: ${{ env.ENV_NAME }}
         lock_file: ${{ env.LOCK_FILE }}
 
@@ -132,7 +130,8 @@ jobs:
     name: "publish to test.pypi"
     runs-on: ubuntu-latest
     # upload to Test PyPI for every commit on main branch
-    if: github.event_name == 'push' && github.event.ref == 'refs/heads/main'
+    # and check for the SciTools repo
+    if: github.event_name == 'push' && github.event.ref == 'refs/heads/main' && github.repository_owner == 'SciTools'
     steps:
     - uses: actions/download-artifact@v3
       with:
@@ -152,7 +151,7 @@ jobs:
     name: "publish to pypi"
     runs-on: ubuntu-latest
     # upload to PyPI for every tag starting with 'v'
-    if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v')
+    if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v') && github.repository_owner == 'SciTools'
     steps:
     - uses: actions/download-artifact@v3
       with:

.github/workflows/refresh-lockfiles.yml

@@ -14,5 +14,5 @@ on:
 
 jobs:
   refresh_lockfiles:
-    uses: scitools/workflows/.github/workflows/refresh-lockfiles.yml@main
+    uses: scitools/workflows/.github/workflows/refresh-lockfiles.yml@2023.04.3
     secrets: inherit

.github/workflows/stale.yml

@@ -14,7 +14,7 @@ jobs:
     if: "github.repository == 'SciTools/iris'"
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/stale@v7
+      - uses: actions/stale@v8
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
 

.gitignore

@@ -77,4 +77,3 @@ docs/iris_image_test_output/
 
 # Files generated during test runs.
 lib/iris/tests/results/**/*.dot
-lib/iris/tests/results/PP/extra_char_data.w_data_loaded.pp.txt

.pre-commit-config.yaml

@@ -28,8 +28,15 @@ repos:
         # Don't commit to main branch.
     -   id: no-commit-to-branch
 
+-   repo: https://github.com/codespell-project/codespell
+    rev: "v2.2.2"
+    hooks:
+    -   id: codespell
+        types_or: [asciidoc, python, markdown, rst]
+        additional_dependencies: [tomli]
+
 -   repo: https://github.com/psf/black
-    rev: 23.1.0
+    rev: 23.3.0
     hooks:
     -   id: black
         pass_filenames: false
@@ -40,7 +47,6 @@ repos:
     hooks:
     -   id: flake8
         types: [file, python]
-        args: [--config=./setup.cfg]
 
 -   repo: https://github.com/pycqa/isort
     rev: 5.12.0

.readthedocs.yml

@@ -19,7 +19,7 @@ build:
       - git stash pop
 
 conda:
-  environment: requirements/ci/readthedocs.yml
+  environment: requirements/readthedocs.yml
 
 sphinx:
   configuration: docs/src/conf.py

MANIFEST.in

@@ -1,20 +1,29 @@
-# Top-level files
-include CHANGES COPYING COPYING.LESSER
 prune .github
+prune benchmarks
+prune docs
+prune etc
+recursive-include lib *.cdl *.cml *.json *.md *.py *.template *.txt *.xml
+prune requirements
+recursive-include requirements *.txt
+prune tools
+exclude .flake8
+exclude .git-blame-ignore-revs
+exclude .git_archival.txt
+exclude .gitattributes
 exclude .gitignore
+exclude .mailmap
+exclude .pre-commit-config.yaml
+exclude .readthedocs.yml
+exclude CHANGES
+exclude codecov.yml
+include COPYING
+include COPYING.LESSER
+exclude Makefile
+exclude noxfile.py
 
-# Files required for conda package management
-recursive-include requirements *
-
-# Files required to build docs
-recursive-include docs *
-prune docs/src/_build
-prune docs/src/generated
-prune docs/gallery_tests
-
-# Files required to build std_names module
-include tools/generate_std_names.py
+# files required to build iris.std_names module
 include etc/cf-standard-name-table.xml
+include tools/generate_std_names.py
 
 global-exclude *.py[cod]
 global-exclude __pycache__

Makefile

@@ -1,2 +1,2 @@
 lockfiles:
-	python tools/update_lockfiles.py -o requirements/ci/nox.lock requirements/ci/py*.yml
+	python tools/update_lockfiles.py -o requirements/locks requirements/py*.yml

benchmarks/README.md

@@ -15,9 +15,20 @@ shifts in performance being flagged in a new GitHub issue.
 installed, as well as Nox (see
 [Benchmark environments](#benchmark-environments)).
 
-[Iris' noxfile](../noxfile.py) includes a `benchmarks` session that provides
-conveniences for setting up before benchmarking, and can also replicate the
-automated overnight run locally. See the session docstring for detail.
+The benchmark runner ([bm_runner.py](./bm_runner.py)) provides conveniences for
+common benchmark setup and run tasks, including replicating the automated 
+overnight run locally. See `python bm_runner.py --help` for detail.
+
+A significant portion of benchmark run time is environment management. Run-time
+can be reduced by placing the benchmark environment on the same file system as
+your
+[Conda package cache](https://conda.io/projects/conda/en/latest/user-guide/configuration/use-condarc.html#specify-pkg-directories),
+if it is not already. You can achieve this by either:
+
+- Temporarily reconfiguring `delegated_env_commands` and `delegated_env_parent` 
+  in [asv.conf.json](asv.conf.json) to reference a location on the same file
+  system as the Conda package cache.
+- Moving your Iris repo to the same file system as the Conda package cache.
 
 ### Environment variables
 
@@ -26,8 +37,8 @@ automated overnight run locally. See the session docstring for detail.
 benchmark scripts.
 * `DATA_GEN_PYTHON` - required - path to a Python executable that can be
 used to generate benchmark test objects/files; see
-[Data generation](#data-generation). The Nox session sets this automatically,
-but will defer to any value already set in the shell.
+[Data generation](#data-generation). The benchmark runner sets this 
+automatically, but will defer to any value already set in the shell.
 * `BENCHMARK_DATA` - optional - path to a directory for benchmark synthetic
 test data, which the benchmark scripts will create if it doesn't already
 exist. Defaults to `<root>/benchmarks/.data/` if not set. Note that some of
@@ -36,7 +47,7 @@ plan accordingly.
 * `ON_DEMAND_BENCHMARKS` - optional - when set (to any value): benchmarks
 decorated with `@on_demand_benchmark` are included in the ASV run. Usually
 coupled with the ASV `--bench` argument to only run the benchmark(s) of
-interest. Is set during the Nox `cperf` and `sperf` sessions.
+interest. Is set during the benchmark runner `cperf` and `sperf` sub-commands.
 
 ## Writing benchmarks
 

benchmarks/asv.conf.json

@@ -19,8 +19,7 @@
     //  * No build-time environment variables.
     //  * Is run in the same environment as the ASV install itself.
     "delegated_env_commands": [
-        "sed -i 's/_PY_VERSIONS_ALL/_PY_VERSION_LATEST/g' noxfile.py",
-        "nox --envdir={conf_dir}/.asv/env/nox01 --session=tests --install-only --no-error-on-external-run --verbose"
+        "PY_VER=3.11 nox --envdir={conf_dir}/.asv/env/nox01 --session=tests --install-only --no-error-on-external-run --verbose"
     ],
     // The parent directory of the above environment.
     //  The most recently modified environment in the directory will be used.