From 16f0cc4232c4b1f88e626003e1f46a3fa5451c58 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Apr 2024 09:25:18 +0100 Subject: [PATCH 1/9] Bump scitools/workflows from 2024.04.2 to 2024.04.3 (#5930) Bumps [scitools/workflows](https://github.com/scitools/workflows) from 2024.04.2 to 2024.04.3. - [Release notes](https://github.com/scitools/workflows/releases) - [Commits](https://github.com/scitools/workflows/compare/2024.04.2...2024.04.3) --- updated-dependencies: - dependency-name: scitools/workflows dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci-manifest.yml | 2 +- .github/workflows/refresh-lockfiles.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-manifest.yml b/.github/workflows/ci-manifest.yml index b04317164d..e490804aba 100644 --- a/.github/workflows/ci-manifest.yml +++ b/.github/workflows/ci-manifest.yml @@ -23,4 +23,4 @@ concurrency: jobs: manifest: name: "check-manifest" - uses: scitools/workflows/.github/workflows/ci-manifest.yml@2024.04.2 + uses: scitools/workflows/.github/workflows/ci-manifest.yml@2024.04.3 diff --git a/.github/workflows/refresh-lockfiles.yml b/.github/workflows/refresh-lockfiles.yml index f5800a3537..849be5e1f1 100644 --- a/.github/workflows/refresh-lockfiles.yml +++ b/.github/workflows/refresh-lockfiles.yml @@ -14,5 +14,5 @@ on: jobs: refresh_lockfiles: - uses: scitools/workflows/.github/workflows/refresh-lockfiles.yml@2024.04.2 + uses: scitools/workflows/.github/workflows/refresh-lockfiles.yml@2024.04.3 secrets: inherit From e6f518a00b667c66fdc804fd72ffe9d51f3a6965 Mon Sep 17 00:00:00 2001 From: Ruth Comer <10599679+rcomer@users.noreply.github.com> Date: Mon, 29 Apr 2024 14:56:54 +0100 Subject: [PATCH 2/9] FIX: weighted percentile works with 1D weights (#5825) --- docs/src/whatsnew/latest.rst | 5 ++++- lib/iris/analysis/__init__.py | 17 +++++++++----- .../tests/unit/analysis/test_WPERCENTILE.py | 22 ++++++++++++++++++- 3 files changed, 37 insertions(+), 7 deletions(-) diff --git a/docs/src/whatsnew/latest.rst b/docs/src/whatsnew/latest.rst index a7ac314758..00e6220c28 100644 --- a/docs/src/whatsnew/latest.rst +++ b/docs/src/whatsnew/latest.rst @@ -39,6 +39,9 @@ This document explains the changes made to Iris for this release #. `@bouweandela`_ updated the ``chunktype`` of Dask arrays, so it corresponds to the array content. (:pull:`5801`) +#. `@rcomer`_ made the :obj:`~iris.analysis.WPERCENTILE` aggregator work with + :func:`~iris.cube.Cube.rolling_window`. (:issue:`5777`, :pull:`5825`) + 💣 Incompatible Changes ======================= @@ -70,7 +73,7 @@ This document explains the changes made to Iris for this release 🔗 Dependencies =============== -#. `@tkknight`_ removed the pin for ``sphinx <=5.3``, so the latest should +#. `@tkknight`_ removed the pin for ``sphinx <=5.3``, so the latest should now be used, currently being v7.2.6. (:pull:`5901`) diff --git a/lib/iris/analysis/__init__.py b/lib/iris/analysis/__init__.py index 6678237c1c..dc2a09d93e 100644 --- a/lib/iris/analysis/__init__.py +++ b/lib/iris/analysis/__init__.py @@ -1467,7 +1467,8 @@ def _weighted_percentile(data, axis, weights, percent, returned=False, **kwargs) axis : int Axis to calculate percentiles over. weights : ndarray - Array with the weights. Must have same shape as data. + Array with the weights. Must have same shape as data or the shape of + data along axis. percent : float or sequence of floats Percentile rank/s at which to extract value/s. returned : bool, default=False @@ -1475,12 +1476,18 @@ def _weighted_percentile(data, axis, weights, percent, returned=False, **kwargs) first element and the sum of the weights as the second element. """ - # Ensure that data and weights arrays are same shape. - if data.shape != weights.shape: - raise ValueError("_weighted_percentile: weights wrong shape.") + # Ensure that weights array is the same shape as data, or the shape of data along + # axis. + if data.shape != weights.shape and data.shape[axis : axis + 1] != weights.shape: + raise ValueError( + f"For data array of shape {data.shape}, weights should be {data.shape} or {data.shape[axis : axis + 1]}" + ) # Ensure that the target axis is the last dimension. data = np.rollaxis(data, axis, start=data.ndim) - weights = np.rollaxis(weights, axis, start=data.ndim) + if weights.ndim > 1: + weights = np.rollaxis(weights, axis, start=data.ndim) + elif data.ndim > 1: + weights = np.broadcast_to(weights, data.shape) quantiles = np.array(percent) / 100.0 # Add data mask to weights if necessary. if ma.isMaskedArray(data): diff --git a/lib/iris/tests/unit/analysis/test_WPERCENTILE.py b/lib/iris/tests/unit/analysis/test_WPERCENTILE.py index a0e6b860ce..0fb64445d0 100644 --- a/lib/iris/tests/unit/analysis/test_WPERCENTILE.py +++ b/lib/iris/tests/unit/analysis/test_WPERCENTILE.py @@ -8,6 +8,8 @@ # importing anything else. import iris.tests as tests # isort:skip +import re + import numpy as np import numpy.ma as ma @@ -26,7 +28,9 @@ def test_missing_mandatory_kwargs(self): def test_wrong_weights_shape(self): data = np.arange(11) weights = np.ones(10) - emsg = "_weighted_percentile: weights wrong shape." + emsg = re.escape( + "For data array of shape (11,), weights should be (11,) or (11,)" + ) with self.assertRaisesRegex(ValueError, emsg): WPERCENTILE.aggregate(data, axis=0, percent=50, weights=weights) @@ -158,6 +162,22 @@ def test_masked_2d_multi_unequal(self): self.assertTupleEqual(weight_total.shape, (shape[-1],)) self.assertArrayEqual(weight_total, np.repeat(4, shape[-1])) + def test_2d_multi_weight1d_unequal(self): + shape = (3, 10) + data = np.arange(np.prod(shape)).reshape(shape) + weights1d = np.ones(shape[-1]) + weights1d[::3] = 3 + weights2d = np.broadcast_to(weights1d, shape) + percent = np.array([30, 50, 75, 80]) + result_1d, wt_total_1d = WPERCENTILE.aggregate( + data, axis=1, percent=percent, weights=weights1d, returned=True + ) + result_2d, wt_total_2d = WPERCENTILE.aggregate( + data, axis=1, percent=percent, weights=weights2d, returned=True + ) + # Results should be the same whether we use 1d or 2d weights. + self.assertArrayAllClose(result_1d, result_2d) + class Test_name(tests.IrisTest): def test(self): From fe22039d02899beefbd5a475a143327878929657 Mon Sep 17 00:00:00 2001 From: Martin Yeo <40734014+trexfeathers@users.noreply.github.com> Date: Mon, 29 Apr 2024 16:28:45 +0100 Subject: [PATCH 3/9] Temporary fix for airspeed-velocity/asv#1396 (#5931) * Temporary fix for airspeed-velocity/asv#1396 . * Add missing return statement. --- benchmarks/asv_delegated_conda.py | 5 +++++ docs/src/whatsnew/latest.rst | 6 ++++++ 2 files changed, 11 insertions(+) diff --git a/benchmarks/asv_delegated_conda.py b/benchmarks/asv_delegated_conda.py index 7d8b6e109c..d53c111ab9 100644 --- a/benchmarks/asv_delegated_conda.py +++ b/benchmarks/asv_delegated_conda.py @@ -192,6 +192,11 @@ def copy_asv_files(src_parent: Path, dst_parent: Path) -> None: # Record new environment information in properties. self._update_info() + def _run_conda(self, args, env=None): + # TODO: remove after airspeed-velocity/asv#1397 is merged and released. + args = ["--yes" if arg == "--force" else arg for arg in args] + return super()._run_conda(args, env) + def checkout_project(self, repo: Repo, commit_hash: str) -> None: """Check out the working tree of the project at given commit hash.""" super().checkout_project(repo, commit_hash) diff --git a/docs/src/whatsnew/latest.rst b/docs/src/whatsnew/latest.rst index 00e6220c28..448b528c38 100644 --- a/docs/src/whatsnew/latest.rst +++ b/docs/src/whatsnew/latest.rst @@ -90,6 +90,10 @@ This document explains the changes made to Iris for this release #. `@bouweandela`_ removed a workaround in :meth:`~iris.cube.CubeList.merge` for an issue with :func:`dask.array.stack` which has been solved since 2017. (:pull:`5923`) +#. `@trexfeathers`_ introduced a temporary fix for Airspeed Velocity's + deprecated use of the ``conda --force`` argument. To be removed once + `airspeed-velocity/asv#1397`_ is merged and released. (:pull:`5931`) + .. comment Whatsnew author names (@github name) in alphabetical order. Note that, @@ -100,3 +104,5 @@ This document explains the changes made to Iris for this release .. comment Whatsnew resources in alphabetical order: + +.. _airspeed-velocity/asv#1397: https://github.com/airspeed-velocity/asv/pull/1397 From 8b41ca8a0cf64fea51fd719fc39c02b614589e36 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 29 Apr 2024 21:54:27 +0100 Subject: [PATCH 4/9] [pre-commit.ci] pre-commit autoupdate (#5932) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.4.1 → v0.4.2](https://github.com/astral-sh/ruff-pre-commit/compare/v0.4.1...v0.4.2) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9ffb386fcb..d183f9bb00 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,7 +29,7 @@ repos: - id: no-commit-to-branch - repo: https://github.com/astral-sh/ruff-pre-commit - rev: "v0.4.1" + rev: "v0.4.2" hooks: - id: ruff types: [file, python] From 5c9640a167622b9bfcd0732f2d2d7b60d3f025c4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 7 May 2024 11:24:05 +0100 Subject: [PATCH 5/9] [pre-commit.ci] pre-commit autoupdate (#5938) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.4.2 → v0.4.3](https://github.com/astral-sh/ruff-pre-commit/compare/v0.4.2...v0.4.3) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d183f9bb00..a500635619 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,7 +29,7 @@ repos: - id: no-commit-to-branch - repo: https://github.com/astral-sh/ruff-pre-commit - rev: "v0.4.2" + rev: "v0.4.3" hooks: - id: ruff types: [file, python] From 1d064e394d5766d52c3c05de7e6a0deda6df5435 Mon Sep 17 00:00:00 2001 From: Bill Little Date: Wed, 8 May 2024 14:02:35 +0100 Subject: [PATCH 6/9] codecov-action gha with token (#5941) * codecov-action gha with token * target coverage.xml only --- .github/workflows/ci-tests.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 52e45f41f5..0970718b1a 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -141,6 +141,8 @@ jobs: run: | nox --session ${{ matrix.session }} -- --verbose ${{ matrix.coverage }} - - name: Upload coverage report - uses: codecov/codecov-action@v4 + - name: "upload coverage report" if: ${{ matrix.coverage }} + uses: codecov/codecov-action@v4 + with: + token: ${{ secrets.CODECOV_TOKEN }} \ No newline at end of file From a56f939384f0b43c10ca0cfd28942882af61618a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 May 2024 10:08:42 +0100 Subject: [PATCH 7/9] Bump scitools/workflows from 2024.04.3 to 2024.05.0 (#5942) Bumps [scitools/workflows](https://github.com/scitools/workflows) from 2024.04.3 to 2024.05.0. - [Release notes](https://github.com/scitools/workflows/releases) - [Commits](https://github.com/scitools/workflows/compare/2024.04.3...2024.05.0) --- updated-dependencies: - dependency-name: scitools/workflows dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci-manifest.yml | 2 +- .github/workflows/refresh-lockfiles.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-manifest.yml b/.github/workflows/ci-manifest.yml index e490804aba..888fc6e478 100644 --- a/.github/workflows/ci-manifest.yml +++ b/.github/workflows/ci-manifest.yml @@ -23,4 +23,4 @@ concurrency: jobs: manifest: name: "check-manifest" - uses: scitools/workflows/.github/workflows/ci-manifest.yml@2024.04.3 + uses: scitools/workflows/.github/workflows/ci-manifest.yml@2024.05.0 diff --git a/.github/workflows/refresh-lockfiles.yml b/.github/workflows/refresh-lockfiles.yml index 849be5e1f1..d2c1fcdcff 100644 --- a/.github/workflows/refresh-lockfiles.yml +++ b/.github/workflows/refresh-lockfiles.yml @@ -14,5 +14,5 @@ on: jobs: refresh_lockfiles: - uses: scitools/workflows/.github/workflows/refresh-lockfiles.yml@2024.04.3 + uses: scitools/workflows/.github/workflows/refresh-lockfiles.yml@2024.05.0 secrets: inherit From c5e0353da11c263656c6c77e5e56eafefcede077 Mon Sep 17 00:00:00 2001 From: Martin Yeo <40734014+trexfeathers@users.noreply.github.com> Date: Mon, 13 May 2024 15:17:41 +0100 Subject: [PATCH 8/9] Remove unit benchmarks (#5949) * Create unit_style directory. * Simple lift-and-shift fully unit-style modules. * Consistent import naming. * Restored experimental/ugrid/__init__.py * Remove ARTIFICIAL_DIM_SIZE. * Restore iterate benchmarks to first class. * Lift-and-shift unit-style cube benchmarks. * Add stock.realistic_4d_w_everything(). * Restore lost coverage. * Favour integration-style benchmarks. * What's New entries. * Clearer README phrasing. --------- Co-authored-by: Elias <110238618+ESadek-MO@users.noreply.github.com> --- benchmarks/README.md | 21 ++ benchmarks/benchmarks/__init__.py | 2 - benchmarks/benchmarks/cube.py | 346 ++++++------------ .../benchmarks/experimental/ugrid/__init__.py | 183 --------- .../experimental/ugrid/regions_combine.py | 8 - benchmarks/benchmarks/generate_data/stock.py | 34 ++ benchmarks/benchmarks/iterate.py | 17 +- benchmarks/benchmarks/load/__init__.py | 11 +- benchmarks/benchmarks/load/ugrid.py | 10 +- benchmarks/benchmarks/merge_concat.py | 52 +++ benchmarks/benchmarks/plot.py | 4 +- benchmarks/benchmarks/save.py | 10 +- .../benchmarks/unit_style/__init__disabled.py | 16 + .../{ => unit_style}/aux_factory.py | 6 +- .../benchmarks/{ => unit_style}/coords.py | 8 +- benchmarks/benchmarks/unit_style/cube.py | 252 +++++++++++++ .../metadata_manager_factory.py | 2 +- .../benchmarks/{ => unit_style}/mixin.py | 6 +- benchmarks/benchmarks/unit_style/ugrid.py | 188 ++++++++++ docs/src/whatsnew/latest.rst | 8 + lib/iris/tests/stock/__init__.py | 267 ++++++++++++++ 21 files changed, 958 insertions(+), 493 deletions(-) create mode 100644 benchmarks/benchmarks/merge_concat.py create mode 100644 benchmarks/benchmarks/unit_style/__init__disabled.py rename benchmarks/benchmarks/{ => unit_style}/aux_factory.py (93%) rename benchmarks/benchmarks/{ => unit_style}/coords.py (94%) create mode 100644 benchmarks/benchmarks/unit_style/cube.py rename benchmarks/benchmarks/{ => unit_style}/metadata_manager_factory.py (97%) rename benchmarks/benchmarks/{ => unit_style}/mixin.py (94%) create mode 100644 benchmarks/benchmarks/unit_style/ugrid.py diff --git a/benchmarks/README.md b/benchmarks/README.md index 316c8f9e32..54e580c8b1 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -62,6 +62,23 @@ interest. Is set during the benchmark runner `cperf` and `sperf` sub-commands. [See the ASV docs](https://asv.readthedocs.io/) for full detail. +### What benchmarks to write + +It is not possible to maintain a full suite of 'unit style' benchmarks: + +* Benchmarks take longer to run than tests. +* Small benchmarks are more vulnerable to noise - they report a lot of false +positive regressions. + +We therefore recommend writing benchmarks representing scripts or single +operations that are likely to be run at the user level. + +The drawback of this approach: a reported regression is less likely to reveal +the root cause (e.g. if a commit caused a regression in coordinate-creation +time, but the only benchmark covering this was for file-loading). Be prepared +for manual investigations; and consider committing any useful benchmarks as +[on-demand benchmarks](#on-demand-benchmarks) for future developers to use. + ### Data generation **Important:** be sure not to use the benchmarking environment to generate any test objects/files, as this environment changes with each commit being @@ -86,6 +103,10 @@ estimate run-time, and these will still be subject to the original problem. ### Scaling / non-Scaling Performance Differences +**(We no longer advocate the below for benchmarks run during CI, given the +limited available runtime and risk of false-positives. It remains useful for +manual investigations).** + When comparing performance between commits/file-type/whatever it can be helpful to know if the differences exist in scaling or non-scaling parts of the Iris functionality in question. This can be done using a size parameter, setting diff --git a/benchmarks/benchmarks/__init__.py b/benchmarks/benchmarks/__init__.py index 14b28b3070..73f5cd6399 100644 --- a/benchmarks/benchmarks/__init__.py +++ b/benchmarks/benchmarks/__init__.py @@ -7,8 +7,6 @@ from os import environ import resource -ARTIFICIAL_DIM_SIZE = int(10e3) # For all artificial cubes, coords etc. - def disable_repeat_between_setup(benchmark_object): """Benchmark where object persistence would be inappropriate (decorator). diff --git a/benchmarks/benchmarks/cube.py b/benchmarks/benchmarks/cube.py index 4548d4c28d..bc053e8301 100644 --- a/benchmarks/benchmarks/cube.py +++ b/benchmarks/benchmarks/cube.py @@ -4,249 +4,111 @@ # See LICENSE in the root of the repository for full licensing details. """Cube benchmark tests.""" -import numpy as np - -from iris import analysis, aux_factory, coords, cube - -from . import ARTIFICIAL_DIM_SIZE, disable_repeat_between_setup -from .generate_data.stock import sample_meshcoord - - -def setup(*params): - """General variables needed by multiple benchmark classes.""" - global data_1d - global data_2d - global general_cube - - data_2d = np.zeros((ARTIFICIAL_DIM_SIZE,) * 2) - data_1d = data_2d[0] - general_cube = cube.Cube(data_2d) - - -class ComponentCommon: - # TODO: once https://github.com/airspeed-velocity/asv/pull/828 is released: - # * make class an ABC - # * remove NotImplementedError - # * combine setup_common into setup - """Run a generalised suite of benchmarks for cubes. - - A base class running a generalised suite of benchmarks for cubes that - include a specified component (e.g. Coord, CellMeasure etc.). Component to - be specified in a subclass. - - ASV will run the benchmarks within this class for any subclasses. - - Should only be instantiated within subclasses, but cannot enforce this - since ASV cannot handle classes that include abstract methods. - """ - - def setup(self): - """Prevent ASV instantiating (must therefore override setup() in any subclasses.).""" - raise NotImplementedError - - def create(self): - """Create a cube (generic). - - cube_kwargs allow dynamic inclusion of different components; - specified in subclasses. - """ - return cube.Cube(data=data_2d, **self.cube_kwargs) - - def setup_common(self): - """Shared setup code that can be called by subclasses.""" - self.cube = self.create() - - def time_create(self): - """Create a cube that includes an instance of the benchmarked component.""" - self.create() - - def time_add(self): - """Add an instance of the benchmarked component to an existing cube.""" - # Unable to create the copy during setup since this needs to be re-done - # for every repeat of the test (some components disallow duplicates). - general_cube_copy = general_cube.copy(data=data_2d) - self.add_method(general_cube_copy, *self.add_args) - - -class Cube: - def time_basic(self): - cube.Cube(data_2d) - - def time_rename(self): - general_cube.name = "air_temperature" - - -class AuxCoord(ComponentCommon): - def setup(self): - self.coord_name = "test" - coord_bounds = np.array([data_1d - 1, data_1d + 1]).transpose() - aux_coord = coords.AuxCoord( - long_name=self.coord_name, - points=data_1d, - bounds=coord_bounds, - units="days since 1970-01-01", - climatological=True, +from iris import coords +from iris.cube import Cube + +from .generate_data.stock import realistic_4d_w_everything + + +class CubeCreation: + params = [[False, True], ["instantiate", "construct"]] + param_names = ["Cube has mesh", "Cube creation strategy"] + + cube_kwargs: dict + + def setup(self, w_mesh: bool, _) -> None: + # Loaded as two cubes due to the hybrid height. + source_cube = realistic_4d_w_everything(w_mesh=w_mesh) + + def get_coords_and_dims( + coords_tuple: tuple[coords._DimensionalMetadata, ...], + ) -> list[tuple[coords._DimensionalMetadata, tuple[int, ...]]]: + return [(c, c.cube_dims(source_cube)) for c in coords_tuple] + + self.cube_kwargs = dict( + data=source_cube.data, + standard_name=source_cube.standard_name, + long_name=source_cube.long_name, + var_name=source_cube.var_name, + units=source_cube.units, + attributes=source_cube.attributes, + cell_methods=source_cube.cell_methods, + dim_coords_and_dims=get_coords_and_dims(source_cube.dim_coords), + aux_coords_and_dims=get_coords_and_dims(source_cube.aux_coords), + aux_factories=source_cube.aux_factories, + cell_measures_and_dims=get_coords_and_dims(source_cube.cell_measures()), + ancillary_variables_and_dims=get_coords_and_dims( + source_cube.ancillary_variables() + ), ) - # Variables needed by the ComponentCommon base class. - self.cube_kwargs = {"aux_coords_and_dims": [(aux_coord, 0)]} - self.add_method = cube.Cube.add_aux_coord - self.add_args = (aux_coord, (0)) - - self.setup_common() - - def time_return_coords(self): - self.cube.coords() - - def time_return_coord_dims(self): - self.cube.coord_dims(self.coord_name) - - -class AuxFactory(ComponentCommon): - def setup(self): - coord = coords.AuxCoord(points=data_1d, units="m") - self.hybrid_factory = aux_factory.HybridHeightFactory(delta=coord) - - # Variables needed by the ComponentCommon base class. - self.cube_kwargs = { - "aux_coords_and_dims": [(coord, 0)], - "aux_factories": [self.hybrid_factory], - } - - self.setup_common() - - # Variables needed by the overridden time_add benchmark in this subclass. - cube_w_coord = self.cube.copy() - [cube_w_coord.remove_aux_factory(i) for i in cube_w_coord.aux_factories] - self.cube_w_coord = cube_w_coord - - def time_add(self): - # Requires override from super().time_add because the cube needs an - # additional coord. - self.cube_w_coord.add_aux_factory(self.hybrid_factory) - - -class CellMeasure(ComponentCommon): - def setup(self): - cell_measure = coords.CellMeasure(data_1d) - - # Variables needed by the ComponentCommon base class. - self.cube_kwargs = {"cell_measures_and_dims": [(cell_measure, 0)]} - self.add_method = cube.Cube.add_cell_measure - self.add_args = (cell_measure, 0) - - self.setup_common() - - -class CellMethod(ComponentCommon): - def setup(self): - cell_method = coords.CellMethod("test") - - # Variables needed by the ComponentCommon base class. - self.cube_kwargs = {"cell_methods": [cell_method]} - self.add_method = cube.Cube.add_cell_method - self.add_args = [cell_method] - - self.setup_common() - - -class AncillaryVariable(ComponentCommon): - def setup(self): - ancillary_variable = coords.AncillaryVariable(data_1d) - - # Variables needed by the ComponentCommon base class. - self.cube_kwargs = {"ancillary_variables_and_dims": [(ancillary_variable, 0)]} - self.add_method = cube.Cube.add_ancillary_variable - self.add_args = (ancillary_variable, 0) - - self.setup_common() - - -class MeshCoord: + def time_create(self, _, cube_creation_strategy: str) -> None: + if cube_creation_strategy == "instantiate": + _ = Cube(**self.cube_kwargs) + + elif cube_creation_strategy == "construct": + new_cube = Cube(data=self.cube_kwargs["data"]) + new_cube.standard_name = self.cube_kwargs["standard_name"] + new_cube.long_name = self.cube_kwargs["long_name"] + new_cube.var_name = self.cube_kwargs["var_name"] + new_cube.units = self.cube_kwargs["units"] + new_cube.attributes = self.cube_kwargs["attributes"] + new_cube.cell_methods = self.cube_kwargs["cell_methods"] + for coord, dims in self.cube_kwargs["dim_coords_and_dims"]: + coord: coords.DimCoord # Type hint to help linters. + new_cube.add_dim_coord(coord, dims) + for coord, dims in self.cube_kwargs["aux_coords_and_dims"]: + new_cube.add_aux_coord(coord, dims) + for aux_factory in self.cube_kwargs["aux_factories"]: + new_cube.add_aux_factory(aux_factory) + for cell_measure, dims in self.cube_kwargs["cell_measures_and_dims"]: + new_cube.add_cell_measure(cell_measure, dims) + for ancillary_variable, dims in self.cube_kwargs[ + "ancillary_variables_and_dims" + ]: + new_cube.add_ancillary_variable(ancillary_variable, dims) + + else: + message = f"Unknown cube creation strategy: {cube_creation_strategy}" + raise NotImplementedError(message) + + +class CubeEquality: params = [ - 6, # minimal cube-sphere - int(1e6), # realistic cube-sphere size - ARTIFICIAL_DIM_SIZE, # To match size in :class:`AuxCoord` + [False, True], + [False, True], + ["metadata_inequality", "coord_inequality", "data_inequality", "all_equal"], ] - param_names = ["number of faces"] - - def setup(self, n_faces): - mesh_kwargs = dict(n_nodes=n_faces + 2, n_edges=n_faces * 2, n_faces=n_faces) - - self.mesh_coord = sample_meshcoord(sample_mesh_kwargs=mesh_kwargs) - self.data = np.zeros(n_faces) - self.cube_blank = cube.Cube(data=self.data) - self.cube = self.create() - - def create(self): - return cube.Cube(data=self.data, aux_coords_and_dims=[(self.mesh_coord, 0)]) - - def time_create(self, n_faces): - _ = self.create() - - @disable_repeat_between_setup - def time_add(self, n_faces): - self.cube_blank.add_aux_coord(self.mesh_coord, 0) - - @disable_repeat_between_setup - def time_remove(self, n_faces): - self.cube.remove_coord(self.mesh_coord) - - -class Merge: - def setup(self): - self.cube_list = cube.CubeList() - for i in np.arange(2): - i_cube = general_cube.copy() - i_coord = coords.AuxCoord([i]) - i_cube.add_aux_coord(i_coord) - self.cube_list.append(i_cube) - - def time_merge(self): - self.cube_list.merge() - - -class Concatenate: - def setup(self): - dim_size = ARTIFICIAL_DIM_SIZE - self.cube_list = cube.CubeList() - for i in np.arange(dim_size * 2, step=dim_size): - i_cube = general_cube.copy() - i_coord = coords.DimCoord(np.arange(dim_size) + (i * dim_size)) - i_cube.add_dim_coord(i_coord, 0) - self.cube_list.append(i_cube) - - def time_concatenate(self): - self.cube_list.concatenate() - - -class Equality: - def setup(self): - self.cube_a = general_cube.copy() - self.cube_b = general_cube.copy() - - aux_coord = coords.AuxCoord(data_1d) - self.cube_a.add_aux_coord(aux_coord, 0) - self.cube_b.add_aux_coord(aux_coord, 1) - - def time_equality(self): - self.cube_a == self.cube_b - - -class Aggregation: - def setup(self): - repeat_number = 10 - repeat_range = range(int(ARTIFICIAL_DIM_SIZE / repeat_number)) - array_repeat = np.repeat(repeat_range, repeat_number) - array_unique = np.arange(len(array_repeat)) - - coord_repeat = coords.AuxCoord(points=array_repeat, long_name="repeat") - coord_unique = coords.DimCoord(points=array_unique, long_name="unique") - - local_cube = general_cube.copy() - local_cube.add_aux_coord(coord_repeat, 0) - local_cube.add_dim_coord(coord_unique, 0) - self.cube = local_cube - - def time_aggregated_by(self): - self.cube.aggregated_by("repeat", analysis.MEAN) + param_names = ["Cubes are lazy", "Cubes have meshes", "Scenario"] + + cube_1: Cube + cube_2: Cube + coord_name = "surface_altitude" + + def setup(self, lazy: bool, w_mesh: bool, scenario: str) -> None: + self.cube_1 = realistic_4d_w_everything(w_mesh=w_mesh, lazy=lazy) + # Using Cube.copy() produces different results due to sharing of the + # Mesh instance. + self.cube_2 = realistic_4d_w_everything(w_mesh=w_mesh, lazy=lazy) + + match scenario: + case "metadata_inequality": + self.cube_2.long_name = "different" + case "coord_inequality": + coord = self.cube_2.coord(self.coord_name) + coord.points = coord.core_points() * 2 + case "data_inequality": + self.cube_2.data = self.cube_2.core_data() * 2 + case "all_equal": + pass + case _: + message = f"Unknown scenario: {scenario}" + raise NotImplementedError(message) + + def time_equality(self, lazy: bool, __, ___) -> None: + _ = self.cube_1 == self.cube_2 + if lazy: + for cube in (self.cube_1, self.cube_2): + # Confirm that this benchmark is safe for repetition. + assert cube.coord(self.coord_name).has_lazy_points() + assert cube.has_lazy_data() diff --git a/benchmarks/benchmarks/experimental/ugrid/__init__.py b/benchmarks/benchmarks/experimental/ugrid/__init__.py index c2335990aa..4976054178 100644 --- a/benchmarks/benchmarks/experimental/ugrid/__init__.py +++ b/benchmarks/benchmarks/experimental/ugrid/__init__.py @@ -3,186 +3,3 @@ # This file is part of Iris and is released under the BSD license. # See LICENSE in the root of the repository for full licensing details. """Benchmark tests for the experimental.ugrid module.""" - -from copy import deepcopy - -import numpy as np - -from iris.experimental import ugrid - -from ... import ARTIFICIAL_DIM_SIZE, disable_repeat_between_setup -from ...generate_data.stock import sample_mesh - - -class UGridCommon: - """Run a generalised suite of benchmarks for any ugrid object. - - A base class running a generalised suite of benchmarks for any ugrid object. - Object to be specified in a subclass. - - ASV will run the benchmarks within this class for any subclasses. - - ASV will not benchmark this class as setup() triggers a NotImplementedError. - (ASV has not yet released ABC/abstractmethod support - asv#838). - - """ - - params = [ - 6, # minimal cube-sphere - int(1e6), # realistic cube-sphere size - ] - param_names = ["number of faces"] - - def setup(self, *params): - self.object = self.create() - - def create(self): - raise NotImplementedError - - def time_create(self, *params): - """Create an instance of the benchmarked object. - - create() method is specified in the subclass. - """ - self.create() - - -class Connectivity(UGridCommon): - def setup(self, n_faces): - self.array = np.zeros([n_faces, 3], dtype=int) - super().setup(n_faces) - - def create(self): - return ugrid.Connectivity(indices=self.array, cf_role="face_node_connectivity") - - def time_indices(self, n_faces): - _ = self.object.indices - - def time_location_lengths(self, n_faces): - # Proofed against the Connectivity name change (633ed17). - if getattr(self.object, "src_lengths", False): - meth = self.object.src_lengths - else: - meth = self.object.location_lengths - _ = meth() - - def time_validate_indices(self, n_faces): - self.object.validate_indices() - - -@disable_repeat_between_setup -class ConnectivityLazy(Connectivity): - """Lazy equivalent of :class:`Connectivity`.""" - - def setup(self, n_faces): - super().setup(n_faces) - self.array = self.object.lazy_indices() - self.object = self.create() - - -class Mesh(UGridCommon): - def setup(self, n_faces, lazy=False): - #### - # Steal everything from the sample mesh for benchmarking creation of a - # brand new mesh. - source_mesh = sample_mesh( - n_nodes=n_faces + 2, - n_edges=n_faces * 2, - n_faces=n_faces, - lazy_values=lazy, - ) - - def get_coords_and_axes(location): - search_kwargs = {f"include_{location}s": True} - return [ - (source_mesh.coord(axis=axis, **search_kwargs), axis) - for axis in ("x", "y") - ] - - self.mesh_kwargs = dict( - topology_dimension=source_mesh.topology_dimension, - node_coords_and_axes=get_coords_and_axes("node"), - connectivities=source_mesh.connectivities(), - edge_coords_and_axes=get_coords_and_axes("edge"), - face_coords_and_axes=get_coords_and_axes("face"), - ) - #### - - super().setup(n_faces) - - self.face_node = self.object.face_node_connectivity - self.node_x = self.object.node_coords.node_x - # Kwargs for reuse in search and remove methods. - self.connectivities_kwarg = dict(cf_role="edge_node_connectivity") - self.coords_kwarg = dict(include_faces=True) - - # TODO: an opportunity for speeding up runtime if needed, since - # eq_object is not needed for all benchmarks. Just don't generate it - # within a benchmark - the execution time is large enough that it - # could be a significant portion of the benchmark - makes regressions - # smaller and could even pick up regressions in copying instead! - self.eq_object = deepcopy(self.object) - - def create(self): - return ugrid.Mesh(**self.mesh_kwargs) - - def time_add_connectivities(self, n_faces): - self.object.add_connectivities(self.face_node) - - def time_add_coords(self, n_faces): - self.object.add_coords(node_x=self.node_x) - - def time_connectivities(self, n_faces): - _ = self.object.connectivities(**self.connectivities_kwarg) - - def time_coords(self, n_faces): - _ = self.object.coords(**self.coords_kwarg) - - def time_eq(self, n_faces): - _ = self.object == self.eq_object - - def time_remove_connectivities(self, n_faces): - self.object.remove_connectivities(**self.connectivities_kwarg) - - def time_remove_coords(self, n_faces): - self.object.remove_coords(**self.coords_kwarg) - - -@disable_repeat_between_setup -class MeshLazy(Mesh): - """Lazy equivalent of :class:`Mesh`.""" - - def setup(self, n_faces, lazy=True): - super().setup(n_faces, lazy=lazy) - - -class MeshCoord(UGridCommon): - # Add extra parameter value to match AuxCoord benchmarking. - params = UGridCommon.params + [ARTIFICIAL_DIM_SIZE] - - def setup(self, n_faces, lazy=False): - self.mesh = sample_mesh( - n_nodes=n_faces + 2, - n_edges=n_faces * 2, - n_faces=n_faces, - lazy_values=lazy, - ) - - super().setup(n_faces) - - def create(self): - return ugrid.MeshCoord(mesh=self.mesh, location="face", axis="x") - - def time_points(self, n_faces): - _ = self.object.points - - def time_bounds(self, n_faces): - _ = self.object.bounds - - -@disable_repeat_between_setup -class MeshCoordLazy(MeshCoord): - """Lazy equivalent of :class:`MeshCoord`.""" - - def setup(self, n_faces, lazy=True): - super().setup(n_faces, lazy=lazy) diff --git a/benchmarks/benchmarks/experimental/ugrid/regions_combine.py b/benchmarks/benchmarks/experimental/ugrid/regions_combine.py index 6657e70056..409d6961c3 100644 --- a/benchmarks/benchmarks/experimental/ugrid/regions_combine.py +++ b/benchmarks/benchmarks/experimental/ugrid/regions_combine.py @@ -7,14 +7,6 @@ Benchmarks stages of operation of the function :func:`iris.experimental.ugrid.utils.recombine_submeshes`. -Where possible benchmarks should be parameterised for two sizes of input data: - -* minimal: enables detection of regressions in parts of the run-time that do - NOT scale with data size. - -* large: large enough to exclusively detect regressions in parts of the - run-time that scale with data size. - """ import os diff --git a/benchmarks/benchmarks/generate_data/stock.py b/benchmarks/benchmarks/generate_data/stock.py index 17f3b23f92..8b29597629 100644 --- a/benchmarks/benchmarks/generate_data/stock.py +++ b/benchmarks/benchmarks/generate_data/stock.py @@ -7,10 +7,12 @@ See :mod:`benchmarks.generate_data` for an explanation of this structure. """ +from contextlib import nullcontext from hashlib import sha256 import json from pathlib import Path +import iris from iris.experimental.ugrid import PARSE_UGRID_ON_LOAD, load_mesh from . import BENCHMARK_DATA, REUSE_DATA, load_realised, run_function_elsewhere @@ -149,3 +151,35 @@ def _external(sample_mesh_kwargs_, save_path_): source_mesh = load_mesh(str(save_path)) # Regenerate MeshCoord from its Mesh, which we saved. return source_mesh.to_MeshCoord(location=location, axis=axis) + + +def realistic_4d_w_everything(w_mesh=False, lazy=False): + """Run :func:`iris.tests.stock.realistic_4d_w_everything` in ``DATA_GEN_PYTHON``. + + Parameters + ---------- + w_mesh : bool + See :func:`iris.tests.stock.realistic_4d_w_everything` for details. + lazy : bool + If True, the Cube will be returned with all arrays as they would + normally be loaded from file (i.e. most will still be lazy Dask + arrays). If False, all arrays will be realised NumPy arrays. + + """ + + def _external(w_mesh_: str, save_path_: str): + import iris + from iris.tests.stock import realistic_4d_w_everything + + cube = realistic_4d_w_everything(w_mesh=bool(w_mesh_)) + iris.save(cube, save_path_) + + save_path = (BENCHMARK_DATA / f"realistic_4d_w_everything_{w_mesh}").with_suffix( + ".nc" + ) + if not REUSE_DATA or not save_path.is_file(): + _ = run_function_elsewhere(_external, w_mesh_=w_mesh, save_path_=str(save_path)) + with PARSE_UGRID_ON_LOAD.context(): + context = nullcontext() if lazy else load_realised() + with context: + return iris.load_cube(save_path, "air_potential_temperature") diff --git a/benchmarks/benchmarks/iterate.py b/benchmarks/benchmarks/iterate.py index 3716602be1..664bcf8ba2 100644 --- a/benchmarks/benchmarks/iterate.py +++ b/benchmarks/benchmarks/iterate.py @@ -8,23 +8,12 @@ from iris import coords, cube, iterate -from . import ARTIFICIAL_DIM_SIZE - - -def setup(): - """General variables needed by multiple benchmark classes.""" - global data_1d - global data_2d - global general_cube - - data_2d = np.zeros((ARTIFICIAL_DIM_SIZE,) * 2) - data_1d = data_2d[0] - general_cube = cube.Cube(data_2d) - class IZip: def setup(self): - local_cube = general_cube.copy() + data_2d = np.zeros((1000,) * 2) + data_1d = data_2d[0] + local_cube = cube.Cube(data_2d) coord_a = coords.AuxCoord(points=data_1d, long_name="a") coord_b = coords.AuxCoord(points=data_1d, long_name="b") self.coord_names = (coord.long_name for coord in (coord_a, coord_b)) diff --git a/benchmarks/benchmarks/load/__init__.py b/benchmarks/benchmarks/load/__init__.py index c977e924af..80d80df384 100644 --- a/benchmarks/benchmarks/load/__init__.py +++ b/benchmarks/benchmarks/load/__init__.py @@ -2,16 +2,7 @@ # # This file is part of Iris and is released under the BSD license. # See LICENSE in the root of the repository for full licensing details. -"""File loading benchmark tests. - -Where applicable benchmarks should be parameterised for two sizes of input data: - * minimal: enables detection of regressions in parts of the run-time that do - NOT scale with data size. - * large: large enough to exclusively detect regressions in parts of the - run-time that scale with data size. Size should be _just_ large - enough - don't want to bloat benchmark runtime. - -""" +"""File loading benchmark tests.""" from iris import AttributeConstraint, Constraint, load, load_cube from iris.cube import Cube diff --git a/benchmarks/benchmarks/load/ugrid.py b/benchmarks/benchmarks/load/ugrid.py index 626b746412..47e23dc050 100644 --- a/benchmarks/benchmarks/load/ugrid.py +++ b/benchmarks/benchmarks/load/ugrid.py @@ -2,15 +2,7 @@ # # This file is part of Iris and is released under the BSD license. # See LICENSE in the root of the repository for full licensing details. -"""Mesh data loading benchmark tests. - -Where possible benchmarks should be parameterised for two sizes of input data: - * minimal: enables detection of regressions in parts of the run-time that do - NOT scale with data size. - * large: large enough to exclusively detect regressions in parts of the - run-time that scale with data size. - -""" +"""Mesh data loading benchmark tests.""" from iris import load_cube as iris_load_cube from iris.experimental.ugrid import PARSE_UGRID_ON_LOAD diff --git a/benchmarks/benchmarks/merge_concat.py b/benchmarks/benchmarks/merge_concat.py new file mode 100644 index 0000000000..315d90f575 --- /dev/null +++ b/benchmarks/benchmarks/merge_concat.py @@ -0,0 +1,52 @@ +# Copyright Iris contributors +# +# This file is part of Iris and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. +"""Benchmarks relating to :meth:`iris.cube.CubeList.merge` and ``concatenate``.""" + +import numpy as np + +from iris.cube import CubeList + +from .generate_data.stock import realistic_4d_w_everything + + +class Merge: + # TODO: Improve coverage. + + cube_list: CubeList + + def setup(self): + source_cube = realistic_4d_w_everything() + + # Merge does not yet fully support cell measures and ancillary variables. + for cm in source_cube.cell_measures(): + source_cube.remove_cell_measure(cm) + for av in source_cube.ancillary_variables(): + source_cube.remove_ancillary_variable(av) + + second_cube = source_cube.copy() + scalar_coord = second_cube.coords(dimensions=[])[0] + scalar_coord.points = scalar_coord.points + 1 + self.cube_list = CubeList([source_cube, second_cube]) + + def time_merge(self): + _ = self.cube_list.merge_cube() + + +class Concatenate: + # TODO: Improve coverage. + + cube_list: CubeList + + def setup(self): + source_cube = realistic_4d_w_everything() + second_cube = source_cube.copy() + first_dim_coord = second_cube.coord(dimensions=0, dim_coords=True) + first_dim_coord.points = ( + first_dim_coord.points + np.ptp(first_dim_coord.points) + 1 + ) + self.cube_list = CubeList([source_cube, second_cube]) + + def time_concatenate(self): + _ = self.cube_list.concatenate_cube() diff --git a/benchmarks/benchmarks/plot.py b/benchmarks/benchmarks/plot.py index 681d8ef9dd..e8fbb5372d 100644 --- a/benchmarks/benchmarks/plot.py +++ b/benchmarks/benchmarks/plot.py @@ -9,8 +9,6 @@ from iris import coords, cube, plot -from . import ARTIFICIAL_DIM_SIZE - mpl.use("agg") @@ -18,7 +16,7 @@ class AuxSort: def setup(self): # Manufacture data from which contours can be derived. # Should generate 10 distinct contours, regardless of dim size. - dim_size = int(ARTIFICIAL_DIM_SIZE / 5) + dim_size = 200 repeat_number = int(dim_size / 10) repeat_range = range(int((dim_size**2) / repeat_number)) data = np.repeat(repeat_range, repeat_number) diff --git a/benchmarks/benchmarks/save.py b/benchmarks/benchmarks/save.py index f2a2611eae..9936331621 100644 --- a/benchmarks/benchmarks/save.py +++ b/benchmarks/benchmarks/save.py @@ -2,15 +2,7 @@ # # This file is part of Iris and is released under the BSD license. # See LICENSE in the root of the repository for full licensing details. -"""File saving benchmarks. - -Where possible benchmarks should be parameterised for two sizes of input data: - * minimal: enables detection of regressions in parts of the run-time that do - NOT scale with data size. - * large: large enough to exclusively detect regressions in parts of the - run-time that scale with data size. - -""" +"""File saving benchmarks.""" from iris import save from iris.experimental.ugrid import save_mesh diff --git a/benchmarks/benchmarks/unit_style/__init__disabled.py b/benchmarks/benchmarks/unit_style/__init__disabled.py new file mode 100644 index 0000000000..d7f84c2b91 --- /dev/null +++ b/benchmarks/benchmarks/unit_style/__init__disabled.py @@ -0,0 +1,16 @@ +# Copyright Iris contributors +# +# This file is part of Iris and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. +"""Small-scope benchmarks that can help with performance investigations. + +By renaming ``__init__.py`` these are all disabled by default: + +- They bloat benchmark run-time. +- They are too vulnerable to 'noise' due to their small scope - small objects, + short operations - they report a lot of false positive regressions. +- We rely on the wider-scope integration-style benchmarks to flag performance + changes, upon which we expect to do some manual investigation - these + smaller benchmarks can be run then. + +""" diff --git a/benchmarks/benchmarks/aux_factory.py b/benchmarks/benchmarks/unit_style/aux_factory.py similarity index 93% rename from benchmarks/benchmarks/aux_factory.py rename to benchmarks/benchmarks/unit_style/aux_factory.py index 2da93351ee..329a2b0bda 100644 --- a/benchmarks/benchmarks/aux_factory.py +++ b/benchmarks/benchmarks/unit_style/aux_factory.py @@ -2,14 +2,12 @@ # # This file is part of Iris and is released under the BSD license. # See LICENSE in the root of the repository for full licensing details. -"""AuxFactory benchmark tests.""" +"""Small-scope AuxFactory benchmark tests.""" import numpy as np from iris import aux_factory, coords -from . import ARTIFICIAL_DIM_SIZE - class FactoryCommon: # TODO: once https://github.com/airspeed-velocity/asv/pull/828 is released: @@ -45,7 +43,7 @@ def time_create(self): class HybridHeightFactory(FactoryCommon): def setup(self): - data_1d = np.zeros(ARTIFICIAL_DIM_SIZE) + data_1d = np.zeros(1000) self.coord = coords.AuxCoord(points=data_1d, units="m") self.setup_common() diff --git a/benchmarks/benchmarks/coords.py b/benchmarks/benchmarks/unit_style/coords.py similarity index 94% rename from benchmarks/benchmarks/coords.py rename to benchmarks/benchmarks/unit_style/coords.py index d1f7631e00..704746f190 100644 --- a/benchmarks/benchmarks/coords.py +++ b/benchmarks/benchmarks/unit_style/coords.py @@ -2,20 +2,20 @@ # # This file is part of Iris and is released under the BSD license. # See LICENSE in the root of the repository for full licensing details. -"""Coord benchmark tests.""" +"""Small-scope Coord benchmark tests.""" import numpy as np from iris import coords -from . import ARTIFICIAL_DIM_SIZE, disable_repeat_between_setup +from .. import disable_repeat_between_setup def setup(): """General variables needed by multiple benchmark classes.""" global data_1d - data_1d = np.zeros(ARTIFICIAL_DIM_SIZE) + data_1d = np.zeros(1000) class CoordCommon: @@ -52,7 +52,7 @@ def time_create(self): class DimCoord(CoordCommon): def setup(self): - point_values = np.arange(ARTIFICIAL_DIM_SIZE) + point_values = np.arange(1000) bounds = np.array([point_values - 1, point_values + 1]).transpose() self.create_kwargs = { diff --git a/benchmarks/benchmarks/unit_style/cube.py b/benchmarks/benchmarks/unit_style/cube.py new file mode 100644 index 0000000000..780418aa14 --- /dev/null +++ b/benchmarks/benchmarks/unit_style/cube.py @@ -0,0 +1,252 @@ +# Copyright Iris contributors +# +# This file is part of Iris and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. +"""Small-scope Cube benchmark tests.""" + +import numpy as np + +from iris import analysis, aux_factory, coords, cube + +from .. import disable_repeat_between_setup +from ..generate_data.stock import sample_meshcoord + + +def setup(*params): + """General variables needed by multiple benchmark classes.""" + global data_1d + global data_2d + global general_cube + + data_2d = np.zeros((1000,) * 2) + data_1d = data_2d[0] + general_cube = cube.Cube(data_2d) + + +class ComponentCommon: + # TODO: once https://github.com/airspeed-velocity/asv/pull/828 is released: + # * make class an ABC + # * remove NotImplementedError + # * combine setup_common into setup + """Run a generalised suite of benchmarks for cubes. + + A base class running a generalised suite of benchmarks for cubes that + include a specified component (e.g. Coord, CellMeasure etc.). Component to + be specified in a subclass. + + ASV will run the benchmarks within this class for any subclasses. + + Should only be instantiated within subclasses, but cannot enforce this + since ASV cannot handle classes that include abstract methods. + """ + + def setup(self): + """Prevent ASV instantiating (must therefore override setup() in any subclasses.).""" + raise NotImplementedError + + def create(self): + """Create a cube (generic). + + cube_kwargs allow dynamic inclusion of different components; + specified in subclasses. + """ + return cube.Cube(data=data_2d, **self.cube_kwargs) + + def setup_common(self): + """Shared setup code that can be called by subclasses.""" + self.cube = self.create() + + def time_create(self): + """Create a cube that includes an instance of the benchmarked component.""" + self.create() + + def time_add(self): + """Add an instance of the benchmarked component to an existing cube.""" + # Unable to create the copy during setup since this needs to be re-done + # for every repeat of the test (some components disallow duplicates). + general_cube_copy = general_cube.copy(data=data_2d) + self.add_method(general_cube_copy, *self.add_args) + + +class Cube: + def time_basic(self): + cube.Cube(data_2d) + + def time_rename(self): + general_cube.name = "air_temperature" + + +class AuxCoord(ComponentCommon): + def setup(self): + self.coord_name = "test" + coord_bounds = np.array([data_1d - 1, data_1d + 1]).transpose() + aux_coord = coords.AuxCoord( + long_name=self.coord_name, + points=data_1d, + bounds=coord_bounds, + units="days since 1970-01-01", + climatological=True, + ) + + # Variables needed by the ComponentCommon base class. + self.cube_kwargs = {"aux_coords_and_dims": [(aux_coord, 0)]} + self.add_method = cube.Cube.add_aux_coord + self.add_args = (aux_coord, (0)) + + self.setup_common() + + def time_return_coords(self): + self.cube.coords() + + def time_return_coord_dims(self): + self.cube.coord_dims(self.coord_name) + + +class AuxFactory(ComponentCommon): + def setup(self): + coord = coords.AuxCoord(points=data_1d, units="m") + self.hybrid_factory = aux_factory.HybridHeightFactory(delta=coord) + + # Variables needed by the ComponentCommon base class. + self.cube_kwargs = { + "aux_coords_and_dims": [(coord, 0)], + "aux_factories": [self.hybrid_factory], + } + + self.setup_common() + + # Variables needed by the overridden time_add benchmark in this subclass. + cube_w_coord = self.cube.copy() + [cube_w_coord.remove_aux_factory(i) for i in cube_w_coord.aux_factories] + self.cube_w_coord = cube_w_coord + + def time_add(self): + # Requires override from super().time_add because the cube needs an + # additional coord. + self.cube_w_coord.add_aux_factory(self.hybrid_factory) + + +class CellMeasure(ComponentCommon): + def setup(self): + cell_measure = coords.CellMeasure(data_1d) + + # Variables needed by the ComponentCommon base class. + self.cube_kwargs = {"cell_measures_and_dims": [(cell_measure, 0)]} + self.add_method = cube.Cube.add_cell_measure + self.add_args = (cell_measure, 0) + + self.setup_common() + + +class CellMethod(ComponentCommon): + def setup(self): + cell_method = coords.CellMethod("test") + + # Variables needed by the ComponentCommon base class. + self.cube_kwargs = {"cell_methods": [cell_method]} + self.add_method = cube.Cube.add_cell_method + self.add_args = [cell_method] + + self.setup_common() + + +class AncillaryVariable(ComponentCommon): + def setup(self): + ancillary_variable = coords.AncillaryVariable(data_1d) + + # Variables needed by the ComponentCommon base class. + self.cube_kwargs = {"ancillary_variables_and_dims": [(ancillary_variable, 0)]} + self.add_method = cube.Cube.add_ancillary_variable + self.add_args = (ancillary_variable, 0) + + self.setup_common() + + +class MeshCoord: + params = [ + 6, # minimal cube-sphere + int(1e6), # realistic cube-sphere size + 1000, # To match size in :class:`AuxCoord` + ] + param_names = ["number of faces"] + + def setup(self, n_faces): + mesh_kwargs = dict(n_nodes=n_faces + 2, n_edges=n_faces * 2, n_faces=n_faces) + + self.mesh_coord = sample_meshcoord(sample_mesh_kwargs=mesh_kwargs) + self.data = np.zeros(n_faces) + self.cube_blank = cube.Cube(data=self.data) + self.cube = self.create() + + def create(self): + return cube.Cube(data=self.data, aux_coords_and_dims=[(self.mesh_coord, 0)]) + + def time_create(self, n_faces): + _ = self.create() + + @disable_repeat_between_setup + def time_add(self, n_faces): + self.cube_blank.add_aux_coord(self.mesh_coord, 0) + + @disable_repeat_between_setup + def time_remove(self, n_faces): + self.cube.remove_coord(self.mesh_coord) + + +class Merge: + def setup(self): + self.cube_list = cube.CubeList() + for i in np.arange(2): + i_cube = general_cube.copy() + i_coord = coords.AuxCoord([i]) + i_cube.add_aux_coord(i_coord) + self.cube_list.append(i_cube) + + def time_merge(self): + self.cube_list.merge() + + +class Concatenate: + def setup(self): + dim_size = 1000 + self.cube_list = cube.CubeList() + for i in np.arange(dim_size * 2, step=dim_size): + i_cube = general_cube.copy() + i_coord = coords.DimCoord(np.arange(dim_size) + (i * dim_size)) + i_cube.add_dim_coord(i_coord, 0) + self.cube_list.append(i_cube) + + def time_concatenate(self): + self.cube_list.concatenate() + + +class Equality: + def setup(self): + self.cube_a = general_cube.copy() + self.cube_b = general_cube.copy() + + aux_coord = coords.AuxCoord(data_1d) + self.cube_a.add_aux_coord(aux_coord, 0) + self.cube_b.add_aux_coord(aux_coord, 1) + + def time_equality(self): + self.cube_a == self.cube_b + + +class Aggregation: + def setup(self): + repeat_number = 10 + repeat_range = range(int(1000 / repeat_number)) + array_repeat = np.repeat(repeat_range, repeat_number) + array_unique = np.arange(len(array_repeat)) + + coord_repeat = coords.AuxCoord(points=array_repeat, long_name="repeat") + coord_unique = coords.DimCoord(points=array_unique, long_name="unique") + + local_cube = general_cube.copy() + local_cube.add_aux_coord(coord_repeat, 0) + local_cube.add_dim_coord(coord_unique, 0) + self.cube = local_cube + + def time_aggregated_by(self): + self.cube.aggregated_by("repeat", analysis.MEAN) diff --git a/benchmarks/benchmarks/metadata_manager_factory.py b/benchmarks/benchmarks/unit_style/metadata_manager_factory.py similarity index 97% rename from benchmarks/benchmarks/metadata_manager_factory.py rename to benchmarks/benchmarks/unit_style/metadata_manager_factory.py index cd50a767a1..0af055fa82 100644 --- a/benchmarks/benchmarks/metadata_manager_factory.py +++ b/benchmarks/benchmarks/unit_style/metadata_manager_factory.py @@ -2,7 +2,7 @@ # # This file is part of Iris and is released under the BSD license. # See LICENSE in the root of the repository for full licensing details. -"""Metadata manager factory benchmark tests.""" +"""Small-scope metadata manager factory benchmark tests.""" from iris.common import ( AncillaryVariableMetadata, diff --git a/benchmarks/benchmarks/mixin.py b/benchmarks/benchmarks/unit_style/mixin.py similarity index 94% rename from benchmarks/benchmarks/mixin.py rename to benchmarks/benchmarks/unit_style/mixin.py index 90fb017b12..92de5e7ad9 100644 --- a/benchmarks/benchmarks/mixin.py +++ b/benchmarks/benchmarks/unit_style/mixin.py @@ -2,15 +2,13 @@ # # This file is part of Iris and is released under the BSD license. # See LICENSE in the root of the repository for full licensing details. -"""Mixin benchmark tests.""" +"""Small-scope CFVariableMixin benchmark tests.""" import numpy as np from iris import coords from iris.common.metadata import AncillaryVariableMetadata -from . import ARTIFICIAL_DIM_SIZE - LONG_NAME = "air temperature" STANDARD_NAME = "air_temperature" VAR_NAME = "air_temp" @@ -29,7 +27,7 @@ class CFVariableMixin: def setup(self): - data_1d = np.zeros(ARTIFICIAL_DIM_SIZE) + data_1d = np.zeros(1000) # These benchmarks are from a user perspective, so using a user-level # subclass of CFVariableMixin to test behaviour. AncillaryVariable is diff --git a/benchmarks/benchmarks/unit_style/ugrid.py b/benchmarks/benchmarks/unit_style/ugrid.py new file mode 100644 index 0000000000..f29ae09015 --- /dev/null +++ b/benchmarks/benchmarks/unit_style/ugrid.py @@ -0,0 +1,188 @@ +# Copyright Iris contributors +# +# This file is part of Iris and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. +"""Benchmark tests for the experimental.ugrid module.""" + +from copy import deepcopy + +import numpy as np + +from iris.experimental import ugrid + +from .. import disable_repeat_between_setup +from ..generate_data.stock import sample_mesh + + +class UGridCommon: + """Run a generalised suite of benchmarks for any ugrid object. + + A base class running a generalised suite of benchmarks for any ugrid object. + Object to be specified in a subclass. + + ASV will run the benchmarks within this class for any subclasses. + + ASV will not benchmark this class as setup() triggers a NotImplementedError. + (ASV has not yet released ABC/abstractmethod support - asv#838). + + """ + + params = [ + 6, # minimal cube-sphere + int(1e6), # realistic cube-sphere size + ] + param_names = ["number of faces"] + + def setup(self, *params): + self.object = self.create() + + def create(self): + raise NotImplementedError + + def time_create(self, *params): + """Create an instance of the benchmarked object. + + create() method is specified in the subclass. + """ + self.create() + + +class Connectivity(UGridCommon): + def setup(self, n_faces): + self.array = np.zeros([n_faces, 3], dtype=int) + super().setup(n_faces) + + def create(self): + return ugrid.Connectivity(indices=self.array, cf_role="face_node_connectivity") + + def time_indices(self, n_faces): + _ = self.object.indices + + def time_location_lengths(self, n_faces): + # Proofed against the Connectivity name change (633ed17). + if getattr(self.object, "src_lengths", False): + meth = self.object.src_lengths + else: + meth = self.object.location_lengths + _ = meth() + + def time_validate_indices(self, n_faces): + self.object.validate_indices() + + +@disable_repeat_between_setup +class ConnectivityLazy(Connectivity): + """Lazy equivalent of :class:`Connectivity`.""" + + def setup(self, n_faces): + super().setup(n_faces) + self.array = self.object.lazy_indices() + self.object = self.create() + + +class Mesh(UGridCommon): + def setup(self, n_faces, lazy=False): + #### + # Steal everything from the sample mesh for benchmarking creation of a + # brand new mesh. + source_mesh = sample_mesh( + n_nodes=n_faces + 2, + n_edges=n_faces * 2, + n_faces=n_faces, + lazy_values=lazy, + ) + + def get_coords_and_axes(location): + search_kwargs = {f"include_{location}s": True} + return [ + (source_mesh.coord(axis=axis, **search_kwargs), axis) + for axis in ("x", "y") + ] + + self.mesh_kwargs = dict( + topology_dimension=source_mesh.topology_dimension, + node_coords_and_axes=get_coords_and_axes("node"), + connectivities=source_mesh.connectivities(), + edge_coords_and_axes=get_coords_and_axes("edge"), + face_coords_and_axes=get_coords_and_axes("face"), + ) + #### + + super().setup(n_faces) + + self.face_node = self.object.face_node_connectivity + self.node_x = self.object.node_coords.node_x + # Kwargs for reuse in search and remove methods. + self.connectivities_kwarg = dict(cf_role="edge_node_connectivity") + self.coords_kwarg = dict(include_faces=True) + + # TODO: an opportunity for speeding up runtime if needed, since + # eq_object is not needed for all benchmarks. Just don't generate it + # within a benchmark - the execution time is large enough that it + # could be a significant portion of the benchmark - makes regressions + # smaller and could even pick up regressions in copying instead! + self.eq_object = deepcopy(self.object) + + def create(self): + return ugrid.Mesh(**self.mesh_kwargs) + + def time_add_connectivities(self, n_faces): + self.object.add_connectivities(self.face_node) + + def time_add_coords(self, n_faces): + self.object.add_coords(node_x=self.node_x) + + def time_connectivities(self, n_faces): + _ = self.object.connectivities(**self.connectivities_kwarg) + + def time_coords(self, n_faces): + _ = self.object.coords(**self.coords_kwarg) + + def time_eq(self, n_faces): + _ = self.object == self.eq_object + + def time_remove_connectivities(self, n_faces): + self.object.remove_connectivities(**self.connectivities_kwarg) + + def time_remove_coords(self, n_faces): + self.object.remove_coords(**self.coords_kwarg) + + +@disable_repeat_between_setup +class MeshLazy(Mesh): + """Lazy equivalent of :class:`Mesh`.""" + + def setup(self, n_faces, lazy=True): + super().setup(n_faces, lazy=lazy) + + +class MeshCoord(UGridCommon): + # Add extra parameter value to match AuxCoord benchmarking. + params = UGridCommon.params + [1000] + + def setup(self, n_faces, lazy=False): + self.mesh = sample_mesh( + n_nodes=n_faces + 2, + n_edges=n_faces * 2, + n_faces=n_faces, + lazy_values=lazy, + ) + + super().setup(n_faces) + + def create(self): + return ugrid.MeshCoord(mesh=self.mesh, location="face", axis="x") + + def time_points(self, n_faces): + _ = self.object.points + + def time_bounds(self, n_faces): + _ = self.object.bounds + + +@disable_repeat_between_setup +class MeshCoordLazy(MeshCoord): + """Lazy equivalent of :class:`MeshCoord`.""" + + def setup(self, n_faces, lazy=True): + super().setup(n_faces, lazy=lazy) diff --git a/docs/src/whatsnew/latest.rst b/docs/src/whatsnew/latest.rst index 448b528c38..206bf0ba9e 100644 --- a/docs/src/whatsnew/latest.rst +++ b/docs/src/whatsnew/latest.rst @@ -94,6 +94,14 @@ This document explains the changes made to Iris for this release deprecated use of the ``conda --force`` argument. To be removed once `airspeed-velocity/asv#1397`_ is merged and released. (:pull:`5931`) +#. `@trexfeathers`_ created :func:`iris.tests.stock.realistic_4d_w_everything`; + providing a :class:`~iris.cube.Cube` aimed to exercise as much of Iris as + possible. (:pull:`5949`) + +#. `@trexfeathers`_ deactivated any small 'unit-style' benchmarks for default + benchmark runs, and introduced larger more 'real world' benchmarks where + coverage was needed. (:pull:`5949`). + .. comment Whatsnew author names (@github name) in alphabetical order. Note that, diff --git a/lib/iris/tests/stock/__init__.py b/lib/iris/tests/stock/__init__.py index ea513b967f..822dd6a6c1 100644 --- a/lib/iris/tests/stock/__init__.py +++ b/lib/iris/tests/stock/__init__.py @@ -8,17 +8,22 @@ from datetime import datetime import os.path +from typing import NamedTuple +from cartopy.crs import CRS from cf_units import Unit import numpy as np import numpy.ma as ma +from iris.analysis import cartography import iris.aux_factory from iris.coord_systems import GeogCS, RotatedGeogCS import iris.coords import iris.coords as icoords from iris.coords import AncillaryVariable, AuxCoord, CellMeasure, CellMethod, DimCoord from iris.cube import Cube +from iris.experimental import ugrid +from iris.util import mask_cube from ._stock_2d_latlons import ( # noqa make_bounds_discontiguous_at_point, @@ -713,6 +718,268 @@ def realistic_4d_w_missing_data(): return cube +def realistic_4d_w_everything(w_mesh=False): + """Returns a cube that will exercise as much of Iris as possible. + + Uses :func:`realistic_4d` as a basis, then modifies accordingly. + + Parameters + ---------- + w_mesh : bool, optional + If True, the horizontal grid will be replaced with a mesh representation. + """ + cube = realistic_4d() + + grid_lon, grid_lat = [cube.coord(n) for n in ("grid_longitude", "grid_latitude")] + (lon_dim,), (lat_dim,) = [c.cube_dims(cube) for c in (grid_lon, grid_lat)] + horizontal_shape = (cube.shape[lon_dim], cube.shape[lat_dim]) + + # Mask a corner of the cube. + mask = np.ones(np.array(cube.shape) // 2) + padding = np.stack( + [np.subtract(cube.shape, mask.shape), np.zeros_like(cube.shape)], axis=1 + ) + mask = np.pad(mask, padding) + cube = mask_cube(cube, mask) + + ################ + # Add various missing types of metadata. + + cube.long_name = "Air Potential Temperature" + cube.var_name = "air_temp" + + cell_method = CellMethod("mean", coords="time", intervals="1 hour") + cube.add_cell_method(cell_method) + + cell_areas = cartography.area_weights(cube, normalize=True) + # Index cell_areas to just get the lat and lon dimensions. + slices = tuple( + slice(None) if i in (lat_dim, lon_dim) else 0 for i in range(cube.ndim) + ) + cell_areas = cell_areas[slices] + cell_measure = CellMeasure( + data=cell_areas, + standard_name="cell_area", + ) + cube.add_cell_measure(cell_measure, (lat_dim, lon_dim)) + + ancillary_variable = AncillaryVariable( + data=np.remainder(cube.data.astype(int), 2), + standard_name="quality_flag", + ) + cube.add_ancillary_variable(ancillary_variable, np.arange(cube.ndim)) + + ################ + # Add 2-dimensional coordinates for lon/lat in the default coord system. + + class XY(NamedTuple): + """Syntactic sugar for storing x and y components.""" + + x: np.ndarray | int | float + y: np.ndarray | int | float + + def get_default_lat_lon(lat: np.ndarray, lon: np.ndarray) -> XY: + """Represent the given lon-lat points/bounds in the default CRS. + + The original coordinates are rotated so the output is therefore arrays + for constructing a 2D coordinate. + """ + default_cs = GeogCS(cartography.DEFAULT_SPHERICAL_EARTH_RADIUS) + default_crs: CRS = default_cs.as_cartopy_crs() + + mesh = np.meshgrid(lat, lon) + transformed = default_crs.transform_points( + cube.coord_system().as_cartopy_crs(), + *mesh, + ) + no_z = transformed[..., :2] + return XY(*no_z.T) + + default_points = get_default_lat_lon(grid_lat.points, grid_lon.points) + corners = [ + get_default_lat_lon( + grid_lat.bounds[:, corner.y], + grid_lon.bounds[:, corner.x], + ) + for corner in [XY(0, 0), XY(1, 0), XY(1, 1), XY(0, 1)] + ] + default_bounds = XY( + np.stack([c.x for c in corners], axis=-1), + np.stack([c.y for c in corners], axis=-1), + ) + + default_lon = AuxCoord( + default_points.x, + bounds=default_bounds.x, + standard_name="longitude", + units="degrees", + ) + default_lat = AuxCoord( + default_points.y, + bounds=default_bounds.y, + standard_name="latitude", + units="degrees", + ) + cube.add_aux_coord(default_lon, (lat_dim, lon_dim)) + cube.add_aux_coord(default_lat, (lat_dim, lon_dim)) + + ################ + # Optionally convert the horizontal grid to a mesh representation. + + def flatten_dim_metadata(dim_metadata: icoords._DimensionalMetadata): + flat_values = dim_metadata._values.flatten() + kwargs = dim_metadata.metadata._asdict() + if getattr(dim_metadata, "bounds", None) is not None: + flat_bounds = dim_metadata.bounds.reshape( + [len(flat_values), dim_metadata.bounds.shape[-1]] + ) + kwargs["bounds"] = flat_bounds + new_instance = dim_metadata.__class__(flat_values, **kwargs) + return new_instance + + def remove_duplicate_nodes(mesh: ugrid.Mesh): + """Remove duplicate nodes from a mesh. + + Mesh.from_coords() does not do this due to complications like lazy + arrays. Not a problem here. + """ + # TODO: + # Contained in a function because this _could_ be generalised into a + # public function. Would need to make it handle Dask arrays and masked + # indices. + + # Example nodes: [a, b, c, a, c, b, d] + # (Real nodes are X-Y pairs so a 2d array). + # Example faces: [[0, 1, 2, 6], [3, 4, 5, 6]] + # I.e. faces made by connecting: a-b-c-d , a-c-b-d + # Processed nodes: [a, b, c, d] + # Processed faces: [[0, 1, 2, 3], [0, 2, 1, 3]] + + nodes = np.stack([c.points for c in mesh.node_coords]) + face_node = mesh.face_node_connectivity + + # first_instances = a full length array but always with the index of + # the first instance of each node e.g.: [0, 1, 2, 0, 2, 1, 3] + nodes_unique, first_instances = np.unique( + nodes, + return_inverse=True, + axis=1, + ) + # E.g. indexing [0, 1, 2, 0, 2, 1, 3] with [[0, 1, 2, 6], [3, 4, 5, 6]] + # -> [[0, 1, 2, 3], [0, 2, 1, 3]] + indices_unique = first_instances[face_node.indices] + # Connectivity indices expected to be a masked array. + indices_unique = np.ma.masked_array(indices_unique, mask=face_node.indices.mask) + + # Replace the original node coords and face-node connectivity with the + # unique-d versions. + node_x, node_y = [ + AuxCoord(nodes_unique[i], **c.metadata._asdict()) + for i, c in enumerate(mesh.node_coords) + ] + mesh.add_coords(node_x=node_x, node_y=node_y) + conn_kwargs = dict(indices=indices_unique, start_index=0) + mesh.add_connectivities( + ugrid.Connectivity(**(face_node.metadata._asdict() | conn_kwargs)) + ) + + return mesh + + new_mesh = ugrid.Mesh.from_coords( + flatten_dim_metadata(default_lon), + flatten_dim_metadata(default_lat), + ) + new_mesh = remove_duplicate_nodes(new_mesh) + + # Create a new Cube with the horizontal (XY) dimensions flattened into a + # UGRID mesh. + def reshape_for_mesh(shape: tuple): + new_shape = [] + for dim, len in enumerate(shape): + if dim not in (lat_dim, lon_dim): + new_shape.append(len) + new_shape.append(np.prod(horizontal_shape)) + return new_shape + + mesh_cube = Cube( + cube.data.reshape(reshape_for_mesh(cube.shape)), + **cube.metadata._asdict(), + ) + for mesh_coord in new_mesh.to_MeshCoords("face"): + mesh_cube.add_aux_coord(mesh_coord, mesh_cube.ndim - 1) + + # Add all appropriate dimensional metadata from the original cube, reshaped + # where necessary. + dim_metadata_groups = [ + cube.dim_coords, + cube.aux_coords, + cube.cell_measures(), + cube.ancillary_variables(), + ] + add_methods = { + AncillaryVariable: Cube.add_ancillary_variable, + AuxCoord: Cube.add_aux_coord, + CellMeasure: Cube.add_cell_measure, + DimCoord: Cube.add_dim_coord, + } + for dim_metadata_group in dim_metadata_groups: + for dim_metadata in dim_metadata_group: + add_method = add_methods[type(dim_metadata)] + cube_dims = dim_metadata.cube_dims(cube) + + if cube_dims == (lat_dim,) or cube_dims == (lon_dim,): + # For realistic_4d() this is just lat and lon, and we're using + # the mesh coords for those. + continue + + elif cube_dims == (lat_dim, lon_dim): + dim_metadata = flatten_dim_metadata(dim_metadata) + add_method(mesh_cube, dim_metadata, mesh_cube.ndim - 1) + + elif {lat_dim, lon_dim}.issubset(cube_dims): + # Simplify implementation by not handling bounds. + assert getattr(dim_metadata, "bounds", None) is None + new_shape = reshape_for_mesh(dim_metadata.shape) + new_dims = list(cube_dims) + for dim in (lat_dim, lon_dim): + new_dims.remove(dim) + new_dims.append(mesh_cube.ndim - 1) + dim_metadata = dim_metadata.__class__( + dim_metadata._values.reshape(new_shape), + **dim_metadata.metadata._asdict(), + ) + add_method(mesh_cube, dim_metadata, new_dims) + + else: + try: + add_method(mesh_cube, dim_metadata, cube_dims) + except iris.exceptions.CannotAddError as err: + if isinstance(dim_metadata, DimCoord): + mesh_cube.add_aux_coord(dim_metadata, cube_dims) + else: + raise err + + for cell_method in cube.cell_methods: + if cell_method not in mesh_cube.cell_methods: + # Think some get copied across implicitly? Not sure how. + mesh_cube.add_cell_method(cell_method) + + for aux_factory in cube.aux_factories: + coord_mapping = { + id(dep): mesh_cube.coord(dep.name()) + for key, dep in aux_factory.dependencies.items() + if dep.name() in [c.name() for c in mesh_cube.coords()] + } + aux_factory = aux_factory.updated(coord_mapping) + mesh_cube.add_aux_factory(aux_factory) + + if w_mesh: + result = mesh_cube + else: + result = cube + return result + + def ocean_sigma_z(): """Return a sample cube with an :class:`iris.aux_factory.OceanSigmaZFactory` vertical coordinate. From 25b685c1490bf8938fa4fb0d6e13cb4c350a1f10 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 14 May 2024 09:43:58 +0100 Subject: [PATCH 9/9] [pre-commit.ci] pre-commit autoupdate (#5952) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.4.3 → v0.4.4](https://github.com/astral-sh/ruff-pre-commit/compare/v0.4.3...v0.4.4) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a500635619..fb33180953 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,7 +29,7 @@ repos: - id: no-commit-to-branch - repo: https://github.com/astral-sh/ruff-pre-commit - rev: "v0.4.3" + rev: "v0.4.4" hooks: - id: ruff types: [file, python]