Review changes.

docs/src/userguide/real_and_lazy_data.rst

@@ -240,18 +240,18 @@ for more information on setting dask processing options.
 
 .. _delayed_netcdf_save:
 
-Delayed netCDF Saving
+Delayed NetCDF Saving
 ---------------------
 
-When saving data to netCDF files, it is possible to *delay* writing lazy content to the
+When saving data to NetCDF files, it is possible to *delay* writing lazy content to the
 output file, to be performed by `Dask <https://docs.dask.org/en/stable/>`_ later,
 thus enabling parallel save operations.
 
 This works in the following way :
- 1. an :func:`iris.save` call is made, with a netcdf file output and the additional
+ 1. an :func:`iris.save` call is made, with a NetCDF file output and the additional
  keyword ``compute=False``.
- This is currently *only* available when saving to netCDF, so it is documented in
- the netCDF file format API. See : :func:`iris.fileformats.netcdf.save`.
+ This is currently *only* available when saving to NetCDF, so it is documented in
+ the Iris NetCDF file format API. See: :func:`iris.fileformats.netcdf.save`.
 
  2. the call creates the output file, but does not fill in variables' data, where
  the data is a lazy array in the Iris object. Instead, these variables are
@@ -270,6 +270,6 @@ from shared lazy input data, these can be computed in parallel efficiently by Da
 can do.
 
 .. note::
- This feature does **not** enable parallel writes to the *same* netCDF output file.
+ This feature does **not** enable parallel writes to the *same* NetCDF output file.
  That can only be done on certain operating systems, with a specially configured
- build of the netCDF C library, and is not supported by Iris at present.
+ build of the NetCDF C library, and is not supported by Iris at present.

docs/src/whatsnew/latest.rst

@@ -53,8 +53,9 @@ This document explains the changes made to Iris for this release
  enable lazy computation of rotated wind vector components (:issue:`4934`,
  :pull:`4972`)
 
-#. `@pp-mo`_ supported delayed saving of lazy data, when writing to the netCDF file
- format. See : :ref:`delayed netCDF saves <delayed_netcdf_save>`.
+#. `@pp-mo`_ and `@lbdreyer`_ supported delayed saving of lazy data, when writing to
+ the netCDF file format. See : :ref:`delayed netCDF saves <delayed_netcdf_save>`.
+ Also with significant input from `@fnattino`_.
  (:pull:`5191`)
 
 

lib/iris/fileformats/netcdf/saver.py

@@ -308,6 +308,10 @@ def _data_fillvalue_check(arraylib, data, check_value):
  return is_masked, contains_value
 
 
+class SaverFillValueWarning(UserWarning):
+ pass
+
+
 def _fillvalue_report(fill_info, is_masked, contains_fill_value, warn=False):
  """
  From the given information, work out whether there was a possible or actual
@@ -336,7 +340,7 @@ def _fillvalue_report(fill_info, is_masked, contains_fill_value, warn=False):
  is_byte_data = fill_info.dtype.itemsize == 1
  result = None
  if is_byte_data and is_masked and user_value is None:
- result = UserWarning(
+ result = SaverFillValueWarning(
  f"CF var '{varname}' contains byte data with masked points, but "
  "no fill_value keyword was given. As saved, these "
  "points will read back as valid values. To save as "
@@ -345,7 +349,7 @@ def _fillvalue_report(fill_info, is_masked, contains_fill_value, warn=False):
  "keyword during saving, otherwise use ncedit/equivalent."
  )
  elif contains_fill_value:
- result = UserWarning(
+ result = SaverFillValueWarning(
  f"CF var '{varname}' contains unmasked data points equal to the "
  f"fill-value, {check_value}. As saved, these points will read back "
  "as missing data. To save these as normal values, "

lib/iris/io/__init__.py

@@ -473,7 +473,6 @@ def save(source, target, saver=None, **kwargs):
  # Force append=True for the tail cubes. Don't modify the incoming
  # kwargs.
  kwargs = kwargs.copy()
- result = []
  for i, cube in enumerate(source):
  if i != 0:
  kwargs["append"] = True

lib/iris/tests/integration/netcdf/test_delayed_save.py

@@ -18,6 +18,7 @@
 
 import iris
 from iris.fileformats.netcdf._thread_safe_nc import default_fillvals
+from iris.fileformats.netcdf.saver import SaverFillValueWarning
 import iris.tests
 from iris.tests.stock import realistic_4d
 
@@ -39,6 +40,7 @@ def make_testcube(
  include_lazy_content=True,
  ensure_fillvalue_collision=False,
  data_is_maskedbytes=False,
+ include_extra_coordlikes=False,
  ):
  cube = realistic_4d()
 
@@ -80,6 +82,24 @@ def fix_array(array):
  cube.data = fix_array(cube.data)
  auxcoord = cube.coord("surface_altitude")
  auxcoord.points = fix_array(auxcoord.points)
+
+ if include_extra_coordlikes:
+ # Also concoct + attach an ancillary variable and a cell-measure, so we can
+ # check that they behave the same as coordinates.
+ ancil_dims = [0, 2]
+ cm_dims = [0, 3]
+ ancil_shape = [cube.shape[idim] for idim in ancil_dims]
+ cm_shape = [cube.shape[idim] for idim in cm_dims]
+ from iris.coords import AncillaryVariable, CellMeasure
+
+ ancil = AncillaryVariable(
+ fix_array(np.zeros(ancil_shape)), long_name="sample_ancil"
+ )
+ cube.add_ancillary_variable(ancil, ancil_dims)
+ cm = CellMeasure(
+ fix_array(np.zeros(cm_shape)), long_name="sample_cm"
+ )
+ cube.add_cell_measure(cm, cm_dims)
  return cube
 
  def test_realfile_loadsave_equivalence(self, save_is_delayed, output_path):
@@ -106,8 +126,8 @@ def test_realfile_loadsave_equivalence(self, save_is_delayed, output_path):
  reloaded_cubes = iris.load(output_path)
  reloaded_cubes = sorted(reloaded_cubes, key=lambda cube: cube.name())
  assert reloaded_cubes == original_cubes
- # NOTE: it might be nicer to do assertCDL, but I', not sure how to access that
- # from pytest-style test code ?
+ # NOTE: it might be nicer to use assertCDL, but unfortunately importing
+ # unitest.TestCase seems to lose us the ability to use fixtures.
 
  @staticmethod
  def getmask(cube_or_coord):
@@ -122,8 +142,11 @@ def test_time_of_writing(self, save_is_delayed, output_path):
  # Check when lazy data is actually written :
  # - in 'immediate' mode, on initial file write
  # - in 'delayed' mode, only when delayed-write is executed.
- original_cube = self.make_testcube()
+ original_cube = self.make_testcube(include_extra_coordlikes=True)
  assert original_cube.has_lazy_data()
+ assert original_cube.coord("surface_altitude").has_lazy_points()
+ assert original_cube.cell_measure("sample_cm").has_lazy_data()
+ assert original_cube.ancillary_variable("sample_ancil").has_lazy_data()
 
  result = iris.save(
  original_cube, output_path, compute=not save_is_delayed
@@ -139,20 +162,34 @@ def test_time_of_writing(self, save_is_delayed, output_path):
  # If 'delayed', the lazy content should all be masked, otherwise none of it.
  data_mask = self.getmask(readback_cube)
  coord_mask = self.getmask(readback_cube.coord("surface_altitude"))
+ ancil_mask = self.getmask(
+ readback_cube.ancillary_variable("sample_ancil")
+ )
+ cm_mask = self.getmask(readback_cube.cell_measure("sample_cm"))
  if save_is_delayed:
  assert np.all(data_mask)
  assert np.all(coord_mask)
+ assert np.all(ancil_mask)
+ assert np.all(cm_mask)
  else:
  assert np.all(~data_mask)
  assert np.all(~coord_mask)
+ assert np.all(~ancil_mask)
+ assert np.all(~cm_mask)
 
  if save_is_delayed:
  result.compute()
  # Re-fetch the arrays. The data is **no longer masked**.
  data_mask = self.getmask(readback_cube)
  coord_mask = self.getmask(readback_cube.coord("surface_altitude"))
+ ancil_mask = self.getmask(
+ readback_cube.ancillary_variable("sample_ancil")
+ )
+ cm_mask = self.getmask(readback_cube.cell_measure("sample_cm"))
  assert np.all(~data_mask)
  assert np.all(~coord_mask)
+ assert np.all(~ancil_mask)
+ assert np.all(~cm_mask)
 
  @pytest.mark.parametrize(
  "warning_type", ["WarnMaskedBytes", "WarnFillvalueCollision"]
@@ -181,12 +218,16 @@ def test_fill_warnings(self, warning_type, output_path, save_is_delayed):
  result = iris.save(cube, output_path, compute=not save_is_delayed)
 
  if not save_is_delayed:
- result_warnings = [log.message for log in logged_warnings]
+ result_warnings = [
+ log.message
+ for log in logged_warnings
+ if isinstance(log.message, SaverFillValueWarning)
+ ]
  else:
  assert len(logged_warnings) == 0
  # Complete the operation now
  # NOTE: warnings should not be *issued* here, instead they are returned.
- warnings.simplefilter("error")
+ warnings.simplefilter("error", category=SaverFillValueWarning)
  result_warnings = result.compute()
 
  # Either way, we should now have 2 similar warnings.

lib/iris/tests/unit/fileformats/netcdf/saver/test_Saver__lazy_stream_data.py

@@ -26,7 +26,7 @@ class Test__lazy_stream_data:
  @staticmethod
  @pytest.fixture(autouse=True)
  def saver_patch():
- # Install patches, so we can creata a Saver without opening a real output file.
+ # Install patches, so we can create a Saver without opening a real output file.
  # Mock just enough of Dataset behaviour to allow a 'Saver.complete()' call.
  mock_dataset = mock.MagicMock(_isopen=False)
  mock_dataset_class = mock.Mock(return_value=mock_dataset)
@@ -63,9 +63,9 @@ def saver(compute) -> Saver:
  )
 
  @staticmethod
- def mock_var(shape, dtype=np.float32):
+ def mock_var(shape):
  # Create a test cf_var object
- return mock.MagicMock(shape=tuple(shape), dtype=np.dtype(dtype))
+ return mock.MagicMock(shape=tuple(shape), dtype=np.dtype(np.float32))
 
  def test_data_save(self, compute, data_is_lazy):
  """Real data is transferred immediately, lazy data creates a delayed write."""
@@ -81,9 +81,9 @@ def test_data_save(self, compute, data_is_lazy):
  assert cf_var.__setitem__.call_count == (0 if data_is_lazy else 1)
  assert len(saver._delayed_writes) == (1 if data_is_lazy else 0)
  if data_is_lazy:
- result_data, result__write, fill_info = saver._delayed_writes[0]
+ result_data, result_writer, fill_info = saver._delayed_writes[0]
  assert result_data is data
- assert isinstance(result__write, nc_threadsafe.NetCDFWriteProxy)
+ assert isinstance(result_writer, nc_threadsafe.NetCDFWriteProxy)
  assert isinstance(fill_info, _FillvalueCheckInfo)
  else:
  cf_var.__setitem__.assert_called_once_with(slice(None), data)

lib/iris/tests/unit/fileformats/netcdf/saver/test_save.py

@@ -4,6 +4,10 @@
 # See COPYING and COPYING.LESSER in the root of the repository for full
 # licensing details.
 """Unit tests for the :func:`iris.fileformats.netcdf.save` function."""
+# Import iris.tests first so that some things can be initialised before
+# importing anything else.
+import iris.tests as tests # isort:skip
+
 from pathlib import Path
 from shutil import rmtree
 from tempfile import mkdtemp
@@ -25,10 +29,6 @@
 from iris.tests.stock import lat_lon_cube
 from iris.tests.stock.mesh import sample_mesh_cube
 
-# Import iris.tests first so that some things can be initialised before
-# importing anything else.
-import iris.tests as tests # isort:skip
-
 
 class Test_conventions(tests.IrisTest):
  def setUp(self):