Assorted bug fixes and tidying you might want (#34)

* Minor comment tidying.

* Allow subsetting of HDF_SDS fields when reading.

* Hack the MODIS_L2 plugin to deal with 3D data, returning a UngriddedDataList. A temporary measure until more robust integration with Iris cubes.

* Fix MODIS plugin. Dimensions vary in length and aren't uniformly ordered.

* The MODIS data objects should be independent.

* Allow HyperPoint(val=) to accept lists and arrays.

* Remove Py3.8 SyntaxWarnings for use of 'is'

* Minor tweaks for compatibility with Matplotlib 3.3

* The matplotlib time unit seems to have changed.

* ABC have moved in collections v3.3

* Allow listify() to accept generators as well.

* Errant whitespace

* Import cf_units.Unit in a consistent manner throughout code

* Use modern cf-units/cftime Julian date interface

Iris 3.0.0 requires cftime>1.5.0, which inspired cf-units to update
their API. Both changes remove the Julian date routines, replacing
them with cftime.datetime.toordinal() and .fromordinal().

Co-authored-by: Duncan Watson-Parris <[email protected]>

cis/data_io/common_data.py

@@ -229,7 +229,6 @@ class CommonDataList(list):
 
  Note that all objects in a CommonDataList must have the same coordinates and coordinate values.
  """
- filenames = []
 
  def __init__(self, iterable=()):
  super(CommonDataList, self).__init__()
@@ -339,6 +338,14 @@ def filenames(self):
  filenames.extend(data.filenames)
  return filenames
 
+ @filenames.setter
+ def filenames(self, names):
+ """
+ Add these filenames to this data list
+ """
+ for data in self:
+ data.filenames.extend(names)
+
  def add_history(self, new_history):
  """
  Appends to, or creates, the metadata history attribute using the supplied history string.

cis/data_io/hdf_sd.py

@@ -46,9 +46,12 @@ class HDF_SDS(object):
  _filename = None
  _variable = None
 
- def __init__(self, filename, variable):
+ def __init__(self, filename, variable, start=None, count=None, stride=None):
  self._filename = filename
  self._variable = variable
+ self._start = start
+ self._count = count
+ self._stride = stride
 
  def _open_sds(self):
  """
@@ -72,13 +75,19 @@ def _close_sds(self):
  if self._sd is not None:
  self._sd.end()
 
- def get(self):
+ def get(self, start=None, count=None, stride=None):
  """
  Call pyhdf.SD.SDS.get(), opening and closing the file
  """
+ if start is None:
+ start = self._start
+ if count is None:
+ count = self._count
+ if stride is None:
+ stride = self._stride
  try:
  self._open_sds()
- data = self._sds.get()
+ data = self._sds.get(start, count, stride)
  return data
  finally:
  self._close_sds()

cis/data_io/hyperpoint.py

@@ -22,11 +22,15 @@ def __new__(cls, lat=None, lon=None, alt=None, pres=None, t=None, val=None):
  Small constructor for the HyperPoint named tuple to allow optional arguments
  and set-up value list.
  """
+ from numpy import ndarray
 
  # If no value was specified create an empty list, otherwise create a list with one entry
- if val is None or val == []:
- val = []
- else:
+ try:
+ if val is None or len(val) == 0:
+ val = []
+ else:
+ val = list(val)
+ except TypeError:
  val = [val]
 
  # If t was given as a datetime we need to convert it into our standard time

cis/data_io/products/MODIS.py

@@ -2,7 +2,7 @@
 from cis.data_io import hdf as hdf
 from cis.data_io.Coord import CoordList, Coord
 from cis.data_io.products import AProduct
-from cis.data_io.ungridded_data import UngriddedCoordinates, UngriddedData
+from cis.data_io.ungridded_data import UngriddedCoordinates, UngriddedData, UngriddedDataList
 
 
 def _get_MODIS_SDS_data(sds):
@@ -18,6 +18,10 @@ def _get_MODIS_SDS_data(sds):
  data = sds.get()
  attributes = sds.attributes()
 
+ # Squeeze dimensions that have been sliced
+ if sds._count is not None and any(np.array(sds._count) == 1):
+ data = data.squeeze()
+
  # Apply Fill Value
  missing_value = attributes.get('_FillValue', None)
  if missing_value is not None:
@@ -75,7 +79,7 @@ def _parse_datetime(self, metadata_dict, keyword):
  matches = re.findall('".*"', ssubsub)
  if len(matches) > 0:
  res = matches[0].replace('\"', '')
- if res is not "":
+ if res != "":
  break
  return res
 
@@ -233,7 +237,7 @@ def __field_interpolate(self, data, factor=5):
  """
  Interpolates the given 2D field by the factor,
  edge pixels are defined by the ones in the centre,
- odd factords only!
+ odd factors only!
  """
  import numpy as np
 
@@ -262,7 +266,7 @@ def _create_coord_list(self, filenames, variable=None):
  apply_interpolation = False
  if variable is not None:
  scale = self.__get_data_scale(filenames[0], variable)
- apply_interpolation = True if scale is "1km" else False
+ apply_interpolation = True if scale == "1km" else False
 
  lat = sdata['Latitude']
  sd_lat = hdf.read_data(lat, _get_MODIS_SDS_data)
@@ -292,6 +296,8 @@ def create_coords(self, filenames, variable=None):
  return UngriddedCoordinates(self._create_coord_list(filenames))
 
  def create_data_object(self, filenames, variable):
+ from itertools import product
+
  logging.debug("Creating data object for variable " + variable)
 
  # reading coordinates
@@ -305,8 +311,25 @@ def create_data_object(self, filenames, variable):
  var = sdata[variable]
  metadata = hdf.read_metadata(var, "SD")
 
- return UngriddedData(var, metadata, coords, _get_MODIS_SDS_data)
+ # Check the dimension of this variable
+ _, ndim, dim_len, _, _ = var[0].info()
+ if ndim == 2:
+ return UngriddedData(var, metadata, coords, _get_MODIS_SDS_data)
 
+ elif ndim < 2:
+ raise NotImplementedError("1D field in MODIS L2 data.")
+
+ else:
+ result = UngriddedDataList()
+
+ # Iterate over all but the last two dimensions
+ ranges = [range(n) for n in dim_len[:-2]]
+ for indices in product(*ranges):
+ for manager in var:
+ manager._start = list(indices) + [0, 0]
+ manager._count = [1] * len(indices) + manager.info()[2][-2:]
+ result.append(UngriddedData(var, metadata, coords.copy(), _get_MODIS_SDS_data))
+ return result
 
  def get_file_format(self, filenames):
  """

cis/data_io/products/NCAR_NetCDF_RAF.py

@@ -234,7 +234,7 @@ def _best_coordinates_setup(self):
  if coordinates is not None:
  coordinates_vars = coordinates.split() # split on whitespace
 
- if len(coordinates_vars) is not 4:
+ if len(coordinates_vars) != 4:
  raise InvalidVariableError('The coordinate attribute does not have four entries. '
  'It should be space separated "longitude latitude altitude time"')
 

cis/data_io/products/gridded_NetCDF.py

@@ -23,7 +23,7 @@ def load_single_file_callback(cube, field, filename):
 
  def get_variable_names(self, filenames, data_type=None):
  import iris
- import cf_units as unit
+ from cf_units import Unit
  from cis.utils import single_warnings_only
 
  variables = []
@@ -40,7 +40,7 @@ def get_variable_names(self, filenames, data_type=None):
  not units.is_time() and \
  not units.is_time_reference() and \
  not units.is_vertical() and \
- not units.is_convertible(unit.Unit('degrees')):
+ not units.is_convertible(Unit('degrees')):
  is_time_lat_lon_pressure_altitude_or_has_only_1_point = False
  break
  if is_time_lat_lon_pressure_altitude_or_has_only_1_point:

cis/data_io/ungridded_data.py

@@ -115,7 +115,7 @@ def standard_name(self, standard_name):
  if standard_name is None or standard_name in STD_NAMES:
  # If the standard name is actually changing from one to another then log the fact
  if self.standard_name is not None \
- and self.standard_name.strip() is not "" \
+ and self.standard_name.strip() != "" \
  and self.standard_name != standard_name:
  logging.debug("Changing standard name for dataset from '{}' to '{}'".format(self.standard_name,
  standard_name))

cis/maths.py

@@ -7,9 +7,10 @@
 import math
 import operator
 
-import cf_units
 import numpy as np
 
+from cf_units import Unit
+
 
 def abs(ungridded_data, in_place=False):
  """
@@ -242,7 +243,7 @@ def exp(ungridded_data, in_place=False):
  An instance of :class:`cis.data_io.ungridded_data.LazyData`.
 
  """
- return _math_op_common(ungridded_data, np.exp, cf_units.Unit('1'), in_place=in_place)
+ return _math_op_common(ungridded_data, np.exp, Unit('1'), in_place=in_place)
 
 
 def log(ungridded_data, in_place=False):

cis/plotting/genericplot.py

@@ -87,7 +87,7 @@ def __init__(self, packed_data, *args, **kwargs):
 
  if self.xaxis.standard_name == 'time':
  # Convert to matplotlib datetime
- self.x = self.xaxis.units.convert(self.xaxis.points, 'days since 001-01-01 00:00:00')
+ self.x = self.xaxis.units.convert(self.xaxis.points, 'days since 1970-01-01 00:00:00')
  else:
  self.x = self.xaxis.points
 
@@ -268,4 +268,4 @@ def _cube_manipulation(cube, x, y):
  except ValueError as e:
  logging.warn('Unable to add cyclic data point for y-axis. Error was: ' + e.args[0])
  y, x = np.meshgrid(y, x)
- return data, x, y
+ return data, x, y

cis/plotting/scatterplot.py

@@ -46,8 +46,8 @@ def __call__(self, ax):
  self.mplkwargs["edgecolors"] = self.edgecolor
  else:
  # For 2D scatter plots set the edgecolors off by default
- self.mplkwargs["edgecolors"] = ''
- 
+ self.mplkwargs["edgecolors"] = None
+
  self.mplkwargs["c"] = self.data
 
  self.mappable = ax.scatter(self.x, self.y, *self.mplargs, **self.mplkwargs)

cis/time_util.py

@@ -157,7 +157,7 @@ def convert_time_using_time_stamp_info_to_std_time(time_array, units, time_stamp
  """
  from cf_units import Unit
  units = str(units).split()
- if len(units) is 0:
+ if len(units) == 0:
  raise ValueError("Units is empty when converting time")
 
  units_in_since_form = Unit(units[0] + " since " + time_stamp_info)

cis/utils.py

@@ -198,7 +198,7 @@ def get_coord(data_object, variable, data):
  Find a specified coord
 
  :param data_object:
- :param variable: 
+ :param variable:
  :param data:
  :return:
  """
@@ -659,7 +659,8 @@ def listify(item):
  :param item: Item which may or may not be a list
  :return: List
  """
- if isinstance(item, tuple):
+ from types import GeneratorType
+ if isinstance(item, (tuple, GeneratorType)):
  return list(item)
  if not isinstance(item, list):
  return [item]

conda_requirements.txt

@@ -3,6 +3,7 @@
 
 # Python dependencies
 # Match the iris requirement for matplotlib<3
+cf-units>=3.0.0
 matplotlib>=2
 numpy>=1.10
 iris>=2.0.0