Syntax update using numpy and scipy (#196)

* fix: np.product() is deprecated

Replaced by np.prod()

* fix: gaussian_filter() moving to scipy.ndimage

The `scipy.ndimage.filters` namespace is deprecated and will be removed in SciPy 2.0.0

* Explicit requirement on numpy and scipy

I just noticed that although numpy and scipy are used explicitly in
sup3r, there was no explicit dependency on those. The respective minimum
versions where defined based on the syntax update in the previous
commit.

* Holding TensorFlow to <2.16

There is something going wrong with TF-2.16 (see PR #198).

requirements.txt

@@ -6,9 +6,11 @@ NREL-farms>=1.0.4
 google-auth-oauthlib==0.5.3
 pytest>=5.2
 pillow>=10.0
-tensorflow>2.4
+tensorflow>2.4,<2.16
 xarray>=2023.0
 netCDF4==1.5.8
+numpy>=1.7.0
 dask>=2022.0
 sphinx>=7.0
+scipy>=1.0.0
 pandas>=2.0

sup3r/bias/bias_calc.py

@@ -15,7 +15,7 @@
 import rex
 from rex.utilities.fun_utils import get_fun_call_str
 from scipy import stats
-from scipy.ndimage.filters import gaussian_filter
+from scipy.ndimage import gaussian_filter
 from scipy.spatial import KDTree
 
 import sup3r.preprocessing.data_handling

sup3r/bias/bias_transforms.py

@@ -6,7 +6,7 @@
 
 import numpy as np
 from rex import Resource
-from scipy.ndimage.filters import gaussian_filter
+from scipy.ndimage import gaussian_filter
 
 logger = logging.getLogger(__name__)
 

sup3r/models/abstract.py

@@ -114,7 +114,7 @@ def get_s_enhance_from_layers(self):
  if hasattr(self, '_gen'):
  s_enhancements = [getattr(layer, '_spatial_mult', 1)
  for layer in self._gen.layers]
- s_enhance = int(np.product(s_enhancements))
+ s_enhance = int(np.prod(s_enhancements))
  return s_enhance
 
  # pylint: disable=E1101
@@ -125,7 +125,7 @@ def get_t_enhance_from_layers(self):
  if hasattr(self, '_gen'):
  t_enhancements = [getattr(layer, '_temporal_mult', 1)
  for layer in self._gen.layers]
- t_enhance = int(np.product(t_enhancements))
+ t_enhance = int(np.prod(t_enhancements))
  return t_enhance
 
  @property

sup3r/models/multi_step.py

@@ -449,7 +449,7 @@ def preflight(self):
  msg = ('Solar and wind spatial enhancements must be equivalent but '
  'received models that do spatial enhancements of '
  '{} (solar) and {} (wind)'.format(s_enh, w_enh))
- assert np.product(s_enh) == np.product(w_enh), msg
+ assert np.prod(s_enh) == np.prod(w_enh), msg
 
  s_t_feat = self.spatial_solar_models.lr_features
  s_o_feat = self.spatial_solar_models.hr_out_features

sup3r/pipeline/forward_pass.py

@@ -100,8 +100,8 @@ def __init__(self,
  self.time_steps = time_steps
  self.s_enhancements = s_enhancements
  self.t_enhancements = t_enhancements
- self.s_enhance = np.product(self.s_enhancements)
- self.t_enhance = np.product(self.t_enhancements)
+ self.s_enhance = np.prod(self.s_enhancements)
+ self.t_enhance = np.prod(self.t_enhancements)
  self.dummy_time_index = np.arange(time_steps)
  self.temporal_slice = temporal_slice
  self.temporal_pad = temporal_pad
@@ -775,8 +775,8 @@ def __init__(self,
  models = getattr(model, 'models', [model])
  self.s_enhancements = [model.s_enhance for model in models]
  self.t_enhancements = [model.t_enhance for model in models]
- self.s_enhance = np.product(self.s_enhancements)
- self.t_enhance = np.product(self.t_enhancements)
+ self.s_enhance = np.prod(self.s_enhancements)
+ self.t_enhance = np.prod(self.t_enhancements)
  self.output_features = model.hr_out_features
  assert len(self.output_features) > 0, 'No output features!'
 
@@ -841,7 +841,7 @@ def preflight(self):
  hr_data_shape = (self.grid_shape[0] * self.s_enhance,
  self.grid_shape[1] * self.s_enhance,
  )
- self.gids = np.arange(np.product(hr_data_shape))
+ self.gids = np.arange(np.prod(hr_data_shape))
  self.gids = self.gids.reshape(hr_data_shape)
 
  out = self.fwp_slicer.get_spatial_slices()
@@ -1475,15 +1475,15 @@ def _get_step_enhance(self, step):
  s_enhance = 1
  t_enhance = 1
  else:
- s_enhance = np.product(
+ s_enhance = np.prod(
  self.strategy.s_enhancements[:model_step])
- t_enhance = np.product(
+ t_enhance = np.prod(
  self.strategy.t_enhancements[:model_step])
 
  elif combine_type.lower() in ('output', 'layer'):
- s_enhance = np.product(
+ s_enhance = np.prod(
  self.strategy.s_enhancements[:model_step + 1])
- t_enhance = np.product(
+ t_enhance = np.prod(
  self.strategy.t_enhancements[:model_step + 1])
  return s_enhance, t_enhance
 

sup3r/postprocessing/file_handling.py

@@ -677,7 +677,7 @@ def invert_uv_features(cls, data, features, lat_lon, max_workers=None):
  logger.debug('Found heights {} for output features {}'
  .format(heights, features))
 
- proc_mem = 4 * np.product(data.shape[:-1])
+ proc_mem = 4 * np.prod(data.shape[:-1])
  n_procs = len(heights)
  max_workers = estimate_max_workers(max_workers, proc_mem, n_procs)
 
@@ -797,7 +797,7 @@ def _write_output(cls, data, features, lat_lon, times, out_file,
  data, features = cls._transform_output(data.copy(), features, lat_lon,
  max_workers)
  gids = (gids if gids is not None
- else np.arange(np.product(lat_lon.shape[:-1])))
+ else np.arange(np.prod(lat_lon.shape[:-1])))
  meta = pd.DataFrame({'gid': gids.flatten(),
  'latitude': lat_lon[..., 0].flatten(),
  'longitude': lat_lon[..., 1].flatten()})

sup3r/preprocessing/batch_handling.py

@@ -10,7 +10,7 @@
 
 import numpy as np
 from rex.utilities import log_mem
-from scipy.ndimage.filters import gaussian_filter
+from scipy.ndimage import gaussian_filter
 
 from sup3r.preprocessing.data_handling.h5_data_handling import (
  DataHandlerDCforH5,

sup3r/preprocessing/data_handling/base.py

@@ -679,7 +679,7 @@ def grid_mem(self):
  int
  Number of bytes for a single feature array at a single time step
  """
- grid_mem = np.product(self.grid_shape)
+ grid_mem = np.prod(self.grid_shape)
  # assuming feature arrays are float32 (4 bytes)
  return 4 * grid_mem
 
@@ -986,7 +986,7 @@ def size(self):
  size : int
  Number of total elements contained in data array
  """
- return np.product(self.requested_shape)
+ return np.prod(self.requested_shape)
 
  def cache_data(self, cache_file_paths):
  """Cache feature data to file and delete from memory

sup3r/preprocessing/data_handling/dual_data_handling.py

@@ -383,7 +383,7 @@ def grid_mem(self):
  int
  Number of bytes for a single feature array at a single time step
  """
- grid_mem = np.product(self.lr_grid_shape)
+ grid_mem = np.prod(self.lr_grid_shape)
  # assuming feature arrays are float32 (4 bytes)
  return 4 * grid_mem
 
@@ -452,7 +452,7 @@ def shape(self):
  @property
  def size(self):
  """Get low_res size"""
- return np.product(self.shape)
+ return np.prod(self.shape)
 
  @property
  def hr_required_shape(self):

sup3r/preprocessing/data_handling/exo_extraction.py

@@ -505,7 +505,7 @@ def get_data(self):
 
  df = pd.DataFrame({'topo': self.source_data.flatten(),
  'gid_target': self.nn})
- n_target = np.product(self.hr_shape[:-1])
+ n_target = np.prod(self.hr_shape[:-1])
  df = df[df['gid_target'] != n_target]
  df = df.sort_values('gid_target')
  df = df.groupby('gid_target').mean()

sup3r/preprocessing/data_handling/exogenous_data_handling.py

@@ -508,12 +508,12 @@ def _get_single_step_enhance(self, step):
  s_enhance = 1
  t_enhance = 1
  else:
- s_enhance = np.product(s_enhancements[:model_step])
- t_enhance = np.product(t_enhancements[:model_step])
+ s_enhance = np.prod(s_enhancements[:model_step])
+ t_enhance = np.prod(t_enhancements[:model_step])
 
  elif combine_type.lower() in ('output', 'layer'):
- s_enhance = np.product(s_enhancements[:model_step + 1])
- t_enhance = np.product(t_enhancements[:model_step + 1])
+ s_enhance = np.prod(s_enhancements[:model_step + 1])
+ t_enhance = np.prod(t_enhancements[:model_step + 1])
 
  else:
  msg = ('Received exo_kwargs entry without valid combine_type '

sup3r/preprocessing/data_handling/nc_data_handling.py

@@ -13,7 +13,7 @@
 import pandas as pd
 import xarray as xr
 from rex import Resource
-from scipy.ndimage.filters import gaussian_filter
+from scipy.ndimage import gaussian_filter
 from scipy.spatial import KDTree
 from scipy.stats import mode
 

sup3r/qa/stats.py

@@ -8,7 +8,7 @@
 import pandas as pd
 import psutil
 from rex.utilities.fun_utils import get_fun_call_str
-from scipy.ndimage.filters import gaussian_filter
+from scipy.ndimage import gaussian_filter
 
 from sup3r.preprocessing.feature_handling import Feature
 from sup3r.qa.utilities import (

sup3r/utilities/interpolate_log_profile.py

@@ -543,7 +543,7 @@ def _get_timestep_interp_input(cls, lev_array, var_array, idt):
 
  """
  array_shape = var_array.shape
- shape = (array_shape[-3], np.product(array_shape[-2:]))
+ shape = (array_shape[-3], np.prod(array_shape[-2:]))
  h_t = lev_array[idt].reshape(shape).T
  var_t = var_array[idt].reshape(shape).T
  mask = ~np.isnan(h_t) & ~np.isnan(var_t)
@@ -648,7 +648,7 @@ def interp_var_to_height(cls,
  array_shape = var_array.shape
 
  # Flatten h_array and var_array along lat, long axis
- shape = (len(levels), array_shape[-4], np.product(array_shape[-2:]))
+ shape = (len(levels), array_shape[-4], np.prod(array_shape[-2:]))
  out_array = np.zeros(shape, dtype=np.float32).T
  total_checks = []
 

sup3r/utilities/interpolation.py

@@ -347,12 +347,12 @@ def interp_to_level(cls, var_array, lev_array, levels):
  array_shape = var_array.shape
 
  # Flatten h_array and var_array along lat, long axis
- shape = (len(levels), array_shape[-4], np.product(array_shape[-2:]))
+ shape = (len(levels), array_shape[-4], np.prod(array_shape[-2:]))
  out_array = np.zeros(shape, dtype=np.float32).T
 
  # iterate through time indices
  for idt in range(array_shape[0]):
- shape = (array_shape[-3], np.product(array_shape[-2:]))
+ shape = (array_shape[-3], np.prod(array_shape[-2:]))
  h_tmp = lev_array[idt].reshape(shape).T
  var_tmp = var_array[idt].reshape(shape).T
  not_nan = ~np.isnan(h_tmp) & ~np.isnan(var_tmp)

sup3r/utilities/pytest.py

@@ -166,7 +166,7 @@ def make_fake_h5_chunks(td):
  lon, lat = np.meshgrid(lon, lat)
  low_res_lat_lon = np.dstack((lat, lon))
 
- gids = np.arange(np.product(shape[:2]))
+ gids = np.arange(np.prod(shape[:2]))
  gids = gids.reshape(shape[:2])
 
  low_res_times = pd_date_range(

sup3r/utilities/regridder.py

@@ -161,7 +161,7 @@ def weights(self):
  dists = np.array(self.distances, dtype=np.float32)
  mask = dists < self.MIN_DISTANCE
  if mask.sum() > 0:
- logger.info(f'{np.sum(mask)} of {np.product(mask.shape)} '
+ logger.info(f'{np.sum(mask)} of {np.prod(mask.shape)} '
  'distances are zero.')
  dists[mask] = self.MIN_DISTANCE
  weights = 1 / dists
@@ -360,7 +360,7 @@ def interpolate(cls, distance_chunk, values):
  dists = np.array(distance_chunk, dtype=np.float32)
  mask = dists < cls.MIN_DISTANCE
  if mask.sum() > 0:
- logger.info(f'{np.sum(mask)} of {np.product(mask.shape)} '
+ logger.info(f'{np.sum(mask)} of {np.prod(mask.shape)} '
  'distances are zero.')
  dists[mask] = cls.MIN_DISTANCE
  weights = 1 / dists

sup3r/utilities/utilities.py

@@ -21,7 +21,7 @@
 from scipy import ndimage as nd
 from scipy.interpolate import RegularGridInterpolator, interp1d
 from scipy.ndimage import zoom
-from scipy.ndimage.filters import gaussian_filter
+from scipy.ndimage import gaussian_filter
 
 np.random.seed(42)