Merge pull request #92 from NREL/gb/solar_module

Gb/solar module

requirements.txt

@@ -1,7 +1,8 @@
 matplotlib>=3.1
-NREL-rex>=0.2.71
+NREL-rex>=0.2.73
 NREL-phygnn>=0.0.20
 NREL-rev>=0.6.6
+NREL-farms>=1.0.4
 pytest>=5.2
 pillow
 tensorflow>2.4

setup.py

@@ -53,6 +53,7 @@ def run(self):
  "sup3r-pipeline=sup3r.pipeline.pipeline_cli:main",
  "sup3r-batch=sup3r.batch.batch_cli:main",
  "sup3r-qa=sup3r.qa.qa_cli:main",
+ "sup3r-solar=sup3r.solar.solar_cli:main",
  ("sup3r-forward-pass=sup3r.pipeline."
  "forward_pass_cli:main"),
  ("sup3r-extract=sup3r.preprocessing."

sup3r/cli.py

@@ -6,7 +6,8 @@
 import logging
 
 from sup3r.version import __version__
-from sup3r.pipeline.forward_pass_cli import from_config as fp_cli
+from sup3r.pipeline.forward_pass_cli import from_config as fwp_cli
+from sup3r.solar.solar_cli import from_config as solar_cli
 from sup3r.preprocessing.data_extract_cli import from_config as dh_cli
 from sup3r.postprocessing.data_collect_cli import from_config as dc_cli
 from sup3r.qa.qa_cli import from_config as qa_cli
@@ -113,7 +114,56 @@ def forward_pass(ctx, verbose):
  """
  config_file = ctx.obj['CONFIG_FILE']
  verbose = any([verbose, ctx.obj['VERBOSE']])
- ctx.invoke(fp_cli, config_file=config_file, verbose=verbose)
+ ctx.invoke(fwp_cli, config_file=config_file, verbose=verbose)
+
+
+@main.command()
+@click.option('-v', '--verbose', is_flag=True,
+ help='Flag to turn on debug logging.')
+@click.pass_context
+def solar(ctx, verbose):
+ """Sup3r solar module to convert GAN output clearsky ratio to irradiance
+
+ Typically we train solar GAN's on clearsky ratio to remove the dependence
+ on known variables like solar position and clearsky irradiance. This module
+ converts the clearsky ratio output by the GAN in the forward-pass step to
+ actual irradiance values (ghi, dni, and dhi). This should be run after the
+ forward-pass but before the data-collect step.
+
+ You can call the solar module via the sup3r-pipeline CLI, or call it
+ directly with either of these equivelant commands::
+
+ $ sup3r -c config_solar.json solar
+
+ $ sup3r-solar from-config -c config_solar.json
+
+ A sup3r solar config.json file can contain any arguments or keyword
+ arguments required to call the
+ :meth:`sup3r.solar.solar.Solar.run_temporal_chunk` method. You do not need
+ to include the ``i_t_chunk`` input, this is added by the CLI. The config
+ also has several optional arguments: ``log_pattern``, ``log_level``, and
+ ``execution_control``. Here's a small example forward pass config::
+
+ {
+ "fp_pattern": "./chunks/sup3r*.h5",
+ "nsrdb_fp": "/datasets/NSRDB/current/nsrdb_2015.h5",
+ "execution_control": {
+ "option": "local"
+ },
+ "execution_control_eagle": {
+ "option": "eagle",
+ "walltime": 4,
+ "alloc": "sup3r"
+ }
+ }
+
+ Note that the ``execution_control`` block will run the job locally, while
+ the ``execution_control_eagle`` block are kwargs that would be required to
+ distribute the job on multiple nodes on the NREL HPC.
+ """
+ config_file = ctx.obj['CONFIG_FILE']
+ verbose = any([verbose, ctx.obj['VERBOSE']])
+ ctx.invoke(solar_cli, config_file=config_file, verbose=verbose)
 
 
 @main.command()

sup3r/pipeline/forward_pass.py

@@ -907,8 +907,8 @@ def max_nodes(self):
  work to, equal to either the specified max number of nodes or total
  number of temporal chunks"""
  nodes = (self._max_nodes if self._max_nodes is not None
- and not self._max_nodes > self.chunks
  else self.fwp_slicer.n_temporal_chunks)
+ nodes = np.min((nodes, self.chunks))
  return nodes
 
  @property

sup3r/postprocessing/file_handling.py

@@ -42,6 +42,24 @@
  'chunks': (2000, 500),
  'min': 0,
  'max': 1},
+ 'dhi': {'scale_factor': 1.0,
+ 'units': 'W/m2',
+ 'dtype': 'uint16',
+ 'chunks': (2000, 500),
+ 'min': 0,
+ 'max': 1350},
+ 'dni': {'scale_factor': 1.0,
+ 'units': 'W/m2',
+ 'dtype': 'uint16',
+ 'chunks': (2000, 500),
+ 'min': 0,
+ 'max': 1350},
+ 'ghi': {'scale_factor': 1.0,
+ 'units': 'W/m2',
+ 'dtype': 'uint16',
+ 'chunks': (2000, 500),
+ 'min': 0,
+ 'max': 1350},
  'temperature': {'scale_factor': 100.0,
  'units': 'C',
  'dtype': 'int16',
@@ -506,7 +524,7 @@ def _write_output(cls, data, features, lat_lon, times, out_file,
  flat_data = np.transpose(flat_data, (1, 0))
  fh.add_dataset(out_file, f, flat_data, dtype=attrs['dtype'],
  attrs=attrs, chunks=attrs['chunks'])
- logger.debug(f'Added {f} to output file')
+ logger.info(f'Added {f} to output file.')
 
  if meta_data is not None:
  fh.run_attrs = {'gan_meta': json.dumps(meta_data)}

sup3r/preprocessing/feature_handling.py

@@ -140,8 +140,7 @@ def compute(cls, data, height=None):
  """
 
  cs_ratio = data['rsds'] / data['clearsky_ghi']
- if cs_ratio.max() > 1:
- cs_ratio /= cs_ratio.max()
+ cs_ratio = np.minimum(cs_ratio, 1)
  cs_ratio = np.maximum(cs_ratio, 0)
 
  return cs_ratio

sup3r/qa/qa.py

@@ -26,6 +26,7 @@ class Sup3rQa:
 
  def __init__(self, source_file_paths, out_file_path, s_enhance, t_enhance,
  temporal_coarsening_method,
+ features=None,
  temporal_slice=slice(None),
  target=None,
  shape=None,
@@ -63,6 +64,11 @@ def __init__(self, source_file_paths, out_file_path, s_enhance, t_enhance,
  Subsample will take every t_enhance-th time step, average will
  average over t_enhance time steps, total will sum over t_enhance
  time steps
+ features : list | dict | None
+ Explicit list of features to validate. Can be a list of string
+ feature names, a dictionary mapping the sup3r output feature name
+ to the source_handler feature name (e.g. {'ghi': 'rsds'}), or None
+ for all features found in the out_file_path.
  temporal_slice : slice | tuple | list
  Slice defining size of full temporal domain. e.g. If we have 5
  files each with 5 time steps then temporal_slice = slice(None) will
@@ -134,14 +140,15 @@ def __init__(self, source_file_paths, out_file_path, s_enhance, t_enhance,
  self.t_enhance = t_enhance
  self._t_meth = temporal_coarsening_method
  self._out_fp = out_file_path
+ self._features = features
  self.qa_fp = qa_fp
  self.save_sources = save_sources
  self.output_handler = self.output_handler_class(self._out_fp)
 
  HandlerClass = get_input_handler_class(source_file_paths,
  input_handler)
  self.source_handler = HandlerClass(source_file_paths,
- self.features,
+ self.source_features,
  target=target,
  shape=shape,
  temporal_slice=temporal_slice,
@@ -160,7 +167,6 @@ def __enter__(self):
 
  def __exit__(self, type, value, traceback):
  self.close()
-
  if type is not None:
  raise
 
@@ -206,19 +212,37 @@ def features(self):
  -------
  list
  """
-
  # all lower case
  ignore = ('meta', 'time_index', 'times', 'xlat', 'xlong')
 
- if self.output_type == 'nc':
- features = list(self.output_handler.variables.keys())
- elif self.output_type == 'h5':
- features = self.output_handler.dsets
+ if self._features is None:
+ if self.output_type == 'nc':
+ features = list(self.output_handler.variables.keys())
+ elif self.output_type == 'h5':
+ features = self.output_handler.dsets
+ features = [f for f in features if f.lower() not in ignore]
 
- features = [f for f in features if f.lower() not in ignore]
+ elif isinstance(self._features, (list, tuple)):
+ features = self._features
+
+ elif isinstance(self._features, dict):
+ features = sorted(self._features.keys())
 
  return features
 
+ @property
+ def source_features(self):
+ """Get a list of feature names from the source input file, excluding
+ meta and time index datasets. This property considers the features
+ input mapping if a dictionary was provided, e.g. if
+ features={'ghi': 'rsds'}, this property will return ['rsds']"""
+ if isinstance(self._features, dict):
+ source_features = [self._features[f] for f in self.features]
+ else:
+ source_features = self.features
+
+ return source_features
+
  @property
  def output_type(self):
  """Get output data type

sup3r/solar/__init__.py

@@ -0,0 +1,5 @@
+# -*- coding: utf-8 -*-
+"""Custom sup3r solar module. This primarily converts GAN output clearsky ratio
+to GHI, DNI, and DHI using NSRDB data and utility modules like DISC"""
+
+from .solar import Solar