Merge pull request #168 from NREL/pp/gaps_integration

GAPs integration

requirements.txt

@@ -1,8 +1,7 @@
 matplotlib>=3.1
 NREL-rex>=0.2.84
 NREL-phygnn>=0.0.23
-NREL-rev<0.8.0
-NREL-gaps>=0.4.0
+NREL-gaps>=0.6.0
 NREL-farms>=1.0.4
 google-auth-oauthlib==0.5.3
 pytest>=5.2

sup3r/__init__.py

@@ -2,6 +2,7 @@
 """Super Resolving Renewable Energy Resource Data (SUP3R)"""
 import os
 from sup3r.version import __version__
+import sup3r.cli # import sets up CLI commands
 
 __author__ = """Brandon Benton"""
 __email__ = "[email protected]"

sup3r/batch/batch_cli.py

@@ -5,12 +5,8 @@
 """
 import click
 
-from rex.utilities.utilities import get_class_properties
-
-from reV.config.batch import BatchConfig
-
 from sup3r import __version__
-from sup3r.batch.batch import BatchJob
+from gaps.batch import BatchJob
 
 
 @click.group()
@@ -24,14 +20,6 @@ def main(ctx, verbose):
  ctx.obj['VERBOSE'] = verbose
 
 
-@main.command()
-def valid_config_keys():
- """
- Echo the valid Batch config keys
- """
- click.echo(', '.join(get_class_properties(BatchConfig)))
-
-
 @main.command()
 @click.option('--config_file', '-c', required=True,
  type=click.Path(exists=True),
@@ -52,18 +40,18 @@ def valid_config_keys():
  help='Flag to turn on debug logging. Default is not verbose.')
 @click.pass_context
 def from_config(ctx, config_file, dry_run, cancel, delete, monitor_background,
- verbose):
+ verbose=False):
  """Run Sup3r batch from a config file."""
- verbose = any([verbose, ctx.obj['VERBOSE']])
+ ctx.ensure_object(dict)
+ ctx.obj['VERBOSE'] = verbose or ctx.obj.get('VERBOSE', False)
+ batch = BatchJob(config_file)
 
  if cancel:
- BatchJob.cancel_all(config_file, verbose=verbose)
+ batch.cancel()
  elif delete:
- BatchJob.delete_all(config_file, verbose=verbose)
+ batch.delete()
  else:
- BatchJob.run(config_file, dry_run=dry_run,
- monitor_background=monitor_background,
- verbose=verbose)
+ batch.run(dry_run=dry_run, monitor_background=monitor_background)
 
 
 if __name__ == '__main__':

sup3r/bias/bias_calc.py

@@ -182,7 +182,7 @@ def get_node_cmd(cls, config):
  initialize the class and call run() on a single node.
  """
  import_str = 'import time;\n'
- import_str += 'from sup3r.pipeline import Status;\n'
+ import_str += 'from gaps import Status;\n'
  import_str += 'from rex import init_logger;\n'
  import_str += f'from sup3r.bias.bias_calc import {cls.__name__};\n'
 

sup3r/bias/bias_calc_cli.py

@@ -10,7 +10,7 @@
 
 import sup3r.bias.bias_calc
 from sup3r.utilities import ModuleName
-from sup3r.utilities.cli import BaseCLI
+from sup3r.utilities.cli import AVAILABLE_HARDWARE_OPTIONS, BaseCLI
 from sup3r.version import __version__
 
 logger = logging.getLogger(__name__)
@@ -34,7 +34,7 @@ def main(ctx, verbose):
 @click.option('-v', '--verbose', is_flag=True,
  help='Flag to turn on debug logging. Default is not verbose.')
 @click.pass_context
-def from_config(ctx, config_file, verbose):
+def from_config(ctx, config_file, verbose=False, **__):
  """Run sup3r bias correction calculation from a config file."""
  config = BaseCLI.from_config_preflight(ModuleName.BIAS_CALC, ctx,
  config_file, verbose)
@@ -62,7 +62,7 @@ def from_config(ctx, config_file, verbose):
  cmd_log = '\n\t'.join(cmd.split('\n'))
  logger.debug(f'Running command:\n\t{cmd_log}')
 
- if hardware_option.lower() in ('eagle', 'slurm'):
+ if hardware_option.lower() in AVAILABLE_HARDWARE_OPTIONS:
  kickoff_slurm_job(ctx, cmd, **exec_kwargs)
  else:
  kickoff_local_job(ctx, cmd)

sup3r/cli.py

@@ -5,7 +5,10 @@
 import click
 import logging
 
+from gaps import Pipeline
+
 from sup3r.version import __version__
+from sup3r.utilities import ModuleName
 from sup3r.pipeline.forward_pass_cli import from_config as fwp_cli
 from sup3r.solar.solar_cli import from_config as solar_cli
 from sup3r.preprocessing.data_extract_cli import from_config as dh_cli
@@ -14,7 +17,6 @@
 from sup3r.qa.visual_qa_cli import from_config as visual_qa_cli
 from sup3r.qa.stats_cli import from_config as stats_cli
 from sup3r.pipeline.pipeline_cli import from_config as pipe_cli
-from sup3r.pipeline.pipeline_cli import valid_config_keys as pipeline_keys
 from sup3r.batch.batch_cli import from_config as batch_cli
 from sup3r.bias.bias_calc_cli import from_config as bias_calc_cli
 from sup3r.utilities.regridder_cli import from_config as regrid_cli
@@ -114,18 +116,15 @@ def forward_pass(ctx, verbose):
  },
  },
  "execution_control": {
- "option": "local"
- },
- "execution_control_eagle": {
- "option": "eagle",
+ "option": "kestrel",
  "walltime": 4,
  "alloc": "sup3r"
  }
  }
 
- Note that the ``execution_control`` block will run the job locally, while
- the ``execution_control_eagle`` block are kwargs that would be required to
- distribute the job on multiple nodes on the NREL HPC.
+ Note that the ``execution_control`` block contains kwargs that would
+ be required to distribute the job on multiple nodes on the NREL HPC.
+ To run the job locally, use ``execution_control: {"option": "local"}``.
  """
  config_file = ctx.obj['CONFIG_FILE']
  verbose = any([verbose, ctx.obj['VERBOSE']])
@@ -163,18 +162,15 @@ def solar(ctx, verbose):
  "fp_pattern": "./chunks/sup3r*.h5",
  "nsrdb_fp": "/datasets/NSRDB/current/nsrdb_2015.h5",
  "execution_control": {
- "option": "local"
- },
- "execution_control_eagle": {
- "option": "eagle",
+ "option": "kestrel",
  "walltime": 4,
  "alloc": "sup3r"
  }
  }
 
-  Note that the ``execution_control`` block will run the job locally, while
- the ``execution_control_eagle`` block are kwargs that would be required to
- distribute the job on multiple nodes on the NREL HPC.
+ Note that the ``execution_control`` block contains kwargs that would
+ be required to distribute the job on multiple nodes on the NREL HPC.
+ To run the job locally, use ``execution_control: {"option": "local"}``.
  """
  config_file = ctx.obj['CONFIG_FILE']
  verbose = any([verbose, ctx.obj['VERBOSE']])
@@ -227,18 +223,15 @@ def bias_calc(ctx, verbose):
  }
  ],
  "execution_control": {
- "option": "local"
- },
- "execution_control_eagle": {
- "option": "eagle",
+ "option": "kestrel",
  "walltime": 4,
  "alloc": "sup3r"
  }
  }
 
- Note that the ``execution_control`` block will run the job locally, while
- the ``execution_control_eagle`` block are kwargs that would be required to
- distribute the job on multiple nodes on the NREL HPC.
+ Note that the ``execution_control`` block contains kwargs that would
+ be required to distribute the job on multiple nodes on the NREL HPC.
+ To run the job locally, use ``execution_control: {"option": "local"}``.
  """
  config_file = ctx.obj['CONFIG_FILE']
  verbose = any([verbose, ctx.obj['VERBOSE']])
@@ -277,7 +270,7 @@ def data_extract(ctx, verbose):
  }
 
  Note that the ``execution_control`` has the same options as forward-pass
- and you can set ``"option": "eagle"`` to run on the NREL HPC.
+ and you can set ``"option": "kestrel"`` to run on the NREL HPC.
  """
  config_file = ctx.obj['CONFIG_FILE']
  verbose = any([verbose, ctx.obj['VERBOSE']])
@@ -316,7 +309,7 @@ def data_collect(ctx, verbose):
  }
 
  Note that the ``execution_control`` has the same options as forward-pass
- and you can set ``"option": "eagle"`` to run on the NREL HPC.
+ and you can set ``"option": "kestrel"`` to run on the NREL HPC.
  """
  config_file = ctx.obj['CONFIG_FILE']
  verbose = any([verbose, ctx.obj['VERBOSE']])
@@ -357,7 +350,7 @@ def qa(ctx, verbose):
  }
 
  Note that the ``execution_control`` has the same options as forward-pass
- and you can set ``"option": "eagle"`` to run on the NREL HPC.
+ and you can set ``"option": "kestrel"`` to run on the NREL HPC.
  """
  config_file = ctx.obj['CONFIG_FILE']
  verbose = any([verbose, ctx.obj['VERBOSE']])
@@ -397,7 +390,7 @@ def visual_qa(ctx, verbose):
  }
 
  Note that the ``execution_control`` has the same options as forward-pass
- and you can set ``"option": "eagle"`` to run on the NREL HPC.
+ and you can set ``"option": "kestrel"`` to run on the NREL HPC.
  """
  config_file = ctx.obj['CONFIG_FILE']
  verbose = any([verbose, ctx.obj['VERBOSE']])
@@ -441,7 +434,7 @@ def stats(ctx, verbose):
  }
 
  Note that the ``execution_control`` has the same options as forward-pass
- and you can set ``"option": "eagle"`` to run on the NREL HPC.
+ and you can set ``"option": "kestrel"`` to run on the NREL HPC.
  """
  config_file = ctx.obj['CONFIG_FILE']
  verbose = any([verbose, ctx.obj['VERBOSE']])
@@ -484,7 +477,7 @@ def regrid(ctx, verbose):
  }
 
  Note that the ``execution_control`` has the same options as forward-pass
- and you can set ``"option": "eagle"`` to run on the NREL HPC.
+ and you can set ``"option": "kestrel"`` to run on the NREL HPC.
  """
  config_file = ctx.obj['CONFIG_FILE']
  verbose = any([verbose, ctx.obj['VERBOSE']])
@@ -539,13 +532,6 @@ def pipeline(ctx, cancel, monitor, background, verbose):
  monitor=monitor, background=background, verbose=verbose)
 
 
-@pipeline.command()
-@click.pass_context
-def valid_pipeline_keys(ctx):
- """Print the valid pipeline config keys"""
- ctx.invoke(pipeline_keys)
-
-
 @main.group(invoke_without_command=True)
 @click.option('--dry-run', is_flag=True,
  help='Flag to do a dry run (make batch dirs without running).')
@@ -598,6 +584,17 @@ def batch(ctx, dry_run, cancel, delete, monitor_background, verbose):
  verbose=verbose)
 
 
+Pipeline.COMMANDS[ModuleName.FORWARD_PASS] = fwp_cli
+Pipeline.COMMANDS[ModuleName.SOLAR] = solar_cli
+Pipeline.COMMANDS[ModuleName.DATA_EXTRACT] = dh_cli
+Pipeline.COMMANDS[ModuleName.DATA_COLLECT] = dc_cli
+Pipeline.COMMANDS[ModuleName.QA] = qa_cli
+Pipeline.COMMANDS[ModuleName.VISUAL_QA] = visual_qa_cli
+Pipeline.COMMANDS[ModuleName.STATS] = stats_cli
+Pipeline.COMMANDS[ModuleName.BIAS_CALC] = bias_calc_cli
+Pipeline.COMMANDS[ModuleName.REGRID] = regrid_cli
+
+
 if __name__ == '__main__':
  try:
  main(obj={})

sup3r/pipeline/__init__.py

@@ -2,6 +2,3 @@
 """
 Sup3r data pipeline architecture.
 """
-from gaps.legacy import Status
-
-from .pipeline import Sup3rPipeline