changed i_t_chunk arg -> temporal_id which makes a lot more sense

sup3r/solar/solar.py

@@ -8,7 +8,6 @@
 import glob
 import json
 import os
-import pandas as pd
 import numpy as np
 import logging
 from scipy.spatial import KDTree
@@ -550,7 +549,7 @@ def run_temporal_chunk(cls, fp_pattern, nsrdb_fp,
  fp_out_suffix='irradiance', tz=-6, agg_factor=1,
  nn_threshold=0.5, cloud_threshold=0.99,
  features=('ghi', 'dni', 'dhi'),
- i_t_chunk=None):
+ temporal_id=None):
  """Run the solar module on all spatial chunks for a single temporal
  chunk corresponding to the fp_pattern. This typically gets run from the
  CLI.
@@ -593,22 +592,22 @@ def run_temporal_chunk(cls, fp_pattern, nsrdb_fp,
  features : list | tuple
  List of features to write to disk. These have to be attributes of
  the Solar class (ghi, dni, dhi).
- i_t_chunk : int | None
- Index of the sorted list of unique temporal indices to run (parsed
- from the files matching fp_pattern). This input typically gets set
- from the CLI. If None, this will run all temporal indices.
+ temporal_id : str | None
+ One of the unique zero-padded temporal id's from the file chunks
+ that match fp_pattern. This input typically gets set from the CLI.
+ If None, this will run all temporal indices.
  """
 
  temp = cls.get_sup3r_fps(fp_pattern, ignore=f'_{fp_out_suffix}.h5')
  fp_sets, t_slices, temporal_ids, spatial_ids, target_fps = temp
 
- if i_t_chunk is not None:
+ if temporal_id is not None:
  fp_sets = [fp_set for i, fp_set in enumerate(fp_sets)
- if temporal_ids[i] == temporal_ids[i_t_chunk]]
+ if temporal_ids[i] == temporal_id]
  t_slices = [t_slice for i, t_slice in enumerate(t_slices)
- if temporal_ids[i] == temporal_ids[i_t_chunk]]
+ if temporal_ids[i] == temporal_id]
  target_fps = [target_fp for i, target_fp in enumerate(target_fps)
- if temporal_ids[i] == temporal_ids[i_t_chunk]]
+ if temporal_ids[i] == temporal_id]
 
  zip_iter = zip(fp_sets, t_slices, target_fps)
  for i, (fp_set, t_slice, fp_target) in enumerate(zip_iter):

sup3r/solar/solar_cli.py

@@ -76,7 +76,7 @@ def from_config(ctx, config_file, verbose):
  'on. Submitting to {} nodes based on the number of temporal '
  'chunks'.format(len(fp_sets), len(set(temporal_ids))))
 
- for i_node, _ in enumerate(temporal_ids):
+ for i_node, temporal_id in enumerate(sorted(set(temporal_ids))):
  node_config = copy.deepcopy(config)
  node_config['log_file'] = (
  log_pattern if log_pattern is None
@@ -86,7 +86,7 @@ def from_config(ctx, config_file, verbose):
  ctx.obj['NAME'] = name
  node_config['job_name'] = name
 
- node_config['i_t_chunk'] = i_node
+ node_config['temporal_id'] = temporal_id
  cmd = Solar.get_node_cmd(node_config)
 
  cmd_log = '\n\t'.join(cmd.split('\n'))

tests/test_solar_module.py

@@ -44,6 +44,7 @@ def make_cs_ratio_files(td, low_res_times, low_res_lat_lon):
  """
  fps = []
  chunk_dir = os.path.join(td, 'chunks/')
+ fp_pattern = os.path.join(chunk_dir, 'sup3r_chunk_*.h5')
  os.makedirs(chunk_dir)
 
  for idt, timestamp in enumerate(low_res_times):
@@ -60,7 +61,7 @@ def make_cs_ratio_files(td, low_res_times, low_res_lat_lon):
  [timestamp],
  out_file, max_workers=1,
  meta_data=GAN_META)
- return fps
+ return fps, fp_pattern
 
 
 def test_solar_module(plot=False):
@@ -71,13 +72,14 @@ def test_solar_module(plot=False):
 
  with tempfile.TemporaryDirectory() as td:
 
- fps = make_cs_ratio_files(td, LOW_RES_TIMES, LOW_RES_LAT_LON)
+ fps, _ = make_cs_ratio_files(td, LOW_RES_TIMES, LOW_RES_LAT_LON)
 
  with Resource(fps[1]) as res:
  meta_base = res.meta
  attrs_base = res.global_attrs
 
- with Solar(fps, NSRDB_FP, t_slice=t_slice, nn_threshold=0.4) as solar:
+ with Solar(fps, NSRDB_FP, t_slice=t_slice,
+ nn_threshold=0.4) as solar:
  ghi = solar.ghi
  dni = solar.dni
  dhi = solar.dhi
@@ -168,8 +170,8 @@ def test_solar_cli(runner):
  """Test the solar CLI. This test is here and not in the test_cli.py file
  because it uses some common test utilities stored here."""
  with tempfile.TemporaryDirectory() as td:
- fps = make_cs_ratio_files(td, LOW_RES_TIMES, LOW_RES_LAT_LON)
- fp_pattern = os.path.join(td, 'chunks/sup3r*.h5')
+ fps, fp_pattern = make_cs_ratio_files(td, LOW_RES_TIMES,
+  LOW_RES_LAT_LON)
  config = {'fp_pattern': fp_pattern,
  'nsrdb_fp': NSRDB_FP,
  'log_level': 'DEBUG',