For peter

.circleci/config.yml

@@ -17,7 +17,7 @@ aliases:
  name: create_conda_env
  environment:
  PKGS: "vcs vcsaddons mesalib matplotlib scipy cia testsrunner 'proj4<5' 'vtk-cdat>8.1'"
- CHANNELS: "-c cdat/label/nightly -c cdat-forge -c conda-forge -c cdat -c pcmdi"
+ CHANNELS: "-c cdat/label/nightly -c conda-forge -c pcmdi"
  command: |
  export PATH=$WORKDIR/miniconda/bin:$PATH
  conda config --set always_yes yes --set changeps1 no
@@ -26,7 +26,7 @@ aliases:
  if [[ $PY_VER = "py2" ]]; then
  conda create -q -n $PY_VER $CHANNELS "python<3" $PKGS $FFMPEG
  else
- conda create -q -n $PY_VER $CHANNELS "python>3" $PKGS $FFMPEG $COVERAGE_PKGS
+ conda create -q -n $PY_VER $CHANNELS "python=3.6" $PKGS $FFMPEG $COVERAGE_PKGS
  fi
 
  - &setup_pmp

pcmdi_metrics/driver/observation.py

@@ -13,11 +13,9 @@
 
 class OBS(Base):
  ''' Creates an output the netCDF file for an observation. '''
+
  def __init__(self, root, var, obs_dict, obs='default',
  file_mask_template=None):
- template = "%(realm)/%(frequency)/%(variable)/" +\
- "%(reference)/%(ac)/%(filename)"
- super(OBS, self).__init__(root, template, file_mask_template)
 
  logging.getLogger("pcmdi_metrics").setLevel(LOG_LEVEL)
 
@@ -29,6 +27,13 @@ def __init__(self, root, var, obs_dict, obs='default',
  if isinstance(obs_name, dict):
  obs_name = obs
 
+ template = obs_dict[var][obs_name].get("template", "%(realm)/%(frequency)/%(variable)/" +
+ "%(reference)/%(ac)/%(filename)")
+ print("OBFDDF:", obs_dict[var])
+ print("TEMPLATE:", root, template)
+
+ super(OBS, self).__init__(root, template, file_mask_template)
+
  obs_table = obs_dict[var][obs_name]['CMIP_CMOR_TABLE']
  self.realm = ''
  self.frequency = ''
@@ -39,6 +44,9 @@ def __init__(self, root, var, obs_dict, obs='default',
  self.reference = obs_name
  self.variable = var
 
+ if "subpath" in obs_dict[var][obs_name]:
+ self.template = obs_dict[var][obs_name]["subpath"]
+
  def setup_based_on_obs_table(self, obs_table):
  ''' Set the realm, frequency, ac based on the
  CMIP_CMOR_TABLE value in the obs dict.'''
@@ -59,6 +67,7 @@ def setup_based_on_obs_table(self, obs_table):
 class Observation(DataSet):
  ''' Handles all the computation (setting masking, target grid, etc)
  and some file I/O related to observations. '''
+
  def __init__(self, parameter, var_name_long, region,
  obs, obs_dict, data_path, sftlf):
  super(Observation, self).__init__(parameter, var_name_long, region,
@@ -98,7 +107,8 @@ def get_obs_from_obs_dict(self):
  dictionary for self.var and self.obs_or_model. '''
  if isinstance(self.obs_dict[self.var][self.obs_or_model], basestring):
  obs_from_obs_dict = \
- self.obs_dict[self.var][self.obs_dict[self.var][self.obs_or_model]]
+ self.obs_dict[self.var][self.obs_dict[self.var]
+ [self.obs_or_model]]
  else:
  obs_from_obs_dict = self.obs_dict[self.var][self.obs_or_model]
  return obs_from_obs_dict
@@ -128,12 +138,12 @@ def get(self):
  except Exception as e:
  if self.level is not None:
  logging.getLogger("pcmdi_metrics").error("{} {} {} {}".format('Failed opening 4D OBS',
- self.var, self.obs_or_model,
- e))
+  self.var, self.obs_or_model,
+  e))
  else:
  logging.getLogger("pcmdi_metrics").error("{} {} {} {}".format('Failed opening 3D OBS',
- self.var,
- self.obs_or_model, e))
+  self.var,
+  self.obs_or_model, e))
 
  def hash(self):
  ''' Return a hash of the file. '''

pcmdi_metrics/driver/pmp_parameter.py

@@ -25,18 +25,20 @@ def process_templated_argument(self, name, default_value="*", extras=None):
  """
 
  process = getattr(self, name, None)
+ print("PROCESS IS:", process)
  if process is None: # Ok not an argument from arg_parse maybe a template or string constructor itself
  if isinstance(name, basestring):
  process = name
  elif isinstance(name, genutil.StringConstructor):
  process = name.template
  else:
- raise RuntimeError("Could not figure out how to process argument {}".format(name))
-
- if not isinstance(process, basestring):
  raise RuntimeError(
  "Could not figure out how to process argument {}".format(name))
 
+ if not isinstance(process, basestring):
+ raise RuntimeError(
+ "Could not figure out how to process argument {}".format(name))
+
  if extras is None:
  sources = []
  elif not isinstance(extras, (list, tuple)):
@@ -97,7 +99,8 @@ def check_str(self, str_var, str_var_name):
  )
 
  if str_var == '':
- logging.getLogger("pcmdi_metrics").warning("%s is blank." % str_var_name)
+ logging.getLogger("pcmdi_metrics").warning(
+ "%s is blank." % str_var_name)
 
  def check_str_seq_in_str_list(self, str_sequence,
  str_sequence_name, str_vars_list):
@@ -116,15 +119,15 @@ def check_str_seq_in_str_list(self, str_sequence,
 
  def check_str_var_in_str_list(self, str_var, str_var_name, str_vars_list):
  if type(str_var) is not str:
-  raise TypeError(
-  "%s is the wrong type. It must be a string." % str_var_name
-  )
+ raise TypeError(
+ "%s is the wrong type. It must be a string." % str_var_name
+ )
 
  if str_var not in str_vars_list:
-  logging.getLogger("pcmdi_metrics").warning(
-  ("%s might not be a valid value in %s."
-  % (str_var, str_var_name))
-  )
+ logging.getLogger("pcmdi_metrics").warning(
+ ("%s might not be a valid value in %s."
+ % (str_var, str_var_name))
+ )
 
  def check_case_id(self):
  self.check_str(self.case_id, 'case_id')
@@ -138,7 +141,8 @@ def check_reference_data_set(self):
  )
 
  if self.reference_data_set == [] or self.reference_data_set == ():
- logging.getLogger("pcmdi_metrics").error("reference_data_set is blank.")
+ logging.getLogger("pcmdi_metrics").error(
+ "reference_data_set is blank.")
 
  def check_test_data_set(self):
  if type(self.test_data_set) is not list \
@@ -170,7 +174,8 @@ def check_vars(self):
 
  def check_ref(self):
  ref_values = ['default', 'all', 'alternate', 'ref3']
- self.check_str_seq_in_str_list(self.reference_data_set, 'reference_data_set', ref_values)
+ self.check_str_seq_in_str_list(
+ self.reference_data_set, 'reference_data_set', ref_values)
 
  def check_target_grid(self):
  self.check_str_var_in_str_list(

pcmdi_metrics/misc/scripts/parallelize_driver.py

@@ -17,6 +17,7 @@
 parser.add_argument("--driver", help="driver to prallelize")
 parser.add_argument("--bash", help="generate a bash script rather than running in parallel", action="store_true")
 parser.use("num_workers")
+parser.add_argument("--granularize")
 p = parser.get_parameter()
 
 param_name = parser.view_args().parameters

pcmdi_metrics/pcmdi/mean_climate_metrics_driver.py

@@ -79,6 +79,8 @@ def load_obs_dict(self):
 
  if hasattr(self.parameter, 'custom_observations'):
  # Can't use load_path_as_file_obj() b/c might not be in /share/
+ print("WE ARE USING CUSTOM OBS:",
+ self.parameter.custom_observations)
  cust_obs_json_file = open(self.parameter.custom_observations)
  obs_dict.update(json.load(cust_obs_json_file))
  cust_obs_json_file.close()