Merge 376 and 378

.travis.yml

@@ -5,27 +5,22 @@ language: python
  - "2.7"
 
 before_install:
- - if [ "$TRAVIS_OS_NAME" = "linux" ]; then sudo apt-get update -qq; sudo apt-get install -y gfortran xvfb; fi
+ - if [ "$TRAVIS_OS_NAME" = "linux" ]; then sudo apt-get update -qq; sudo apt-get install -y gfortran ; fi
  - if [ "$TRAVIS_OS_NAME" = "linux" ] && ! [ -d ${HOME}/miniconda] ; then wget https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh; fi
- - if [ "$TRAVIS_OS_NAME" = "osx" ]; then brew update ; fi 
- - if [ "$TRAVIS_OS_NAME" = "osx" ] && ! [ -d ${HOME}/miniconda ]; then wget https://repo.continuum.io/miniconda/Miniconda-latest-MacOSX-x86_64.sh -O miniconda.sh; fi
+# - if [ "$TRAVIS_OS_NAME" = "osx" ]; then brew update ; fi 
+# - if [ "$TRAVIS_OS_NAME" = "osx" ] && ! [ -d ${HOME}/miniconda ]; then wget https://repo.continuum.io/miniconda/Miniconda-latest-MacOSX-x86_64.sh -O miniconda.sh; fi
  - if [ "$TRAVIS_OS_NAME" = "osx" ] && ! [ -d /usr/local/gfortran ]; then wget https://github.com/UV-CDAT/uvcdat/releases/download/v2.4.1/gfortran-4.9.2-Mac.tar.gz -O ~/gfortran-4.9.2-Mac.tar.gz ; pushd / ; sudo tar xzvf ~/gfortran-4.9.2-Mac.tar.gz ; pushd ; fi
  - export PATH="$HOME/miniconda/bin:$PATH"
- - if [ ! -d ${HOME}/miniconda ]; then bash miniconda.sh -b -p $HOME/miniconda ; conda install --yes --channel uvcdat python=2.7 setuptools cdms2 cdutil genutil unidata cdtime vcs ; fi
+ - if [ ! -d ${HOME}/miniconda ]; then bash miniconda.sh -b -p $HOME/miniconda ; conda install --yes --channel uvcdat/label/nightly -c uvcdat uvcdat-nox  ; fi
  - hash -r
  - conda config --set always_yes yes --set changeps1 no
  - conda update -q conda
- - conda update -q cdms2 cdutil genutil unidata cdtime vcs -c uvcdat
  - conda install openssl=1.0.2d
- - conda update -f -c doutriaux1/label/doutriaux1 -c cdoutrix -c uvcdat --show-channel-urls -q -y vtk
  # Useful for debugging any issues with conda
  - conda info -a
 
 install:
  - python setup.py install
 
-before_script:
- - if [ "$TRAVIS_OS_NAME" = "linux" ]; then export DISPLAY=:99.0 ; sh -e /etc/init.d/xvfb start ; sleep 3 ; fi
-
 script:
  - python test/test_suite.py # --no-graphics

doc/parameter_files/input_parameter_file_all_options.py

@@ -75,6 +75,23 @@
 # metrics calculations)
 save_mod_clims = True
 
+
+# REGIONAL STUDIES
+# USER CAN CREATE CUSTOM REGIONS
+import cdutil
+regions_specs = {"Nino34":
+ {"value":0.,
+ "domain":cdutil.region.domain(latitude=(-5.,5.,"ccb"), longitude=(190.,240.,"ccb"))},
+ "NAM": { "value":0.,
+ "domain": {'latitude':(0.,45.), 'longitude':(210.,310.)},
+ }
+ }
+# REGIONS ON WHICH WE WANT TO RUN METRICS (var specific)
+# Here we run "all" default regions (glb, NHEX, SHEX, TROP) for both
+# but also ocean and user defined Nino34 and NAME for tas (only)
+regions = {"tas": [None, "ocean","Nino34","NAM"],"tos":None}
+
+
 # DATA LOCATION: MODELS, OBS AND METRICS OUTPUT - AND TEMPLATES FOR MODEL OUTPUT CLIMATOLOGY FILES
 # Following are custom keys specific to your local system
 # it is a python dictionary

setup.py

@@ -87,7 +87,8 @@
  ('share/graphics/vcs', portrait_files),
  ('share/pcmdi', ('doc/obs_info_dictionary.json',
  'share/pcmdi_metrics_table',
- 'share/disclaimer.txt')),
+ 'share/disclaimer.txt',
+ 'share/default_regions.py')),
  ]
 
 if install_dev:

share/default_regions.py

@@ -0,0 +1,18 @@
+import cdutil
+
+regions_specs = {
+ 'NHEX' : {'domain':cdutil.region.domain(latitude=(30.,90,'ccb'))},
+ 'SHEX' : {'domain':cdutil.region.domain(latitude=(-90.,-30,'ccb'))},
+ 'TROPICS': {'domain':cdutil.region.domain(latitude=(-30.,30,'ccb'))},
+ "global" : {},
+ 'land_NHEX' : {'value':100, 'domain':cdutil.region.domain(latitude=(30.,90,'ccb'))},
+ 'land_SHEX' : {'value':100, 'domain':cdutil.region.domain(latitude=(-90.,-30,'ccb'))},
+ 'land_TROPICS': {'value':100, 'domain':cdutil.region.domain(latitude=(-30.,30,'ccb'))},
+ "land" : {'value':100, },
+ 'ocean_NHEX' : {'value':0, 'domain':cdutil.region.domain(latitude=(30.,90,'ccb'))},
+ 'ocean_SHEX' : {'value':0, 'domain':cdutil.region.domain(latitude=(-90.,-30,'ccb'))},
+ 'ocean_TROPICS': {'value':0, 'domain':cdutil.region.domain(latitude=(30.,30,'ccb'))},
+ "ocean" : {'value':0, },
+ }
+
+default_regions = ['global','NHEX','SHEX','TROPICS']

src/python/io/base.py

@@ -1,6 +1,7 @@
 import json
 import cdms2
 import MV2
+import cdutil
 import genutil
 import os
 import pcmdi_metrics
@@ -13,19 +14,30 @@
 # where value is a integer between 0 and 9 included
 cdms2.setNetcdfDeflateLevelFlag(value)
 
+#cdutil region object need a serializer
+class CDMSDomainsEncoder(json.JSONEncoder):
+ def default(self, o):
+ cmp = o.components()[0].kargs
+ args=",".join(["%s=%s" % (k,v) for k,v in cmp.iteritems()])
+ return {o.id:"cdutil.region.domain(%s)"%args}
 
 class Base(genutil.StringConstructor):
 
- def __init__(self, root, file_template):
+ def __init__(self, root, file_template, file_mask_template=None):
  genutil.StringConstructor.__init__(self, root + "/" + file_template)
  self.targetGrid = None
  self.mask = None
  self.targetMask = None
+ self.file_mask_template = file_mask_template
+ self.root = root
 
  def __call__(self):
  return os.path.abspath(genutil.StringConstructor.__call__(self))
 
- def get(self, var, varInFile=None, *args, **kargs):
+ def get(self, var, varInFile=None, region={}, *args, **kargs):
+ if region is None:
+ region = {}
+
  self.variable = var
  if varInFile is None:
  varInFile = var
@@ -34,12 +46,31 @@ def get(self, var, varInFile=None, *args, **kargs):
  out = f(varInFile, *args, **kargs)
  f.close()
 
- # Now are we looking at a region in particular?
- if self.mask is not None:
+ # Now are we masking anything?
+ value = region.get("value",None)
+ if value is not None: # Indeed we are
+ if self.mask is None:
+ if isinstance(self.file_mask_template, basestring):
+ self.file_mask_template = Base(self.root,self.file_mask_template,{"domain":self.region.get("domain",None)})
+ try:
+ oMask = self.file_mask_template.get("sftlf")
+ # ok that failed falling back on autogenerate
+ except:
+ #dup.tb = args.traceback
+ #dup("Could not find obs mask, generating")
+ #dup.tb = False
+ oMask = cdutil.generateLandSeaMask(
+ out,
+ regridTool=self.regridTool).filled(1.) * 100.
+ oMask = MV2.array(oMask)
+ oMask.setAxis(-1, out.getLongitude())
+ oMask.setAxis(-2, out.getLatitude())
+ self.mask = oMask
  if self.mask.shape != out.shape:
  dum, msk = genutil.grower(out, self.mask)
  else:
  msk = self.mask
+ msk = MV2.not_equal(msk,value)
  out = MV2.masked_where(msk, out)
  if self.targetGrid is not None:
  out = out.regrid(
@@ -55,6 +86,16 @@ def get(self, var, varInFile=None, *args, **kargs):
  else:
  msk = self.targetMask
  out = MV2.masked_where(msk, out)
+ # Now are we looking at a region in particular?
+ domain = region.get("domain",None)
+ if domain is not None: # Ok we are subsetting 
+ if isinstance(domain,dict):
+ out=out(**domain)
+ elif isinstance(domain,(list,tuple)):
+ out=out(*domain)
+ elif isinstance(domain,cdms2.selectors.Selector):
+ domain.id=region.get("id","region")
+ out=out(*[domain])
  return out
 
  def setTargetGrid(self, target, regridTool="esmf", regridMethod="linear"):
@@ -89,7 +130,7 @@ def write(self, data, type="json", mode="w", *args, **kargs):
  f = open(fnm, mode)
  data["metrics_git_sha1"] = pcmdi_metrics.__git_sha1__
  data["uvcdat_version"] = cdat_info.get_version()
- json.dump(data, f, *args, **kargs)
+ json.dump(data, f, cls=CDMSDomainsEncoder, *args, **kargs)
  f.close()
  elif type.lower() in ["asc", "ascii", "txt"]:
  f = open(fnm, mode)

src/python/pcmdi/io.py

@@ -3,11 +3,11 @@
 
 class OBS(pcmdi_metrics.io.base.Base):
 
- def __init__(self, root, var, obs_dic, reference="default"):
+ def __init__(self, root, var, obs_dic, reference="default", file_mask_template=None):
 
  template = "%(realm)/%(frequency)/%(variable)/" +\
  "%(reference)/%(ac)/%(filename)"
- pcmdi_metrics.io.base.Base.__init__(self, root, template)
+ pcmdi_metrics.io.base.Base.__init__(self, root, template,file_mask_template)
  obs_name = obs_dic[var][reference]
  # usually send "default", "alternate", etc
  # but some case (sftlf) we send the actual name