Update to latest master

.gitignore

@@ -1,9 +1,90 @@
-*.pyc
-build
-*.pyo
-*~
-*swp*
-*.o
-test_portrait
-PCMDI_METRICS
-obs_info_dictionary.json
+led / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+# Usually these files are written by a python script from a template
+# before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# OSX
+.DS_Store
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# IPython Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# dotenv
+.env
+
+# virtualenv
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+
+# Rope project settings
+.ropeproject

.travis.yml

@@ -1,26 +1,30 @@
 os:
  - linux
+# - osx
 language: python
  - "2.7"
 
 sudo: false
 
 before_install:
  - export UVCDAT_FEATURE=""
- - if [ "$TRAVIS_OS_NAME" = "linux" ]; then export UVCDAT_FEATURE="-nox"; fi
- - if [ "$TRAVIS_OS_NAME" = "linux" ] && ! [ -d ${HOME}/miniconda] ; then wget https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh; fi
+ - if [ "$TRAVIS_OS_NAME" = "linux" ]; then wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh; fi
+ - if [ "$TRAVIS_OS_NAME" = "osx" ]; then curl -o miniconda.sh https://repo.continuum.io/miniconda/Miniconda2-latest-MacOSX-x86_64.sh ; fi
  - export PATH="$HOME/miniconda/bin:$PATH"
- - if [ ! -d ${HOME}/miniconda ]; then bash miniconda.sh -b -p $HOME/miniconda ; conda install --yes --channel uvcdat/label/nightly -c uvcdat uvcdat${UVCDAT_FEATURE} hdf5==1.8.16 ; fi
+ - if [ ! -d ${HOME}/miniconda ]; then bash miniconda.sh -b -p $HOME/miniconda ; fi
  - hash -r
  - conda config --set always_yes yes --set changeps1 no
- - conda update -q conda
- - conda install openssl=1.0.2d
+ - conda install -f -q "conda<4.3.13"
+ - conda install --show-channel-urls --yes vcs vcsaddons mesalib cdp nose flake8 image-compare -c conda-forge -c uvcdat
+ # - conda install openssl=1.0.2d
  # Useful for debugging any issues with conda
  - conda info -a
 
 install:
  - python setup.py install
 
 script:
- - export HDF5_DISABLE_VERSION_CHECK=1
- - python test/test_suite.py # --no-graphics
+ - python run_tests.py -v2
+
+after_success:
+ - if [ "$TRAVIS_BRANCH" == "master" -a "$TRAVIS_PULL_REQUEST" == "false" ]; then conda install conda-build && conda install anaconda-client && bash -x recipes/pcmdi_metrics/conda_upload.sh; fi

README.md

@@ -1,9 +1,9 @@
 PCMDI metrics package (PMP)
 ======
-[![stable version](http:https://img.shields.io/badge/stable version-1.1-brightgreen.svg)](https://github.com/PCMDI/pcmdi_metrics/releases/tag/1.1)
+[![stable version](https:https://img.shields.io/badge/stable%20version-1.1.2-brightgreen.svg)](https://github.com/PCMDI/pcmdi_metrics/releases/tag/1.1.2)
 ![repo size](https://reposs.herokuapp.com/?path=PCMDI/pcmdi_metrics)
-![platforms](http:https://img.shields.io/badge/platforms-linux | osx-lightgrey.svg)
-[![DOI](http:https://img.shields.io/badge/DOI-10.5281/zenodo.58439-orange.svg)](http:https://doi.org/10.5281/zenodo.58439)
+![platforms](https:https://img.shields.io/badge/platforms-linux%20|%20osx-lightgrey.svg)
+[![DOI](https:https://zenodo.org/badge/DOI/10.5281/zenodo.809463.svg)](https:https://doi.org/10.5281/zenodo.809463)
 [![Anaconda-Server Badge](https://anaconda.org/pcmdi/pcmdi_metrics/badges/installer/conda.svg)](https://conda.anaconda.org/pcmdi)
 [![Anaconda-Server Badge](https://anaconda.org/pcmdi/pcmdi_metrics/badges/downloads.svg)](https://anaconda.org/pcmdi/pcmdi_metrics)
 
@@ -13,7 +13,7 @@ The PCMDI metrics package is used to objectively compare results from climate mo
 The metrics package consists of four parts: 1) Analysis software, 2) an observationally-based database of global (or near global, land or ocean) annual cycle climatologies, 3) a database of performance metrics computed for CMIP models and 4) package documentation.
 The package expects model data to be [CF-compliant](http:https://cfconventions.org/). To successfully use the package some input data "conditioning" may be required. We provide several demo scripts within the package to facilitate new users.
 
-**Users of the current release (v1.1) will need to contact the PMP developers ([email protected]) to obtain supporting datasets and get started using the package.**
+**Users of the current release (v1.1.2) will need to contact the PMP developers ([email protected]) to obtain supporting datasets and get started using the package.**
 
 
 GETTING STARTED
@@ -32,3 +32,5 @@ PMP versions
 v1.0 - Prototype version of the PMP
 
 v1.1 - First public release, emphasizing climatological statistics, with development branches for ENSO and regional monsoon precipitation indices
+
+v1.1.2 - Now managed through Anaconda, and tied to UV-CDAT 2.10. Weights on bias statistic added. Extensive provenance information incorporated into json files.

demo/parameter_files/pmp_input_parameters_demo1.py

@@ -21,26 +21,26 @@
 # non the less it is still a 'reserved' variable for the PMP
 case_id = 'simple-test1'
 # Now we will defined the list of models to use
-# For this we use the 'reserved' python variable: model_versions
+# For this we use the 'reserved' python variable: test_data_set
 # THIS IS A MANDATORY ENTRY
 # This is our first list of a python list, in this case there is only one entry
-model_versions = ['ACCESS1-0'] 
+test_data_set = ['ACCESS1-0']
 
 # The following lines will tell PMP where the data reside on your system
 # Note that we could use the 'templating' filename system here as well
 # Also note that these path are 'relative' to our current working path
 # But one could use absolute paths as well
 ## MODELS DATA LOCATION
-mod_data_path = 'pmp_demo/cmip5clims_metrics_package-amip/'
+test_data_path = 'pmp_demo/cmip5clims_metrics_package-amip/'
 ## ROOT PATH FOR OBSERVATIONS
-obs_data_path = 'pmp_demo/obs/'
+reference_data_path = 'pmp_demo/obs/'
 
 ## DIRECTORY WHERE TO SAVE RESULTS
 # USER CHOOSES, RESULTS STORED IN metrics_output_path + case_id 
 metrics_output_path = './pmp_demo/%(case_id)/' 
 
 # OBSERVATIONS TO USE: CHOICES INCLUDE 'default','alternate1','alternate2',... AND ARE VARIABLE DEPENDENT 
-ref = ['default'] #,'alternate1','alternate2']
+reference_data_set = ['default'] #,'alternate1','alternate2']
 
 ## A PYTHON LIST OF VARIABLES TO COMPUTE STATISTICS
 # THIS EXAMPLE ONLY INCLUDES ONE FIELD, PRECIPICATION
@@ -49,7 +49,7 @@
 # INTERPOLATION (REGRIDDING) OPTIONS
 # First our target grid, i.e the final grid onto which both model and obs will be put
 # OPTIONS: '2.5x2.5' or an actual cdms2 grid object
-targetGrid = '2.5x2.5' 
+target_grid = '2.5x2.5'
 # Now let's select which cdms2 regrid tool we will use
 regrid_tool = 'esmf' 
 # Some regrid tools also require to specify which method of regriding to use

demo/parameter_files/pmp_input_parameters_demo2_save_interpolated_clims.py

@@ -19,30 +19,30 @@
 
 
 ## FIRST USE OF A PYTHON LIST, IN THIS CASE IT HAS ONLY ONE ENTRY 
-model_versions = ['ACCESS1-0'] # THIS IS A MANDETORY ENTRY FOR DOCUMENTING RESULTS
+test_data_set = ['ACCESS1-0'] # THIS IS A MANDETORY ENTRY FOR DOCUMENTING RESULTS
 
 ###############################################################################
 ## DATA LOCATION: MODELS, OBS AND METRICS OUTPUT
 ## ROOT PATH FOR MODELS CLIMATOLOGIES
 #mod_data_path = '/work/metricspackage/mod_clims/cmip5-amip'
-mod_data_path = './demo_obs_model_data/mods/'
+test_data_path = './demo_obs_model_data/mods/'
 filename_template = "pr_Amon_ACCESS1-0_amip_r1i1p1_197901-198912-clim-ac.nc"
 ## ROOT PATH FOR OBSERVATIONS
-obs_data_path = './demo_obs_model_data/obs/'
+reference_data_path = './demo_obs_model_data/obs/'
 
 ## DIRECTORY WHERE TO SAVE RESULTS
 case_id = 'simple-test1'
 metrics_output_path = './pmp-test/' # USER CHOOSES, RESULTS STORED IN metrics_output_path + case_id 
 ###############################################################################
 
 # OBSERVATIONS TO USE: CHOICES INCLUDE 'default','alternate1','alternate2',... AND ARE VARIABLE DEPENDENT 
-ref = ['default'] #,'alternate1','alternate2']
+reference_data_set = ['default'] #,'alternate1','alternate2']
 
 ## A PYTHON LIST OF VARIABLES TO COMPUTE STATISTICS
 vars = ['pr'] # THIS EXAMPLE ONLY INCLUDES ONE FIELD, PRECIPICATION
 
 # INTERPOLATION OPTIONS
-targetGrid = '2.5x2.5' # OPTIONS: '2.5x2.5' or an actual cdms2 grid object
+target_grid = '2.5x2.5' # OPTIONS: '2.5x2.5' or an actual cdms2 grid object
 regrid_tool = 'esmf' #'regrid2' # OPTIONS: 'regrid2','esmf'
 regrid_method = 'linear' # OPTIONS: 'linear','conservative', only if tool is esmf
 
@@ -60,9 +60,9 @@
  } 
 
 generate_sftlf = True # IF MODEL LAND SEA MASK NOT AVAILABLE AUTOMATICALLY (APPROXIMATE!) GENERATE IN AT TARGET RESoLUTION 
-save_mod_clims = True # True - output interpolated model climatology False - don't output
+save_test_clims = True # True - output interpolated model climatology False - don't output
 
-model_clims_interpolated_output = './interpolated-output/'
+test_clims_interpolated_output = './interpolated-output/'
 ext = '.nc'
 
 if domain == 'land':

demo/pmp_demo.py

@@ -8,7 +8,7 @@
 import sys
 import shlex
 import genutil
-
+import argparse
 
 class bgcolor:
  HEADER = '\033[95m'
@@ -59,6 +59,11 @@ def describe(demo_file,colorized=True):
  comment(bc+"This ends this parameter file"+bgcolor.ENDC)
 
 def demo(demo_file,title,colorized=True):
+ parser = argparse.ArgumentParser(description='Just used for pointing to demo data')
+ parser.add_argument('-d', dest='demo_data_path',
+ help='Path to the tarball of demo data')
+ args = parser.parse_args()
+
  comment("""
  PMP Demo: %s 
 
@@ -78,34 +83,41 @@ def demo(demo_file,title,colorized=True):
  if not os.path.exists(demo_pth):
  os.makedirs(demo_pth)
  # http:https://oceanonly.llnl.gov/gleckler1/pmp-demo-data/pmpv1.1_demodata.tar
- tar_filename = "pmpv1.1_demodata.tar"
- tar_pth = os.path.join(demo_pth,tar_filename)
-
- good_md5 = "a6ef8f15457378ff36fd46e8fbf5f157"
-
- attempts = 0
- while attempts < 3:
- md5 = hashlib.md5()
- if os.path.exists(tar_filename):
- f = open(tar_filename)
- md5.update(f.read())
+
+ if args.demo_data_path:
+ tar_pth = os.path.abspath(args.demo_data_path)
+ if not os.path.exists(tar_pth):
+ raise IOError('%s does not exist' % tar_pth)
+
+ else:
+ tar_filename = "pmpv1.1_demodata.tar" 
+ tar_pth = os.path.join(demo_pth,tar_filename)
+
+ good_md5 = "a6ef8f15457378ff36fd46e8fbf5f157"
+
+ attempts = 0
+ while attempts < 3:
+ md5 = hashlib.md5()
+ if os.path.exists(tar_filename):
+ f = open(tar_filename)
+ md5.update(f.read())
+ if md5.hexdigest() == good_md5:
+ attempts = 5
+ continue
+ print "Downloading: ", tar_filename
+ r = requests.get("http:https://oceanonly.llnl.gov/gleckler1/pmp-demo-data/pmpv1.1_demodata.tar", stream=True)
+ with open(tar_pth, 'wb') as f:
+ for chunk in r.iter_content(chunk_size=1024):
+ if chunk: # filter local_filename keep-alive new chunks
+ f.write(chunk)
+ md5.update(chunk)
+ f.close()
  if md5.hexdigest() == good_md5:
  attempts = 5
- continue
- print "Downloading: ", tar_filename
- r = requests.get("http:https://oceanonly.llnl.gov/gleckler1/pmp-demo-data/pmpv1.1_demodata.tar", stream=True)
- with open(tar_pth, 'wb') as f:
- for chunk in r.iter_content(chunk_size=1024):
- if chunk: # filter local_filename keep-alive new chunks
- f.write(chunk)
- md5.update(chunk)
- f.close()
- if md5.hexdigest() == good_md5:
- attempts = 5
- else:
- attempts += 1
+ else:
+ attempts += 1
 
- comment("Successfuly downloaded demo tarball\nNow untarring it", None)
+  comment("Successfuly downloaded demo tarball\nNow untarring it", None)
 
  tar_process = subprocess.Popen(shlex.split("tar xvf %s"%tar_pth),cwd=demo_pth)
  tar_process.wait()