Merge pull request #386 from PCMDI/343_pjg_simple_demos

343 pjg simple demos

.travis.yml

@@ -6,12 +6,14 @@ language: python
 
 before_install:
  - if [ "$TRAVIS_OS_NAME" = "linux" ]; then sudo apt-get update -qq; sudo apt-get install -y gfortran ; fi
+ - export UVCDAT_FEATURE=""
+ - if [ "$TRAVIS_OS_NAME" = "linux" ]; then export UVCDAT_FEATURE="-nox"; fi
  - if [ "$TRAVIS_OS_NAME" = "linux" ] && ! [ -d ${HOME}/miniconda] ; then wget https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh; fi
 # - if [ "$TRAVIS_OS_NAME" = "osx" ]; then brew update ; fi 
 # - if [ "$TRAVIS_OS_NAME" = "osx" ] && ! [ -d ${HOME}/miniconda ]; then wget https://repo.continuum.io/miniconda/Miniconda-latest-MacOSX-x86_64.sh -O miniconda.sh; fi
  - if [ "$TRAVIS_OS_NAME" = "osx" ] && ! [ -d /usr/local/gfortran ]; then wget https://github.com/UV-CDAT/uvcdat/releases/download/v2.4.1/gfortran-4.9.2-Mac.tar.gz -O ~/gfortran-4.9.2-Mac.tar.gz ; pushd / ; sudo tar xzvf ~/gfortran-4.9.2-Mac.tar.gz ; pushd ; fi
  - export PATH="$HOME/miniconda/bin:$PATH"
- - if [ ! -d ${HOME}/miniconda ]; then bash miniconda.sh -b -p $HOME/miniconda ; conda install --yes --channel uvcdat/label/nightly -c uvcdat uvcdat-nox ; fi
+ - if [ ! -d ${HOME}/miniconda ]; then bash miniconda.sh -b -p $HOME/miniconda ; conda install --yes --channel uvcdat/label/nightly -c uvcdat uvcdat${UVCDAT_FEATURE} hdf5==1.8.16 ; fi
  - hash -r
  - conda config --set always_yes yes --set changeps1 no
  - conda update -q conda
@@ -23,4 +25,5 @@ install:
  - python setup.py install
 
 script:
+ - export HDF5_DISABLE_VERSION_CHECK=1
  - python test/test_suite.py # --no-graphics

demo/parameter_files/pmp_input_parameters_demo1.py

@@ -1,49 +1,73 @@
 ################################################################################
-# SAMPLE INPUT PARAMETER FILE FOR THE PCMDI METRICS PACKAGE (PMP V1.1)
-# THIS IS A VERY SIMPLE EXAMPLE, ONLY COMPUTING STATISTICS FOR ONE MODEL VERSION AND ONE VARIABLE 
-# SAMPLE MODEL AND OBSERVATIONAL DATA TO RUN THIS CODE ARE AVAILABLE FROM: 
-# 
+# THIS IS A SIMPLE EXAMPLE mean climate "parameter file" USED IN A DEMO EXECUTION OF THE PMP
+
+# For more info, see https://github.com/PCMDI/pcmdi_metrics/wiki/Using-the-package
+#
+# THIS PARAMETER FILE IS USED TO COMPUTE STATISTICS FOR ONE MODEL VERSION AND ONE VARIABLE ONLY
+
 # NOTES FOR PYTHON NEWBIES: ON ANY GIVEN LINE, ANTHING TO THE RIGHT OF A "#" IS CONSIDERED A COMMENT IN PYTHON 
 # IN THIS SIMPLE EXAMPLE WE DEFINE CHARACTER STRINGS, BUT ALSO ONE OF THE MOST BASIC AND POWERFUL PYTHON OBJECTS KNOWN AS A LIST. 
 # PYTHON LISTS ARE DEFINED WITH SQUARE BRACKETS [] ... FOR MORE INFO SEE: https://docs.python.org/2/tutorial/datastructures.html
 ################################################################################
+#
+# First, we define the 'template' used by the PMP to construct file names and paths that correspond to the location of the model and observatinal data.
+# Keywords in between %() will be automatically filled by PMP
+# In this example we only use four of the 'official' keys: 'variable', 'model_version', 'realization' and 'period'
+# some of these are defined later in the parameter file
+# For a complete list of official keys see: 
+filename_template = "%(variable)_%(model_version)_Amon_amip_%(realization)_%(period)-clim.nc"
+# First lets define an case id to help us differentiate between many se of parameter files
+# The python 'case_id' variable is optional in the parameter file
+# non the less it is still a 'reserved' variable for the PMP
+case_id = 'simple-test1'
+# Now we will defined the list of models to use
+# For this we use the 'reserved' python variable: model_versions
+# THIS IS A MANDATORY ENTRY
+# This is our first list of a python list, in this case there is only one entry
+model_versions = ['ACCESS1-0'] 
 
-## FIRST USE OF A PYTHON LIST, IN THIS CASE IT HAS ONLY ONE ENTRY 
-model_versions = ['ACCESS1-0'] # THIS IS A MANDETORY ENTRY FOR DOCUMENTING RESULTS
-
-###############################################################################
-## DATA LOCATION: MODELS, OBS AND METRICS OUTPUT
-## ROOT PATH FOR MODELS CLIMATOLOGIES
-#mod_data_path = '/work/metricspackage/mod_clims/cmip5-amip'
-mod_data_path = './demo_obs_model_data/mods/'
-filename_template = "pr_Amon_ACCESS1-0_amip_r1i1p1_197901-198912-clim-ac.nc"
+# The following lines will tell PMP where the data reside on your system
+# Note that we could use the 'templating' filename system here as well
+# Also note that these path are 'relative' to our current working path
+# But one could use absolute paths as well
+## MODELS DATA LOCATION
+mod_data_path = 'pmp_demo/cmip5clims_metrics_package-amip/'
 ## ROOT PATH FOR OBSERVATIONS
-obs_data_path = './demo_obs_model_data/obs/'
+obs_data_path = 'pmp_demo/obs/'
 
 ## DIRECTORY WHERE TO SAVE RESULTS
-case_id = 'simple-test1'
-metrics_output_path = './pmp-test/' # USER CHOOSES, RESULTS STORED IN metrics_output_path + case_id 
-###############################################################################
+# USER CHOOSES, RESULTS STORED IN metrics_output_path + case_id 
+metrics_output_path = './pmp_demo/%(case_id)/' 
 
 # OBSERVATIONS TO USE: CHOICES INCLUDE 'default','alternate1','alternate2',... AND ARE VARIABLE DEPENDENT 
 ref = ['default'] #,'alternate1','alternate2']
 
 ## A PYTHON LIST OF VARIABLES TO COMPUTE STATISTICS
-vars = ['pr'] # THIS EXAMPLE ONLY INCLUDES ONE FIELD, PRECIPICATION
+# THIS EXAMPLE ONLY INCLUDES ONE FIELD, PRECIPICATION
+vars = ['pr'] 
 
-# INTERPOLATION OPTIONS
-targetGrid = '2.5x2.5' # OPTIONS: '2.5x2.5' or an actual cdms2 grid object
-regrid_tool = 'esmf' #'regrid2' # OPTIONS: 'regrid2','esmf'
-regrid_method = 'linear' # OPTIONS: 'linear','conservative', only if tool is esmf
+# INTERPOLATION (REGRIDDING) OPTIONS
+# First our target grid, i.e the final grid onto which both model and obs will be put
+# OPTIONS: '2.5x2.5' or an actual cdms2 grid object
+targetGrid = '2.5x2.5' 
+# Now let's select which cdms2 regrid tool we will use
+regrid_tool = 'esmf' 
+# Some regrid tools also require to specify which method of regriding to use
+# OPTIONS: 'linear','conservative', only if tool is esmf
+regrid_method = 'linear'
 
 # SIMULATION PARAMETERS (required in PMP v1.1)
-period = '1979-1989' # PERIOD OF CLIMATOLOGY
-realization = 'r1i1p1' # REALIZATION
-
-
-
-
+# These are manadatory in any PMP parameter file
+# Beside remember that these are actually using in our templating system defined above
+# PERIOD OF CLIMATOLOGY
+period = '198001-200512'
+# MODEL REALIZATION
+realization = 'r1i1p1'
 
+### DONE!
 
+# THIS DEMO PARAMETER FILE IS AVAILABLE AT: https://github.com/PCMDI/pcmdi_metrics/blob/master/demo/parameter_files/pmp_input_parameters_demo1.py 
+# FOR A COMPLETE LISTING OF ALL AVAILABLE KEYS IN THE PMP PARAMETER FILES SEE: https://github.com/PCMDI/pcmdi_metrics/blob/master/doc/parameter_files/input_parameter_file_all_options.py
 
+
 

demo/pmp_demo.py

@@ -0,0 +1,129 @@
+#!/usr/bin/env python
+
+# This is a very simple demo for new users
+import subprocess
+import requests
+import hashlib
+import os
+import sys
+import shlex
+import genutil
+
+
+class bgcolor:
+ HEADER = '\033[95m'
+ OKBLUE = '\033[94m'
+ OKGREEN = '\033[92m'
+ WARNING = '\033[93m'
+ FAIL = '\033[91m'
+ ENDC = '\033[0m'
+ BOLD = '\033[1m'
+ UNDERLINE = '\033[4m'
+
+
+def comment(text,cont="[Press enter]"):
+ print text
+ if cont is None:
+ return
+ else:
+ return raw_input(cont)
+
+def describe(demo_file,colorized=True):
+ f = open(demo_file)
+ for l in f.xreadlines(): # goes thru the file
+ ln = l.strip()
+ if ln=="":
+ continue
+ if ln[0]=="#": # comment
+ while ln[0]=="#":
+ ln=ln[1:]
+ if ln=="":
+ break
+ if colorized:
+ bc = bgcolor.OKBLUE
+ else:
+ bc = ""
+ comment(bc+ln.strip()+bgcolor.ENDC,None)
+ else:
+ if colorized:
+ bc = bgcolor.BOLD+bgcolor.UNDERLINE+bgcolor.HEADER
+ else:
+ bc = ""
+ print
+ comment(bc+ln+bgcolor.ENDC)
+ print
+ if colorized:
+ bc = bgcolor.OKBLUE
+ else:
+ bc = ""
+ comment(bc+"This ends this parameter file"+bgcolor.ENDC)
+
+def demo(demo_file,title,colorized=True):
+ comment("""
+ PMP Demo: %s 
+
+ This is a demonstration of the PMP
+ It will download some observation and model data for you
+ It will then demonstrate how to setup a parameter file to execute PMP on these
+ It will run the PMP 
+ It will show you where to find the results and how to look at them""" % title)
+
+ cont = comment("""We will now download and untar a small set of data for the demo
+ Data will be untarred in the 'pmp_demo' directory created in the current directory""","Continue? [Y/n]")
+ if cont.strip().lower() not in ["","y","yes"]:
+ sys.exit()
+
+ ## Download data
+ demo_pth = os.path.join(os.getcwd(),"pmp_demo")
+ if not os.path.exists(demo_pth):
+ os.makedirs(demo_pth)
+ # http:https://oceanonly.llnl.gov/gleckler1/pmp-demo-data/pmpv1.1_demodata.tar
+ tar_filename = "pmpv1.1_demodata.tar"
+ tar_pth = os.path.join(demo_pth,tar_filename)
+
+ good_md5 = "a6ef8f15457378ff36fd46e8fbf5f157"
+
+ attempts = 0
+ while attempts < 3:
+ md5 = hashlib.md5()
+ if os.path.exists(tar_filename):
+ f = open(tar_filename)
+ md5.update(f.read())
+ if md5.hexdigest() == good_md5:
+ attempts = 5
+ continue
+ print "Downloading: ", tar_filename
+ r = requests.get("http:https://oceanonly.llnl.gov/gleckler1/pmp-demo-data/pmpv1.1_demodata.tar", stream=True)
+ with open(tar_pth, 'wb') as f:
+ for chunk in r.iter_content(chunk_size=1024):
+ if chunk: # filter local_filename keep-alive new chunks
+ f.write(chunk)
+ md5.update(chunk)
+ f.close()
+ if md5.hexdigest() == good_md5:
+ attempts = 5
+ else:
+ attempts += 1
+
+ comment("Successfuly downloaded demo tarball\nNow untarring it", None)
+
+ tar_process = subprocess.Popen(shlex.split("tar xvf %s"%tar_pth),cwd=demo_pth)
+ tar_process.wait()
+
+ comment("Success! Files are now untarred in %s\nLet's run this demo!\n" % demo_pth,None)
+ comment("""The PMP package runs off a 'parameter' file which needs to be edited by the user
+ Please kindly take a look at our sample parameter file in: %s""" % demo_file)
+
+ describe(demo_file)
+ cmd = "pcmdi_metrics_driver.py -p %s" % demo_file
+ comment("We will now run the pmp using this parameter file\nTo do so we are using the follwoing command\n%s" % cmd)
+ pmp = subprocess.Popen(shlex.split(cmd))
+ sys.path.insert(0,os.path.dirname(demo_file))
+ exec("import %s as pmp_param" % os.path.basename(demo_file)[:-3])
+ pmp.wait()
+ loc = genutil.StringConstructor(os.path.join(pmp_param.metrics_output_path))
+ for att in ["case_id","model_version","period","realization","period"]:
+ if hasattr(pmp_param,att):
+ setattr(loc,att,getattr(pmp_param,att))
+ comment("You can now look at the results in: %s%s%s" % (bgcolor.HEADER+bgcolor.BOLD,loc(),bgcolor.ENDC))
+

demo/pmp_demo_1.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+import sys
+import os
+
+bd,nm=os.path.split(sys.argv[0])
+print bd,nm
+sys.path.insert(0,bd)
+import pmp_demo
+pmp_demo.demo(os.path.join(sys.prefix,"share","pmp","demo","pmp_input_parameters_demo1.py"),"simple-test1")