Merge pull request #212 from durack1/issue161_durack1_1p0p0FinalTweaks

Issue161 durack1 1p0p0 final tweaks

demo/GFDL/createXmlAndNcs.py

@@ -1,4 +1,4 @@
-#!/home/p1d/140922_metrics/PCMDI_METRICS/bin/python
+#!/home/p1d/150113_metrics/PCMDI_METRICS/bin/python
 
 """
 Created on Tue Jan 28 14:07:12 2014
@@ -26,6 +26,10 @@
 PJD 2 Sep 2014 - Updated to latest build (UV-CDAT v2.0 beta default/metrics devel branch)
 PJD 22 Sep 2014 - Updated to latest build (including VCS/portraits)
 PJD 24 Sep 2014 - Updated to include GFDL-CM4 data
+PJD 11 Dec 2014 - Updated to latest build (including jsons)
+PJD 31 Dec 2014 - Updated to latest build (jsons, and test cases)
+PJD 7 Jan 2015 - Added sftlf creation
+PJD 13 Jan 2015 - Updated to latest build
 
 @author: durack1
 """
@@ -45,44 +49,52 @@
 #cdm.setAutoBounds(1) ; # Use with caution
 
 # Set build info once
-buildDate = '140922'
+buildDate = '150113'
 
 # Create input variable lists 
 uvcdatInstall = ''.join(['/home/p1d/',buildDate,'_metrics/PCMDI_METRICS/bin/'])
 data = [
- ['ocean','GFDL-ESM2G','*0001-0100*','/archive/esm2g/fre/postriga_esm_20110506/ESM2G/ESM2G_pi-control_C2/gfdl.default-prod/pp/ocean_z/av/monthly_100yr/'],
- ['ocean','GFDL-CM4','*0001-0005*','/archive/jpk/mdt/20140829/tikal_201403_awgUpdates_mom6_2014.08.29/CM4i_c96L48_am4a1r1_1860forc/gfdl.ncrc2-default-prod-openmp/pp/ocean_z/av/monthly_5yr/'],
- ['atmos','GFDL-ESM2G','*0001-0100*','/archive/esm2g/fre/postriga_esm_20110506/ESM2G/ESM2G_pi-control_C2/gfdl.default-prod/pp/atmos/av/monthly_100yr/'],
- ['atmos','GFDL-CM4','*0001-0005*','/archive/jpk/mdt/20140829/tikal_201403_awgUpdates_mom6_2014.08.29/CM4i_c96L48_am4a1r1_1860forc/gfdl.ncrc2-default-prod-openmp/pp/atmos/av/monthly_5yr/']
+ ['ocean','GFDL-ESM2G','experiment','*0001-0100*','/archive/esm2g/fre/postriga_esm_20110506/ESM2G/ESM2G_pi-control_C2/gfdl.default-prod/pp/ocean_z/av/monthly_100yr/'],
+ ['ocean','GFDL-CM4','experiment','*0001-0005*','/archive/jpk/mdt/20140829/tikal_201403_awgUpdates_mom6_2014.08.29/CM4i_c96L48_am4a1r1_1860forc/gfdl.ncrc2-default-prod-openmp/pp/ocean_z/av/monthly_5yr/'],
+ ['atmos','GFDL-ESM2G','experiment','*0001-0100*','/archive/esm2g/fre/postriga_esm_20110506/ESM2G/ESM2G_pi-control_C2/gfdl.default-prod/pp/atmos/av/monthly_100yr/'],
+ ['atmos','GFDL-CM4','experiment','*0001-0005*','/archive/jpk/mdt/20140829/tikal_201403_awgUpdates_mom6_2014.08.29/CM4i_c96L48_am4a1r1_1860forc/gfdl.ncrc2-default-prod-openmp/pp/atmos/av/monthly_5yr/'],
+ ['fx','GFDL-ESM2G','experiment','*0001-0100*','/archive/esm2g/fre/postriga_esm_20110506/ESM2G/ESM2G_pi-control_C2/gfdl.default-prod/pp/atmos/'],
+ ['fx','GFDL-CM4','experiment','*0001-0005*','/archive/Charles.Seman/tikal_201409_awgUpdates_mom6_2014.08.29/CM4_c96L48_am4b2r1_2000climo/gfdl.ncrc2-intel-prod-openmp/pp/atmos/']
 ]
-inVars = [['temp'],['hght','olr','olr_clr','precip','slp','swup_toa','swup_toa_clr','t_ref','temp','u_ref','ucomp','v_ref','vcomp']]
-outVars = [['tos'],['zg','rlut','rlutcs','pr','psl','rsut','rsutcs','tas','ta','uas','ua','vas','va']]
+inVars = [['temp'],['hght','olr','olr_clr','precip','slp','swup_toa','swup_toa_clr','t_ref','temp','u_ref','ucomp','v_ref','vcomp'],['land_mask']]
+outVars = [['tos'],['zg','rlut','rlutcs','pr','psl','rsut','rsutcs','tas','ta','uas','ua','vas','va'],['sftlf']]
 
 for count1,realm in enumerate(data):
  realmId = realm[0]
  modelId = realm[1]
- timeAve = realm[2]
- dataPath = realm[3]
+ experimentId = realm[2]
+ timeAve = realm[3]
+ dataPath = realm[4]
  #print realmId,modelId,dataPath
- # Create input xml file
- command = "".join([uvcdatInstall,'cdscan -x test_',modelId,'_',realmId,'.xml ',dataPath,timeAve,'.nc'])
+ # Create input xml file conditional on realm
+ if realmId == 'fx':
+ command = "".join([uvcdatInstall,'cdscan -x test_',modelId,'_',experimentId,'_',realmId,'.xml ',dataPath,'atmos.static.nc'])
+ else:
+ command = "".join([uvcdatInstall,'cdscan -x test_',modelId,'_',experimentId,'_',realmId,'.xml ',dataPath,timeAve,'.nc'])
  #print command
  fnull = open(os.devnull,'w') ; # Create dummy to write stdout output
  p = subprocess.call(command,stdout=fnull,shell=True)
  fnull.close() ; # Close dummy
- print 'XML spanning file created for model/realm:',modelId,realmId
+ print 'XML spanning file created for model/experiment/realm:',modelId,experimentId,realmId
  #sys.exit()
 
  # Open xml file to read
- infile = ''.join(['test_',modelId,'_',realmId,'.xml'])
+ infile = ''.join(['test_',modelId,'_',experimentId,'_',realmId,'.xml'])
  #print infile
  fIn = cdm.open(infile)
 
  # Deal with variables
- if count1<2:
+ if count1 < 2:
  realmIndex = 0
- else:
+ elif count1 < 4:
  realmIndex = 1
+ else:
+ realmIndex = 2
  inVarList = inVars[realmIndex]
  outVarList = outVars[realmIndex]
 
@@ -94,13 +106,15 @@
  #print varRead,varWrite
  if realmId == 'atmos':
  tableId = 'Amon'
+ elif realmId == 'fx':
+ tableId = 'fx'
  else:
  tableId = 'Omon' 
  data = fIn(varRead)
  data.id = varWrite
  print "".join(['** Writing variable: ',varRead,' to ',varWrite,' **'])
  #outfile = ".".join(['cmip5.GFDL-ESM2G.piControl.r1i1p1.mo',tableId,varWrite,'ver-1.latestX.000101-010012.AC.nc'])
- outfile = "_".join([varWrite,modelId,tableId,'historical_r1i1p1_01-12-clim.nc'])
+ outfile = "_".join([varWrite,modelId,experimentId,tableId,'r1i1p1_01-12-clim.nc'])
  print "".join(['** Writing file: ',outfile])
  if os.path.isfile(outfile):
  os.remove(outfile) ; # purge existing file
@@ -120,5 +134,5 @@
  fIn.close()
 
 # Execute shell command
-# source /home/p1d/140922_metrics/PCMDI_METRICS/bin/setup_runtime.csh
+# source /home/p1d/150113_metrics/PCMDI_METRICS/bin/setup_runtime.csh
 # > pcmdi_metrics_driver.py -p gfdl_input_parameters_test.py

demo/GFDL/gfdl_input_parameters_test.py

@@ -1,6 +1,6 @@
 import getpass
 
-buildDate = '140922' ; # Must be set to allow correct metrics install to be picked up
+buildDate = '150113' ; # Must be set to allow correct metrics install to be picked up
 
 ################################################################################
 # OPTIONS ARE SET BY USER IN THIS FILE AS INDICATED BELOW BY: 
@@ -14,13 +14,17 @@
 model_versions = ['GFDL-CM4','GFDL-ESM2G',]
 
 ### VARIABLES AND OBSERVATIONS TO USE
-vars = ['pr','tos']
-#vars = ['pr','tas','rlut','rsut','hus_850']
-#vars = ['ta_850','ta_200','ua_850','ua_200','va_850','va_200','zg_500','rlut','rsut','rlutcs','rsutcs','tas']
-#vars = ['pr','psl','rlut','rlutcs','rsut','rsutcs','ta_200','ta_850','tas','tauu','tauv','ua_200','ua_850','va_200','va_850','vas','zg_500']
+# Variable acronyms are described in the CMIP5 standard output document - http:https://cmip-pcmdi.llnl.gov/cmip5/docs/standard_output.pdf
+#vars = ['pr','psl','rlut','rlutcs','rsut','rsutcs','ta_200','ta_850','tas','ua_200','ua_850','va_200','va_850','zg_500'] ; # GFDL atmos test suite
+#vars = ['clt','hfss','pr','prw','psl','rlut','rlutcs','rsdt','rsut','rsutcs','tas','tauu','tauv','ts','uas','vas'] ; # 2d atmos variables
+#vars = ['hur','hus','huss','ta','ua','va','zg'] ; # 3d atmos variables
+#vars = ['hus_850','ta_850','ta_200','ua_850','ua_200','va_850','va_200','zg_500'] ; # 3d atmos variables - example heights
+#vars = ['sos','tos','zos'] ; # 2d ocean variables
+#vars = ['rlwcrf','rswcrf'] ; # Non-standard CMIP5 variables (available from obs output)
+vars = ['tos','zg_500','rlut','rlutcs','pr','psl','rsut','rsutcs','tas','ta_850','uas','ua_200','ua_850','vas','va_200','va_850'] ; # Full GFDL test suite
 
-# Observations to use at the moment "default" or "alternate"
-ref = 'default'  #'all'
+# Observations to use "default", "alternate" or "all" or a specific obs reference e.g. "ref3"
+ref = 'all' ; # 'default' ; 'all' ; # Selecting 'default' uses a single obs dataset, 'all' processes against all available datasets
 ext = '.nc' ; # '.xml'
 
 # INTERPOLATION OPTIONS
@@ -40,8 +44,8 @@
 ## DATA LOCATION: MODELS, OBS AND METRICS OUTPUT
 
 ## Templates for climatology files
-## TEMPLATE EXAMPLE: cmip5.GFDL-ESM2G.historical.r1i1p1.mo.atm.Amon.rlut.ver-1.1980-1999.AC.nc
-filename_template = "%(variable)_%(model_version)_%(table)_historical_%(realization)_%(period)-clim.nc"
+## TEMPLATE EXAMPLE: tas_GFDL-ESM2G_experiment_Amon_r1i1p1_198001-199912-clim.nc
+filename_template = "%(variable)_%(model_version)_experiment_%(table)_%(realization)_%(period)-clim.nc"
 
 ## dictionary for custom %(keyword) designed by user
 # Driver will match each key to its value defined by a variable name OR all if variable name is not present, OR "" if "all" is not defined
@@ -54,9 +58,9 @@
 ## ROOT PATH FOR OBSERVATIONS
 #obs_data_path = '/home/p1d/obs/'
 obs_data_path = ''.join(['/home/',getpass.getuser(),'/obs/'])
-## DIRECTORY WHERE TO PUT RESULTS
+## DIRECTORY WHERE TO PUT RESULTS - case_id will be appended to this path
 metrics_output_path = './metrics_output_path'
-## DIRECTORY WHERE TO PUT INTERPOLATED MODELS' CLIMATOLOGIES
+## DIRECTORY WHERE TO PUT INTERPOLATED MODELS' CLIMATOLOGIES - case_id will be appended to this path
 model_clims_interpolated_output = './metrics_output_path/Interpolation_Output'
 ## FILENAME FOR INTERPOLATED CLIMATOLOGIES OUTPUT
-filename_output_template = "%(model_version).historical.r1i1p1.mo.%(table_realm).%(variable)%(level).ver-1.%(period).interpolated.%(regridMethod).%(targetGridName).AC%(ext)"
+filename_output_template = "%(model_version)_experiment_%(table)_%(realization)_%(variable)%(level)_%(period)_interpolated_%(regridMethod)_%(targetGridName)-AC%(ext)"

demo/GFDL/portrait-3obs.py

@@ -14,16 +14,15 @@
 except:
  raise RuntimeError("Sorry your python is not build with VCS support and cannot generate portrait plots")
 
-# PATH WHERE METRICS RESULTS FOR GFDL SIT
-gfdl_pth="/work/durack1/Shared/140808_metrics-gfdl/metrics_output_path/sampletest"
+# GFDL PATH FOR METRICS RESULTS
+gfdl_pth = "./metrics_output_path/sampletest"
 
 # STANDARD PYTHON MODULES
 import glob,json,os,sys
 import numpy as np
 # CDAT MODULES
 import pcmdi_metrics.graphics.portraits
 import MV2
-from genutil import statistics
 
 # CREATE VCS OBJECT AS A PORTAIT PLOT AND LOAD PLOT SETTINGS FOR TEST CASE 
 x=vcs.init()
@@ -69,8 +68,7 @@
 
 # CMIP5 METRICS RESULTS - CURRENTLY USING FOR CONTROL SIMULATIONS
 json_files = glob.glob(os.path.join(sys.prefix,"share","CMIP_metrics_results","CMIP5","piControl","*.json")) 
-# ADD GFDL JSON FILES... 
-# This is pretty hard coded might want to consider more magic
+# ADD GFDL JSON FILES... This is hard coded might want to consider more magic
 json_files += glob.glob(os.path.join(gfdl_pth,'*.json'))
 
 # CONSTRUCT PYTHON DICTIONARY WITH RESULTS METRICS USED IN PORTRAIT 
@@ -96,8 +94,8 @@
 # ORGANIZE METRICS INTO A VARIABLES X MODELS MATRIX 
 out1_rel,out2_rel,out3_rel = [np.ma.masked_all((len(vars),len(mods)),np.float32) for _ in range(3)] ; # Define arrays to fill
 
+# LOOP OVER VARIABLE
 for vn, var in enumerate(vars):
- # LOOP OVER VARIABLE
  # LOOP OVER MODEL
  for mn,mod in enumerate(mods):
  try:

demo/GFDL/portrait-4seas.py

@@ -14,16 +14,15 @@
 except:
  raise RuntimeError("Sorry your python is not build with VCS support and cannot generate portrait plots")
 
-# PATH WHERE METRICS RESULTS FOR GFDL SIT
-gfdl_pth="/work/durack1/Shared/140808_metrics-gfdl/metrics_output_path/sampletest"
+# GFDL PATH FOR METRICS RESULTS
+gfdl_pth = "./metrics_output_path/sampletest"
 
 # STANDARD PYTHON MODULES
 import glob,json,os,sys
 import numpy as np
 # CDAT MODULES
 import pcmdi_metrics.graphics.portraits
 import MV2
-from genutil import statistics
 
 # CREATE VCS OBJECT AS A PORTAIT PLOT AND LOAD PLOT SETTINGS FOR TEST CASE 
 x=vcs.init()
@@ -96,10 +95,9 @@
 # ORGANIZE METRICS INTO A VARIABLES X MODELS MATRIX 
 out1_rel,out2_rel,out3_rel,out4_rel = [np.ma.masked_all((len(vars),len(mods)),np.float32) for _ in range(4)] ; # Define arrays to fill
 
+# LOOP OVER VARIABLE
 for vn, var in enumerate(vars):
- # LOOP OVER VARIABLE
  vals1,vals2,vals3,vals4 = [[] for _ in range(4)]
-
  # LOOP OVER MODEL
  for mn,mod in enumerate(mods):
  try:

install_metrics.bash

@@ -428,7 +428,7 @@ main() {
  echo "Metrics - ${metrics_checkout} - Install Success"
  echo "*******************************"
  echo "Please test as follows:"
- echo "source ${install_prefix}/bin/setup_runtime.sh"
+ echo "source ${install_prefix}/bin/setup_runtime.sh or .csh"
  echo "python ${metrics_build_directory}/test/test_suite.py"
  echo "*******************************"
  echo "Create your customized input_parameters.py (inspire yourself from examples in ${install_prefix}/doc/pcmdi_input_parameters_sample.py"