diff --git a/demo/GFDL/createXmlAndNcs.py b/demo/GFDL/createXmlAndNcs.py index 8e4266e55..87a2e05f6 100755 --- a/demo/GFDL/createXmlAndNcs.py +++ b/demo/GFDL/createXmlAndNcs.py @@ -1,4 +1,4 @@ -#!/home/p1d/140922_metrics/PCMDI_METRICS/bin/python +#!/home/p1d/150113_metrics/PCMDI_METRICS/bin/python """ Created on Tue Jan 28 14:07:12 2014 @@ -26,6 +26,10 @@ PJD 2 Sep 2014 - Updated to latest build (UV-CDAT v2.0 beta default/metrics devel branch) PJD 22 Sep 2014 - Updated to latest build (including VCS/portraits) PJD 24 Sep 2014 - Updated to include GFDL-CM4 data +PJD 11 Dec 2014 - Updated to latest build (including jsons) +PJD 31 Dec 2014 - Updated to latest build (jsons, and test cases) +PJD 7 Jan 2015 - Added sftlf creation +PJD 13 Jan 2015 - Updated to latest build @author: durack1 """ @@ -45,44 +49,52 @@ #cdm.setAutoBounds(1) ; # Use with caution # Set build info once -buildDate = '140922' +buildDate = '150113' # Create input variable lists uvcdatInstall = ''.join(['/home/p1d/',buildDate,'_metrics/PCMDI_METRICS/bin/']) data = [ - ['ocean','GFDL-ESM2G','*0001-0100*','/archive/esm2g/fre/postriga_esm_20110506/ESM2G/ESM2G_pi-control_C2/gfdl.default-prod/pp/ocean_z/av/monthly_100yr/'], - ['ocean','GFDL-CM4','*0001-0005*','/archive/jpk/mdt/20140829/tikal_201403_awgUpdates_mom6_2014.08.29/CM4i_c96L48_am4a1r1_1860forc/gfdl.ncrc2-default-prod-openmp/pp/ocean_z/av/monthly_5yr/'], - ['atmos','GFDL-ESM2G','*0001-0100*','/archive/esm2g/fre/postriga_esm_20110506/ESM2G/ESM2G_pi-control_C2/gfdl.default-prod/pp/atmos/av/monthly_100yr/'], - ['atmos','GFDL-CM4','*0001-0005*','/archive/jpk/mdt/20140829/tikal_201403_awgUpdates_mom6_2014.08.29/CM4i_c96L48_am4a1r1_1860forc/gfdl.ncrc2-default-prod-openmp/pp/atmos/av/monthly_5yr/'] + ['ocean','GFDL-ESM2G','experiment','*0001-0100*','/archive/esm2g/fre/postriga_esm_20110506/ESM2G/ESM2G_pi-control_C2/gfdl.default-prod/pp/ocean_z/av/monthly_100yr/'], + ['ocean','GFDL-CM4','experiment','*0001-0005*','/archive/jpk/mdt/20140829/tikal_201403_awgUpdates_mom6_2014.08.29/CM4i_c96L48_am4a1r1_1860forc/gfdl.ncrc2-default-prod-openmp/pp/ocean_z/av/monthly_5yr/'], + ['atmos','GFDL-ESM2G','experiment','*0001-0100*','/archive/esm2g/fre/postriga_esm_20110506/ESM2G/ESM2G_pi-control_C2/gfdl.default-prod/pp/atmos/av/monthly_100yr/'], + ['atmos','GFDL-CM4','experiment','*0001-0005*','/archive/jpk/mdt/20140829/tikal_201403_awgUpdates_mom6_2014.08.29/CM4i_c96L48_am4a1r1_1860forc/gfdl.ncrc2-default-prod-openmp/pp/atmos/av/monthly_5yr/'], + ['fx','GFDL-ESM2G','experiment','*0001-0100*','/archive/esm2g/fre/postriga_esm_20110506/ESM2G/ESM2G_pi-control_C2/gfdl.default-prod/pp/atmos/'], + ['fx','GFDL-CM4','experiment','*0001-0005*','/archive/Charles.Seman/tikal_201409_awgUpdates_mom6_2014.08.29/CM4_c96L48_am4b2r1_2000climo/gfdl.ncrc2-intel-prod-openmp/pp/atmos/'] ] -inVars = [['temp'],['hght','olr','olr_clr','precip','slp','swup_toa','swup_toa_clr','t_ref','temp','u_ref','ucomp','v_ref','vcomp']] -outVars = [['tos'],['zg','rlut','rlutcs','pr','psl','rsut','rsutcs','tas','ta','uas','ua','vas','va']] +inVars = [['temp'],['hght','olr','olr_clr','precip','slp','swup_toa','swup_toa_clr','t_ref','temp','u_ref','ucomp','v_ref','vcomp'],['land_mask']] +outVars = [['tos'],['zg','rlut','rlutcs','pr','psl','rsut','rsutcs','tas','ta','uas','ua','vas','va'],['sftlf']] for count1,realm in enumerate(data): realmId = realm[0] modelId = realm[1] - timeAve = realm[2] - dataPath = realm[3] + experimentId = realm[2] + timeAve = realm[3] + dataPath = realm[4] #print realmId,modelId,dataPath - # Create input xml file - command = "".join([uvcdatInstall,'cdscan -x test_',modelId,'_',realmId,'.xml ',dataPath,timeAve,'.nc']) + # Create input xml file conditional on realm + if realmId == 'fx': + command = "".join([uvcdatInstall,'cdscan -x test_',modelId,'_',experimentId,'_',realmId,'.xml ',dataPath,'atmos.static.nc']) + else: + command = "".join([uvcdatInstall,'cdscan -x test_',modelId,'_',experimentId,'_',realmId,'.xml ',dataPath,timeAve,'.nc']) #print command fnull = open(os.devnull,'w') ; # Create dummy to write stdout output p = subprocess.call(command,stdout=fnull,shell=True) fnull.close() ; # Close dummy - print 'XML spanning file created for model/realm:',modelId,realmId + print 'XML spanning file created for model/experiment/realm:',modelId,experimentId,realmId #sys.exit() # Open xml file to read - infile = ''.join(['test_',modelId,'_',realmId,'.xml']) + infile = ''.join(['test_',modelId,'_',experimentId,'_',realmId,'.xml']) #print infile fIn = cdm.open(infile) # Deal with variables - if count1<2: + if count1 < 2: realmIndex = 0 - else: + elif count1 < 4: realmIndex = 1 + else: + realmIndex = 2 inVarList = inVars[realmIndex] outVarList = outVars[realmIndex] @@ -94,13 +106,15 @@ #print varRead,varWrite if realmId == 'atmos': tableId = 'Amon' + elif realmId == 'fx': + tableId = 'fx' else: tableId = 'Omon' data = fIn(varRead) data.id = varWrite print "".join(['** Writing variable: ',varRead,' to ',varWrite,' **']) #outfile = ".".join(['cmip5.GFDL-ESM2G.piControl.r1i1p1.mo',tableId,varWrite,'ver-1.latestX.000101-010012.AC.nc']) - outfile = "_".join([varWrite,modelId,tableId,'historical_r1i1p1_01-12-clim.nc']) + outfile = "_".join([varWrite,modelId,experimentId,tableId,'r1i1p1_01-12-clim.nc']) print "".join(['** Writing file: ',outfile]) if os.path.isfile(outfile): os.remove(outfile) ; # purge existing file @@ -120,5 +134,5 @@ fIn.close() # Execute shell command -# source /home/p1d/140922_metrics/PCMDI_METRICS/bin/setup_runtime.csh +# source /home/p1d/150113_metrics/PCMDI_METRICS/bin/setup_runtime.csh # > pcmdi_metrics_driver.py -p gfdl_input_parameters_test.py diff --git a/demo/GFDL/gfdl_input_parameters_test.py b/demo/GFDL/gfdl_input_parameters_test.py index 40f892c63..d2828f70e 100644 --- a/demo/GFDL/gfdl_input_parameters_test.py +++ b/demo/GFDL/gfdl_input_parameters_test.py @@ -1,6 +1,6 @@ import getpass -buildDate = '140922' ; # Must be set to allow correct metrics install to be picked up +buildDate = '150113' ; # Must be set to allow correct metrics install to be picked up ################################################################################ # OPTIONS ARE SET BY USER IN THIS FILE AS INDICATED BELOW BY: @@ -14,13 +14,17 @@ model_versions = ['GFDL-CM4','GFDL-ESM2G',] ### VARIABLES AND OBSERVATIONS TO USE -vars = ['pr','tos'] -#vars = ['pr','tas','rlut','rsut','hus_850'] -#vars = ['ta_850','ta_200','ua_850','ua_200','va_850','va_200','zg_500','rlut','rsut','rlutcs','rsutcs','tas'] -#vars = ['pr','psl','rlut','rlutcs','rsut','rsutcs','ta_200','ta_850','tas','tauu','tauv','ua_200','ua_850','va_200','va_850','vas','zg_500'] +# Variable acronyms are described in the CMIP5 standard output document - http://cmip-pcmdi.llnl.gov/cmip5/docs/standard_output.pdf +#vars = ['pr','psl','rlut','rlutcs','rsut','rsutcs','ta_200','ta_850','tas','ua_200','ua_850','va_200','va_850','zg_500'] ; # GFDL atmos test suite +#vars = ['clt','hfss','pr','prw','psl','rlut','rlutcs','rsdt','rsut','rsutcs','tas','tauu','tauv','ts','uas','vas'] ; # 2d atmos variables +#vars = ['hur','hus','huss','ta','ua','va','zg'] ; # 3d atmos variables +#vars = ['hus_850','ta_850','ta_200','ua_850','ua_200','va_850','va_200','zg_500'] ; # 3d atmos variables - example heights +#vars = ['sos','tos','zos'] ; # 2d ocean variables +#vars = ['rlwcrf','rswcrf'] ; # Non-standard CMIP5 variables (available from obs output) +vars = ['tos','zg_500','rlut','rlutcs','pr','psl','rsut','rsutcs','tas','ta_850','uas','ua_200','ua_850','vas','va_200','va_850'] ; # Full GFDL test suite -# Observations to use at the moment "default" or "alternate" -ref = 'default' #'all' +# Observations to use "default", "alternate" or "all" or a specific obs reference e.g. "ref3" +ref = 'all' ; # 'default' ; 'all' ; # Selecting 'default' uses a single obs dataset, 'all' processes against all available datasets ext = '.nc' ; # '.xml' # INTERPOLATION OPTIONS @@ -40,8 +44,8 @@ ## DATA LOCATION: MODELS, OBS AND METRICS OUTPUT ## Templates for climatology files -## TEMPLATE EXAMPLE: cmip5.GFDL-ESM2G.historical.r1i1p1.mo.atm.Amon.rlut.ver-1.1980-1999.AC.nc -filename_template = "%(variable)_%(model_version)_%(table)_historical_%(realization)_%(period)-clim.nc" +## TEMPLATE EXAMPLE: tas_GFDL-ESM2G_experiment_Amon_r1i1p1_198001-199912-clim.nc +filename_template = "%(variable)_%(model_version)_experiment_%(table)_%(realization)_%(period)-clim.nc" ## dictionary for custom %(keyword) designed by user # Driver will match each key to its value defined by a variable name OR all if variable name is not present, OR "" if "all" is not defined @@ -54,9 +58,9 @@ ## ROOT PATH FOR OBSERVATIONS #obs_data_path = '/home/p1d/obs/' obs_data_path = ''.join(['/home/',getpass.getuser(),'/obs/']) -## DIRECTORY WHERE TO PUT RESULTS +## DIRECTORY WHERE TO PUT RESULTS - case_id will be appended to this path metrics_output_path = './metrics_output_path' -## DIRECTORY WHERE TO PUT INTERPOLATED MODELS' CLIMATOLOGIES +## DIRECTORY WHERE TO PUT INTERPOLATED MODELS' CLIMATOLOGIES - case_id will be appended to this path model_clims_interpolated_output = './metrics_output_path/Interpolation_Output' ## FILENAME FOR INTERPOLATED CLIMATOLOGIES OUTPUT -filename_output_template = "%(model_version).historical.r1i1p1.mo.%(table_realm).%(variable)%(level).ver-1.%(period).interpolated.%(regridMethod).%(targetGridName).AC%(ext)" +filename_output_template = "%(model_version)_experiment_%(table)_%(realization)_%(variable)%(level)_%(period)_interpolated_%(regridMethod)_%(targetGridName)-AC%(ext)" diff --git a/demo/GFDL/portrait-3obs.py b/demo/GFDL/portrait-3obs.py index 94b8482c6..523ae465d 100644 --- a/demo/GFDL/portrait-3obs.py +++ b/demo/GFDL/portrait-3obs.py @@ -14,8 +14,8 @@ except: raise RuntimeError("Sorry your python is not build with VCS support and cannot generate portrait plots") -# PATH WHERE METRICS RESULTS FOR GFDL SIT -gfdl_pth="/work/durack1/Shared/140808_metrics-gfdl/metrics_output_path/sampletest" +# GFDL PATH FOR METRICS RESULTS +gfdl_pth = "./metrics_output_path/sampletest" # STANDARD PYTHON MODULES import glob,json,os,sys @@ -23,7 +23,6 @@ # CDAT MODULES import pcmdi_metrics.graphics.portraits import MV2 -from genutil import statistics # CREATE VCS OBJECT AS A PORTAIT PLOT AND LOAD PLOT SETTINGS FOR TEST CASE x=vcs.init() @@ -69,8 +68,7 @@ # CMIP5 METRICS RESULTS - CURRENTLY USING FOR CONTROL SIMULATIONS json_files = glob.glob(os.path.join(sys.prefix,"share","CMIP_metrics_results","CMIP5","piControl","*.json")) -# ADD GFDL JSON FILES... -# This is pretty hard coded might want to consider more magic +# ADD GFDL JSON FILES... This is hard coded might want to consider more magic json_files += glob.glob(os.path.join(gfdl_pth,'*.json')) # CONSTRUCT PYTHON DICTIONARY WITH RESULTS METRICS USED IN PORTRAIT @@ -96,8 +94,8 @@ # ORGANIZE METRICS INTO A VARIABLES X MODELS MATRIX out1_rel,out2_rel,out3_rel = [np.ma.masked_all((len(vars),len(mods)),np.float32) for _ in range(3)] ; # Define arrays to fill +# LOOP OVER VARIABLE for vn, var in enumerate(vars): - # LOOP OVER VARIABLE # LOOP OVER MODEL for mn,mod in enumerate(mods): try: diff --git a/demo/GFDL/portrait-4seas.py b/demo/GFDL/portrait-4seas.py old mode 100755 new mode 100644 index 3c6bf0709..3da4104b1 --- a/demo/GFDL/portrait-4seas.py +++ b/demo/GFDL/portrait-4seas.py @@ -14,8 +14,8 @@ except: raise RuntimeError("Sorry your python is not build with VCS support and cannot generate portrait plots") -# PATH WHERE METRICS RESULTS FOR GFDL SIT -gfdl_pth="/work/durack1/Shared/140808_metrics-gfdl/metrics_output_path/sampletest" +# GFDL PATH FOR METRICS RESULTS +gfdl_pth = "./metrics_output_path/sampletest" # STANDARD PYTHON MODULES import glob,json,os,sys @@ -23,7 +23,6 @@ # CDAT MODULES import pcmdi_metrics.graphics.portraits import MV2 -from genutil import statistics # CREATE VCS OBJECT AS A PORTAIT PLOT AND LOAD PLOT SETTINGS FOR TEST CASE x=vcs.init() @@ -96,10 +95,9 @@ # ORGANIZE METRICS INTO A VARIABLES X MODELS MATRIX out1_rel,out2_rel,out3_rel,out4_rel = [np.ma.masked_all((len(vars),len(mods)),np.float32) for _ in range(4)] ; # Define arrays to fill +# LOOP OVER VARIABLE for vn, var in enumerate(vars): - # LOOP OVER VARIABLE vals1,vals2,vals3,vals4 = [[] for _ in range(4)] - # LOOP OVER MODEL for mn,mod in enumerate(mods): try: diff --git a/demo/GFDL/test_3obsRMS.png b/demo/GFDL/test_3obsRMS.png index a37a51bbf..a0579397c 100644 Binary files a/demo/GFDL/test_3obsRMS.png and b/demo/GFDL/test_3obsRMS.png differ diff --git a/demo/GFDL/test_4seasRMS.png b/demo/GFDL/test_4seasRMS.png index 54e49fc5b..a775d2096 100644 Binary files a/demo/GFDL/test_4seasRMS.png and b/demo/GFDL/test_4seasRMS.png differ diff --git a/install_metrics.bash b/install_metrics.bash index e7e509c79..e61f48e8d 100755 --- a/install_metrics.bash +++ b/install_metrics.bash @@ -428,7 +428,7 @@ main() { echo "Metrics - ${metrics_checkout} - Install Success" echo "*******************************" echo "Please test as follows:" - echo "source ${install_prefix}/bin/setup_runtime.sh" + echo "source ${install_prefix}/bin/setup_runtime.sh or .csh" echo "python ${metrics_build_directory}/test/test_suite.py" echo "*******************************" echo "Create your customized input_parameters.py (inspire yourself from examples in ${install_prefix}/doc/pcmdi_input_parameters_sample.py"