diff --git a/demo/external-setups/NCAR/ncar_cmip_parameterfile_06262016.py b/demo/external-setups/NCAR/ncar_cmip_parameterfile_06262016.py new file mode 100755 index 000000000..4630000f4 --- /dev/null +++ b/demo/external-setups/NCAR/ncar_cmip_parameterfile_06262016.py @@ -0,0 +1,101 @@ +################################################################################ +# SAMPLE INPUT PARAMETER FILE FOR THE PCMDI METRICS PACKAGE (PMP V1.1) +# + +import os + +### USER SETTING ######################################### +# CHANGE SETTING OF domain BY COMMENTING OUT UNWANTED OPTIONS + +domain = None +#domain = "land" +#domain = "ocean" + +########################################################## + +## FIRST USE OF A PYTHON LIST, IN THIS CASE IT HAS ONLY ONE ENTRY +model_versions = ['30L_cam5301_FAMIP.002','30L_cam5301_FAMIP.003'] # THIS IS A MANDETORY ENTRY FOR DOCUMENTING RESULTS + +model_versions = ['eul128x256_d67iamip.ES01','f40.1979_amip.track1.1deg.001','30L_cam5301_FAMIP.002','30L_cam5301_FAMIP.003','f40.1979_amip.track1.T31.001','30L_cam5301_B03F2_taper2_D05_FAMIP','60Lcam5301_B11F2_FAMIP','f.e10.FAMIPC3.f09_f09.001','46L_cam5301_B03F2_taper2_D05_FAMIP'] + +model_versions = os.listdir('~/processed_data/ncar_clims-picontrol_processed/') + +model_versios = ['b.e15.B1850.f09_g16.pi_control.36', 'b.e15.B1850.f09_g16.pi_control.41', 'b.e15.B1850.f09_g16.pi_control.all.58', 'b.e15.B1850.f09_g16.pi_control.all.66', 'b.e15.B1850.f09_g16.pi_control.all.79', 'b.e15.B1850G.f09_g16.pi_control.01', 'b.e15.B1850G.f09_g16.pi_control.15', 'b.e15.B1850G.f09_g16.pi_control.15b', 'b.e15.B1850G.f09_g16.pi_control.28'] + +############################################################################### +## DATA LOCATION: MODELS, OBS AND METRICS OUTPUT +## ROOT PATH FOR MODELS CLIMATOLOGIES +#mod_data_path = '/work/metricspackage/mod_clims/cmip5-amip' +mod_data_path = '/work/gleckler1/processed_data/ncar_clims-picontrol_processed/%(model_version)/' + +filename_template = "30L_cam5301_FAMIP.002.xml" +filename_template = "%(variable).nc" +##filename_template = "%(variable)_%(model_version)_Amon_picontrol_%(exp)r1i1p1_01-12-clim.nc" + +## ROOT PATH FOR OBSERVATIONS +obs_data_path = '/work/gleckler1/processed_data/obs/' + +## DIRECTORY WHERE TO SAVE RESULTS +case_id = 'simple-test1' +metrics_output_path = './pmp-test-control/' # USER CHOOSES, RESULTS STORED IN metrics_output_path + case_id +############################################################################### + +ncar_cmip_direct_name_map = {'psl':'PSL','tas':'TREFHT','huss':'QREFHT','ua':'U','va':'V','ta':'T','pr':'PRECC','rlut':'FLUT','rsut':'SOLIN','rlutcs':'FLUTC','rsdtcs':'SOLIN','rsds':'FSDS','rlds':'FLDS','prw':'TMQ','zg':'Z3','tauu':'TAUX','tauv':'TAUY'} + +## REGIONS ON WHICH WE WANT TO RUN METRICS (var specific) +regions = {"tas" : [None,"land","ocean"], + "uas" : [None,"land","ocean"], + "vas" : [None,"land","ocean"], + "pr" : [None,"land","ocean"], + "psl": [None,"land","ocean",], + "huss": [None,"land","ocean"], + "prw": [None,"land","ocean"], + } #terre + +# OBSERVATIONS TO USE: CHOICES INCLUDE 'default','alternate1','alternate2',... AND ARE VARIABLE DEPENDENT +ref = ['default'] #,'alternate1','alternate2'] + +## A PYTHON LIST OF VARIABLES TO COMPUTE STATISTICS +vars = ['pr','tas','rlut','prw','ua_850','ua_200','va_850','va_200','ta_200','ta_850','zg_500'] # THIS EXAMPLE ONLY INCLUDES ONE FIELD, PRECIPICATION +#vars = ['psl'] + + +# INTERPOLATION OPTIONS +targetGrid = '2.5x2.5' # OPTIONS: '2.5x2.5' or an actual cdms2 grid object +regrid_tool = 'esmf' #'regrid2' # OPTIONS: 'regrid2','esmf' +regrid_method = 'linear' # OPTIONS: 'linear','conservative', only if tool is esmf + +# SIMULATION PARAMETERS (required in PMP v1.1) +period = '1979-1989' # PERIOD OF CLIMATOLOGY +realization = 'r1i1p1' # REALIZATION + +### SAME AS DEMO1 ABOVE +######################################## +### BELOW IS DEMO2 + +## REGIONS ON WHICH WE WANT TO RUN METRICS (var specific) +regions = {"tas" : [None,"land","ocean"], + "psl" : [None,"land","ocean"], + "pr" : [domain], + } + +generate_sftlf = True # IF MODEL LAND SEA MASK NOT AVAILABLE AUTOMATICALLY (APPROXIMATE!) GENERATE IN AT TARGET RESoLUTION +save_mod_clims = True # True - output interpolated model climatology False - don't output + +model_clims_interpolated_output = './interpolated-output/' +ext = '.nc' + +if domain == 'land': + filename_output_template = "%(variable)_%(model_version)_%_historical_" +\ + "%(realization)_%(period)_interpolated_%(regridMethod)_%(targetGridName)-clim-land%(ext)" + +if domain == 'ocean': + filename_output_template = "%(variable)_%(model_version)_%_historical_" +\ + "%(realization)_%(period)_interpolated_%(regridMethod)_%(targetGridName)-clim-ocean%(ext)" + filename_output_template = variable + '_crap.nc' + +if domain == None: + filename_output_template = "%(variable)_%(model_version)_%_historical_" +\ + "%(realization)_%(period)_interpolated_%(regridMethod)_%(targetGridName)-clim%(ext)" + + diff --git a/demo/external-setups/NCAR/ncar_input_parameters_test.py b/demo/external-setups/NCAR/ncar_input_parameters_test.py deleted file mode 100644 index d6e07e8fb..000000000 --- a/demo/external-setups/NCAR/ncar_input_parameters_test.py +++ /dev/null @@ -1,65 +0,0 @@ -import os - -########################################################################## -# OPTIONS ARE SET BY USER IN THIS FILE AS INDICATED BELOW: -########################################################################## - -# METRICS RUN IDENTIFICATION -# Defines a subdirectory to output metrics results so multiple runs can be -# compared -case_id = 'sampletest_141125' - -# LIST OF MODEL VERSIONS TO BE TESTED - WHICH ARE EXPECTED TO BE PART OF -# CLIMATOLOGY FILENAME -model_versions = ['NCAR-CAM5_1deg', 'NCAR-CAM5_0p25deg'] # Model identifier -period = '01-12' # Model climatological period (if relevant) -realization = 'r1i1p1' # Model run identifier (if relevant) - -# VARIABLES AND OBSERVATIONS TO USE -# Variable acronyms are described in the CMIP5 standard output document - -# http://cmip-pcmdi.llnl.gov/cmip5/docs/standard_output.pdf -vars = ['pr', 'prw', 'tas'] # NCAR atmos subsuite test - -# Observations to use 'default', 'alternate' or specific enumerated -# climatology e.g. 'ref3' -ref = ['default'] # ,'all','alternate','ref3' - -# INTERPOLATION OPTIONS -targetGrid = '2.5x2.5' # Options: '2.5x2.5' or an actual cdms2 grid object -regrid_tool = 'regrid2' # Options: 'regrid2','esmf' -# Options: 'linear','conservative', only if tool is esmf -regrid_method = 'linear' -# Options: 'regrid2','esmf' - Note regrid2 will fail with non lat/lon grids -regrid_tool_ocn = 'esmf' -# Options: 'linear','conservative', only if tool is esmf -regrid_method_ocn = 'linear' -# Options: True or False (Save interpolated model climatologies used in -# metrics calculations) -save_mod_clims = True - -# DATA LOCATION: MODELS, OBS AND METRICS OUTPUT - AND TEMPLATES FOR MODEL -# OUTPUT CLIMATOLOGY FILES -base_dir = '/glade/u/home/durack1/141104_metrics/' -# Template example: tas_GFDL-ESM2G_Amon_historical_r1i1p1_198001-199912-clim.nc -filename_template = "%(variable)_%(model_version)_%(table)_%(period)-clim.nc" -# ROOT PATH FOR MODELS CLIMATOLOGIES -mod_data_path = os.path.join(base_dir, "%(model_version)") -# ROOT PATH FOR OBSERVATIONS -obs_data_path = '/glade/u/home/durack1/obs' -# DIRECTORY WHERE TO PUT RESULTS - will create case_id subdirectory -metrics_output_path = base_dir -# DIRECTORY WHERE TO PUT INTERPOLATED MODELS' CLIMATOLOGIES - will create -# case_id subdirectory -model_clims_interpolated_output = metrics_output_path -# FILENAME FOR INTERPOLATED CLIMATOLOGIES OUTPUT -filename_output_template = "%(variable)%(level)_%(model_version)_%(table)_%(realization)_" +\ - "%(model_period)interpolated%(regrid_method)_%(targetGridName)-clim" - -# DICTIONARY FOR CUSTOM %(keyword) IMPLEMENTED BY USER FOR CUSTOM METRICS -# Driver will match each key to its value defined by a variable name OR -# all if variable name is not present, OR "" if "all" is not defined -custom_keys = { - "key1": { - "all": "key1_value_for_all_var", - "tas": "key1_value_for_tas"}, -} diff --git a/demo/external-setups/NCAR/time_model_xml_generation.py b/demo/external-setups/NCAR/time_model_xml_generation.py new file mode 100644 index 000000000..df3bf539c --- /dev/null +++ b/demo/external-setups/NCAR/time_model_xml_generation.py @@ -0,0 +1,73 @@ +#### USED TO GENERATE XML FILES FOR NCAR CLIMATOLGIES + +# NCAR OUTPUT PRODUCES CLIMATOLOGIES FOR EACH CALENDAR MONTH AS SEPERATE FILES +# XMLS PRODUCED BY THIS CODES COMBINE THESE SO THAT THE MONTHLY CLIMATOLOGICAL ANNUAL CYCLE CAN BE READ BY A SINGLE (XML) FILE +# THE VARIABLES "filenamea" AND "filenameb" BELOW ARE NOT LIKELY TO WORK FOR THE GENERAL CASE. THE LOGIC TO TRAP months "01", "02"... and "10", "11" and "12" MAY NEED TO BE MODIFIED DEPENDING ON THE FILES AVAILABLE. + +# LAST UPDATE 6/29/16 PJG + +#### + +import cdms2 as cdms +import os, string +import time +import sys +import argparse + +# Set cdms preferences - no compression, no shuffling, no complaining +cdms.setNetcdfDeflateFlag(1) +cdms.setNetcdfDeflateLevelFlag(9) ; # 1-9, min to max - Comes at heavy IO (read/write time cost) +cdms.setNetcdfShuffleFlag(0) +cdms.setCompressionWarnings(0) ; # Turn off nag messages +# Set bounds automagically +#cdm.setAutoBounds(1) ; # Use with caution + +parser = argparse.ArgumentParser(description='Given a list of directories with simulation clims, use cdscan to produce xmls for annual cycle climatologies ') +parser.add_argument("-b","--basedir",help="root directory below which subdirectories of individual simulations are expected") #,default="*.nc",nargs="*") +parser.add_argument("-o","--outdir",help="output directory",default=None) + +args =parser.parse_args(sys.argv[1:]) + +pathin = args.basedir + '/' # Get string from first index of list - only one directory expected + +xmldir = args.outdir + '/' + +try: + os.mkdir(xmldir) +except: + pass + +print 'pathin is...', pathin +print 'pathout is...', xmldir + +# It is assummed that each subdirectory (run) with the path xmldir includes climatological files +runs = os.listdir(pathin) + +print runs + +for run in runs: + + filename = run + +# In this case the clim files are in the run directory + try: + filenamea = filename + '.001_0*_climo.nc' + filenameb = filename + '.001_1*_climo.nc' + cmd = 'cdscan -x ' + xmldir + filename + '.xml ' + pathin + filename + '/' + filenamea + ' ' + pathin + filename + '/' + filenameb + print cmd + os.popen(cmd).readlines() + except: + pass + +# In this case the clim files are in the run/climo directory + try: + filenamea = filename + '_0*_climo.nc' + filenameb = filename + '_1*_climo.nc' + + cmd = 'cdscan -x ' + xmldir + filename + '.xml ' + pathin + filename + '/climo/' + filenamea + ' ' + pathin + filename + '/climo/' + filenameb + os.popen(cmd).readlines() + except: + pass + + + diff --git a/demo/external-setups/NCAR/transform_ncar_clims_to_CMIP5_structure.py b/demo/external-setups/NCAR/transform_ncar_clims_to_CMIP5_structure.py new file mode 100644 index 000000000..2d6f92753 --- /dev/null +++ b/demo/external-setups/NCAR/transform_ncar_clims_to_CMIP5_structure.py @@ -0,0 +1,174 @@ +##### +# CONVERT NCAR CLIMATOLOGIES TO BE CONSISTENT WITH CMIP STRUCTURE FOR PMP USE +# PLANNED IMPROVEMENTS: +# CURRENTLY ASSUMES cdscan XML FILES HAVE BEEN CREATED IN ADVANCE +# CURRENTLY ASSUMES LOCATION OF XML FILES AND PRODUCES CLIMS FOR ALL SIMULATIONS WITH XMLS +# CURRENTLY HAS OUTPUT "data" DIRECTORY +# argparse WILL BE USED TO SELECT ALL OF THE ABOVE VIA COMMAND LINE EXECUTION + +# CMOR WILL BE USED TO WRITE CLIMATOLGIES - Time/bounds CURRENTLY HARDWIRED +##### LAST UPDATE 6/29/16 PJG + + +import cdms2 as cdms +import os, string +import MV2 as MV +import cdutil +import sys + +# Set cdms preferences - no compression, no shuffling, no complaining +cdms.setNetcdfDeflateFlag(1) +cdms.setNetcdfDeflateLevelFlag(9) ; # 1-9, min to max - Comes at heavy IO (read/write time cost) +cdms.setNetcdfShuffleFlag(0) +cdms.setCompressionWarnings(0) ; # Turn off nag messages +# Set bounds automagically +#cdms.setAutoBounds(1) ; # Use with caution + + +### DICTIONARY MAPPING NCAR AND CMIP VARIABLE ID'S +ncar_cmip_direct_name_map = {'psl':'PSL','tas':'TREFHT','huss':'QREFHT','ua':'U','va':'V','ta':'T','pr':'PRECC','rlut':'FSNTOA','rsut':'SOLIN','rlutcs':'FLUTC','rsutcs':'FSNTOAC','rsds':'FSDS','rlds':'FLDS','prw':'TMQ','zg':'Z3','tauu':'TAUX','tauv':'TAUY','swcre':'swcf'} + + +# Code currently + +lst1 = os.popen('ls xmls/*.xml').readlines() + +lst = [] +for l in lst1: +# if string.find(l,'FAMIPC5_ne120_79to05_03_omp2') == -1: + lst.append(l) + +#lst = os.popen('ls xmls/f.e11.FAMIPC5.f19_f19.topo_2d_control.001.xml') + +vars = ['psl','tas','ua','va','pr','rlut','prw','zg','tauu','tauv'] +vars = ['rsutcs'] +#vars = ['rlutcs','rsdtcs','rsds','rlds'] + +# CLIMATOLOGY TIME MODEL + +timel = [15.5, 45.5, 75.5, 106, 136.5, 167, 197.5, 228.5, 259, 289.5, 320, 350.5] +timelbds = [(0, 31), (31, 60), (60, 91), (91, 121), (121, 152), (152, 182,), (182, 213), (213, 244), (244, 274), (274, 305), (305, 335), (335, 366)] +ta = cdms.createAxis(timel,id='time') +tb = MV.array(timelbds) +tb = tb.astype('float64') +ta.setBounds(tb) +ta.climatology = "climatology_bnds" +ta.units = "days since 0" +ta.calendar = 'gregorian' +ta.axis = 'T' +ta.long_name = 'time' +ta.standard_name = 'time' + + +print 'BEGIN PROCESSING...' + +for l in lst: + l = l[:-1] + + f = cdms.open(l) + +# TRAP EXP NAME + tmp1 = string.split(l,'/')[1] + expname = string.split(tmp1,'.xml')[0] + + try: + os.mkdir('data/' + expname) + except: + pass + try: + dirout = 'data/' + expname + except: + pass + + for var in vars: + var_ncar = ncar_cmip_direct_name_map[var] + d = f(var_ncar) + + if var in ['ua','va','ta']: + ps1=f('PS') + po1=f('P0') + ha1=f('hyam') + hb1=f('hybm') + print 'before vertical interpolation of ', var,' ', `d.shape` + p1=cdutil.vertical.reconstructPressureFromHybrid(ps1,ha1,hb1,po1) + dall = cdutil.vertical.linearInterpolation(d,p1) + print 'finished vertical interpolation of ', var + levs = dall.getLevel() + print levs + d = MV.multiply(dall(level = slice(0,2)),0.) + d1 = dall(level=(85000,85000)) + d2 = dall(level=(20000,20000)) + d[:,0,:,:] = d1[:,0,:,:] + d[:,1,:,:] = d2[:,0,:,:] + levs = [85000.,20000.] + levs_tv = cdms.createAxis(levs,id = 'level') + levs_tv.units = 'Pa' + levs_tv.axis = 'Z' + d.setAxis(1,levs_tv) + + if var in ['zg']: + ps1=f('PS') + po1=f('P0') + ha1=f('hyam') + hb1=f('hybm') + print 'before vertical interpolation of ', var,' ', `d.shape` + p1=cdutil.vertical.reconstructPressureFromHybrid(ps1,ha1,hb1,po1) + dall = cdutil.vertical.linearInterpolation(d,p1) + print 'finished vertical interpolation of ', var + levs = dall.getLevel() + print levs + d = MV.multiply(dall(level = slice(0,1)),0.) + d1 = dall(level=(50000,50000)) + d[:,0,:,:] = d1[:,0,:,:] + levs = [50000.] + levs_tv = cdms.createAxis(levs,id = 'level') + levs_tv.units = 'Pa' + levs_tv.axis = 'Z' + d.setAxis(1,levs_tv) + + if var == 'pr': + d1 = f('PRECL') + d = (d + d1)*1000. + d.units = 'kg m-2 s-1' + + if var == 'rlut': + d1 = f('FSNTOA') + d2 = f('FSNT') + d3 = f('FLNT') + d = d1-d2+d3 ; #FSNTOA - FSNT + FLNT + d.id = 'rlut' + d.long_name = 'toa_outgoing_longwave_flux' + d.history = 'Converted to RLUT from FSNTOA - FSNT + FLNT' + d.units = d.units + + if var == 'rsut': + d1 = f('SOLIN') + d2 = f('FSNTOA') + d = d1-d2 ; #FSNTOA - FSNT + FLNT + d.id = 'rsut' + d.long_name = 'toa_reflected_shortwave_flux' + d.history = 'Converted to RSUT from SOLIN - FSNTOA' + d.units = d.units + + if var == 'rsutcs': + d1 = f('SOLIN') + d2 = f('FSNTOAC') + d = d1-d2 ; #FSNTOA - FSNT + FLNT + d.id = 'rsutcs' + d.long_name = 'toa_reflected_shortwave_flux clear sky' + d.history = 'Converted to RSUT from SOLIN - FSNTOA clear sky' + d.units = d.units + + print f,' ', var_ncar, ' ', d.shape + + d.id = var + d.setAxis(0,ta) + + + g = cdms.open(dirout + '/' + var + '.nc','w+') + g.write(d) + g.close() + f.close() + + +