Make e3sm_to_cmip task

tests/integration/generated/test_complete_run_chrysalis.cfg

@@ -35,7 +35,6 @@ years = "1850:1854:2",
  frequency = "monthly"
  input_files = "eam.h0"
  input_subdir = "archive/atm/hist"
- ts_fmt = "cmip"
 
  [[ atm_daily_180x360_aave ]]
  frequency = "daily"
@@ -56,7 +55,6 @@ years = "1850:1854:2",
  input_files = "elm.h0"
  input_subdir = "archive/lnd/hist"
  vars = "FSH,LAISHA,LAISUN,RH2M"
- ts_fmt = "cmip"
 
  [[ rof_monthly ]]
  extra_vars = 'areatotal2'
@@ -66,6 +64,23 @@ years = "1850:1854:2",
  mapping_file = ""
  vars = "RIVER_DISCHARGE_OVER_LAND_LIQ"
 
+[e3sm_to_cmip]
+active = True
+walltime = "00:30:00"
+years = "1850:1854:2"
+
+ [[ atm_monthly_180x360_aave ]]
+ frequency = "monthly"
+ input_files = "eam.h0"
+ input_subdir = "archive/atm/hist"
+
+ [[ land_monthly ]]
+ extra_vars = "landfrac"
+ frequency = "monthly"
+ input_files = "elm.h0"
+ input_subdir = "archive/lnd/hist"
+ vars = "FSH,LAISHA,LAISUN,RH2M"
+
 [tc_analysis]
 active = True
 scratch = "/lcrc/globalscratch/ac.forsyth2/"

tests/integration/template_complete_run.cfg

@@ -35,7 +35,6 @@ years = "1850:1854:2",
  frequency = "monthly"
  input_files = "eam.h0"
  input_subdir = "archive/atm/hist"
- ts_fmt = "cmip"
 
  [[ atm_daily_180x360_aave ]]
  frequency = "daily"
@@ -56,7 +55,6 @@ years = "1850:1854:2",
  input_files = "elm.h0"
  input_subdir = "archive/lnd/hist"
  vars = "FSH,LAISHA,LAISUN,RH2M"
- ts_fmt = "cmip"
 
  [[ rof_monthly ]]
  extra_vars = 'areatotal2'
@@ -66,6 +64,23 @@ years = "1850:1854:2",
  mapping_file = ""
  vars = "RIVER_DISCHARGE_OVER_LAND_LIQ"
 
+[e3sm_to_cmip]
+active = True
+walltime = "00:30:00"
+years = "1850:1854:2"
+
+ [[ atm_monthly_180x360_aave ]]
+ frequency = "monthly"
+ input_files = "eam.h0"
+ input_subdir = "archive/atm/hist"
+
+ [[ land_monthly ]]
+ extra_vars = "landfrac"
+ frequency = "monthly"
+ input_files = "elm.h0"
+ input_subdir = "archive/lnd/hist"
+ vars = "FSH,LAISHA,LAISUN,RH2M"
+
 [tc_analysis]
 active = True
 scratch = "#expand scratch#"

tests/test_sections.py

@@ -98,12 +98,10 @@ def test_sections(self):
  expected_section = {
  "active": "True",
  "area_nm": "area",
- "cmip_metadata": "e3sm_to_cmip/default_metadata.json",
  "dpf": 30,
  "extra_vars": "",
  "mapping_file": "MAPPING_FILE_TS",
  "tpd": 1,
- "ts_fmt": "ts_only",
  "vars": "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT",
  "years": ["0001:0020:5"],
  }
@@ -118,7 +116,6 @@ def test_sections(self):
  "area_nm": "area",
  "campaign": "none",
  "case": "CASE",
- "cmip_metadata": "e3sm_to_cmip/default_metadata.json",
  "constraint": "",
  "debug": False,
  "dpf": 30,
@@ -140,7 +137,6 @@ def test_sections(self):
  "subsection": None,
  "templateDir": "zppy/templates",
  "tpd": 1,
- "ts_fmt": "ts_only",
  "vars": "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT",
  "walltime": "02:00:00",
  "www": "WWWW",
@@ -251,29 +247,23 @@ def test_subsections(self):
  expected_section = {
  "active": "True",
  "area_nm": "area",
- "cmip_metadata": "e3sm_to_cmip/default_metadata.json",
  "dpf": 30,
  "extra_vars": "",
  "tpd": 1,
- "ts_fmt": "ts_only",
  "ts_grid1": {
  "area_nm": None,
- "cmip_metadata": None,
  "dpf": None,
  "extra_vars": None,
  "mapping_file": "MAPPING_FILE_TS_GRID1",
  "tpd": None,
- "ts_fmt": None,
  "years": ["0001:0020:5"],
  },
  "ts_grid2": {
  "area_nm": None,
- "cmip_metadata": None,
  "dpf": None,
  "extra_vars": None,
  "mapping_file": "MAPPING_FILE_TS_GRID2",
  "tpd": None,
- "ts_fmt": None,
  "years": ["0001:0020:10"],
  },
  "vars": "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT",
@@ -289,7 +279,6 @@ def test_subsections(self):
  "bundle": "",
  "campaign": "none",
  "case": "CASE",
- "cmip_metadata": "e3sm_to_cmip/default_metadata.json",
  "constraint": "",
  "debug": False,
  "dpf": 30,
@@ -311,7 +300,6 @@ def test_subsections(self):
  "subsection": "ts_grid1",
  "templateDir": "zppy/templates",
  "tpd": 1,
- "ts_fmt": "ts_only",
  "vars": "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT",
  "walltime": "02:00:00",
  "www": "WWWW",
@@ -326,7 +314,6 @@ def test_subsections(self):
  "bundle": "",
  "campaign": "none",
  "case": "CASE",
- "cmip_metadata": "e3sm_to_cmip/default_metadata.json",
  "constraint": "",
  "debug": False,
  "dpf": 30,
@@ -348,7 +335,6 @@ def test_subsections(self):
  "subsection": "ts_grid2",
  "templateDir": "zppy/templates",
  "tpd": 1,
- "ts_fmt": "ts_only",
  "vars": "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT",
  "walltime": "02:00:00",
  "www": "WWWW",

zppy/__main__.py

@@ -12,6 +12,7 @@
 from zppy.bundle import Bundle, predefined_bundles
 from zppy.climo import climo
 from zppy.e3sm_diags import e3sm_diags
+from zppy.e3sm_to_cmip import e3sm_to_cmip
 from zppy.global_time_series import global_time_series
 from zppy.ilamb import ilamb
 from zppy.mpas_analysis import mpas_analysis
@@ -188,6 +189,9 @@ def main(): # noqa: C901
  # time series tasks
  existing_bundles = ts(config, scriptDir, existing_bundles, job_ids_file)
 
+ # e3sm_to_cmip tasks
+ existing_bundles = e3sm_to_cmip(config, scriptDir, existing_bundles, job_ids_file)
+
  # tc_analysis tasks
  existing_bundles = tc_analysis(config, scriptDir, existing_bundles, job_ids_file)
 

zppy/e3sm_diags.py

@@ -222,7 +222,7 @@ def e3sm_diags(config, scriptDir, existing_bundles, job_ids_file): # noqa: C901
  p.pprint(c)
  p.pprint(s)
 
- export = "NONE"
+ export = "ALL"
  existing_bundles = handle_bundles(
  c,
  scriptFile,

zppy/e3sm_to_cmip.py

@@ -0,0 +1,144 @@
+import os
+import pprint
+import re
+
+import jinja2
+
+from zppy.bundle import handle_bundles
+from zppy.utils import (
+ checkStatus,
+ getComponent,
+ getTasks,
+ getYears,
+ makeExecutable,
+ setMappingFile,
+ submitScript,
+)
+
+
+# -----------------------------------------------------------------------------
+def e3sm_to_cmip(config, scriptDir, existing_bundles, job_ids_file):
+
+ # --- Initialize jinja2 template engine ---
+ templateLoader = jinja2.FileSystemLoader(
+ searchpath=config["default"]["templateDir"]
+ )
+ templateEnv = jinja2.Environment(loader=templateLoader)
+ template = templateEnv.get_template("e3sm_to_cmip.bash")
+
+ # --- List of tasks ---
+ tasks = getTasks(config, "e3sm_to_cmip")
+ print(config["e3sm_to_cmip"])
+ if len(tasks) == 0:
+ return existing_bundles
+
+ # --- Generate and submit e3sm_to_cmip scripts ---
+ dependencies = []
+
+ for c in tasks:
+
+ setMappingFile(c)
+
+ # Grid name (if not explicitly defined)
+ # 'native' if no remapping
+ # or extracted from mapping filename
+ if c["grid"] == "":
+ if c["mapping_file"] == "":
+ c["grid"] = "native"
+ elif c["mapping_file"] == "glb":
+ c["grid"] = "glb"
+ else:
+ tmp = os.path.basename(c["mapping_file"])
+ # FIXME: W605 invalid escape sequence '\.'
+ tmp = re.sub("\.[^.]*\.nc$", "", tmp) # noqa: W605
+ tmp = tmp.split("_")
+ if tmp[0] == "map":
+ c["grid"] = "%s_%s" % (tmp[-2], tmp[-1])
+ else:
+ raise ValueError(
+ "Cannot extract target grid name from mapping file %s"
+ % (c["mapping_file"])
+ )
+
+ # Component
+ c["component"] = getComponent(c["input_files"])
+
+ c["cmor_tables_prefix"] = c["diagnostics_base_path"]
+
+ # Loop over year sets
+ year_sets = getYears(c["years"])
+ for s in year_sets:
+
+ c["yr_start"] = s[0]
+ c["yr_end"] = s[1]
+ if ("last_year" in c.keys()) and (c["yr_end"] > c["last_year"]):
+ continue # Skip this year set
+ c["ypf"] = s[1] - s[0] + 1
+ c["scriptDir"] = scriptDir
+ if c["subsection"]:
+ sub = c["subsection"]
+ else:
+ sub = c["grid"]
+
+ # List of dependencies
+ dependencies.append(
+ os.path.join(
+ scriptDir,
+ "ts_%s_%04d-%04d-%04d.status"
+ % (
+ c[
+ "subsection"
+ ], # Depends on the ts subtask with the same name.
+ c["yr_start"],
+ c["yr_end"],
+ c["ypf"],
+ ),
+ ),
+ )
+
+ prefix = "e3sm_to_cmip_%s_%04d-%04d-%04d" % (
+ sub,
+ c["yr_start"],
+ c["yr_end"],
+ c["ypf"],
+ )
+ c["prefix"] = prefix
+ scriptFile = os.path.join(scriptDir, "%s.bash" % (prefix))
+ statusFile = os.path.join(scriptDir, "%s.status" % (prefix))
+ settingsFile = os.path.join(scriptDir, "%s.settings" % (prefix))
+ skip = checkStatus(statusFile)
+ if skip:
+ continue
+
+ # Create script
+ with open(scriptFile, "w") as f:
+ f.write(template.render(**c))
+ makeExecutable(scriptFile)
+
+ with open(settingsFile, "w") as sf:
+ p = pprint.PrettyPrinter(indent=2, stream=sf)
+ p.pprint(c)
+ p.pprint(s)
+
+ export = "ALL"
+ existing_bundles = handle_bundles(
+ c,
+ scriptFile,
+ export,
+ dependFiles=dependencies,
+ existing_bundles=existing_bundles,
+ )
+ if not c["dry_run"]:
+ if c["bundle"] == "":
+ # Submit job
+ submitScript(
+ scriptFile,
+ statusFile,
+ export,
+ job_ids_file,
+ dependFiles=dependencies,
+ )
+ else:
+ print("...adding to bundle '%s'" % (c["bundle"]))
+
+ return existing_bundles

zppy/ilamb.py

@@ -64,6 +64,18 @@ def ilamb(config, scriptDir, existing_bundles, job_ids_file):
  c["year2"],
  c["ts_num_years"],
  ),
+ )
+ )
+ dependencies.append(
+ os.path.join(
+ scriptDir,
+ "e3sm_to_cmip_%s_%04d-%04d-%04d.status"
+ % (
+ c["ts_land_subsection"],
+ c["year1"],
+ c["year2"],
+ c["ts_num_years"],
+ ),
  ),
  )
  if not c["land_only"]:
@@ -79,6 +91,18 @@ def ilamb(config, scriptDir, existing_bundles, job_ids_file):
  ),
  ),
  )
+ dependencies.append(
+ os.path.join(
+ scriptDir,
+ "e3sm_to_cmip_%s_%04d-%04d-%04d.status"
+ % (
+ c["ts_atm_subsection"],
+ c["year1"],
+ c["year2"],
+ c["ts_num_years"],
+ ),
+ ),
+ )
 
  prefix = "ilamb_%04d-%04d" % (
  c["year1"],