Merge branch 'main' into 900_msa_precip_variability_demo

.github/workflows/build_workflow.yml

@@ -25,21 +25,27 @@ jobs:
  needs: check-jobs-to-skip
  if: ${{ needs.check-jobs-to-skip.outputs.should_skip != 'true'}} || ${{ github.event_name == 'push' }}
  runs-on: ubuntu-latest
- timeout-minutes: 2
+ timeout-minutes: 10
  steps:
+ - uses: actions/setup-node@v3
+ with:
+ node-version: '16'
  - name: Checkout Code Repository
  uses: actions/checkout@v2
-
+ with:
+ node-version: '16'
  - name: Set up Python 3.9
  uses: actions/setup-python@v2
  with:
+ node-version: '16'
  python-version: 3.9
-
  # Run all pre-commit hooks on all the files.
  # Getting only staged files can be tricky in case a new PR is opened
  # since the action is run on a branch in detached head state
  - name: Install and Run Pre-commit
  uses: pre-commit/[email protected]
+ with:
+ node-version: '16'
 
  build:
  needs: check-jobs-to-skip
@@ -50,14 +56,17 @@ jobs:
  shell: bash -l {0}
  timeout-minutes: 10
  steps:
+ - uses: actions/setup-node@v3
+ with:
+ node-version: '16'
  - uses: actions/checkout@v2
-
  - name: Cache Conda
  uses: actions/cache@v2
  env:
  # Increase this value to reset cache if conda/dev.yml has not changed in the workflow
  CACHE_NUMBER: 0
  with:
+ node-version: '16'
  path: ~/conda_pkgs_dir
  key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-publish
 

.pre-commit-config.yaml

@@ -4,15 +4,15 @@ fail_fast: true
 
 repos:
  - repo: https://github.com/pre-commit/pre-commit-hooks
- rev: v4.3.0
+ rev: v4.4.0
  hooks:
  - id: trailing-whitespace
  args: [--markdown-linebreak-ext=md]
  - id: end-of-file-fixer
  - id: check-yaml
 
  - repo: https://github.com/psf/black
- rev: 22.10.0
+ rev: 23.1.0
  hooks:
  - id: black
 
@@ -24,7 +24,7 @@ repos:
  # Need to use flake8 GitHub mirror due to CentOS git issue with GitLab
  # https://github.com/pre-commit/pre-commit/issues/1206
  - repo: https://github.com/pycqa/flake8
- rev: 5.0.4
+ rev: 6.0.0
  hooks:
  - id: flake8
  args: ["--config=setup.cfg"]

cmec/scripts/pmp_param_generator.py

@@ -24,7 +24,6 @@ def check_for_opt(key, settings):
 
 
 if __name__ == "__main__":
-
  config_json = sys.argv[1]
  out_file_name = sys.argv[2]
  pmp_config = sys.argv[3]

conda-env/dev.yml

@@ -19,11 +19,13 @@ dependencies:
  - eofs=1.4.0
  - seaborn=0.11.1
  - enso_metrics=1.1.1
+ - xcdat=0.4.0
+ - xmltodict=0.13.0
+ - setuptools=65.5.0
  - netcdf4=1.6.0
  - regionmask=0.9.0
  - rasterio=1.2.10
  - shapely=1.8.0
- - xcdat=0.4.0
  # Testing
  # ==================
  - pre_commit=2.20.0

pcmdi_metrics/__init__.py

@@ -15,5 +15,6 @@
 plog.addHandler(ch)
 plog.setLevel(LOG_LEVEL)
 from . import io # noqa
-from . import pcmdi # noqa
+#from . import pcmdi # noqa
+#from . import mean_climate # noqa
 from .version import __git_sha1__, __git_tag_describe__, __version__ # noqa

pcmdi_metrics/io/__init__.py

@@ -1,3 +1,6 @@
 # init for pcmdi_metrics.io
+from .xcdat_openxml import xcdat_open # noqa # isort:skip
 from . import base # noqa
 from .base import MV2Json # noqa
+from .default_regions_define import load_regions_specs # noqa
+from .default_regions_define import region_subset # noqa

pcmdi_metrics/io/base.py

@@ -14,9 +14,11 @@
 import genutil
 import MV2
 import numpy
+import xcdat
 
 import pcmdi_metrics
 from pcmdi_metrics import LOG_LEVEL
+from pcmdi_metrics.io import xcdat_open
 
 value = 0
 cdms2.setNetcdfShuffleFlag(value) # where value is either 0 or 1
@@ -80,7 +82,7 @@ def update_dict(d, u):
 
 
 def generateProvenance():
- extra_pairs = {"matplotlib": "matplotlib ", "scipy": "scipy"}
+ extra_pairs = {"matplotlib": "matplotlib ", "scipy": "scipy", "xcdat": "xcdat", "xarray": "xarray"}
  prov = cdat_info.generateProvenance(extra_pairs=extra_pairs)
  prov["packages"]["PMP"] = pcmdi_metrics.version.__git_tag_describe__
  prov["packages"][
@@ -164,6 +166,7 @@ def write(
  type="json",
  mode="w",
  include_YAML=False,
+ include_history=False,
  include_script=False,
  *args,
  **kwargs,
@@ -174,7 +177,7 @@ def write(
 
  if not os.path.exists(dir_path):
  try:
- os.makedirs(dir_path)
+ os.makedirs(dir_path, exist_ok=True)
  except Exception:
  logging.getLogger("pcmdi_metrics").error(
  "Could not create output directory: %s" % dir_path
@@ -213,7 +216,9 @@ def write(
  if not include_script:
  if "script" in out_dict["provenance"].keys():
  del out_dict["provenance"]["script"]
-
+ if not include_history:
+ if "history" in out_dict["provenance"].keys():
+ del out_dict["provenance"]["history"]
  json.dump(out_dict, f, cls=CDMSDomainsEncoder, *args, **kwargs)
  f.close()
 
@@ -224,11 +229,14 @@ def write(
  f.close()
 
  elif self.type == "nc":
+ """
  f = cdms2.open(file_name, "w")
  f.write(data, *args, **kwargs)
  f.metrics_git_sha1 = pcmdi_metrics.__git_sha1__
  f.uvcdat_version = cdat_info.get_version()
  f.close()
+ """
+ data.to_netcdf(file_name)
 
  else:
  logging.getLogger("pcmdi_metrics").error("Unknown type: %s" % type)
@@ -352,7 +360,7 @@ def get(self, var, var_in_file=None, region={}, *args, **kwargs):
  if self.is_masking():
  self.var_from_file = self.mask_var(self.var_from_file)
 
- self.var_from_file = self.set_target_grid_and_mask_in_var(self.var_from_file)
+ self.var_from_file = self.set_target_grid_and_mask_in_var(self.var_from_file, var)
 
  self.var_from_file = self.set_domain_in_var(self.var_from_file, self.region)
 
@@ -361,13 +369,18 @@ def get(self, var, var_in_file=None, region={}, *args, **kwargs):
  def extract_var_from_file(self, var, var_in_file, *args, **kwargs):
  if var_in_file is None:
  var_in_file = var
- # self.extension = 'nc'
- var_file = cdms2.open(self(), "r")
- for att in ["var_in_file,", "varInFile"]:
- if att in kwargs:
- del kwargs[att]
- extracted_var = var_file(var_in_file, *args, **kwargs)
- var_file.close()
+
+ try:
+ ds = xcdat_open(self(), data_var=var_in_file, decode_times=True)
+ except Exception:
+ ds = xcdat_open(self(), data_var=var_in_file, decode_times=False) # Temporary part to read in cdms written obs4MIP AC files
+
+ if 'level' in list(kwargs.keys()):
+ level = kwargs['level']
+ ds = ds.sel(plev=level)
+
+ extracted_var = ds
+
  return extracted_var
 
  def is_masking(self):
@@ -377,46 +390,51 @@ def is_masking(self):
  return False
 
  def mask_var(self, var):
+ """
+ self: <pcmdi_metrics.io.base.Base object at 0x7f24a0768a60>
+ var: <xarray.Dataset>
+ """
+ var_shape = tuple(var.dims[d] for d in ['lat', 'lon'])
+
  if self.mask is None:
  self.set_file_mask_template()
  self.mask = self.get_mask_from_var(var)
- if self.mask.shape != var.shape:
+ # if self.mask.shape != var.shape:
+ if self.mask.shape != var_shape:
  dummy, mask = genutil.grower(var, self.mask)
  else:
  mask = self.target_mask
  mask = MV2.not_equal(mask, self.value)
  return MV2.masked_where(mask, var)
 
- def set_target_grid_and_mask_in_var(self, var):
+ def set_target_grid_and_mask_in_var(self, var, var_in_file):
+ """
+ self: <class 'pcmdi_metrics.io.base.Base'> object
+ self(): string, path to input file
+ """
  if self.target_grid is not None:
- var = var.regrid(
- self.target_grid,
- regridTool=self.regrid_tool,
- regridMethod=self.regrid_method,
- coordSys="deg",
- diag={},
- periodicity=1,
- )
-
+ var = var.regridder.horizontal(var_in_file, self.target_grid, tool=self.regrid_tool)
  if self.target_mask is not None:
- if self.target_mask.shape != var.shape:
+ # if self.target_mask.shape != var.shape:
+ if self.target_mask.shape != var[var_in_file].shape:
  dummy, mask = genutil.grower(var, self.target_mask)
  else:
  mask = self.target_mask
  var = MV2.masked_where(mask, var)
-
  return var
 
  def set_domain_in_var(self, var, region):
- domain = region.get("domain", None)
- if domain is not None:
- if isinstance(domain, dict):
- var = var(**domain)
- elif isinstance(domain, (list, tuple)):
- var = var(*domain)
- elif isinstance(domain, cdms2.selectors.Selector):
- domain.id = region.get("id", "region")
- var = var(*[domain])
+ """
+ self: <class 'pcmdi_metrics.io.base.Base'>
+ var: <xarray.Dataset>
+ region: <class 'dict'>, e.g., {'domain': Selector(<cdutil.region.DomainComponent object at 0x7fdbe2b70760>), 'id': 'NHEX'}
+ """
+ region_id = region['id']
+ from pcmdi_metrics.io import load_regions_specs, region_subset
+ regions_specs = load_regions_specs()
+ if region_id not in ['global', 'land', 'ocean']:
+ var = region_subset(var, regions_specs, region=region_id)
+
  return var
 
  def set_file_mask_template(self):
@@ -429,7 +447,8 @@ def set_file_mask_template(self):
 
  def get_mask_from_var(self, var):
  try:
- o_mask = self.file_mask_template.get("sftlf")
+ # o_mask = self.file_mask_template.get("sftlf")
+ o_mask = self.file_mask_template.get("sftlf", var_in_file="sftlf")
  except Exception:
  o_mask = (
  cdutil.generateLandSeaMask(var, regridTool=self.regrid_tool).filled(1.0)
@@ -444,7 +463,8 @@ def set_target_grid(self, target, regrid_tool="esmf", regrid_method="linear"):
  self.regrid_tool = regrid_tool
  self.regrid_method = regrid_method
  if target == "2.5x2.5":
- self.target_grid = cdms2.createUniformGrid(-88.875, 72, 2.5, 0, 144, 2.5)
+ # self.target_grid = cdms2.createUniformGrid(-88.875, 72, 2.5, 0, 144, 2.5)
+ self.target_grid = xcdat.create_uniform_grid(-88.875, 88.625, 2.5, 0, 357.5, 2.5)
  self.target_grid_name = target
  elif cdms2.isGrid(target):
  self.target_grid = target