Move ci to balfrin

jenkins/Jenkinsfile

@@ -10,7 +10,7 @@ pipeline {
  PATH = "$WORKSPACE/miniconda/bin:$PATH"
  }
  agent {
- label 'tsa'
+ label 'balfrin'
  }
 
  post {
@@ -33,13 +33,13 @@ pipeline {
  sh '''#!/usr/bin/env bash
  set -e
  spack_c2sm_url=https://github.com/C2SM/spack-c2sm.git
- spack_c2sm_tag=v0.20.1.0
+ spack_c2sm_tag=v0.20.1.9
  spack_c2sm_dir=${WORKSPACE}/s
  git clone --depth 1 --recurse-submodules -b ${spack_c2sm_tag} ${spack_c2sm_url} ${spack_c2sm_dir}
  . ${spack_c2sm_dir}/setup-env.sh
  spack env activate -p ${WORKSPACE}/spack-env
- spack mirror add iwf2-mirror /project/s83c/rz+/spack-mirror-test/
- spack install --no-check-signature
+ spack mirror add iwf2-mirror /store_new/mch/msopr/icon_workflow_2/spack-mirror
+ spack install --no-check-signature --no-checksum
  '''
  }
  }

spack-env/spack.yaml

@@ -4,10 +4,10 @@
 # configuration settings.
 spack:
  specs:
- - fdb
- - eckit@1.20.2 ~mpi
- - eccodes@2.25 jp2k=none
- - hdf5 ~mpi
+ - fdb@=5.11.116 +tools
+ - eckit@=1.26.2 ~mpi
+ - eccodes@=2.33.0 jp2k=none
+ - metkit@=1.11.9
  view: true
  concretizer:
  unify: true

src/meteodatalab/data/fdb_config_balfrin.yaml

@@ -1,8 +1,8 @@
 ---
 type: local
 engine: toc
-schema: /scratch/mch/ckanesan/fdb-setup/fdb-schema
+schema: /store_new/mch/msopr/icon_workflow_2/fdb-schema
 spaces:
 - handler: Default
  roots:
- - path: /scratch/mch/ckanesan/fdb-root/
+ - path: /store_new/mch/msopr/icon_workflow_2/fdb_32_39x45_51/

src/meteodatalab/mars.py

@@ -155,12 +155,12 @@ def to_fdb(self) -> dict[str, typing.Any]:
 
  obj = dc.replace(self, levelist=levelist)
  out = typing.cast(dict[str, typing.Any], obj.dump())
- return out | {"param": self._param_id()}
+ return out | {"param": self._param_id(), "model": out["model"].lower()}
 
  def to_polytope(self) -> dict[str, typing.Any]:
  result = self.to_fdb()
  if isinstance(result["param"], list):
  param: str | list[str] = [str(p) for p in result["param"]]
  else:
  param = str(result["param"])
- return result | {"param": param, "model": result["model"].lower()}
+ return result | {"param": param}

tests/test_meteodatalab/conftest.py

@@ -28,23 +28,44 @@ def machine():
 
 
 @pytest.fixture
-def data_dir(request, machine):
+def unpack(work_dir: Path):
+ def f(path: Path) -> Path:
+ src = path.with_suffix(".tar.gz")
+ if not src.exists():
+ return path
+ dst = work_dir
+ tgt = dst / path.name
+ if tgt.exists():
+ return tgt
+ subprocess.run(["tar", "-xf", str(src), "-C", str(dst)], check=True)
+ return tgt
+
+ return f
+
+
+@pytest.fixture
+def data_dir(request, machine, unpack):
  """Base data dir."""
- if machine != "tsa":
+ if machine == "tsa":
+ base_dir = Path("/project/s83c/rz+/icon_data_processing_incubator/")
+ elif machine == "balfrin":
+ base_dir = Path("/store_new/mch/msopr/icon_workflow_2/")
+ else:
  return None
- base_dir = Path("/project/s83c/rz+/icon_data_processing_incubator/")
  marker = request.node.get_closest_marker("data")
  if marker is None:
- return base_dir / "datasets/32_39x45_51"
+ return unpack(base_dir / "datasets/32_39x45_51")
  match marker.args[0]:
  case "original":
  return base_dir / "datasets/original"
  case "reduced":
- return base_dir / "datasets/32_39x45_51"
+ return unpack(base_dir / "datasets/32_39x45_51")
  case "reduced-time":
- return base_dir / "datasets/32_39x45_51/COSMO-1E_time"
+ return unpack(base_dir / "datasets/32_39x45_51") / "COSMO-1E_time"
  case "reduced-ens":
- return base_dir / "datasets/32_39x45_51/COSMO-1E_ens"
+ return unpack(base_dir / "datasets/32_39x45_51") / "COSMO-1E_ens"
+ case "flexpart":
+ return base_dir / "data/flexpart"
  raise RuntimeError(f"No match for data mark {marker.args[0]}")
 
 
@@ -59,7 +80,7 @@ def fieldextra_path(machine):
  """Fieldextra path."""
  conf = {
  "tsa": Path("/project/s83c/fieldextra/tsa"),
- "balfrin": Path("/users/tsm/proj.aare/fieldextra/v14.3.1/"),
+ "balfrin": Path("/users/tsm/proj.aare/fieldextra/v14.4.0/"),
  }
  return conf.get(machine)
 

tests/test_meteodatalab/test_brn.py

@@ -3,16 +3,17 @@
 
 # First-party
 import meteodatalab.operators.brn as mbrn
-from meteodatalab.grib_decoder import GribReader
+from meteodatalab.data_source import DataSource
+from meteodatalab.grib_decoder import load
 from meteodatalab.metadata import set_origin_xy
 
 
 def test_brn(data_dir, fieldextra):
  datafile = data_dir / "COSMO-1E/1h/ml_sl/000/lfff00000000"
  cdatafile = data_dir / "COSMO-1E/1h/const/000/lfff00000000c"
 
- reader = GribReader.from_files([cdatafile, datafile])
- ds = reader.load_fieldnames(["P", "T", "QV", "U", "V", "HHL", "HSURF"])
+ source = DataSource(datafiles=[datafile, cdatafile])
+ ds = load(source, {"param": ["P", "T", "QV", "U", "V", "HHL", "HSURF"]})
  set_origin_xy(ds, "HHL")
 
  brn = mbrn.fbrn(

tests/test_meteodatalab/test_flexpart.py

@@ -1,6 +1,3 @@
-# Standard library
-from pathlib import Path
-
 # Third-party
 import pytest
 import xarray as xr
@@ -12,12 +9,8 @@
 from meteodatalab.grib_decoder import GribReader
 
 
-@pytest.fixture
-def data_dir(data_dir):
- return Path("/project/s83c/rz+/icon_data_processing_incubator/data/flexpart/")
-
-
 @pytest.mark.ifs
+@pytest.mark.data("flexpart")
 def test_flexpart(data_dir, fieldextra):
  datafiles = list(data_dir.glob("efs*"))
  constants = ("z", "lsm", "sdor")

tests/test_meteodatalab/test_mars.py

@@ -14,7 +14,7 @@ def sample():
  "expver": "0001",
  "levtype": "ml",
  "levelist": list(range(1, 81)),
- "model": "COSMO-1E",
+ "model": "cosmo-1e",
  "number": 0,
  "stream": "enfo",
  "param": 500028, # U
@@ -46,7 +46,7 @@ def test_fdb_c2e(sample):
  step=0,
  model=mars.Model.COSMO_2E,
  ).to_fdb()
- expected = sample | {"model": "COSMO-2E", "levelist": list(range(1, 61))}
+ expected = sample | {"model": "cosmo-2e", "levelist": list(range(1, 61))}
 
  assert observed == expected