move ci to balfrin

jenkins/Jenkinsfile

@@ -10,7 +10,7 @@ pipeline {
  PATH = "$WORKSPACE/miniconda/bin:$PATH"
  }
  agent {
- label 'tsa'
+ label 'balfrin'
  }
 
  post {
@@ -33,12 +33,12 @@ pipeline {
  sh '''#!/usr/bin/env bash
  set -e
  spack_c2sm_url=https://github.com/C2SM/spack-c2sm.git
- spack_c2sm_tag=v0.20.1.0
+ spack_c2sm_tag=v0.20.1.9
  spack_c2sm_dir=${WORKSPACE}/s
  git clone --depth 1 --recurse-submodules -b ${spack_c2sm_tag} ${spack_c2sm_url} ${spack_c2sm_dir}
  . ${spack_c2sm_dir}/setup-env.sh
  spack env activate -p ${WORKSPACE}/spack-env
- spack mirror add iwf2-mirror /project/s83c/rz+/spack-mirror-test/
+ spack mirror add iwf2-mirror /store_new/mch/msopr/icon_workflow_2/spack-mirror
  spack install --no-check-signature
  '''
  }

src/meteodatalab/data/fdb_config_balfrin.yaml

@@ -5,4 +5,4 @@ schema: /scratch/mch/ckanesan/fdb-setup/fdb-schema
 spaces:
 - handler: Default
  roots:
- - path: /scratch/mch/ckanesan/fdb-root/
+ - path: /scratch/mch/ckanesan/fdb_32_39x45_51/

tests/test_meteodatalab/conftest.py

@@ -30,9 +30,12 @@ def machine():
 @pytest.fixture
 def data_dir(request, machine):
  """Base data dir."""
- if machine != "tsa":
+ if machine == "tsa":
+ base_dir = Path("/project/s83c/rz+/icon_data_processing_incubator/")
+ elif machine == "balfrin":
+ base_dir = Path("/scratch/mch/ckanesan/")
+ else:
  return None
- base_dir = Path("/project/s83c/rz+/icon_data_processing_incubator/")
  marker = request.node.get_closest_marker("data")
  if marker is None:
  return base_dir / "datasets/32_39x45_51"
@@ -45,6 +48,8 @@ def data_dir(request, machine):
  return base_dir / "datasets/32_39x45_51/COSMO-1E_time"
  case "reduced-ens":
  return base_dir / "datasets/32_39x45_51/COSMO-1E_ens"
+ case "flexpart":
+ return base_dir / "data/flexpart"
  raise RuntimeError(f"No match for data mark {marker.args[0]}")
 
 
@@ -59,7 +64,7 @@ def fieldextra_path(machine):
  """Fieldextra path."""
  conf = {
  "tsa": Path("/project/s83c/fieldextra/tsa"),
- "balfrin": Path("/users/tsm/proj.aare/fieldextra/v14.3.1/"),
+ "balfrin": Path("/users/tsm/proj.aare/fieldextra/v14.4.0/"),
  }
  return conf.get(machine)
 

tests/test_meteodatalab/test_brn.py

@@ -3,16 +3,17 @@
 
 # First-party
 import meteodatalab.operators.brn as mbrn
-from meteodatalab.grib_decoder import GribReader
+from meteodatalab.data_source import DataSource
+from meteodatalab.grib_decoder import load
 from meteodatalab.metadata import set_origin_xy
 
 
 def test_brn(data_dir, fieldextra):
  datafile = data_dir / "COSMO-1E/1h/ml_sl/000/lfff00000000"
  cdatafile = data_dir / "COSMO-1E/1h/const/000/lfff00000000c"
 
- reader = GribReader.from_files([cdatafile, datafile])
- ds = reader.load_fieldnames(["P", "T", "QV", "U", "V", "HHL", "HSURF"])
+ source = DataSource(datafiles=[datafile, cdatafile])
+ ds = load(source, {"param": ["P", "T", "QV", "U", "V", "HHL", "HSURF"]})
  set_origin_xy(ds, "HHL")
 
  brn = mbrn.fbrn(

tests/test_meteodatalab/test_flexpart.py

@@ -1,6 +1,3 @@
-# Standard library
-from pathlib import Path
-
 # Third-party
 import pytest
 import xarray as xr
@@ -12,12 +9,8 @@
 from meteodatalab.grib_decoder import GribReader
 
 
-@pytest.fixture
-def data_dir(data_dir):
- return Path("/project/s83c/rz+/icon_data_processing_incubator/data/flexpart/")
-
-
 @pytest.mark.ifs
+@pytest.mark.data("flexpart")
 def test_flexpart(data_dir, fieldextra):
  datafiles = list(data_dir.glob("efs*"))
  constants = ("z", "lsm", "sdor")