Update validation code, test retry of tide model file access

GeoscienceAustralia · Jan 30, 2023 · f717435 · f717435
1 parent 41c7d04
commit f717435
Show file tree

Hide file tree

Showing 6 changed files with 11,094 additions and 1,481 deletions.
diff --git a/coastlines/raster.py b/coastlines/raster.py
@@ -565,8 +565,14 @@ def generate_rasters(
  # Add this new data as a new variable in our satellite dataset to allow
  # each satellite pixel to be analysed and filtered/masked based on the
  # tide height at the exact moment of satellite image acquisition.
- ds["tide_m"], tides_lowres = pixel_tides(ds, resample=True)
- log.info(f"Study area {study_area}: Finished modelling tide heights")
+ try: 
+ ds["tide_m"], tides_lowres = pixel_tides(ds, resample=True, directory='blah')
+ log.info(f"Study area {study_area}: Finished modelling tide heights")
+
+ except FileNotFoundError:
+
+ log.exception(f"Study area {study_area}: Unable to access tide modelling files")
+ sys.exit(2)
 
  # Based on the entire time-series of tide heights, compute the max
  # and min satellite-observed tide height for each pixel, then

diff --git a/coastlines/validation.py b/coastlines/validation.py
@@ -1800,20 +1800,10 @@ def nearest_features(input_gdf, comp_gdf, cols='erode_v'):
 
  return out_gdf
 
- # Optionally load either Smartline advanced or basic data
- if advanced:
-
- # Join advanced Smartline data
- smartline = gpd.read_file('../input_data/Smartline.gdb',
- bbox=bbox.buffer(100)).to_crs('EPSG:3577').rename(
- {'INTERTD1_V': 'smartline'}, axis=1)
-
- else:
-
- # Join basic Smartline data
- smartline = gpd.read_file('../input_data/Smartline_basic/smartline_basic.shp',
+ # Load Smartline advanced data
+ smartline = gpd.read_file('https://dea-public-data.s3.ap-southeast-2.amazonaws.com/derivative/dea_coastlines/supplementary/Smartline.gpkg',
  bbox=bbox.buffer(100)).to_crs('EPSG:3577').rename(
- {'erode_v': 'smartline'}, axis=1)
+ {'INTERTD1_V': 'smartline'}, axis=1)
 
  # Identify unique profiles (no need to repeat Smartline extraction for each time)
  inds = [i[1][0] for i in val_gdf.groupby('id').groups.items()]
@@ -1857,9 +1847,23 @@ def deacl_validation(val_path,
  maxx, miny = val_df.max().loc[[f'{datum}_x', f'{datum}_y']]
  bbox = gpd.GeoSeries(box(minx, miny, maxx, maxy), crs='EPSG:3577')
 
- # Import corresponding waterline contours
- deacl_gdf = (gpd.read_file(deacl_path,
- bbox=bbox.buffer(100))
+# # Import corresponding waterline contours
+# deacl_gdf = (gpd.read_file(deacl_path,
+# bbox=bbox.buffer(100))
+# .to_crs('EPSG:3577')
+# .dissolve('year')
+# .reset_index())
+
+# deacl_gdf = (gpd.read_file(deacl_path, 
+# bbox=bbox.buffer(100), 
+# layer='coastlines_v0.0.2_shorelines_annual')
+# .to_crs('EPSG:3577')
+# .dissolve('year')
+# .reset_index())
+
+ deacl_gdf = (gpd.read_file(deacl_path, 
+ bbox=bbox.buffer(100), 
+ layer='shorelines_annual')
  .to_crs('EPSG:3577')
  .dissolve('year')
  .reset_index())
@@ -1958,4 +1962,4 @@ def deacl_validation(val_path,
  'error_m']]
 
  # Export data
- results_df.to_csv(out_name, index=False)
+ results_df.to_csv(f"data/validation/processed/{out_name}", index=False)
diff --git a/notebooks/DEACoastlines_generation_CLI.ipynb b/notebooks/DEACoastlines_generation_CLI.ipynb
@@ -88,17 +88,17 @@
  "metadata": {},
  "outputs": [],
  "source": [
- "study_area = 424\n",
- "raster_version = 'testing'\n",
- "vector_version = 'testing'\n",
- "continental_version = 'testing'\n",
- "config_path = 'configs/dea_coastlines_config.yaml'\n",
+ "# study_area = 424\n",
+ "# raster_version = 'testing'\n",
+ "# vector_version = 'testing'\n",
+ "# continental_version = 'testing'\n",
+ "# config_path = 'configs/dea_coastlines_config.yaml'\n",
  "\n",
- "# study_area = 9\n",
- "# raster_version = 'development'\n",
- "# vector_version = 'development'\n",
- "# continental_version = 'development'\n",
- "# config_path = 'configs/dea_coastlines_config_development.yaml'"
+ "study_area = 9\n",
+ "raster_version = 'development'\n",
+ "vector_version = 'development'\n",
+ "continental_version = 'development'\n",
+ "config_path = 'configs/dea_coastlines_config_development.yaml'"
  ]
  },
  {
@@ -190,25 +190,36 @@
  },
  {
  "cell_type": "code",
- "execution_count": 5,
+ "execution_count": 6,
  "id": "47397eb8-ddec-46c6-9f7c-431a50bb7eae",
  "metadata": {},
  "outputs": [
  {
  "name": "stdout",
  "output_type": "stream",
  "text": [
- "<Client: 'tcp:https://127.0.0.1:33009' processes=1 threads=31, memory=237.21 GiB>\n",
- "2023-01-25 02:09:06 INFO Study area 424: Loaded study area grid\n",
- "2023-01-25 02:09:09 INFO Study area 424: Loaded virtual product\n",
+ "/env/lib/python3.8/site-packages/distributed/node.py:180: UserWarning: Port 8787 is already in use.\n",
+ "Perhaps you already have a cluster running?\n",
+ "Hosting the HTTP server on port 38341 instead\n",
+ " warnings.warn(\n",
+ "<Client: 'tcp:https://127.0.0.1:41371' processes=1 threads=7, memory=59.21 GiB>\n",
+ "2023-01-30 02:03:44 INFO Study area 9: Loaded study area grid\n",
+ "2023-01-30 02:04:00 INFO Study area 9: Loaded virtual product\n",
  "Creating reduced resolution tide modelling array\n",
  "Modelling tides using FES2014 tide model\n",
- "Reprojecting tides into original array\n",
- "100%|████████████████████████████████████████| 135/135 [00:01<00:00, 114.61it/s]\n",
- "2023-01-25 02:09:36 INFO Study area 424: Finished modelling tide heights\n",
- "2023-01-25 02:09:36 INFO Study area 424: Calculating low and high tide cutoffs for each pixel\n",
- "2023-01-25 02:09:36 INFO Study area 424: Started exporting raster data\n",
- "2023-01-25 02:10:44 INFO Study area 424: Completed exporting raster data\n"
+ "2023-01-30 02:04:00 ERROR Study area 9: Unable to access tide modelling files\n",
+ "Traceback (most recent call last):\n",
+ " File \"/home/jovyan/Robbi/dea-coastlines/coastlines/raster.py\", line 569, in generate_rasters\n",
+ " ds[\"tide_m\"], tides_lowres = pixel_tides(ds, resample=True, directory='blah')\n",
+ " File \"/env/lib/python3.8/site-packages/dea_tools/coastal.py\", line 592, in pixel_tides\n",
+ " tide_df = model_tides(\n",
+ " File \"/env/lib/python3.8/site-packages/dea_tools/coastal.py\", line 336, in model_tides\n",
+ " model = pyTMD.model(directory, format=\"netcdf\", compressed=False).elevation(model)\n",
+ " File \"/env/lib/python3.8/site-packages/pyTMD/model.py\", line 842, in elevation\n",
+ " self.model_file = self.pathfinder(model_files)\n",
+ " File \"/env/lib/python3.8/site-packages/pyTMD/model.py\", line 1425, in pathfinder\n",
+ " raise FileNotFoundError(output_file)\n",
+ "FileNotFoundError: ['blah/fes2014/ocean_tide/2n2.nc', 'blah/fes2014/ocean_tide/eps2.nc', 'blah/fes2014/ocean_tide/j1.nc', 'blah/fes2014/ocean_tide/k1.nc', 'blah/fes2014/ocean_tide/k2.nc', 'blah/fes2014/ocean_tide/l2.nc', 'blah/fes2014/ocean_tide/la2.nc', 'blah/fes2014/ocean_tide/m2.nc', 'blah/fes2014/ocean_tide/m3.nc', 'blah/fes2014/ocean_tide/m4.nc', 'blah/fes2014/ocean_tide/m6.nc', 'blah/fes2014/ocean_tide/m8.nc', 'blah/fes2014/ocean_tide/mf.nc', 'blah/fes2014/ocean_tide/mks2.nc', 'blah/fes2014/ocean_tide/mm.nc', 'blah/fes2014/ocean_tide/mn4.nc', 'blah/fes2014/ocean_tide/ms4.nc', 'blah/fes2014/ocean_tide/msf.nc', 'blah/fes2014/ocean_tide/msqm.nc', 'blah/fes2014/ocean_tide/mtm.nc', 'blah/fes2014/ocean_tide/mu2.nc', 'blah/fes2014/ocean_tide/n2.nc', 'blah/fes2014/ocean_tide/n4.nc', 'blah/fes2014/ocean_tide/nu2.nc', 'blah/fes2014/ocean_tide/o1.nc', 'blah/fes2014/ocean_tide/p1.nc', 'blah/fes2014/ocean_tide/q1.nc', 'blah/fes2014/ocean_tide/r2.nc', 'blah/fes2014/ocean_tide/s1.nc', 'blah/fes2014/ocean_tide/s2.nc', 'blah/fes2014/ocean_tide/s4.nc', 'blah/fes2014/ocean_tide/sa.nc', 'blah/fes2014/ocean_tide/ssa.nc', 'blah/fes2014/ocean_tide/t2.nc']\n"
  ]
  }
  ],