From 9179ac2f9f67b0d181d417b6c07841aa4640b6be Mon Sep 17 00:00:00 2001 From: jmills-ncar Date: Thu, 7 Jun 2018 17:08:19 -0600 Subject: [PATCH 1/9] fixing complete git mangle with large file addition --- training/Dockerfile | 12 +++++++++-- wps/Dockerfile | 27 ++++++++++++++++++++---- wps/build.sh | 3 +-- wps/make_geogrid.py | 42 ++++++++++++++++++++++++++++++++++++++ wps/wrf_hydro_namelist.wps | 20 +----------------- 5 files changed, 77 insertions(+), 27 deletions(-) diff --git a/training/Dockerfile b/training/Dockerfile index 2299031..105308d 100644 --- a/training/Dockerfile +++ b/training/Dockerfile @@ -17,14 +17,20 @@ RUN mkdir /home/docker/wrf-hydro-training RUN chmod -R 777 /home/docker/wrf-hydro-training COPY ./create_wrfinput.R /home/docker/wrf-hydro-training/wrf_hydro_model_tools/create_wrfinput.R -#Bring in NLDAS data +#get regridding scripts RUN wget https://ral.ucar.edu/sites/default/files/public/ESMFregrid_NLDAS.tar_.gz \ && tar -xf ESMFregrid_NLDAS.tar_.gz \ && mv NLDAS /home/docker/wrf-hydro-training/regridding \ && rm ESMFregrid_NLDAS.tar_.gz #Get NLDAS forcing for regridding exercise -COPY ./nldas_forcing.tar.gz /home/docker/wrf-hydro-training/regridding/data/nldas_forcing.tar.gz +#COPY ./nldas_forcing.tar.gz /home/docker/wrf-hydro-training/regridding/data/nldas_forcing.tar.gz + +COPY ./gdrive_download.py gdrive_download.py +RUN chmod 777 gdrive_download.py + + +RUN python gdrive_download.py --file_id 1PrDTA-hMFNqQrDag1Dm7tX1rT1gQ_25e --dest_file /home/docker/wrf-hydro-training/regridding/data/nldas_forcing.tar.gz # install NCL RUN wget https://www.earthsystemgrid.org/dataset/ncl.640.dap/file/ncl_ncarg-6.4.0-Debian8.6_64bit_gnu492.tar.gz \ @@ -74,5 +80,7 @@ RUN chmod -R 777 /home/docker/wrf-hydro-training/ USER docker WORKDIR /home/docker +#Make a temp directory needed for matplotlib + ENTRYPOINT ["/entrypoint.sh"] CMD ["interactive"] \ No newline at end of file diff --git a/wps/Dockerfile b/wps/Dockerfile index 9535caa..77530b1 100755 --- a/wps/Dockerfile +++ b/wps/Dockerfile @@ -49,6 +49,8 @@ RUN chmod -R 777 /home/docker/WRF_WPS ############################ ## Python +WORKDIR / + #Fetch latest miniconda and install RUN wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \ && bash Miniconda3-latest-Linux-x86_64.sh -b -p /home/docker/miniconda3 \ @@ -69,6 +71,7 @@ RUN pip install wrf-python Image ############################ # Setup stuff for geogrid script +COPY ./namelist.wps_orig /home/docker/WRF_WPS/utilities/namelist.wps_orig RUN mkdir -p /home/docker/WRF_WPS/WPS/utilities/ \ && mv /home/docker/WRF_WPS/WPS/namelist.wps \ @@ -77,10 +80,22 @@ RUN mkdir -p /home/docker/WRF_WPS/WPS/utilities/ \ COPY make_geogrid.py /home/docker/WRF_WPS/utilities/make_geogrid.py COPY wrf_hydro_namelist.wps /home/docker/WRF_WPS/utilities/ -COPY geog_conus.tar.gz . -RUN tar -xf geog_conus.tar.gz && \ - mv geog_conus /home/docker/WRF_WPS/utilities/geog_conus && \ - rm geog_conus.tar.gz +# Get geog data from google drive +## Get download script +COPY gdrive_download.py /gdrive_download.py +RUN chmod 777 /gdrive_download.py + +RUN python gdrive_download.py --file_id 1EICBJ8-pRpkox27Bnq3mrBAqyZQsV662 --dest_file geog_conus.tar.gz \ + && tar -xf geog_conus.tar.gz \ + && mv geog_conus /home/docker/WRF_WPS/utilities/geog_conus \ + && rm geog_conus.tar.gz + +RUN chmod -R 777 /home/docker/WRF_WPS/utilities/geog_conus + +#COPY geog_conus.tar.gz . +#RUN tar -xf geog_conus.tar.gz && \ +# mv geog_conus /home/docker/WRF_WPS/utilities/geog_conus && \ +# rm geog_conus.tar.gz RUN mv /home/docker/WRF_WPS/utilities/geog_conus/GEOGRID.TBL.ARW.wrf_hydro_training \ /home/docker/WRF_WPS/WPS/geogrid/GEOGRID.TBL.ARW @@ -90,6 +105,10 @@ RUN mv /home/docker/WRF_WPS/utilities/geog_conus/GEOGRID.TBL.ARW.wrf_hydro_train #################################### USER docker WORKDIR /home/docker + +RUN mkdir /home/docker/tmp/ \ + && chmod 777 -R /home/docker/tmp/ + #ENV NETCDF=/usr/local ENTRYPOINT ["python", "/home/docker/WRF_WPS/utilities/make_geogrid.py"] diff --git a/wps/build.sh b/wps/build.sh index 3a8a6cc..c154f47 100755 --- a/wps/build.sh +++ b/wps/build.sh @@ -1,5 +1,4 @@ #!/bin/bash - -docker build "$@" -t wrfhydro/wps . +docker build "$@" --no-cache -t wrfhydro/wps . exit $? diff --git a/wps/make_geogrid.py b/wps/make_geogrid.py index 40a3c73..a1ea89d 100644 --- a/wps/make_geogrid.py +++ b/wps/make_geogrid.py @@ -22,11 +22,21 @@ import shutil import os import time +<<<<<<< HEAD +======= import matplotlib.pyplot as plt +>>>>>>> c407c2cdd5c461832ff07b5e679470f4ceae81f0 import cartopy.io.img_tiles as cimgt from wrf import (projection, latlonutils) import math +<<<<<<< HEAD +#Setup matlob lib to not use any xwindows backend +import matplotlib +matplotlib.use('Agg') +import matplotlib.pyplot as plt +======= +>>>>>>> c407c2cdd5c461832ff07b5e679470f4ceae81f0 # Functions def build_projparams(ref_lat=0.0, ref_lon=0.0, dx=1000.0, dy=1000.0, map_proj='lambert', truelat1=0.0, truelat2=0.0, @@ -249,11 +259,19 @@ def main(): default='false', help="Only create a plot of the domain. Geogrid will not be created if " "plot_only = true, only a plot of the domain will be created.") +<<<<<<< HEAD + #parser.add_argument("--display", + # dest="display", + # default='false', + # help="Display the image in python. Warning may cause issues in a terminal" + # "session.") +======= parser.add_argument("--display", dest="display", default='false', help="Display the image in python. Warning may cause issues in a terminal" "session.") +>>>>>>> c407c2cdd5c461832ff07b5e679470f4ceae81f0 # parser.add_argument("--orig_nml_path", # dest="new_nml_path", @@ -269,6 +287,16 @@ def main(): args = parser.parse_args() patch_nml_path = pathlib.Path(args.namelist_path) +<<<<<<< HEAD + output_dir = args.output_dir + plot_only = args.plot_only + #display = args.display + display = 'false' + + # orig_nml_path = args.orig_nml_path + # new_nml_path = args.new_nml_path + orig_nml_path = pathlib.Path('/home/docker/WRF_WPS/utilities/namelist.wps_orig') +======= output_dir = pathlib.Path(args.output_dir) plot_only = args.plot_only display = args.display @@ -276,6 +304,7 @@ def main(): # orig_nml_path = args.orig_nml_path # new_nml_path = args.new_nml_path orig_nml_path = pathlib.Path('/home/docker/WRF_WPS/WPS/namelist.wps_orig') +>>>>>>> c407c2cdd5c461832ff07b5e679470f4ceae81f0 new_nml_path = pathlib.Path('/home/docker/WRF_WPS/WPS/namelist.wps') if display.lower() == 'true': @@ -285,12 +314,20 @@ def main(): if plot_only.lower() == 'true': plot_from_wps(patch_nml_path=str(patch_nml_path), +<<<<<<< HEAD + figFilename=output_dir + '/domain.png', +======= figFilename=str(output_dir) + '/domain.png', +>>>>>>> c407c2cdd5c461832ff07b5e679470f4ceae81f0 display=display) else: print('Plotting domain') plot_from_wps(patch_nml_path=str(patch_nml_path), +<<<<<<< HEAD + figFilename=output_dir + '/domain.png', +======= figFilename=str(output_dir) + '/domain.png', +>>>>>>> c407c2cdd5c461832ff07b5e679470f4ceae81f0 display=display) print('Generating geogrid file') @@ -304,8 +341,13 @@ def main(): subprocess.run(['./geogrid.exe'], cwd=new_nml_path.parent) +<<<<<<< HEAD + shutil.copy(str(new_nml_path.parent / 'geo_em.d01.nc'), + output_dir + '/geo_em.d01.nc') +======= shutil.move(str(new_nml_path.parent / 'geo_em.d01.nc'), str(pathlib.Path(output_dir) / 'geo_em.d01.nc')) +>>>>>>> c407c2cdd5c461832ff07b5e679470f4ceae81f0 if __name__ == '__main__': diff --git a/wps/wrf_hydro_namelist.wps b/wps/wrf_hydro_namelist.wps index 8480d9e..3c5486b 100644 --- a/wps/wrf_hydro_namelist.wps +++ b/wps/wrf_hydro_namelist.wps @@ -1,13 +1,3 @@ -&share - -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -! Specify the number of domains -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - max_dom = 1, - -/ - &geogrid !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! @@ -39,13 +29,5 @@ map_proj = 'lambert', truelat1 = 30.0, truelat2 = 60.0, - stand_lon = -97.00, - -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -! Define the data sources and data path -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - geog_data_res = 'default', - geog_data_path = '/home/docker/WRF_WPS/utilities/geog_conus' - + stand_lon = -97.00 / From bab9834b805420fa7613739a577fe8f4682ed666 Mon Sep 17 00:00:00 2001 From: jmills-ncar Date: Thu, 7 Jun 2018 17:17:13 -0600 Subject: [PATCH 2/9] fixing more conflicts in make_geogrid.py --- wps/Readme.md | 113 +++++++++++++++++++++++++++++++++++++++++ wps/gdrive_download.py | 51 +++++++++++++++++++ wps/namelist.wps_orig | 53 +++++++++++++++++++ 3 files changed, 217 insertions(+) create mode 100644 wps/Readme.md create mode 100644 wps/gdrive_download.py create mode 100755 wps/namelist.wps_orig diff --git a/wps/Readme.md b/wps/Readme.md new file mode 100644 index 0000000..ec42c8a --- /dev/null +++ b/wps/Readme.md @@ -0,0 +1,113 @@ +![](https://ral.ucar.edu/sites/default/files/public/wrf_hydro_symbol_logo_2017_09_150pxby63px.png) WRF-HYDRO + +# Overview +This container is used primarily to create geogrid files for a specified domain to be used with the WRF-Hydro modeling system. + +This container includes the following: + +* Ubuntu base image +* WRF and WPS built with the GNU Fortran compiler ‘gfortran’ +* Python 3.6 command line utility for create WRF-Hydro geogrid files using the WPS geogrid.exe program. +* WRF-WPS Geographical input data for the Continental United States. + +## Where to get help and/or post issues +If you have general questions about Docker, there are ample online resources including the excellent Docker documentation at https://docs.docker.com/. + +The best place ask questions or post issues is via the Issues page of the GitHub repository at +https://github.com/NCAR/wrf_hydro_docker/issues. + +## Data sources +WRF Preprocessing System (WPS) geographical input data are one of the primary datasets used by the NoahMP Land Surface Model (LSM). These datasets can be obtained from the [WPS geographical input data download page](http://www2.mmm.ucar.edu/wrf/users/download/get_sources_wps_geog.html). However, these data are rather large (~50GB) and contain a number of datasets that are not used in most implementations of WRF-Hydro. We have reduced the data by removing various resolutions of the data that are not commonly used by WRF-Hydro. Furthermore, the dataset provided with this container has been subset to the Continental United States (CONUS). Thus, the dataset provided with this container is considerably smaller than the complete WRF-WPS dataset. + +## Creating the geo_em_d01.nc (geogrid) file +### geogrid.exe +The WPS program `geogrid.exe` is used to create the geo_em_d01.nc, hereafter referred to as the 'geogrid' file. The `geogrid.exe` program takes a fortran namelist (`namelist.wps`) and the [WPS geographical input data](http://www2.mmm.ucar.edu/wrf/users/download/get_sources_wps_geog.html) as inputs and creates the geogrid file. However, the `geogrid.exe.` program requires that WRF and WPS be built according to your system specification, and building WRF and WPS can be difficult on some systems. Additionally, much of the functionality of WRF and WPS is not utilized for creating a geogrid file for WRF-Hydro, and many of the options in the `namelist.wps` are not relevant to this process. Therefore, we have created a Docker container and Python command line utility to abstract much of the WRF/WPS complexity and simplify the process of creating a geogrid file for WRF-Hydro users. + +We will cover the steps to create the geogrid file using this method in the section 'Usage and invocations'. For more non-standard, advanced usage please see the WRF-WPS documentation. + +### Defining domain boundaries +WRF-Hydro uses domains boundaries defined by the `namelist.wps` input namelist to the geogrid.exe program. The first step to creating the geogrid file is to define our domain boundaries. The `geogrid.exe` program takes a centerpoint, x and y coordinates, and other projection information to define a bounding box for the domain. There are a number of resources available to assists users with defining this bounding box, including the NOAA supported [DomainWizard](https://esrl.noaa.gov/gsd/wrfportal/DomainWizard.html). If you have no knowledge of your coordinates, the [DomainWizard](https://esrl.noaa.gov/gsd/wrfportal/DomainWizard.html) is the best starting point. + +### Exploring the abbreviated namelist.wps file +The WPS `geogrid.exe` utility is controlled by options set in the `namelist.wps`. As previously stated, there are many options in the `namelist.wps` file that are not relevant to most WRF-Hydro users. Therefore, the Python command line utility supplied with this container accepts an abbreviated `namelist.wps` file. + +`namelist.wps` + +---------- +``` +&geogrid + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Define the extend in west-east (e_we) and south-north (e_sn) directions +! Note: will create a domain of size (e_we-1) x (e_sn-1) +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + e_we = 16, + e_sn = 17, + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Define the center point of your domain +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + ref_lat = 41.47100 + ref_lon = -73.74365 + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Define the domain grid spacing (in meters) +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + dx = 1000, + dy = 1000, + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! Define the map projection +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + map_proj = 'lambert', + truelat1 = 30.0, + truelat2 = 60.0, + stand_lon = -97.00 + +/ +``` +---------- + +### Usage + +** Step 1:** Create a directory to bind-mount to Docker for passing files between your system and docker + +``` +mkdir ~/dockerMount +``` + +**Step 2: Create a namelist.wps file for your domain using the above example as a starting point and save it in your mount directory from step 1.** + +**Step 3: Run Docker invoking the python make_geogrid.py utility with the required arguments.** + +**NOTE THE PATHS LISTED BELOW IN THE ARUGMENT LIST ARE FOR THE DOCKER FILESYSTEM** +You most likely will only need to change the name of your namelist.wps file in the --namelist_path +argument. + +``` +docker run -v :/home/docker/mount \ + wrfhydro/wps \ + --namelist_path /home/docker/mount/namelist.wps \ + --output_dir /home/docker/mount/ \ + --plot_only False +``` + +#### We will now dissect the pieces of this Docker command. + +----------------- + +`docker run -v :/home/docker/mount...` - Run the container with the `-v` argument to bind mount a volume on your local system to a folder in the docker container. It is best to leave the folder in the docker container unchanged. + +`--namelist_path /home/docker/mount/namelist.wps \` - Path to your `namelist` file **ON THE DOCKER FILE SYSTEM** + +`--output_dir /home/docker/mount/ \` - Path to directory **ON THE DOCKER FILE SYSTEM** to hold output. It is easiest to have this be the mounted folder from the `-v` argument + + `--plot_only False` - Only create a plot and do not run geogrid. Useful for making changes to the domain location and boundaries. + +**Note: You can access help on these arguments with the following command** + +`docker run wrfhydro/wps --help` \ No newline at end of file diff --git a/wps/gdrive_download.py b/wps/gdrive_download.py new file mode 100644 index 0000000..e92c197 --- /dev/null +++ b/wps/gdrive_download.py @@ -0,0 +1,51 @@ +import requests +from argparse import ArgumentParser + +def download_file_from_google_drive(id, destination): + URL = "https://docs.google.com/uc?export=download" + + session = requests.Session() + + response = session.get(URL, params = { 'id' : id }, stream = True) + token = get_confirm_token(response) + + if token: + params = { 'id' : id, 'confirm' : token } + response = session.get(URL, params = params, stream = True) + + save_response_content(response, destination) + +def get_confirm_token(response): + for key, value in response.cookies.items(): + if key.startswith('download_warning'): + return value + + return None + +def save_response_content(response, destination): + CHUNK_SIZE = 32768 + + with open(destination, "wb") as f: + for chunk in response.iter_content(CHUNK_SIZE): + if chunk: # filter out keep-alive new chunks + f.write(chunk) + +def main(): + + parser = ArgumentParser() + parser.add_argument("--file_id", + dest="file_id", + help="Google drive file ID. Get from shareable link") + parser.add_argument("--dest_file", + dest="dest_file", + help="Full path including filename for downloaded file.") + + args = parser.parse_args() + file_id = args.file_id + dest_file = args.dest_file + + download_file_from_google_drive(file_id, dest_file) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/wps/namelist.wps_orig b/wps/namelist.wps_orig new file mode 100755 index 0000000..444806a --- /dev/null +++ b/wps/namelist.wps_orig @@ -0,0 +1,53 @@ +&share + wrf_core = 'ARW', + max_dom = 1, + start_date = '2006-08-16_12:00:00', + end_date = '2006-08-16_18:00:00', + interval_seconds = 21600 + io_form_geogrid = 2, +/ + +&geogrid + parent_id = 1, + parent_grid_ratio = 1, + i_parent_start = 1, + j_parent_start = 1, + e_we = 74, + e_sn = 61, + ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!! IMPORTANT NOTE !!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! The default datasets used to produce the HGT_M, GREENFRAC, + ! and LU_INDEX/LANDUSEF fields have changed in WPS v3.8. The HGT_M field + ! is now interpolated from 30-arc-second USGS GMTED2010, the GREENFRAC + ! field is interpolated from MODIS FPAR, and the LU_INDEX/LANDUSEF fields + ! are interpolated from 21-class MODIS. + ! + ! To match the output given by the default namelist.wps in WPS v3.7.1, + ! the following setting for geog_data_res may be used: + ! + ! geog_data_res = 'gtopo_10m+usgs_10m+nesdis_greenfrac+10m','gtopo_2m+usgs_2m+nesdis_greenfrac+2m', + ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!! IMPORTANT NOTE !!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! + dx = 30000, + dy = 30000, + map_proj = 'lambert', + ref_lat = 34.83, + ref_lon = -81.03, + truelat1 = 30.0, + truelat2 = 60.0, + stand_lon = -98.0, + + geog_data_res = 'default', + geog_data_path = '/home/docker/WRF_WPS/utilities/geog_conus' +/ + +&ungrib + out_format = 'WPS', + prefix = 'FILE', +/ + +&metgrid + fg_name = 'FILE' + io_form_metgrid = 2, +/ From 1938d6c47f0a42a5120089da30d8cde33c2f5422 Mon Sep 17 00:00:00 2001 From: jmills-ncar Date: Thu, 7 Jun 2018 17:20:22 -0600 Subject: [PATCH 3/9] fixing more conflicts in make_geogrid.py --- wps/make_geogrid.py | 40 ++-------------------------------------- 1 file changed, 2 insertions(+), 38 deletions(-) diff --git a/wps/make_geogrid.py b/wps/make_geogrid.py index a1ea89d..949aa8c 100644 --- a/wps/make_geogrid.py +++ b/wps/make_geogrid.py @@ -22,21 +22,15 @@ import shutil import os import time -<<<<<<< HEAD -======= -import matplotlib.pyplot as plt ->>>>>>> c407c2cdd5c461832ff07b5e679470f4ceae81f0 import cartopy.io.img_tiles as cimgt from wrf import (projection, latlonutils) import math -<<<<<<< HEAD #Setup matlob lib to not use any xwindows backend import matplotlib matplotlib.use('Agg') import matplotlib.pyplot as plt -======= ->>>>>>> c407c2cdd5c461832ff07b5e679470f4ceae81f0 + # Functions def build_projparams(ref_lat=0.0, ref_lon=0.0, dx=1000.0, dy=1000.0, map_proj='lambert', truelat1=0.0, truelat2=0.0, @@ -259,19 +253,11 @@ def main(): default='false', help="Only create a plot of the domain. Geogrid will not be created if " "plot_only = true, only a plot of the domain will be created.") -<<<<<<< HEAD #parser.add_argument("--display", # dest="display", # default='false', # help="Display the image in python. Warning may cause issues in a terminal" # "session.") -======= - parser.add_argument("--display", - dest="display", - default='false', - help="Display the image in python. Warning may cause issues in a terminal" - "session.") ->>>>>>> c407c2cdd5c461832ff07b5e679470f4ceae81f0 # parser.add_argument("--orig_nml_path", # dest="new_nml_path", @@ -287,7 +273,6 @@ def main(): args = parser.parse_args() patch_nml_path = pathlib.Path(args.namelist_path) -<<<<<<< HEAD output_dir = args.output_dir plot_only = args.plot_only #display = args.display @@ -296,15 +281,6 @@ def main(): # orig_nml_path = args.orig_nml_path # new_nml_path = args.new_nml_path orig_nml_path = pathlib.Path('/home/docker/WRF_WPS/utilities/namelist.wps_orig') -======= - output_dir = pathlib.Path(args.output_dir) - plot_only = args.plot_only - display = args.display - - # orig_nml_path = args.orig_nml_path - # new_nml_path = args.new_nml_path - orig_nml_path = pathlib.Path('/home/docker/WRF_WPS/WPS/namelist.wps_orig') ->>>>>>> c407c2cdd5c461832ff07b5e679470f4ceae81f0 new_nml_path = pathlib.Path('/home/docker/WRF_WPS/WPS/namelist.wps') if display.lower() == 'true': @@ -314,20 +290,12 @@ def main(): if plot_only.lower() == 'true': plot_from_wps(patch_nml_path=str(patch_nml_path), -<<<<<<< HEAD figFilename=output_dir + '/domain.png', -======= - figFilename=str(output_dir) + '/domain.png', ->>>>>>> c407c2cdd5c461832ff07b5e679470f4ceae81f0 display=display) else: print('Plotting domain') plot_from_wps(patch_nml_path=str(patch_nml_path), -<<<<<<< HEAD figFilename=output_dir + '/domain.png', -======= - figFilename=str(output_dir) + '/domain.png', ->>>>>>> c407c2cdd5c461832ff07b5e679470f4ceae81f0 display=display) print('Generating geogrid file') @@ -341,13 +309,9 @@ def main(): subprocess.run(['./geogrid.exe'], cwd=new_nml_path.parent) -<<<<<<< HEAD shutil.copy(str(new_nml_path.parent / 'geo_em.d01.nc'), output_dir + '/geo_em.d01.nc') -======= - shutil.move(str(new_nml_path.parent / 'geo_em.d01.nc'), - str(pathlib.Path(output_dir) / 'geo_em.d01.nc')) ->>>>>>> c407c2cdd5c461832ff07b5e679470f4ceae81f0 + if __name__ == '__main__': From d38af3066a34079457310c73b790706ef78e29e1 Mon Sep 17 00:00:00 2001 From: jmills-ncar Date: Thu, 7 Jun 2018 17:21:30 -0600 Subject: [PATCH 4/9] removed no cache form wps build --- wps/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wps/build.sh b/wps/build.sh index c154f47..94ac2f9 100755 --- a/wps/build.sh +++ b/wps/build.sh @@ -1,4 +1,4 @@ #!/bin/bash -docker build "$@" --no-cache -t wrfhydro/wps . +docker build "$@" -t wrfhydro/wps . exit $? From c9c558477ef7098eb3bc970e4c14b9e56b95e0d0 Mon Sep 17 00:00:00 2001 From: jmills-ncar Date: Fri, 8 Jun 2018 08:36:42 -0600 Subject: [PATCH 5/9] wps fix --- wps/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wps/Dockerfile b/wps/Dockerfile index 77530b1..37e359c 100755 --- a/wps/Dockerfile +++ b/wps/Dockerfile @@ -82,12 +82,12 @@ COPY wrf_hydro_namelist.wps /home/docker/WRF_WPS/utilities/ # Get geog data from google drive ## Get download script -COPY gdrive_download.py /gdrive_download.py -RUN chmod 777 /gdrive_download.py +COPY gdrive_download.py gdrive_download.py +RUN chmod 777 gdrive_download.py RUN python gdrive_download.py --file_id 1EICBJ8-pRpkox27Bnq3mrBAqyZQsV662 --dest_file geog_conus.tar.gz \ && tar -xf geog_conus.tar.gz \ - && mv geog_conus /home/docker/WRF_WPS/utilities/geog_conus \ + && mv geog_training /home/docker/WRF_WPS/utilities/geog_conus \ && rm geog_conus.tar.gz RUN chmod -R 777 /home/docker/WRF_WPS/utilities/geog_conus From efcdc360b02202c50faa0101b9fb951de042125a Mon Sep 17 00:00:00 2001 From: jmills-ncar Date: Fri, 8 Jun 2018 09:12:34 -0600 Subject: [PATCH 6/9] readmes --- README.md | 19 +++++++- dev/README.md | 133 -------------------------------------------------- 2 files changed, 17 insertions(+), 135 deletions(-) delete mode 100644 dev/README.md diff --git a/README.md b/README.md index 62e62cf..8f49244 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,21 @@ [![Build Status](https://travis-ci.org/NCAR/wrf_hydro_docker.svg?branch=master)](https://travis-ci.org/NCAR/wrf_hydro_docker) -##Docker containers for wrf_hydro modeling system. +# Overview -Under construction \ No newline at end of file +This repository contains all Dockerfiles and assosciated container contents for all images +available on the [WRF-Hydro Dockerhub page](https://hub.docker.com/u/wrfhydro/). Some of these +containers require large datasets. These datasets are avilable on a public GoogleDrive folder at +[WRF_HYDRO_DOCKER_DATA](https://drive.google.com/open?id=1NY9YdVLcJMIqE6ibLVyKe1fJ-Eoj74Kr). + +Each subdirectory in this repository cooresponds to a Docker image on [WRF-Hydro Dockerhub page] +(https://hub.docker.com/u/wrfhydro/). + +## Where to get help and/or post issues +If you have general questions about Docker, there are ample online resources including the excellent Docker documentation at https://docs.docker.com/. + +The best place ask questions or post issues with these lessons is via the Issues page of the GitHub +repository at https://github.com/NCAR/wrf_hydro_docker/issues. + +## Usage +Please see usage instructions in each sub-directory. diff --git a/dev/README.md b/dev/README.md deleted file mode 100644 index 802849b..0000000 --- a/dev/README.md +++ /dev/null @@ -1,133 +0,0 @@ -# WRF-Hydro Development Docker - -The intent of this docker container is to enable compiling and running -of WRF-Hydro on the HOST without the need to explicitly enter -the docker container. The target audience is developers of the -WRF-Hydro model. - -More specifically, code and the run directories are mounted to the container -by shell commands which are presented as functions in -`wrf_hydro_tools`. (These functions export working directories and the -HOST user's environment to the docker container without any work on -the user's part). - -This approach allows source code to be edited/developed in -applications native to the HOST. It emulates the development process -on a linux environment very closely. It eliminates the need to install -the source code in the docker container, allowing code changes and -development to persist on the HOST as you'd expect. - -The docker container also has an iteractive mode for flexibilty. - -# Caveats - -Unfortunately, the interaction between the HOST filesystem and the -compilation is not seamless. The code will not currently compile in -place. Therefore the source code is copied into the container where it -is compiled. After successful compilation, the resulting binary is -moved back to the HOST system. However, the intermediate, -c-preprocessed files are not moved back to the host. So the state of -the source repo (the untracked files) remains unchanged. - -No one who cares about windows has yet to contribute to make this -generalizable beyond OSX - -# Requirements - -The `wrf_hydro_tools` repository must be installed with the proper -configuration of the `~/.wrf_hydro_tools file`. To access functions defined in -`wrf_hydro_docker`, the path to `wrf_hydro_docker` must be specifed -in addtion to the requirement of the path to `wrf_hydro_tools` by -itself. E.g. the following are my choice of location on the machine -where I use docker: - -``` -jamesmcc@chimayo[541]:~> cat ~/.wrf_hydro_tools -wrf_hydro_tools=~/WRF_Hydro/wrf_hydro_tools -wrf_hydro_docker=~/WRF_Hydro/wrf_hydro_docker -``` - -And, of course, the `wrf_hydro_tools/utilities/sourceMe.sh` file needs -to be sourced (either manually or by your `~/.bash_profile`) for the -functions to be available in your path. - -# Compile Mode -The function in `wrf_hydro_tools` -``` -compile_docker -``` -calls `wrf_hydro_docker/development/compile_docker.sh`. The header -of that file has details: -``` -# Purpose: Pass the native OS environment variables and -# the working directory to a docker for compile. -# Arguments: -# 1) the name of the docker container image (optional, default=wrf_hydro_dev) -# Dependencies: -# wrf_hydro_tools repository installed locally. -# Usage: -# ./compile_docker.sh [some_other_image] -``` -## Usage Description: -Run as a substitute for the standard `./compile_NoahMP.sh` - 1. Call it from `wrf_hydro_model/trunk/NDHMS` (actually anywhere in a - model repo will work). - 2. NOTE: it does not use setEnvars.sh. Compile options are set in the calling - environment. This can be done easily with the `wrf_hydro_tools` - function `setHenv`. - -# Run Mode -The function in `wrf_hydro_tools` -``` -run_docker -``` -calls `wrf_hydro_docker/development/run_docker.sh`. The header -of that file has details: -``` -# Purpose: Run WRF-Hydro using MPI in a docker container in-place on host. -# -# Arguments: -# 1: number of processors/cores, required. -# 2: the binary, required. -# 3: the name of the docker container image (optional, default=wrf_hydro_dev) -# -# Note: standard error and standard out are both teed, so they appear in terminal and -# on file, to wrf_hydro.stdout and wrf_hydro.stderr, respectively. -# -# Usage: -# ./run_docker.sh 4 wrf_hydro.exe [some_other_image] -``` -## Usage Description: -Run as a substitute for mpiexec... sort of. Syntax is slightly -different (no -n or -np argument and you dong have to `./` the binary) -and standard output and error are handled for you with tee so that you -also see the output on screen while it is running. - -Note that the `run_docker` command currently mounts OSX `/Users/` to `/Users` in -the container. - -# Interactive Mode -The function in `wrf_hydro_tools` -``` -interactive_docker -``` -calls `wrf_hydro_docker/development/interactive_docker.sh`. The header -of that file has details: -``` -# Purpose: Pass the native OS environment variables and -# the working directory to a docker for compile. -# Arguments: -# 1) the name of the docker container image (optional, default=wrf_hydro_dev) -# Dependencies: -# wrf_hydro_tools repository installed locally. -# Usage: -# ./interactive_docker.sh [some_other_image] -``` -## Usage Description: -Basically: enter the container at the same location where the command -is issued. The `interactive_docker` command currently mounts OSX `/Users/` to `/Users` in -the container. The initialization (entrypoint) script puts you in the -working directory from which the command was called. - -Note that you can not currently compile the code in a mounted HOST -directory. See the Caveats section above. From dbc2c9931d4bcbb95f772c2e561004f9db8ad08e Mon Sep 17 00:00:00 2001 From: jmills-ncar Date: Fri, 8 Jun 2018 09:13:01 -0600 Subject: [PATCH 7/9] readmes --- dev/Readme.md | 6 ++ dev/base/Readme.md | 36 +++++++++++ dev/conda/Readme.md | 47 ++++++++++++++ domains/Readme.md | 22 +++++++ domains/croton_NY/Readme.md | 120 ++++++++++++++++++++++++++++++++++++ training/Readme.md | 56 +++++++++++++++++ training/gdrive_download.py | 51 +++++++++++++++ 7 files changed, 338 insertions(+) create mode 100644 dev/Readme.md create mode 100644 dev/base/Readme.md create mode 100644 dev/conda/Readme.md create mode 100644 domains/Readme.md create mode 100644 domains/croton_NY/Readme.md create mode 100644 training/Readme.md create mode 100644 training/gdrive_download.py diff --git a/dev/Readme.md b/dev/Readme.md new file mode 100644 index 0000000..8fbfca6 --- /dev/null +++ b/dev/Readme.md @@ -0,0 +1,6 @@ +![](https://ral.ucar.edu/sites/default/files/public/wrf_hydro_symbol_logo_2017_09_150pxby63px.png) WRF-HYDRO + +# Overview + +Containers used for WRF-Hydro development. These containers are also suitable options for running +WRF-Hydro simulations on a single-node system. See sub-directories for details on each container. diff --git a/dev/base/Readme.md b/dev/base/Readme.md new file mode 100644 index 0000000..685d459 --- /dev/null +++ b/dev/base/Readme.md @@ -0,0 +1,36 @@ +![](https://ral.ucar.edu/sites/default/files/public/wrf_hydro_symbol_logo_2017_09_150pxby63px.png) WRF-HYDRO + +# Overview +This container is used for WRF-Hydro development and single-node WRF-Hydro simulations. + +This container includes the following: + +* Ubuntu base image +* All system libraries required by WRF-Hydro +* Text editors - VIM, Nano, Emacs +* git version control system +* nccmp - NetCDF Compare utility for diffing NetCDF files +* NetCDF C and Fortran libraries +* MPI + +# Usage +**Step 1: Pull the image** +``` +docker pull wrfhydro/dev:base +``` + +**Step 2: Run the image** +``` +docker run -it wrfhydro/dev:base +``` + +**Optionally: Run the image with a volume mount** + +**NOTE: DO NOT COMPILE WRFHYDRO IN THE MOUNTED FOLDER.** +On some filesystems, WRF-Hydro will not compile correctly if compilation occurs in a mounted +directory. WRF-Hydro will run in a mounted directory on most filesystems, but compilation may fail. + +``` +docker run -v : -it wrfhydro/dev:base +``` + diff --git a/dev/conda/Readme.md b/dev/conda/Readme.md new file mode 100644 index 0000000..85ab82f --- /dev/null +++ b/dev/conda/Readme.md @@ -0,0 +1,47 @@ +![](https://ral.ucar.edu/sites/default/files/public/wrf_hydro_symbol_logo_2017_09_150pxby63px.png) WRF-HYDRO + +# Overview +This container is used for WRF-Hydro development and single-node WRF-Hydro simulations. + +This container includes the following: + +* Everything in wrfhydro/dev + +* Miniconda with the following libraries and all dependencies + + jupyterlab + + jupyter_contrib_nbextensions + + boltons + + cartopy + + deepdiff + + f90nml + + netcdf4 + + pytest + + pytest-datadir + + rasterio + + termcolor + + wrfhydropy + + xarray + +# Usage +**Step 1: Pull the image** +``` +docker pull wrfhydro/dev:conda +``` + +**Step 2: Run the image** +``` +docker run -it wrfhydro/dev:conda +``` + +**Optionally: Run the image with a volume mount** + +**NOTE: DO NOT COMPILE WRFHYDRO IN THE MOUNTED FOLDER.** +On some filesystems, WRF-Hydro will not compile correctly if compilation occurs in a mounted +directory. WRF-Hydro will run in a mounted directory on most filesystems, but compilation may fail. + + +``` +docker run -v : -it +wrfhydro/dev:conda +``` + diff --git a/domains/Readme.md b/domains/Readme.md new file mode 100644 index 0000000..a11b157 --- /dev/null +++ b/domains/Readme.md @@ -0,0 +1,22 @@ +# Overview + +These docker data containers are used primarily by the WRF-Hydro development team. If you are +searching for an example test case for general use, please see +https://ral.ucar.edu/projects/wrf_hydro/testcases. + +Generally, these docker domain containers are used with a volume mount to one of our `wrfhydro/dev` +containers to create a modular docker environment that contains a domain and a WRF-Hydro development environment. + +# Usage +For example: + +``` +docker pull wrfhydro/dev:base +docker pull wrfhydro/domains:croton_NY +``` + +``` +docker create --name croton_NY wrfhydro/domains:croton_NY +docker run --volumes-from croton_NY -it wrfhydro/dev:base +``` + diff --git a/domains/croton_NY/Readme.md b/domains/croton_NY/Readme.md new file mode 100644 index 0000000..19771cf --- /dev/null +++ b/domains/croton_NY/Readme.md @@ -0,0 +1,120 @@ +# Overview + +This test case includes prepared geospatial data and input files for a +sample domain (region of interest) and prepared forcing data. This domain is a small region (15km x +16km) encompassing the West Branch +of the Croton River, NY, USA (USGS stream gage 0137462010) during hurricane +Irene, 2011-08-26 to 2011-09-02. The simulation begins with a restart from a +spinup period from 2010-10-01 to 2011-08-26. The forcing data +prepared for this test case is North American Land Data Assimilation System +(NLDAS) hourly data. There are 3 basic routing +configurations included in the test case, National Water Model (NWM), Gridded, +and NCAR Reach. See the WRF-Hydro V5 Technical Description located at +https://ral.ucar.edu/projects/wrf_hydro for a more detailed description of model +physics options, configurations, and input files. However, some non-standard +files will be described below. + +# Usage + +These docker data containers are used primarily by the WRF-Hydro development team. If you are +searching for an example test case for general use, please see +https://ral.ucar.edu/projects/wrf_hydro/testcases. + +Generally, these docker domain containers are used with a volume mount to one of our `wrfhydro/dev` +containers to create a modular docker environment that contains a domain and a WRF-Hydro development environment. + +For example: + +``` +docker pull wrfhydro/dev:base +docker pull wrfhydro/domains:croton_NY +``` + +``` +docker create --name croton_NY wrfhydro/domains:croton_NY +docker run --volumes-from croton_NY -it wrfhydro/dev:base +``` + + +# Example case Directory contents + +example_case: directory containing all input files for the Croton, NY example testcase + +``` + | + -FORCING: Directory containing all NLDAS hrldas formatted hourly + forcing data for the simulation period. + -Gridded: Directory containing all files required for the gridded routing + configuration with lakes included + | + -DOMAIN: Directory containing all geospatial data and input files for + the Gridded routing option. + -lake_shapes: directory containing shape files that define lakes + -RESTART: Directory containing model restart files for the Gridded + routing option. + -referenceSim: Directory containing restart files and a hydrograph from + a successful run for reference + -hydro.namelist: Fortran namelist file for the hydro model. + -namelist.hrldas: Fortran namelist file for the Noah-MP land surface + model. + -croton_frxst_pts_csv.csv: .CSV formatted file of gage locations in + latitude/longitude coordinates (WGS84) + -Gridded_no_lakes: Directory containing all files required for the gridded routing + configuration without lakes + | + -DOMAIN: Directory containing all geospatial data and input files for + the Gridded routing option + -RESTART: Directory containing model restart files for the Gridded + routing option. + -referenceSim: Directory containing restart files and a hydrograph from + a successful run for reference + -hydro.namelist: Fortran namelist file for the hydro model. + -namelist.hrldas: Fortran namelist file for the Noah-MP land surface + model. + -croton_frxst_pts_csv.csv: .CSV formatted file of gage locations in + latitude/longitude coordinates (WGS84) + -NWM: Directory containing all files required for the National Water Model + (NWM) routing configuration + | + -DOMAIN: Directory containing all geospatial data and input files for + the NWM routing option. + | + -RouteLink_nudgeEdit.nc: An edited route link file with one gage + removed from nudging + -RESTART: Directory containing model restart files for the NWM routing + option. + -referenceSim: Directory containing restart files and hydrograph from a + successful run for reference + -nudgingTimeSliceObs: Directory containing nudging "time slice" + observation files. + -hydro.namelist: Fortran namelist file for the hydro model. + -namelist.hrldas: Fortran namelist file for the Noah-MP land surface + model. + -Reach: Directory containing all files required for NCAR reach routing + configuration + | + -DOMAIN: Directory containing all geospatial data and input files for + the NCAR reach routing option. + -stream_network: directory containing files that define the stream + network. + -RESTART: Directory containing model restart files for the NCAR reach + routing option. + -referenceSim: Directory containing restart files and hydrograph from a + successful run for reference + -hydro.namelist: Fortran namelist file for the hydro model. + -namelist.hrldas: Fortran namelist file for the Noah-MP land surface + model. + -Supplemental: Directory containing supplemental files used to create the example + case, including forcing files used for spinup + | + -supplimental_forcing.tar.gz: Tar ball containing additional forcing + data for spinup + -namelist.wps: Reduced namelist.wps used to create the geogrid file using + the WRF-Hydro geogrid Docker utilitiy. + -supp_precip.tar.gz: Supplemental regridded StageIV precipitation data. + -USGS-Obs.csv: csv files containing USGS 15 minute streamflow data for + gages in the domain. + -study_map.PNG: Study area map + -namelist_patches.json: json file used by wrfhydropy python package for + namelist parsing +``` diff --git a/training/Readme.md b/training/Readme.md new file mode 100644 index 0000000..5ad25c8 --- /dev/null +++ b/training/Readme.md @@ -0,0 +1,56 @@ +![](https://ral.ucar.edu/sites/default/files/public/wrf_hydro_symbol_logo_2017_09_150pxby63px.png) WRF-HYDRO + +# Overview +This container is used for WRF-Hydro development and single-node WRF-Hydro simulations. + +This container includes the following: + +* Ubuntu base image +* All system libraries required by WRF-Hydro +* Text editors - VIM, Nano, Emacs +* git version control system + +##Requirements + +The easiest and recommended way to run the training lessons is via the wrfhydro/training Docker container, which has all software dependencies and data pre-installed. + +* Docker >= v.17.12 +* Web browser (Google Chrome recommended) + +## Where to get help and/or post issues +If you have general questions about Docker, there are ample online resources including the excellent Docker documentation at https://docs.docker.com/. + +If you have questions regarding the lessons please contact us here https://ral.ucar.edu/projects/wrf_hydro/contact. + +The best place ask questions or post issues with these lessons is via the Issues page of the GitHub repository at https://github.com/NCAR/wrf_hydro_training/issues. + +## How to run +Make sure you have Docker installed and that it can access your localhost ports. Most out-of-the-box Docker installations accepting all defaults will have this configuration. + +**Step 1: Open a terminal or PowerShell session** + +**Step 2: Pull the wrfhydro/training Docker container for the desired code version** +Each training container is specific to a release version of the WRF-Hydro source code, which can be found at https://github.com/NCAR/wrf_hydro_nwm_public/releases. +Issue the following command in your terminal to pull a specific version of the training corresponding to your code release version. In this example, we will pull the training container for v5.0.0. + +`docker pull wrfhydro/training:v5.0.0` + +**Step 3: Start the training Docker container** +Issue the following command in your terminal session to start the training Docker container. +`docker run --name wrf-hydro-training -p 8888:8888 -it wrfhydro/training` + +**Note: Port forwarding is setup with the -p 8888:8888 argument, which maps your localhost port to +the container port. If you already have something running on port 8888 on your localhost you will +need to change this number** + +The container will start and perform a number of actions before starting the training. +* First, the container will pull the model code corresponding to the specified major version, in this case v5.0.0 +* Second, the container will pull an example test case compatible with the model code release. +* Third, the container will pull the training lessons corresponding to the psecified major version. +* Fourth, the container will launch a Jupyter Notebook server and echo the address to your terminal. + +**Step 4: Open the jupyter notebook lessons** +All lessons for this training are contained in the `~/wrf-hydro-training/lessons` folder. The +lessons are interactive and can execute code commands live. For more information on jupyter +notebooks visit the jupyter project page at http://jupyter.org/. + diff --git a/training/gdrive_download.py b/training/gdrive_download.py new file mode 100644 index 0000000..e92c197 --- /dev/null +++ b/training/gdrive_download.py @@ -0,0 +1,51 @@ +import requests +from argparse import ArgumentParser + +def download_file_from_google_drive(id, destination): + URL = "https://docs.google.com/uc?export=download" + + session = requests.Session() + + response = session.get(URL, params = { 'id' : id }, stream = True) + token = get_confirm_token(response) + + if token: + params = { 'id' : id, 'confirm' : token } + response = session.get(URL, params = params, stream = True) + + save_response_content(response, destination) + +def get_confirm_token(response): + for key, value in response.cookies.items(): + if key.startswith('download_warning'): + return value + + return None + +def save_response_content(response, destination): + CHUNK_SIZE = 32768 + + with open(destination, "wb") as f: + for chunk in response.iter_content(CHUNK_SIZE): + if chunk: # filter out keep-alive new chunks + f.write(chunk) + +def main(): + + parser = ArgumentParser() + parser.add_argument("--file_id", + dest="file_id", + help="Google drive file ID. Get from shareable link") + parser.add_argument("--dest_file", + dest="dest_file", + help="Full path including filename for downloaded file.") + + args = parser.parse_args() + file_id = args.file_id + dest_file = args.dest_file + + download_file_from_google_drive(file_id, dest_file) + + +if __name__ == "__main__": + main() \ No newline at end of file From cef956dc4f17a2d1349fcb925f995f7dc5b19c06 Mon Sep 17 00:00:00 2001 From: jmills-ncar Date: Fri, 8 Jun 2018 09:51:37 -0600 Subject: [PATCH 8/9] readme --- dev/Readme.md | 3 ++- domains/croton_NY/Readme.md | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/dev/Readme.md b/dev/Readme.md index 8fbfca6..bb9f40c 100644 --- a/dev/Readme.md +++ b/dev/Readme.md @@ -3,4 +3,5 @@ # Overview Containers used for WRF-Hydro development. These containers are also suitable options for running -WRF-Hydro simulations on a single-node system. See sub-directories for details on each container. +WRF-Hydro simulations on a single-node system. See sub-tags (e.g. 'dev', 'conda') for details on +each container. diff --git a/domains/croton_NY/Readme.md b/domains/croton_NY/Readme.md index 19771cf..44c2a04 100644 --- a/domains/croton_NY/Readme.md +++ b/domains/croton_NY/Readme.md @@ -1,3 +1,6 @@ +# THESE DOCKER DATA CONTAINERS FOR DOMAINS WILL SOON BE DEPRECATED +These images are a temporary solution until an object store is established. + # Overview This test case includes prepared geospatial data and input files for a From 9a12715228379cf3aaa59cd46e7f45f15b6b78dc Mon Sep 17 00:00:00 2001 From: jmills-ncar Date: Fri, 8 Jun 2018 10:40:30 -0600 Subject: [PATCH 9/9] added tagged training to entrypoint --- README.md | 2 + dev/Readme.md | 91 ++++++++++++++++++++++++++++++++++++++++-- domains/Readme.md | 5 +++ training/Dockerfile | 4 +- training/entrypoint.sh | 6 +-- 5 files changed, 100 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 8f49244..5cccce5 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +![](https://ral.ucar.edu/sites/default/files/public/wrf_hydro_symbol_logo_2017_09_150pxby63px.png) WRF-HYDRO + # wrf_hydro_docker [![Build Status](https://travis-ci.org/NCAR/wrf_hydro_docker.svg?branch=master)](https://travis-ci.org/NCAR/wrf_hydro_docker) diff --git a/dev/Readme.md b/dev/Readme.md index bb9f40c..7e953da 100644 --- a/dev/Readme.md +++ b/dev/Readme.md @@ -1,7 +1,92 @@ ![](https://ral.ucar.edu/sites/default/files/public/wrf_hydro_symbol_logo_2017_09_150pxby63px.png) WRF-HYDRO -# Overview - Containers used for WRF-Hydro development. These containers are also suitable options for running -WRF-Hydro simulations on a single-node system. See sub-tags (e.g. 'dev', 'conda') for details on +WRF-Hydro simulations on a single-node system. See sub-tags (e.g. 'base', 'conda') for details on each container. + +--------------------------------------------------- +## base + +### Overview +This container is used for WRF-Hydro development and single-node WRF-Hydro simulations. + +This container includes the following: + +* Ubuntu base image +* All system libraries required by WRF-Hydro +* Text editors - VIM, Nano, Emacs +* git version control system +* nccmp - NetCDF Compare utility for diffing NetCDF files +* NetCDF C and Fortran libraries +* MPI + +### Usage +**Step 1: Pull the image** +``` +docker pull wrfhydro/dev:base +``` + +**Step 2: Run the image** +``` +docker run -it wrfhydro/dev:base +``` + +**Optionally: Run the image with a volume mount** + +**NOTE: DO NOT COMPILE WRFHYDRO IN THE MOUNTED FOLDER.** +On some filesystems, WRF-Hydro will not compile correctly if compilation occurs in a mounted +directory. WRF-Hydro will run in a mounted directory on most filesystems, but compilation may fail. + +``` +docker run -v : -it wrfhydro/dev:base +``` + +--------------------------------------------------- + +## conda + +### Overview +This container is used for WRF-Hydro development and single-node WRF-Hydro simulations. + +This container includes the following: + +* Everything in wrfhydro/dev + +* Miniconda with the following libraries and all dependencies + + jupyterlab + + jupyter_contrib_nbextensions + + boltons + + cartopy + + deepdiff + + f90nml + + netcdf4 + + pytest + + pytest-datadir + + rasterio + + termcolor + + wrfhydropy + + xarray + +### Usage +**Step 1: Pull the image** +``` +docker pull wrfhydro/dev:conda +``` + +**Step 2: Run the image** +``` +docker run -it wrfhydro/dev:conda +``` + +**Optionally: Run the image with a volume mount** + +**NOTE: DO NOT COMPILE WRFHYDRO IN THE MOUNTED FOLDER.** +On some filesystems, WRF-Hydro will not compile correctly if compilation occurs in a mounted +directory. WRF-Hydro will run in a mounted directory on most filesystems, but compilation may fail. + + +``` +docker run -v : -it +wrfhydro/dev:conda +``` + diff --git a/domains/Readme.md b/domains/Readme.md index a11b157..02a996e 100644 --- a/domains/Readme.md +++ b/domains/Readme.md @@ -1,3 +1,8 @@ +![](https://ral.ucar.edu/sites/default/files/public/wrf_hydro_symbol_logo_2017_09_150pxby63px.png) WRF-HYDRO + +# THESE DOCKER DATA CONTAINERS FOR DOMAINS WILL SOON BE DEPRECATED +These images are a temporary solution until an object store is established. + # Overview These docker data containers are used primarily by the WRF-Hydro development team. If you are diff --git a/training/Dockerfile b/training/Dockerfile index 105308d..8f38fa3 100644 --- a/training/Dockerfile +++ b/training/Dockerfile @@ -29,8 +29,8 @@ RUN wget https://ral.ucar.edu/sites/default/files/public/ESMFregrid_NLDAS.tar_.g COPY ./gdrive_download.py gdrive_download.py RUN chmod 777 gdrive_download.py - -RUN python gdrive_download.py --file_id 1PrDTA-hMFNqQrDag1Dm7tX1rT1gQ_25e --dest_file /home/docker/wrf-hydro-training/regridding/data/nldas_forcing.tar.gz +RUN mkdir /home/docker/wrf-hydro-training/regridding/data/ \ + && python gdrive_download.py --file_id 1PrDTA-hMFNqQrDag1Dm7tX1rT1gQ_25e --dest_file /home/docker/wrf-hydro-training/regridding/data/nldas_forcing.tar.gz # install NCL RUN wget https://www.earthsystemgrid.org/dataset/ncl.640.dap/file/ncl_ncarg-6.4.0-Debian8.6_64bit_gnu492.tar.gz \ diff --git a/training/entrypoint.sh b/training/entrypoint.sh index 755931b..638f944 100644 --- a/training/entrypoint.sh +++ b/training/entrypoint.sh @@ -37,11 +37,11 @@ echo echo -e "\e[0;49;32m-----------------------------------\e[0m" echo -e "\e[7;49;32mRetrieving WRF-Hydro training\e[0m" -release=$(curl -s https://api.github.com/repos/NCAR/wrf_hydro_model_tools/releases/latest) +release=$(curl -s https://api.github.com/repos/NCAR/wrf_hydro_training/releases/latest) version=$(echo "$release" | grep "tag_name" | cut -d : -f 2,3 | tr -d \") version=$(echo $version | tr "," " ") -#git clone --branch $version https://github.com/NCAR/wrf_hydro_training -git clone https://github.com/NCAR/wrf_hydro_training +git clone --branch $version https://github.com/NCAR/wrf_hydro_training +#git clone https://github.com/NCAR/wrf_hydro_training mv /home/docker/wrf_hydro_training/lessons /home/docker/wrf-hydro-training/lessons rm -rf /home/docker/wrf_hydro_training/