From d0c8878f73e60f9b1eef19bedf92625cc4aee8ec Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Tue, 14 May 2024 15:59:59 +0200 Subject: [PATCH 1/4] Initial setup of WRF-runner --- wrf-runner/README.md | 49 ++++++++++++++++ wrf-runner/namelist.input | 120 ++++++++++++++++++++++++++++++++++++++ wrf-runner/namelist.wps | 40 +++++++++++++ wrf-runner/wrf_runner.sh | 50 ++++++++++++++++ 4 files changed, 259 insertions(+) create mode 100644 wrf-runner/README.md create mode 100644 wrf-runner/namelist.input create mode 100644 wrf-runner/namelist.wps create mode 100644 wrf-runner/wrf_runner.sh diff --git a/wrf-runner/README.md b/wrf-runner/README.md new file mode 100644 index 0000000..3fd6dae --- /dev/null +++ b/wrf-runner/README.md @@ -0,0 +1,49 @@ +# WRF Runner + +* Run a complete WPS/WRF experiment in one go +* Make a dedicated directory for each experiment +* The original WRF source code is untouched +* Namelists under version control so you can always go back + +## How to use + +It assumes you have a compiled WRF and WPS with the following folder structure + + * $HOME/wrf-model/WPS # compiled clone of https://github.com/wrf-model/WRF + * $HOME/wrf-model/WRF # compiled clone of https://github.com/wrf-model/WPS + * $HOME/Urban-M4/misc/wrf-runner # clone of https://github.com/Urban-M4/misc + * $HOME/Urban-M4/experiments # destination for input/output of each experiment + +Make sure the paths are set up correctly on your system and match with the paths +in `wrf_runner.sh`. Modify (and commit) the namelists as seen fit, then call the +runner: + +```bash +# Run interactive job +bash wrf_runner.sh + +# Or submit as batch job to slurm queue +sbatch wrf_runner.sh +``` + +This will create a new folder in `$HOME/Urban-M4/experiments/` where +all output and intermediate files are stored, and run all steps for WPS and WRF. + +## Notes + +[f90nml](https://f90nml.readthedocs.io/en/latest/) is used to format and modify +namelists from the command line. Beware that: + + * paths must be in "'double quotes'" (see https://github.com/marshallward/f90nml/issues/126) + * cannot update in place + + +## To do + +* Use MODIS_LCZ landuse with wudapt-to-wrf +* Use the "standard" heatwave case for amsterdam +* Use IFS analysis for initial/boundary conditions +* Use custom WUR landuse from summer in the city (checkout different branch of WRF?) +* Use custom river temperatures from rijkswaterstaat(?) +* Enable additional custom landuse for building albedo/emissivity +* Add some default plots of input/output for quick inspection of results diff --git a/wrf-runner/namelist.input b/wrf-runner/namelist.input new file mode 100644 index 0000000..adcf625 --- /dev/null +++ b/wrf-runner/namelist.input @@ -0,0 +1,120 @@ +&time_control + run_days = 0 + run_hours = 12 + run_minutes = 0 + run_seconds = 0 + start_year = 2024, 2024, 2024, 2024 + start_month = 4, 4, 4, 4 + start_day = 19, 19, 19, 19 + start_hour = 0, 0, 0, 0 + start_minute = 0, 0, 0, 0 + start_second = 0, 0, 0, 0 + end_year = 2024, 2024, 2024, 2024 + end_month = 4, 4, 4, 4 + end_day = 19, 19, 19, 19 + end_hour = 12, 12, 12, 12 + end_minute = 0, 0, 0, 0 + end_second = 0, 0, 0, 0 + interval_seconds = 10800 + input_from_file = .true., .true., .true., .true. + history_interval = 60, 60, 60, 60 + frames_per_outfile = 1000, 1000, 1000, 1000 + restart = .false. + restart_interval = 5000 + io_form_history = 2 + io_form_restart = 2 + io_form_input = 2 + io_form_boundary = 2 + debug_level = 1 +/ +&domains + time_step = 60 + time_step_fract_num = 0 + time_step_fract_den = 1 + max_dom = 1 + e_we = 120, 121, 121, 126 + e_sn = 120, 121, 121, 126 + num_metgrid_levels = 34 + num_metgrid_soil_levels = 4 + dx = 12500.0, 2500.0, 500.0, 100.0 + dy = 12500.0, 2500.0, 500.0, 100.0 + grid_id = 1, 2, 3, 4 + parent_id = 1, 1, 2, 3 + i_parent_start = 1, 48, 50, 55 + j_parent_start = 1, 48, 51, 55 + parent_grid_ratio = 1, 5, 5, 5 + parent_time_step_ratio = 1, 5, 5, 5 + feedback = 1 + smooth_option = 0 + max_ts_locs = 57 + ts_buf_size = 6000 + p_top_requested = 1000 + e_vert = 74, 74, 74, 74 + eta_levels = 1.0, 0.99421191128831299, 0.98958514826812849, 0.98367625218041377, 0.97655838524794414, + 0.96838285571891347, 0.95870112135559438, 0.94744530276601047, 0.93462088213306749, 0.9202339396959881, + 0.90428158484924004, 0.88677817094443068, 0.86778769000747558, 0.84739755793670579, 0.82566728133565903, + 0.80266872663842503, 0.77850745078494887, 0.75332140543234494, 0.72721046598554695, 0.70028138549713437, + 0.67266294542736116, 0.64451821579865431, 0.61596501370545731, 0.58712285073511083, 0.55812618988288065, + 0.52913800149514079, 0.50043977074507839, 0.47231557438325439, 0.44503493645651632, 0.41884565163219539, + 0.39398016446548717, 0.3704105656615998, 0.34809080488412658, 0.32696187390979314, 0.30696745576875156, + 0.28805332668826317, 0.27016755544480436, 0.25326020433590829, 0.23728352853227014, 0.22219187640169447, + 0.20794139048093696, 0.19449020682780963, 0.18179805631696985, 0.16982666334413157, 0.15853924744580114, + 0.14790092200348867, 0.13787789683528531, 0.12843837528033888, 0.1195517567904311, 0.11118883628208323, + 0.10330246698230749, 0.09583304261151257, 0.088729828058808871, 0.081949862945427374, 0.075456366807874403, + 0.069218440069773235, 0.063209568901071531, 0.057410715175679049, 0.051829952653874897, 0.046480538250685273, + 0.041374134064291058, 0.036519112883129826, 0.031920458509843014, 0.027582357338649388, 0.023507899327186641, + 0.019698978320458513, 0.016156491402940443, 0.012879740842262649, 0.0098666334413157249, 0.0071139795664091707, + 0.004616596062795914, 0.002368203339147769, 0.00036052828307999075, 0.0 +/ +&physics + use_wudapt_lcz = 1 + mp_physics = 6, 6, 6, 6, 6 + ra_lw_physics = 4, 4, 4, 4, 4 + ra_sw_physics = 4, 4, 4, 4, 4 + radt = 0, 0, 0, 30, 30 + sf_sfclay_physics = 1, 1, 1, 1, 1 + sf_surface_physics = 2, 2, 2, 2, 2 + bl_pbl_physics = 1, 1, 1, 0, 0 + bldt = 0, 0, 0, 0, 0 + cu_physics = 3, 3, 0, 0, 0 + cudt = 5, 5, 5, 5, 5 + isfflx = 1 + ifsnow = 1 + icloud = 1 + surface_input_source = 1 + num_soil_layers = 4 + num_land_cat = 21 + sf_urban_physics = 1, 1, 1, 1 + +/ +&fdda +/ +&dynamics + w_damping = 0 + diff_opt = 1, 1, 1, 2, 2 + km_opt = 4, 4, 4, 3, 3 + diff_6th_opt = 0, 0, 0, 0, 0 + diff_6th_factor = 0.12, 0.12, 0.12, 0.12, 0.12 + base_temp = 290.0 + damp_opt = 0 + zdamp = 5000.0, 5000.0, 5000.0, 5000.0, 5000.0 + dampcoef = 0.2, 0.2, 0.2, 0.2, 0.2 + khdif = 0, 0, 0, 0, 0 + kvdif = 0, 0, 0, 0, 0 + non_hydrostatic = .true., .true., .true., .true., .true. + moist_adv_opt = 1, 1, 1, 1, 1 + scalar_adv_opt = 1, 1, 1, 1, 1 +/ +&bdy_control + spec_bdy_width = 5 + spec_zone = 1 + relax_zone = 4 + specified = .true., .false., .false., .false., .false. + nested = .false., .true., .true., .true., .true. +/ +&grib2 +/ +&namelist_quilt + nio_tasks_per_group = 0 + nio_groups = 1 +/ diff --git a/wrf-runner/namelist.wps b/wrf-runner/namelist.wps new file mode 100644 index 0000000..07f838f --- /dev/null +++ b/wrf-runner/namelist.wps @@ -0,0 +1,40 @@ +&share + wrf_core = 'ARW', + max_dom = 1, + start_date = '2024-04-19_00:00:00','2024-04-19_00:00:00','2024-04-19_00:00:00','2024-04-19_00:00:00', + end_date = '2024-04-19_12:00:00','2024-04-19_12:00:00','2024-04-19_12:00:00','2024-04-19_12:00:00', + interval_seconds = 10800 + io_form_geogrid = 2, +/ + +&geogrid + parent_id = 1, 1, 2, 3 + parent_grid_ratio = 1, 5, 5, 5 + i_parent_start = 1, 48, 50, 55 + j_parent_start = 1, 48, 51, 55 + e_we = 120, 121, 121, 126 + e_sn = 120, 121, 121, 126 + geog_data_res = '30s','30s','30s','30s' + dx = 12500, + dy = 12500, + map_proj = 'lambert', + ref_lat = 51.964716, + ref_lon = 5.663308, + truelat1 = 30.0, + truelat2 = 60.0, + stand_lon = 4.55, + geog_data_path = '/projects/0/prjs0914/wrf-data/default/static/WPS_GEOG' +/ + +&ungrib + out_format = 'WPS', + prefix = 'FILE', +/ + +&metgrid + fg_name = 'FILE' + io_form_metgrid = 2, +/ + + + diff --git a/wrf-runner/wrf_runner.sh b/wrf-runner/wrf_runner.sh new file mode 100644 index 0000000..7ecb258 --- /dev/null +++ b/wrf-runner/wrf_runner.sh @@ -0,0 +1,50 @@ +#!/bin/bash +#SBATCH --job-name=wrf_experiment # Job name +#SBATCH --partition=thin # Partition or queue name +#SBATCH --nodes=1 # Number of nodes +#SBATCH --ntasks-per-node=4 # Number of tasks per node +#SBATCH --cpus-per-task=1 # Number of CPU cores per task +#SBATCH --time=1:00:00 # Maximum runtime (D-HH:MM:SS) + +# Security; fail on first error; explicit vars only +set -euxo pipefail + +# Load dependencies +module load 2023 +module load netCDF-Fortran/4.6.1-gompi-2023a # also loads gcc and gompi +module load Python/3.11.3-GCCcore-12.3.0 + +# For modifying namelists programmatically +pip install --user f90nml + +# Set some paths +export NETCDF=/sw/arch/RHEL8/EB_production/2023/software/netCDF-Fortran/4.6.1-gompi-2023a +export WPS_HOME=$HOME/wrf-model/WPS +export WRF_HOME=$HOME/wrf-model/WRF +export WRF_RUNNER=$HOME/Urban-M4/misc/wrf-runner +export OUTPUT_DIR=$HOME/Urban-M4/experiments +export DATA_HOME=/projects/0/prjs0914/wrf-data/default + +# Make new run directory +export RUNDIR=$OUTPUT_DIR/$(date +"%Y-%m-%d_%H-%M-%S") +mkdir -p $RUNDIR +cd $RUNDIR +echo $PWD +f90nml $WRF_RUNNER/namelist.wps namelist.wps + +# Run WPS +f90nml -g geogrid -v opt_geogrid_tbl_path="'$WPS_HOME/geogrid/'" namelist.wps patched_nml && mv patched_nml namelist.wps +f90nml -g metgrid -v opt_metgrid_tbl_path="'$WPS_HOME/metgrid'" namelist.wps patched_nml && mv patched_nml namelist.wps +ln -s $WPS_HOME/ungrib/Variable_Tables/Vtable.GFS Vtable +$WPS_HOME/link_grib.csh "${DATA_HOME}/real-time/gfs-data/*" +$WPS_HOME/geogrid.exe +$WPS_HOME/ungrib.exe +$WPS_HOME/metgrid.exe + +# Run WRF +f90nml $WRF_RUNNER/namelist.input namelist.input +$WRF_HOME/run/real.exe +$WRF_HOME/run/wrf.exe + +# Report status +status=$? && [ $status -eq 0 ] && echo "Run successful" || echo "Run failed" From bd4e822c692f97a173a6347030b53cb6816658ff Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Tue, 14 May 2024 16:53:40 +0200 Subject: [PATCH 2/4] Add link to w2w PR --- wrf-runner/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wrf-runner/README.md b/wrf-runner/README.md index 3fd6dae..8287b4c 100644 --- a/wrf-runner/README.md +++ b/wrf-runner/README.md @@ -40,7 +40,7 @@ namelists from the command line. Beware that: ## To do -* Use MODIS_LCZ landuse with wudapt-to-wrf +* Use MODIS_LCZ landuse with wudapt-to-wrf (checkout [this branch](https://github.com/matthiasdemuzere/w2w/pull/127#issuecomment-2066019831)) * Use the "standard" heatwave case for amsterdam * Use IFS analysis for initial/boundary conditions * Use custom WUR landuse from summer in the city (checkout different branch of WRF?) From 73dda732772ffe10bd56a5aa803fb1ca5a05d32f Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Tue, 14 May 2024 17:36:56 +0200 Subject: [PATCH 3/4] Failed branch never reached due to set -e --- wrf-runner/wrf_runner.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wrf-runner/wrf_runner.sh b/wrf-runner/wrf_runner.sh index 7ecb258..aeb8aec 100644 --- a/wrf-runner/wrf_runner.sh +++ b/wrf-runner/wrf_runner.sh @@ -47,4 +47,4 @@ $WRF_HOME/run/real.exe $WRF_HOME/run/wrf.exe # Report status -status=$? && [ $status -eq 0 ] && echo "Run successful" || echo "Run failed" +status=$? && [ $status -eq 0 ] && echo "Run successful" From d1c81317a3b76fa68527c03e3d02bda63062db93 Mon Sep 17 00:00:00 2001 From: Peter Kalverla Date: Fri, 24 May 2024 11:46:41 +0200 Subject: [PATCH 4/4] Explicitly link ARW geogrid table --- wrf-runner/wrf_runner.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/wrf-runner/wrf_runner.sh b/wrf-runner/wrf_runner.sh index aeb8aec..2c37a82 100644 --- a/wrf-runner/wrf_runner.sh +++ b/wrf-runner/wrf_runner.sh @@ -35,7 +35,8 @@ f90nml $WRF_RUNNER/namelist.wps namelist.wps # Run WPS f90nml -g geogrid -v opt_geogrid_tbl_path="'$WPS_HOME/geogrid/'" namelist.wps patched_nml && mv patched_nml namelist.wps f90nml -g metgrid -v opt_metgrid_tbl_path="'$WPS_HOME/metgrid'" namelist.wps patched_nml && mv patched_nml namelist.wps -ln -s $WPS_HOME/ungrib/Variable_Tables/Vtable.GFS Vtable +ln -sf $WPS_HOME/ungrib/Variable_Tables/Vtable.GFS Vtable +ln -sf $WPS_HOME/geogrid/GEOGRID.TBL.ARW $WPS_HOME/geogrid/GEOGRID.TBL # make sure the right geogrid table is linked. $WPS_HOME/link_grib.csh "${DATA_HOME}/real-time/gfs-data/*" $WPS_HOME/geogrid.exe $WPS_HOME/ungrib.exe