From 2786d7b76a18d347c6e42f13d1ee1f2b137503f3 Mon Sep 17 00:00:00 2001 From: Daniel Rosen Date: Tue, 8 Oct 2019 16:20:43 -0600 Subject: [PATCH] Add ability to create multiple hydrology instances in driver. Each instance will represent a member in an ensemble. Hydrology instances will change their working directory to a subdirectory that matches the component name (i.e. HYD-03). WRF-Hydro ensemble is running side-by-side with a new test case sbys_tuolumne_hyd.ens002.ldas. Coupled LND-HYD ensemble work is incomplete. --- .../runconfig/lishydro.runconfig.hyd.ens002 | 50 ++++ .../runsettings/sbys_tuolumne_hyd.ens002.ldas | 10 + compset/setuprun.csh | 29 +- src/driver/driver.F90 | 269 +++++++++++++++--- src/wrf_hydro_nwm | 2 +- 5 files changed, 312 insertions(+), 48 deletions(-) create mode 100644 compset/runconfig/lishydro.runconfig.hyd.ens002 create mode 100644 compset/runsettings/sbys_tuolumne_hyd.ens002.ldas diff --git a/compset/runconfig/lishydro.runconfig.hyd.ens002 b/compset/runconfig/lishydro.runconfig.hyd.ens002 new file mode 100644 index 0000000..5f13a08 --- /dev/null +++ b/compset/runconfig/lishydro.runconfig.hyd.ens002 @@ -0,0 +1,50 @@ +############################################### +#### LISHydro runtime configuration ##### +############################################### + +# optionally turn off a component (options are "yes" and "no") +lnd: no +hyd: yes +med: no + +# PET lists - if not set, use all PETs +#pets_lnd: +pets_hyd: __PETLISTHYD__ +multi_instance_hyd: true +instance_count_hyd: 2 +#pets_med: + +# global clock +time_step: __TIMESTEP__ +start_time: __STARTTIME__ +stop_time: __STOPTIME__ + +# run sequence +runSeq:: + @__TIMESTEP__ + HYD-1 + HYD-2 + @ +:: + +# component attributes +driverAttributes:: + Verbosity = high + Profiling = 0 +:: + +hydAttributes:: + Verbosity = 1 + Diagnostic = 0 + Profiling = 0 + realize_all_export = false + config_file = hydro.namelist + das_config_file = namelist.hrldas + time_step = 0 + forcings_directory = __FORCINGDIRHYD__ + domain_id = 1 + nest_to_nest = false + import_dependency = false + output_directory = HYD_OUTPUT +:: + diff --git a/compset/runsettings/sbys_tuolumne_hyd.ens002.ldas b/compset/runsettings/sbys_tuolumne_hyd.ens002.ldas new file mode 100644 index 0000000..b4bf866 --- /dev/null +++ b/compset/runsettings/sbys_tuolumne_hyd.ens002.ldas @@ -0,0 +1,10 @@ +#!/usr/bin/csh +set RUNCONFIG="hyd.ens002" +set TASKS="512" +set TIME="01:00:00" +set PETLISTHYD="0 255 256 511" +set TIMESTEP="3600" +set STARTTIME="2015 10 01 0 0 0" +set STOPTIME="2015 11 01 0 0 0" +set REMAP="redist" +set FORCINGDIRHYD="WRFHYDRO_FORCING/LDASOUT/tuolumne" diff --git a/compset/setuprun.csh b/compset/setuprun.csh index 887749c..cc15e88 100755 --- a/compset/setuprun.csh +++ b/compset/setuprun.csh @@ -133,13 +133,28 @@ mkdir -p $RUNDIR if ($RUNCONFIG =~ *hyd*) then set DATA_HYD=$DATA_ROOT/WRFHydro/$COMPSET if (-d $DATA_HYD) then - cp $DATA_HYD/namelist.hrldas $RUNDIR - cp $DATA_HYD/hydro.namelist $RUNDIR - cp $DATA_HYD/WRFHYDRO_PARMS/CHANPARM.TBL $RUNDIR - ln -sf $DATA_HYD/WRFHYDRO_DOMAIN $RUNDIR/WRFHYDRO_DOMAIN - ln -sf $DATA_HYD/WRFHYDRO_FORCING $RUNDIR/WRFHYDRO_FORCING - ln -sf $DATA_HYD/WRFHYDRO_PARMS $RUNDIR/WRFHYDRO_PARMS - ln -sf $DATA_HYD/WRFHYDRO_RESTART $RUNDIR/WRFHYDRO_RESTART + set ensemble=`find $DATA_HYD -name 'HYD-*'` + foreach data_hyd_member (${ensemble}) + set member=`basename $data_hyd_member` + set rundir_member=$RUNDIR/$member + mkdir -p $rundir_member + cp $data_hyd_member/namelist.hrldas $rundir_member + cp $data_hyd_member/hydro.namelist $rundir_member + cp $data_hyd_member/WRFHYDRO_PARMS/CHANPARM.TBL $rundir_member + ln -sf $data_hyd_member/WRFHYDRO_DOMAIN $rundir_member/WRFHYDRO_DOMAIN + ln -sf $data_hyd_member/WRFHYDRO_FORCING $rundir_member/WRFHYDRO_FORCING + ln -sf $data_hyd_member/WRFHYDRO_PARMS $rundir_member/WRFHYDRO_PARMS + ln -sf $data_hyd_member/WRFHYDRO_RESTART $rundir_member/WRFHYDRO_RESTART + end + if ( "$ensemble" == "" ) then + cp $DATA_HYD/namelist.hrldas $RUNDIR + cp $DATA_HYD/hydro.namelist $RUNDIR + cp $DATA_HYD/WRFHYDRO_PARMS/CHANPARM.TBL $RUNDIR + ln -sf $DATA_HYD/WRFHYDRO_DOMAIN $RUNDIR/WRFHYDRO_DOMAIN + ln -sf $DATA_HYD/WRFHYDRO_FORCING $RUNDIR/WRFHYDRO_FORCING + ln -sf $DATA_HYD/WRFHYDRO_PARMS $RUNDIR/WRFHYDRO_PARMS + ln -sf $DATA_HYD/WRFHYDRO_RESTART $RUNDIR/WRFHYDRO_RESTART + endif else echo "ERROR: DATA_HYD directory not found [$DATA_HYD]" exit 1 diff --git a/src/driver/driver.F90 b/src/driver/driver.F90 index 54a0dae..7eaecf8 100644 --- a/src/driver/driver.F90 +++ b/src/driver/driver.F90 @@ -86,12 +86,18 @@ subroutine SetModelServices(driver, rc) type(ESMF_Time) :: stopTime type(ESMF_TimeInterval) :: timeStep type(ESMF_Clock) :: internalClock + integer :: i + character(6) :: maxStr + character(3) :: instStrFmt + character(10) :: compName type(ESMF_GridComp) :: child type(ESMF_CplComp) :: connector type(ESMF_Config) :: config type(NUOPC_FreeFormat) :: attrFF logical :: enabledLnd, enabledHyd, enabledMed integer, allocatable :: petList(:) + logical :: multiInst + integer :: instCnt integer :: dt rc = ESMF_SUCCESS @@ -117,7 +123,7 @@ subroutine SetModelServices(driver, rc) if (enabledLnd) then ! get PET lists from config - call getPetListFromConfig(config, "pets_lnd:", petList, rc=rc) + call getPetListFromConfig(config, "pets_lnd:", petList=petList, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, file=__FILE__)) return ! bail out @@ -156,36 +162,126 @@ subroutine SetModelServices(driver, rc) if (enabledHyd) then - ! get PET lists from config - call getPetListFromConfig(config, "pets_hyd:", petList, rc=rc) + ! get instance count from config + call ESMF_ConfigGetAttribute(config, instCnt, & + label="instance_count_hyd:", default=1, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, file=__FILE__)) return ! bail out - ! SetServices for HYD - if (allocated(petList)) then - call NUOPC_DriverAddComp(driver, "HYD", hydSS, petList=petList, comp=child, rc=rc) + if (instCnt.gt.1) then + multiInst = .true. ! default multi instance .true. + else + multiInst = .false. ! default multi instance .false. + endif + call ESMF_ConfigGetAttribute(config, multiInst, & + label="multi_instance_hyd:", default=multiInst, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, file=__FILE__)) return ! bail out + if ((.NOT.multiInst) .AND. (instCnt.gt.1)) then + call ESMF_LogSetError(ESMF_RC_ARG_BAD, & + msg="multi_instance_hyd must be true for instance count gt 1", & + line=__LINE__, file=__FILE__, rcToReturn=rc) + return + elseif (instCnt.gt.999999) then + call ESMF_LogSetError(ESMF_RC_ARG_BAD, & + msg="instance_count_hyd must be less than 999999", & + line=__LINE__, file=__FILE__, rcToReturn=rc) + return + endif + + if (multiInst) then + + ! generate the instance string format descriptor + write(maxStr,"(I0)") instCnt + write(instStrFmt,"(I0,A1,I0)") len_trim(maxStr),".",len_trim(maxStr) + + ! check for overlapping PETs + call checkPetListFromConfig(config, "pets_hyd:", rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, file=__FILE__)) return ! bail out - deallocate(petList) + + do i=1, instCnt + + write(compName,"(A4,I"//trim(instStrFmt)//")") "HYD-",i + + ! get instance PET lists from config + call getPetListFromConfig(config, "pets_hyd:", instance=i, & + petList=petList, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, file=__FILE__)) return ! bail out + + ! SetServices for HYD + if (allocated(petList)) then + call NUOPC_DriverAddComp(driver, compName, hydSS, petList=petList, & + comp=child, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, file=__FILE__)) return ! bail out + deallocate(petList) + else + call NUOPC_DriverAddComp(driver, compName, hydSS, comp=child, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, file=__FILE__)) return ! bail out + endif + call NUOPC_CompAttributeSet(child, name="Verbosity", value="0", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, file=__FILE__)) return ! bail out + + ! read HYD attributes from config file into FreeFormat + attrFF = NUOPC_FreeFormatCreate(config, label="hydAttributes::", & + relaxedflag=.true., rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, file=__FILE__)) return ! bail out + call NUOPC_CompAttributeIngest(child, attrFF, addFlag=.true., rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, file=__FILE__)) return ! bail out + call NUOPC_FreeFormatDestroy(attrFF, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, file=__FILE__)) return ! bail out + call NUOPC_CompAttributeAdd(child, & + attrList=(/"multi_instance_hyd"/), rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, file=__FILE__)) return ! bail out + call ESMF_AttributeSet(child, name="multi_instance_hyd", & + value="true", convention="NUOPC", purpose="Instance", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, file=__FILE__)) return ! bail out + enddo + else - call NUOPC_DriverAddComp(driver, "HYD", hydSS, comp=child, rc=rc) + + ! get PET lists from config + call getPetListFromConfig(config, "pets_hyd:", petList=petList, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, file=__FILE__)) return ! bail out - endif - call NUOPC_CompAttributeSet(child, name="Verbosity", value="0", rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & - line=__LINE__, file=__FILE__)) return ! bail out - ! read HYD attributes from config file into FreeFormat - attrFF = NUOPC_FreeFormatCreate(config, label="hydAttributes::", & - relaxedflag=.true., rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & - line=__LINE__, file=__FILE__)) return ! bail out - call NUOPC_CompAttributeIngest(child, attrFF, addFlag=.true., rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & - line=__LINE__, file=__FILE__)) return ! bail out - call NUOPC_FreeFormatDestroy(attrFF, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & - line=__LINE__, file=__FILE__)) return ! bail out + ! SetServices for HYD + if (allocated(petList)) then + call NUOPC_DriverAddComp(driver, "HYD", hydSS, petList=petList, & + comp=child, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, file=__FILE__)) return ! bail out + deallocate(petList) + else + call NUOPC_DriverAddComp(driver, "HYD", hydSS, comp=child, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, file=__FILE__)) return ! bail out + endif + call NUOPC_CompAttributeSet(child, name="Verbosity", value="0", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, file=__FILE__)) return ! bail out + + ! read HYD attributes from config file into FreeFormat + attrFF = NUOPC_FreeFormatCreate(config, label="hydAttributes::", & + relaxedflag=.true., rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, file=__FILE__)) return ! bail out + call NUOPC_CompAttributeIngest(child, attrFF, addFlag=.true., rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, file=__FILE__)) return ! bail out + call NUOPC_FreeFormatDestroy(attrFF, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, file=__FILE__)) return ! bail out + + endif ! multiple instances endif !enabledHyd @@ -196,7 +292,7 @@ subroutine SetModelServices(driver, rc) if (enabledMed) then ! get PET lists from config - call getPetListFromConfig(config, "pets_med:", petList, rc=rc) + call getPetListFromConfig(config, "pets_med:", petList=petList, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, file=__FILE__)) return ! bail out @@ -368,44 +464,137 @@ subroutine isComponentEnabled(config, label, isEnabled, rc) end subroutine isComponentEnabled - subroutine getPetListFromConfig(config, label, petList, rc) - type(ESMF_Config), intent(inout) :: config - character(len=*), intent(in) :: label - integer, allocatable :: petList(:) - integer, intent(out) :: rc + subroutine getPetListFromConfig(config, label, instance, petList, rc) + type(ESMF_Config), intent(inout) :: config + character(len=*), intent(in) :: label + integer, intent(in), optional :: instance + integer, allocatable, intent(inout) :: petList(:) + integer, intent(out) :: rc ! local - logical :: isPresent - integer :: i, minPet, maxPet + integer :: l_instance + logical :: isPresent + integer :: attrCnt + integer, allocatable :: petBoundsList(:) + integer :: i, minPet, maxPet + + if (present(instance)) then + l_instance = instance + else + l_instance = 1 + endif call ESMF_ConfigFindLabel(config, trim(label), & isPresent=isPresent, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, file=__FILE__)) return ! bail out if (isPresent) then - call ESMF_ConfigGetAttribute(config, minPet, default=-1, rc=rc) + attrCnt = ESMF_ConfigGetLen(config, label=trim(label), rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, file=__FILE__)) return ! bail out - call ESMF_ConfigGetAttribute(config, maxPet, default=-2, rc=rc) + if (attrCnt .ge. (l_instance*2)) then + allocate (petBoundsList(attrCnt), stat=rc) + if (ESMF_LogFoundAllocError(statusToCheck=rc, & + msg="Could not allocate petBoundsList", & + line=__LINE__, file=__FILE__)) return ! bail out + call ESMF_ConfigGetAttribute(config, petBoundsList, & + label=trim(label), rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, file=__FILE__)) return ! bail out + minPet = petBoundsList((2*l_instance)-1) + maxPet = petBoundsList(2*l_instance) + deallocate(petBoundsList, stat=rc) + if (ESMF_LogFoundDeallocError(statusToCheck=rc, & + msg="Could not deallocate petBoundsList", & + line=__LINE__, file=__FILE__)) return ! bail out + if (minPet <= maxPet) then + allocate(petList(maxPet-minPet+1), stat=rc) + do i=1, maxPet-minPet+1 + petList(i) = minPet+i-1 + enddo + else + deallocate (petList, stat=rc) + if (ESMF_LogFoundDeallocError(statusToCheck=rc, & + msg="Could not deallocate petList", & + line=__LINE__, file=__FILE__)) return ! bail out + call ESMF_LogSetError(ESMF_RC_ARG_BAD, & + msg=trim(label)//" min must be <= max", & + line=__LINE__, file=__FILE__, rcToReturn=rc) + return + endif + else + call ESMF_LogSetError(ESMF_RC_ARG_BAD, & + msg=trim(label)//" PET bounds missing", & + line=__LINE__, file=__FILE__, rcToReturn=rc) + return + endif + endif + + end subroutine getPetListFromConfig + + subroutine checkPetListFromConfig(config, label, rc) + type(ESMF_Config), intent(inout) :: config + character(len=*), intent(in) :: label + integer, intent(out) :: rc + + ! local + logical :: isPresent + integer :: attrCnt + integer, allocatable :: petBoundsList(:) + integer :: i, j + + call ESMF_ConfigFindLabel(config, trim(label), & + isPresent=isPresent, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, file=__FILE__)) return ! bail out + if (isPresent) then + attrCnt = ESMF_ConfigGetLen(config, label=trim(label), rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, file=__FILE__)) return ! bail out - if (minPet <= maxPet) then - allocate(petList(maxPet-minPet+1), stat=rc) + if (MOD(attrCnt,2) .eq. 0) then + allocate (petBoundsList(attrCnt), stat=rc) if (ESMF_LogFoundAllocError(statusToCheck=rc, & - msg="Could not allocate petList", & + msg="Could not allocate petBoundsList", & line=__LINE__, file=__FILE__)) return ! bail out - do i=1, maxPet-minPet+1 - petList(i) = minPet+i-1 + call ESMF_ConfigGetAttribute(config, petBoundsList, & + label=trim(label), rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, file=__FILE__)) return ! bail out + do i=1, attrCnt, 2 + if (petBoundsList(i).gt.petBoundsList(i)) then + call ESMF_LogSetError(ESMF_RC_ARG_BAD, & + msg=trim(label)//" min must be <= max", & + line=__LINE__, file=__FILE__, rcToReturn=rc) + return + endif + do j=i+2, attrCnt, 2 + if ((petBoundsList(j).le.petBoundsList(i+1)) .AND. & + (petBoundsList(j).ge.petBoundsList(i))) then + call ESMF_LogSetError(ESMF_RC_ARG_BAD, & + msg=trim(label)//" instances have overlapping PETs", & + line=__LINE__, file=__FILE__, rcToReturn=rc) + return + endif + enddo enddo + deallocate(petBoundsList, stat=rc) + if (ESMF_LogFoundDeallocError(statusToCheck=rc, & + msg="Could not deallocate petBoundsList", & + line=__LINE__, file=__FILE__)) return ! bail out else call ESMF_LogSetError(ESMF_RC_ARG_BAD, & - msg=trim(label)//" min must be <= max", & + msg=trim(label)//" PET bounds missing", & line=__LINE__, file=__FILE__, rcToReturn=rc) return endif + else + call ESMF_LogSetError(ESMF_RC_ARG_BAD, & + msg=trim(label)//" PET bounds missing", & + line=__LINE__, file=__FILE__, rcToReturn=rc) + return endif - end subroutine getPetListFromConfig + end subroutine checkPetListFromConfig subroutine getTimeFromConfig(config, label, tm, rc) type(ESMF_Config), intent(inout) :: config diff --git a/src/wrf_hydro_nwm b/src/wrf_hydro_nwm index 267e97f..5aa19e1 160000 --- a/src/wrf_hydro_nwm +++ b/src/wrf_hydro_nwm @@ -1 +1 @@ -Subproject commit 267e97fdd4f77cfeaca4c206d21c4053946b1940 +Subproject commit 5aa19e1b96e744d147974a5890f3fa02b13394d6