From 363dedac7579bef7b995d52ed82476f10811c49a Mon Sep 17 00:00:00 2001 From: Daniel Rosen Date: Fri, 6 Dec 2019 22:19:44 +0000 Subject: [PATCH 1/6] Update component_HYCOM.mk to support NAVY HYCOM NUOPC cap. --- src/incmake/component_HYCOM.mk | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/src/incmake/component_HYCOM.mk b/src/incmake/component_HYCOM.mk index cf1b9de8..657c35cf 100644 --- a/src/incmake/component_HYCOM.mk +++ b/src/incmake/component_HYCOM.mk @@ -1,15 +1,23 @@ # Location of the ESMF makefile fragment for this component: -hycom_mk=$(HYCOM_BINDIR)/hycom_nuopc.mk +hycom_mk=$(HYCOM_BINDIR)/hycom.mk all_component_mk_files+=$(hycom_mk) # Location of source code and installation HYCOM_SRCDIR?=$(ROOTDIR)/HYCOM -HYCOM_BINDIR?=$(ROOTDIR)/HYCOM-INSTALL +HYCOM_BINDIR?=$(ROOTDIR)/HYCOM/HYCOM-INSTALL ifeq ($(MACHINE_ID),linux_gnu) HYCOM_ARCH=Alinux-gnu-relo + HYCOM_TYPE=nuopc + HYCOM_OPTS=-DEOS_SIG2 -DEOS_17T -DESPC_COUPLE +else ifeq ($(MACHINE_ID),hera) + HYCOM_ARCH=intelsse-impi-sm-relo + HYCOM_TYPE=mpi + HYCOM_OPTS=-DEOS_SIG2 -DEOS_17T -DESPC_COUPLE else HYCOM_ARCH=Aintelrelo + HYCOM_TYPE=nuopc + HYCOM_OPTS=-DEOS_SIG2 -DEOS_17T -DESPC_COUPLE endif # Make sure the expected directories exist and are non-empty: @@ -17,6 +25,7 @@ $(call require_dir,$(HYCOM_SRCDIR),HYCOM source directory) HYCOM_ALL_OPTS=\ HYCOM_ARCH=$(HYCOM_ARCH) \ + HYCOM_TYPE=$(HYCOM_TYPE) \ COMP_BINDIR=$(HYCOM_BINDIR) \ COMP_SRCDIR=$(HYCOM_SRCDIR) @@ -25,11 +34,12 @@ HYCOM_ALL_OPTS=\ # Rules for building this component: $(hycom_mk): configure - +$(MODULE_LOGIC) ; cd "$(HYCOM_SRCDIR)/sorc" ; exec $(MAKE) \ - ARCH="$(HYCOM_ARCH)" TYPE=nuopc nuopc - +-$(MODULE_LOGIC) ; cd "$(HYCOM_SRCDIR)/sorc" ; exec $(MAKE) \ - ARCH="$(HYCOM_ARCH)" TYPE=nuopc DESTDIR=/ \ - INSTDIR="$(HYCOM_BINDIR)" nuopcinstall + +$(MODULE_LOGIC) ; cd "$(HYCOM_SRCDIR)/NUOPC" ; exec $(MAKE) \ + ARCH="$(HYCOM_ARCH)" TYPE="$(HYCOM_TYPE)" CPP_EXTRAS="$(HYCOM_OPTS)" \ + nuopc + +-$(MODULE_LOGIC) ; cd "$(HYCOM_SRCDIR)/NUOPC" ; exec $(MAKE) \ + ARCH="$(HYCOM_ARCH)" TYPE="$(HYCOM_TYPE)" CPP_EXTRAS="$(HYCOM_OPTS)" \ + DESTDIR=/ INSTDIR="$(HYCOM_BINDIR)" nuopcinstall test -d "$(HYCOM_BINDIR)" test -s "$(hycom_mk)" @@ -46,8 +56,9 @@ build_HYCOM: $(hycom_mk) # Rules for cleaning the SRCDIR and BINDIR: clean_HYCOM: - -+cd $(HYCOM_SRCDIR) ; exec $(MAKE) $(HYCOM_ALL_OPTS) \ - ARCH="$$HYCOM_ARCH" TYPE=nuopc clean + +-$(MODULE_LOGIC) ; cd $(HYCOM_SRCDIR)/NUOPC ; exec $(MAKE) \ + ARCH="$(HYCOM_ARCH)" TYPE="$(HYCOM_TYPE)" CPP_EXTRAS="$(HYCOM_OPTS)" \ + nuopcdistclean distclean_HYCOM: clean_HYCOM rm -rf $(HYCOM_BINDIR) $(hycom_mk) From 5992a16aa0086ee079b62b1f4a19a1a9e1245ba6 Mon Sep 17 00:00:00 2001 From: Biju Thomas Date: Tue, 2 Jun 2020 09:05:38 -0500 Subject: [PATCH 2/6] Pointing machine architecture and parallelization strategy for HYCOM/NUOPC for jet/orion/wcoss --- src/incmake/component_HYCOM.mk | 16 ++++++++++++++++ src/incmake/env/wcoss/wcoss_dell_p3.mk | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/incmake/component_HYCOM.mk b/src/incmake/component_HYCOM.mk index 657c35cf..7d30e3c5 100644 --- a/src/incmake/component_HYCOM.mk +++ b/src/incmake/component_HYCOM.mk @@ -14,6 +14,22 @@ else ifeq ($(MACHINE_ID),hera) HYCOM_ARCH=intelsse-impi-sm-relo HYCOM_TYPE=mpi HYCOM_OPTS=-DEOS_SIG2 -DEOS_17T -DESPC_COUPLE +else ifeq ($(MACHINE_ID),jet) + HYCOM_ARCH=intelsse-impi-sm-relo + HYCOM_TYPE=mpi + HYCOM_OPTS=-DEOS_SIG2 -DEOS_17T -DESPC_COUPLE +else ifeq ($(MACHINE_ID),orion) + HYCOM_ARCH=intelsse-impi-sm-relo + HYCOM_TYPE=mpi + HYCOM_OPTS=-DEOS_SIG2 -DEOS_17T -DESPC_COUPLE +else ifeq ($(MACHINE_ID),wcoss_cray) + HYCOM_ARCH=xc40-intel-relo + HYCOM_TYPE=mpi + HYCOM_OPTS=-DEOS_SIG2 -DEOS_17T -DESPC_COUPLE +else ifeq ($(MACHINE_ID),wcoss_dell_p3) + HYCOM_ARCH=intelsse-impi-sm-relo + HYCOM_TYPE=mpi + HYCOM_OPTS=-DEOS_SIG2 -DEOS_17T -DESPC_COUPLE else HYCOM_ARCH=Aintelrelo HYCOM_TYPE=nuopc diff --git a/src/incmake/env/wcoss/wcoss_dell_p3.mk b/src/incmake/env/wcoss/wcoss_dell_p3.mk index 87936226..85030b1c 100644 --- a/src/incmake/env/wcoss/wcoss_dell_p3.mk +++ b/src/incmake/env/wcoss/wcoss_dell_p3.mk @@ -1,4 +1,4 @@ -MACHINE_ID=wcoss +MACHINE_ID=wcoss_dell_p3 FULL_MACHINE_ID=wcoss_dell_p3 PEX=3 USE_MODULES=YES From a8a9c66c92a83539829e96e98a2de8d00da58118 Mon Sep 17 00:00:00 2001 From: Bin Liu Date: Fri, 22 Jan 2021 06:07:43 -0500 Subject: [PATCH 3/6] Feature/hafs couplehycom (#2) Merge feature/hafs_couplehycom branch back into support/HAFS branch Preparations for external PIO (Ufuk) *Added CPP -DPIO to wrap calls to pio library *component_CDEPS.mk added to configure CDEPS build *component_CDEPS_DATM.mk added to configure CDEPS_DATM build *component_CDEPS_DOCN.mk added to configure CDEPS_DOCN build *component_CMEPS.mk CMEPS-INSTALL moved, default INTERNAL_PIO_INIT added *component_HYCOM.mk added extra build call to HYCOM model *module_EARTH_GRID_COMP.F90 added FRONT_CDEPS_DATM and FRONT_CDEPS_DOCN, added pio initialization Added diagnostics (Dan) *module_EARTH_GRID_COMP.F90 added connector diagnostics configuration Co-authored-by: Ufuk Turuncoglu Co-authored-by: Daniel Rosen --- src/MAIN_NEMS.F90 | 18 ++++ src/incmake/component_CDEPS.mk | 93 ++++++++++++++++++++ src/incmake/component_CDEPS_DATM.mk | 43 ++++++++++ src/incmake/component_CDEPS_DOCN.mk | 43 ++++++++++ src/incmake/component_CMEPS.mk | 7 +- src/incmake/component_HYCOM.mk | 3 + src/module_EARTH_GRID_COMP.F90 | 127 ++++++++++++++++++++++++++-- 7 files changed, 324 insertions(+), 10 deletions(-) create mode 100755 src/incmake/component_CDEPS.mk create mode 100644 src/incmake/component_CDEPS_DATM.mk create mode 100644 src/incmake/component_CDEPS_DOCN.mk diff --git a/src/MAIN_NEMS.F90 b/src/MAIN_NEMS.F90 index 7229831c..80463ec4 100644 --- a/src/MAIN_NEMS.F90 +++ b/src/MAIN_NEMS.F90 @@ -66,6 +66,14 @@ PROGRAM MAIN_NEMS USE module_NEMS_Rusage,ONLY: NEMS_Rusage ! !----------------------------------------------------------------------- +!*** This module contains PIO initialization related routines. +!----------------------------------------------------------------------- +! +#if defined PIO + USE shr_pio_mod, ONLY: shr_pio_init1 +#endif +! +!----------------------------------------------------------------------- ! IMPLICIT NONE ! @@ -118,6 +126,7 @@ PROGRAM MAIN_NEMS ! CHARACTER(LEN=MPI_MAX_PROCESSOR_NAME) :: PROCNAME !<-- The processor(host) name INTEGER :: PROCNAME_LEN !<-- Actual PROCRNAME string length + INTEGER :: COMM_WORLD !<-- Copy of MPI_COMM_WORLD ! !----------------------------------------------------------------------- !*********************************************************************** @@ -191,6 +200,15 @@ PROGRAM MAIN_NEMS ! the start succeeded. ! !----------------------------------------------------------------------- +!*** Initialize PIO. 8 is the maximum number of component models +!----------------------------------------------------------------------- +! + COMM_WORLD = MPI_COMM_WORLD +#if defined PIO + call shr_pio_init1(8, "pio_in", COMM_WORLD) +#endif +! +!----------------------------------------------------------------------- !*** Set up the default log. !----------------------------------------------------------------------- ! diff --git a/src/incmake/component_CDEPS.mk b/src/incmake/component_CDEPS.mk new file mode 100755 index 00000000..03b8cf6b --- /dev/null +++ b/src/incmake/component_CDEPS.mk @@ -0,0 +1,93 @@ +# Location of the ESMF makefile fragment for this component: +cdeps_mk = $(ROOTDIR)/CDEPS/CDEPS_INSTALL/cdeps.mk +all_component_mk_files+=$(cdeps_mk) + +# Need this because cmeps_mk is not available at this level +cmeps_mk = $(ROOTDIR)/CMEPS/CMEPS_INSTALL/cmeps.mk + +# Location of source code and installation +CDEPS_SRCDIR?=$(ROOTDIR)/CDEPS +CDEPS_BLDDIR?=$(ROOTDIR)/CDEPS/CDEPS_BUILD +CDEPS_BINDIR?=$(ROOTDIR)/CDEPS/CDEPS_INSTALL + +# Make sure the expected directories exist and are non-empty: +$(call require_dir,$(CDEPS_SRCDIR),CDEPS source directory) + +ifndef CONFIGURE_NEMS_FILE +$(error CONFIGURE_NEMS_FILE not set.) +endif + +include $(CONFIGURE_NEMS_FILE) + +# Rule for building this component: +build_CDEPS: $(cdeps_mk) + +CDEPS_ALL_OPTS=\ + COMP_SRCDIR="$(CDEPS_SRCDIR)" \ + COMP_BINDIR="$(CDEPS_BINDIR)" \ + MACHINE_ID="$(MACHINE_ID)" \ + FC="$(FC)" \ + CC="$(CC)" \ + CXX="$(CXX)" + +# Rule for building CDEPS using cmake build system +$(cdeps_mk): $(cmeps_mk) configure + $(MODULE_LOGIC); export $(CDEPS_ALL_OPTS); \ + set -e; $(MODULE_LOGIC); mkdir -p $(CDEPS_BLDDIR); \ + cd $(CDEPS_BLDDIR); \ + exec cmake -DCMAKE_BUILD_TYPE=Release \ + -DBLD_STANDALONE=ON \ + -DCMAKE_INSTALL_PREFIX=$(CDEPS_BINDIR) \ + -DCMAKE_Fortran_COMPILER=$(FC) \ + -DCMAKE_C_COMPILER=$(CC) \ + -DCMAKE_CXX_COMPILER=$(CXX) \ + -DPIO=$(PIO_ROOT) \ + -DPIO_Fortran_LIBRARY=$(PIO_LIBDIR) \ + -DPIO_Fortran_INCLUDE_DIR=$(PIO_INC) \ + -DPIO_C_LIBRARY=$(PIO_LIBDIR) \ + -DPIO_C_INCLUDE_DIR=$(PIO_INC) ../ + + $(MODULE_LOGIC); export $(CDEPS_ALL_OPTS); \ + set -e; $(MODULE_LOGIC) ; cd $(CDEPS_BLDDIR); \ + exec $(MAKE) -j 1 $(CDEPS_ALL_OPTS) install VERBOSE=1 + + $(MODULE_LOGIC); export $(CDEPS_ALL_OPTS); \ + set -e; $(MODULE_LOGIC) ; cd $(CDEPS_BLDDIR); \ + exec find ./d??? -name "*.a" -exec cp {} ../CDEPS_INSTALL/lib/ \; + + $(MODULE_LOGIC); export $(CDEPS_ALL_OPTS); \ + set -e; $(MODULE_LOGIC) ; cd $(CDEPS_BLDDIR); \ + exec find ./fox/lib -name "*.a" -exec cp {} ../CDEPS_INSTALL/lib/ \; + + $(MODULE_LOGIC); export $(CDEPS_ALL_OPTS); \ + set -e; $(MODULE_LOGIC) ; cd $(CDEPS_BLDDIR); \ + exec find share/ -name "*.a" -exec cp {} ../CDEPS_INSTALL/lib/ \; + + $(MODULE_LOGIC); export $(CDEPS_ALL_OPTS); \ + set -e; $(MODULE_LOGIC) ; cd $(CDEPS_BLDDIR); \ + exec find ./d??? -name "*_comp_nuopc.mod" -exec cp {} ../CDEPS_INSTALL/include/ \; + + rm -f $(CDEPS_BINDIR)/cdeps.mk + @echo "# ESMF self-describing build dependency makefile fragment" > $(CDEPS_BINDIR)/cdeps.mk + @echo "# src location: $(CDEPS_SRCDIR)" >> $(CDEPS_BINDIR)/cdeps.mk + @echo >> $(CDEPS_BINDIR)/cdeps.mk + @echo "ESMF_DEP_FRONT = " >> $(CDEPS_BINDIR)/cdeps.mk + @echo "ESMF_DEP_INCPATH = $(CDEPS_BINDIR)/include" >> $(CDEPS_BINDIR)/cdeps.mk + @echo "ESMF_DEP_CMPL_OBJS = " >> $(CDEPS_BINDIR)/cdeps.mk + @echo "ESMF_DEP_LINK_OBJS = -L$(CDEPS_BINDIR)/lib -ldshr -lstreams -lcdeps_share -lFoX_dom -lFoX_sax -lFoX_common -lFoX_utils -lFoX_fsys -L$(PIO_LIBDIR) -lpiof -lpioc" >> $(CDEPS_BINDIR)/cdeps.mk + + test -d "$(CDEPS_BINDIR)" + test -s "$(cdeps_mk)" + +# Rule for cleaning the SRCDIR and BINDIR: +clean_CDEPS: + cp -n $(MODULE_DIR)/$(CHOSEN_MODULE) $(CONFDIR)/modules.nems ; \ + $(MODULE_LOGIC); export $(CDEPS_ALL_OPTS); \ + set -e; \ + if [ -d "~/Dropbox" ]; then cd $(CDEPS_BLDDIR); \ + exec $(MAKE) clean; fi + +distclean_CDEPS: clean_CDEPS + rm -rf $(CDEPS_BINDIR) + rm -rf $(CDEPS_BLDDIR) + rm -f $(cdeps_mk) diff --git a/src/incmake/component_CDEPS_DATM.mk b/src/incmake/component_CDEPS_DATM.mk new file mode 100644 index 00000000..67a66436 --- /dev/null +++ b/src/incmake/component_CDEPS_DATM.mk @@ -0,0 +1,43 @@ +# Location of the ESMF makefile fragment for this component: +cdeps_datm_mk = $(ROOTDIR)/CDEPS/DATM_INSTALL/cdeps_datm.mk +all_component_mk_files+=$(cdeps_datm_mk) + +# Need this because cmeps_mk is not available at this level +cdeps_mk = $(ROOTDIR)/CDEPS/CDEPS_INSTALL/cdeps.mk + +# Location of source code and installation +CDEPS_DATM_BINDIR?=$(ROOTDIR)/CDEPS/DATM_INSTALL +CDEPS_BINDIR?=$(ROOTDIR)/CDEPS/CDEPS_INSTALL + +# Make sure we're setting CDEPS=Y if DATM is enabled: +ifeq (,$(findstring CDEPS,$(COMPONENTS))) + $(error DATM requires CDEPS) +endif + +# Rule for building this component: +build_CDEPS_DATM: $(cdeps_datm_mk) + +DATM_ALL_FLAGS=\ + COMP_BINDIR="$(CDEPS_DATM_BINDIR)" + +$(cdeps_datm_mk): $(cdeps_mk) configure + mkdir -p $(CDEPS_DATM_BINDIR) + rm -f $(cdeps_datm_mk) + @echo "# ESMF self-describing build dependency makefile fragment" > $(cdeps_datm_mk) + @echo "# src location: $(ROOTDIR)/CDEPS" >> $(cdeps_datm_mk) + @echo >> $(cdeps_datm_mk) + @echo "ESMF_DEP_FRONT = atm_comp_nuopc" >> $(cdeps_datm_mk) + @echo "ESMF_DEP_INCPATH = " >> $(cdeps_datm_mk) + @echo "ESMF_DEP_CMPL_OBJS = " >> $(cdeps_datm_mk) + @echo "ESMF_DEP_LINK_OBJS = -L$(CDEPS_BINDIR)/lib -ldatm -ldshr -lstreams -lcdeps_share -lFoX_dom -lFoX_sax -lFoX_common -lFoX_utils -lFoX_fsys -L$(PIO_LIBDIR) -lpiof -lpioc" >> $(cdeps_datm_mk) + + test -d "$(CDEPS_DATM_BINDIR)" + test -s "$(cdeps_datm_mk)" + +# Rule for cleaning the SRCDIR and BINDIR: +clean_CDEPS_DATM: + @echo "there is nothing to clean for CDEPS provided DATM" + +distclean_CDEPS_DATM: clean_CDEPS_DATM + rm -rf $(CDEPS_DATM_BINDIR) $(cdeps_datm_mk) + diff --git a/src/incmake/component_CDEPS_DOCN.mk b/src/incmake/component_CDEPS_DOCN.mk new file mode 100644 index 00000000..3de4ba9e --- /dev/null +++ b/src/incmake/component_CDEPS_DOCN.mk @@ -0,0 +1,43 @@ +# Location of the ESMF makefile fragment for this component: +cdeps_docn_mk = $(ROOTDIR)/CDEPS/DOCN_INSTALL/cdeps_docn.mk +all_component_mk_files+=$(cdeps_docn_mk) + +# Need this because cmeps_mk is not available at this level +cdeps_mk = $(ROOTDIR)/CDEPS/CDEPS_INSTALL/cdeps.mk + +# Location of source code and installation +CDEPS_DOCN_BINDIR?=$(ROOTDIR)/CDEPS/DOCN_INSTALL +CDEPS_BINDIR?=$(ROOTDIR)/CDEPS/CDEPS_INSTALL + +# Make sure we're setting CDEPS=Y if DOCN is enabled: +ifeq (,$(findstring CDEPS,$(COMPONENTS))) + $(error DOCN requires CDEPS) +endif + +# Rule for building this component: +build_CDEPS_DOCN: $(cdeps_docn_mk) + +DOCN_ALL_FLAGS=\ + COMP_BINDIR="$(CDEPS_DOCN_BINDIR)" + +$(cdeps_docn_mk): $(cdeps_mk) configure + mkdir -p $(CDEPS_DOCN_BINDIR) + rm -f $(cdeps_docn_mk) + @echo "# ESMF self-describing build dependency makefile fragment" > $(cdeps_docn_mk) + @echo "# src location: $(ROOTDIR)/CDEPS" >> $(cdeps_docn_mk) + @echo >> $(cdeps_docn_mk) + @echo "ESMF_DEP_FRONT = ocn_comp_nuopc" >> $(cdeps_docn_mk) + @echo "ESMF_DEP_INCPATH = " >> $(cdeps_docn_mk) + @echo "ESMF_DEP_CMPL_OBJS = " >> $(cdeps_docn_mk) + @echo "ESMF_DEP_LINK_OBJS = -L$(CDEPS_BINDIR)/lib -ldocn -ldshr -lstreams -lcdeps_share -lFoX_dom -lFoX_sax -lFoX_common -lFoX_utils -lFoX_fsys -L$(PIO_LIBDIR) -lpiof -lpioc" >> $(cdeps_docn_mk) + + test -d "$(CDEPS_DOCN_BINDIR)" + test -s "$(cdeps_docn_mk)" + +# Rule for cleaning the SRCDIR and BINDIR: +clean_CDEPS_DOCN: + @echo "there is nothing to clean for CDEPS provided DOCN" + +distclean_CDEPS_DOCN: clean_CDEPS_DOCN + rm -rf $(CDEPS_DOCN_BINDIR) $(cdeps_docn_mk) + diff --git a/src/incmake/component_CMEPS.mk b/src/incmake/component_CMEPS.mk index d5feba66..7ce7a529 100644 --- a/src/incmake/component_CMEPS.mk +++ b/src/incmake/component_CMEPS.mk @@ -4,7 +4,7 @@ all_component_mk_files+=$(cmeps_mk) # Location of source code and installation CMEPS_SRCDIR?=$(ROOTDIR)/CMEPS -CMEPS_BINDIR?=$(ROOTDIR)/CMEPS_INSTALL +CMEPS_BINDIR?=$(ROOTDIR)/CMEPS/CMEPS_INSTALL # Make sure the expected directories exist and are non-empty: $(call require_dir,$(CMEPS_SRCDIR),CMEPS source directory) @@ -24,7 +24,10 @@ CMEPS_ALL_OPTS=\ MACHINE_ID="$(MACHINE_ID)" \ FC="$(FC)" \ CC="$(CC)" \ - CXX="$(CXX)" + CXX="$(CXX)" \ + INTERNAL_PIO_INIT=0 + +$(info "CMEPS_ALL_OPTS = $(CMEPS_ALL_OPTS)") $(cmeps_mk): configure $(MODULE_LOGIC) ; export $(CMEPS_ALL_OPTS) ; \ diff --git a/src/incmake/component_HYCOM.mk b/src/incmake/component_HYCOM.mk index 7d30e3c5..e4c00ccc 100644 --- a/src/incmake/component_HYCOM.mk +++ b/src/incmake/component_HYCOM.mk @@ -50,6 +50,9 @@ HYCOM_ALL_OPTS=\ # Rules for building this component: $(hycom_mk): configure + +$(MODULE_LOGIC) ; cd "$(HYCOM_SRCDIR)" ; exec $(MAKE) \ + ARCH="$(HYCOM_ARCH)" TYPE="$(HYCOM_TYPE)" CPP_EXTRAS="$(HYCOM_OPTS)" \ + esmf +$(MODULE_LOGIC) ; cd "$(HYCOM_SRCDIR)/NUOPC" ; exec $(MAKE) \ ARCH="$(HYCOM_ARCH)" TYPE="$(HYCOM_TYPE)" CPP_EXTRAS="$(HYCOM_OPTS)" \ nuopc diff --git a/src/module_EARTH_GRID_COMP.F90 b/src/module_EARTH_GRID_COMP.F90 index 9a181cbc..c2a903d1 100644 --- a/src/module_EARTH_GRID_COMP.F90 +++ b/src/module_EARTH_GRID_COMP.F90 @@ -70,6 +70,9 @@ MODULE module_EARTH_GRID_COMP #endif #ifdef FRONT_DATM use FRONT_DATM, only: DATM_SS => SetServices +#endif +#ifdef FRONT_CDEPS_DATM + use FRONT_CDEPS_DATM, only: DATM_SS => SetServices #endif ! - Handle build time OCN options: #ifdef FRONT_SOCN @@ -89,6 +92,9 @@ MODULE module_EARTH_GRID_COMP #endif #ifdef FRONT_POM use FRONT_POM, only: POM_SS => SetServices +#endif +#ifdef FRONT_CDEPS_DOCN + use FRONT_CDEPS_DOCN, only: DOCN_SS => SetServices #endif ! - Handle build time ICE options: #ifdef FRONT_SICE @@ -3374,6 +3380,10 @@ END SUBROUTINE EARTH_REGISTER subroutine SetModelServices(driver, rc) #ifdef CMEPS use med_internalstate_mod , only : med_id +#endif +#ifdef PIO + use mpi, only : MPI_COMM_NULL + use shr_pio_mod , only : shr_pio_init2 #endif type(ESMF_GridComp) :: driver integer, intent(out) :: rc @@ -3396,8 +3406,13 @@ subroutine SetModelServices(driver, rc) logical :: read_restart character(ESMF_MAXSTR) :: cvalue character(len=5) :: inst_suffix - logical :: isPresent #endif + logical :: isPresent + integer, allocatable :: comms(:), comps(:) + integer, allocatable :: comp_comm_iam(:) + logical, allocatable :: comp_iamin(:) + type(ESMF_VM) :: vm + integer :: Global_Comm rc = ESMF_SUCCESS ! query the Component for info @@ -3415,11 +3430,16 @@ subroutine SetModelServices(driver, rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, file=trim(name)//":"//__FILE__)) return ! bail out - ! get petCount and config - call ESMF_GridCompGet(driver, petCount=petCount, config=config, rc=rc) + ! get petCount, vm and config + call ESMF_GridCompGet(driver, petCount=petCount, config=config, vm=vm, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, file=trim(name)//":"//__FILE__)) return ! bail out - + + ! get MPI communicator + call ESMF_VMGet(vm, mpiCommunicator=Global_Comm, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, file=trim(name)//":"//__FILE__)) return ! bail out + ! read and ingest free format driver attributes attrFF = NUOPC_FreeFormatCreate(config, label="EARTH_attributes::", & relaxedflag=.true., rc=rc) @@ -3485,8 +3505,19 @@ subroutine SetModelServices(driver, rc) line=__LINE__, file=trim(name)//":"//__FILE__)) return ! bail out #endif + ! allocate arrays required for PIO initialization (phase 2) + if (.not. allocated(comms)) allocate(comms(componentCount+1)) + if (.not. allocated(comps)) allocate(comps(componentCount+1)) + if (.not. allocated(comp_iamin)) allocate(comp_iamin(componentCount)) + if (.not. allocated(comp_comm_iam)) allocate(comp_comm_iam(componentCount)) + + comps(1) = 1 + comms(1) = Global_Comm + ! determine information for each component and add to the driver do i=1, componentCount + comps(i+1) = i+1 + ! construct component prefix prefix=trim(compLabels(i)) ! read in petList bounds @@ -3597,7 +3628,7 @@ subroutine SetModelServices(driver, rc) return ! bail out #endif elseif (trim(model) == "datm") then -#ifdef FRONT_DATM +#if defined FRONT_DATM || defined FRONT_CDEPS_DATM call NUOPC_DriverAddComp(driver, trim(prefix), DATM_SS, & petList=petList, comp=comp, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & @@ -3686,6 +3717,19 @@ subroutine SetModelServices(driver, rc) call ESMF_LogSetError(ESMF_RC_NOT_VALID, msg=msg, line=__LINE__, & file=__FILE__, rcToReturn=rc) return ! bail out +#endif + elseif (trim(model) == "docn") then +#if defined FRONT_CDEPS_DOCN + call NUOPC_DriverAddComp(driver, trim(prefix), DOCN_SS, & + petList=petList, comp=comp, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, file=trim(name)//":"//__FILE__)) return ! bail out +#else + write (msg, *) "Model '", trim(model), "' was requested, "// & + "but is not available in the executable!" + call ESMF_LogSetError(ESMF_RC_NOT_VALID, msg=msg, line=__LINE__, & + file=__FILE__, rcToReturn=rc) + return ! bail out #endif elseif (trim(model) == "sice") then #ifdef FRONT_SICE @@ -3968,6 +4012,24 @@ subroutine SetModelServices(driver, rc) call NUOPC_FreeFormatDestroy(attrFF, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, file=trim(name)//":"//__FILE__)) return ! bail out + + ! read and ingest free format component attributes + call ESMF_ConfigFindNextLabel(config, & + label=trim(prefix)//"_modelio::", isPresent=isPresent, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, file=trim(name)//":"//__FILE__)) return ! bail out + if (isPresent) then + attrFF = NUOPC_FreeFormatCreate(config, & + label=trim(prefix)//"_modelio::", relaxedflag=.true., rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, file=trim(name)//":"//__FILE__)) return ! bail out + call NUOPC_CompAttributeIngest(comp, attrFF, addFlag=.true., rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, file=trim(name)//":"//__FILE__)) return ! bail out + call NUOPC_FreeFormatDestroy(attrFF, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, file=trim(name)//":"//__FILE__)) return ! bail out + endif ! clean-up deallocate(petList) @@ -3981,7 +4043,25 @@ subroutine SetModelServices(driver, rc) call AddAttributes(comp, driver, config, i+1, trim(prefix), inst_suffix, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, file=trim(name)//":"//__FILE__)) return ! bail out -#endif +#endif + if (ESMF_GridCompIsPetLocal(comp, rc=rc)) then + call ESMF_GridCompGet(comp, vm=vm, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, file=trim(name)//":"//__FILE__)) return ! bail out + + call ESMF_VMGet(vm, mpiCommunicator=comms(i+1), localPet=comp_comm_iam(i), rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, file=trim(name)//":"//__FILE__)) return ! bail out + + comp_iamin(i) = .true. + else +#ifdef PIO + comms(i+1) = MPI_COMM_NULL +#else + comms(i+1) = 0 +#endif + comp_iamin(i) = .false. + end if enddo #if ESMF_VERSION_MAJOR < 8 @@ -3992,9 +4072,18 @@ subroutine SetModelServices(driver, rc) line=__LINE__, file=trim(name)//":"//__FILE__)) return ! bail out #endif +#if defined PIO + ! Initialize PIO + call shr_pio_init2(comps(2:), compLabels, comp_iamin, comms(2:), comp_comm_iam) +#endif + ! clean-up deallocate(compLabels) - + if (allocated(comms)) deallocate(comms) + if (allocated(comps)) deallocate(comps) + if (allocated(comp_iamin)) deallocate(comp_iamin) + if (allocated(comp_comm_iam)) deallocate(comp_comm_iam) + end subroutine !----------------------------------------------------------------------------- @@ -4005,6 +4094,9 @@ subroutine SetRunSequence(driver, rc) ! local variables character(ESMF_MAXSTR) :: name + type(ESMF_CplComp), pointer :: connectorList(:) + integer :: i + type(NUOPC_FreeFormat) :: attrFF #if ESMF_VERSION_MAJOR >= 8 type(ESMF_Config) :: config type(NUOPC_FreeFormat) :: runSeqFF @@ -4031,6 +4123,26 @@ subroutine SetRunSequence(driver, rc) autoAddConnectors=.true., rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, file=trim(name)//":"//__FILE__)) return ! bail out + + ! read CONNECTOR attributes from config file into FreeFormat + attrFF = NUOPC_FreeFormatCreate(config, & + label="CONNECTOR_attributes::", relaxedflag=.true., rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, file=__FILE__)) return ! bail out + nullify(connectorList) + call NUOPC_DriverGetComp(driver, compList=connectorList, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, file=__FILE__)) return ! bail out + do i=1, size(connectorList) + call NUOPC_CompAttributeIngest(connectorList(i), attrFF, & + addFlag=.true., rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, file=__FILE__)) return ! bail out + enddo + deallocate(connectorList) + call NUOPC_FreeFormatDestroy(attrFF, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, file=__FILE__)) return ! bail out #else ! access runSeq in the config call SetFromConfig(driver, mode="setRunSequence", rc=rc) @@ -4690,7 +4802,6 @@ subroutine AddAttributes(gcomp, driver, config, compid, compname, inst_suffix, r file=__FILE__)) & return ! bail out else - print*, trim(attrList(n)) call NUOPC_CompAttributeGet(driver, name=trim(attrList(n)), value=cvalue, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, & From 279746ab838119098f167a07c5356c5b02ba5a0d Mon Sep 17 00:00:00 2001 From: Ufuk Turuncoglu Date: Thu, 10 Jun 2021 00:08:20 -0500 Subject: [PATCH 4/6] ad capability to handle modelio attribute in nems.configure --- src/module_EARTH_GRID_COMP.F90 | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/module_EARTH_GRID_COMP.F90 b/src/module_EARTH_GRID_COMP.F90 index 01dd5477..3e91d9a4 100644 --- a/src/module_EARTH_GRID_COMP.F90 +++ b/src/module_EARTH_GRID_COMP.F90 @@ -452,6 +452,20 @@ subroutine SetModelServices(driver, rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return call NUOPC_FreeFormatDestroy(attrFF, rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return + + call ESMF_ConfigFindNextLabel(config, & + label=trim(prefix)//"_modelio::", isPresent=isPresent, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + if (isPresent) then + attrFF = NUOPC_FreeFormatCreate(config, & + label=trim(prefix)//"_modelio::", relaxedflag=.true., rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + call NUOPC_CompAttributeIngest(comp, attrFF, addFlag=.true., rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + call NUOPC_FreeFormatDestroy(attrFF, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + endif + ! clean-up deallocate(petList) From 5e6ad3a864c18230a7fd3e372f8e85d8cadef00b Mon Sep 17 00:00:00 2001 From: Ufuk Turuncoglu Date: Fri, 11 Jun 2021 15:35:33 -0500 Subject: [PATCH 5/6] add cdeps data ocean support --- src/module_EARTH_GRID_COMP.F90 | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/module_EARTH_GRID_COMP.F90 b/src/module_EARTH_GRID_COMP.F90 index 3e91d9a4..867b97e0 100644 --- a/src/module_EARTH_GRID_COMP.F90 +++ b/src/module_EARTH_GRID_COMP.F90 @@ -64,6 +64,9 @@ MODULE module_EARTH_GRID_COMP #endif #ifdef FRONT_MOM6 use FRONT_MOM6, only: MOM6_SS => SetServices +#endif +#ifdef FRONT_CDEPS_DOCN + use FRONT_CDEPS_DOCN, only: DOCN_SS => SetServices #endif ! - Handle build time ICE options: #ifdef FRONT_CICE6 @@ -380,6 +383,14 @@ subroutine SetModelServices(driver, rc) found_comp = .true. end if #endif +#ifdef FRONT_CDEPS_DOCN + if (trim(model) == "docn") then + call NUOPC_DriverAddComp(driver, trim(prefix), DOCN_SS, & + petList=petList, comp=comp, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + found_comp = .true. + end if +#endif #ifdef FRONT_CICE6 if (trim(model) == "cice6") then call NUOPC_DriverAddComp(driver, trim(prefix), CICE6_SS, & From 145033d95bdf194a73c49cd9dc5c98fc4afcae19 Mon Sep 17 00:00:00 2001 From: Ufuk Turuncoglu Date: Sun, 13 Jun 2021 01:15:53 -0500 Subject: [PATCH 6/6] fix undefined variable issue --- src/module_EARTH_GRID_COMP.F90 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/module_EARTH_GRID_COMP.F90 b/src/module_EARTH_GRID_COMP.F90 index 867b97e0..d287fcbe 100644 --- a/src/module_EARTH_GRID_COMP.F90 +++ b/src/module_EARTH_GRID_COMP.F90 @@ -249,12 +249,12 @@ subroutine SetModelServices(driver, rc) integer :: petListBounds(2) integer :: componentCount type(NUOPC_FreeFormat) :: attrFF, fdFF - logical :: found_comp + logical :: found_comp + logical :: isPresent #ifdef CMEPS logical :: read_restart character(ESMF_MAXSTR) :: cvalue character(len=5) :: inst_suffix - logical :: isPresent #endif rc = ESMF_SUCCESS