From 4072f6f51a13a3f1d7199dad9d001d180e31a2e4 Mon Sep 17 00:00:00 2001 From: "G. Dylan Dickerson" Date: Fri, 2 Aug 2024 17:45:29 -0600 Subject: [PATCH 1/4] Prepare atm_set_smlstep_pert_variables_work for OpenACC porting Ensures that lines of code within a loop or conditional are apparent to the reader. This change only affects whitespace to indent code within an if-condition. --- .../dynamics/mpas_atm_time_integration.F | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F index fc8ce795e1..94bafd1553 100644 --- a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F +++ b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F @@ -2069,19 +2069,19 @@ subroutine atm_set_smlstep_pert_variables_work(nCells, nEdges, nCellsSolve, & do iCell=cellSolveStart,cellSolveEnd if (bdyMaskCell(iCell) <= nRelaxZone) then ! no conversion in specified zone, regional_MPAS - do i=1,nEdgesOnCell(iCell) - iEdge = edgesOnCell(i,iCell) + do i=1,nEdgesOnCell(iCell) + iEdge = edgesOnCell(i,iCell) !DIR$ IVDEP - do k = 2, nVertLevels - flux = edgesOnCell_sign(i,iCell) * (fzm(k) * u_tend(k,iEdge) + fzp(k) * u_tend(k-1,iEdge)) - w_tend(k,iCell) = w_tend(k,iCell) & - - (zb_cell(k,i,iCell) + sign(1.0_RKIND, u_tend(k,iEdge)) * zb3_cell(k,i,iCell)) * flux + do k = 2, nVertLevels + flux = edgesOnCell_sign(i,iCell) * (fzm(k) * u_tend(k,iEdge) + fzp(k) * u_tend(k-1,iEdge)) + w_tend(k,iCell) = w_tend(k,iCell) & + - (zb_cell(k,i,iCell) + sign(1.0_RKIND, u_tend(k,iEdge)) * zb3_cell(k,i,iCell)) * flux + end do end do - end do !DIR$ IVDEP - do k = 2, nVertLevels - w_tend(k,iCell) = ( fzm(k) * zz(k,iCell) + fzp(k) * zz(k-1,iCell) ) * w_tend(k,iCell) - end do + do k = 2, nVertLevels + w_tend(k,iCell) = ( fzm(k) * zz(k,iCell) + fzp(k) * zz(k-1,iCell) ) * w_tend(k,iCell) + end do end if ! no conversion in specified zone end do From db991924d9d6a27e2d75442796147d36f1bad7ef Mon Sep 17 00:00:00 2001 From: "G. Dylan Dickerson" Date: Fri, 2 Aug 2024 18:32:32 -0600 Subject: [PATCH 2/4] Copy invariant fields used in atm_set_smlstep_pert_variables_work with OpenACC Ensure that the invariant fields that are used in this work routine are present on the device from model startup to model shutdown. Fields are added to the device with copyin directives and other code within mpas_atm_dynamics_init and fields are deleted from the device with matching delete directives and code in mpas_atm_dynamics_finalize. --- .../dynamics/mpas_atm_time_integration.F | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F index 94bafd1553..3b408aad96 100644 --- a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F +++ b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F @@ -212,6 +212,11 @@ subroutine mpas_atm_dynamics_init(domain) real (kind=RKIND), dimension(:), pointer :: invAreaCell integer, dimension(:), pointer :: bdyMaskCell integer, dimension(:), pointer :: bdyMaskEdge + real (kind=RKIND), dimension(:,:), pointer :: zz + real (kind=RKIND), dimension(:,:,:), pointer :: zb_cell + real (kind=RKIND), dimension(:,:,:), pointer :: zb3_cell + real (kind=RKIND), dimension(:), pointer :: fzm + real (kind=RKIND), dimension(:), pointer :: fzp #endif @@ -271,6 +276,22 @@ subroutine mpas_atm_dynamics_init(domain) call mpas_pool_get_array(mesh, 'bdyMaskEdge', bdyMaskEdge) !$acc enter data copyin(bdyMaskEdge) + + call mpas_pool_get_array(mesh, 'zz', zz) + !$acc enter data copyin(zz) + + call mpas_pool_get_array(mesh, 'zb_cell', zb_cell) + !$acc enter data copyin(zb_cell) + + call mpas_pool_get_array(mesh, 'zb3_cell', zb3_cell) + !$acc enter data copyin(zb3_cell) + + call mpas_pool_get_array(mesh, 'fzm', fzm) + !$acc enter data copyin(fzm) + + call mpas_pool_get_array(mesh, 'fzp', fzp) + !$acc enter data copyin(fzp) + #endif end subroutine mpas_atm_dynamics_init @@ -319,6 +340,11 @@ subroutine mpas_atm_dynamics_finalize(domain) real (kind=RKIND), dimension(:), pointer :: invAreaCell integer, dimension(:), pointer :: bdyMaskCell integer, dimension(:), pointer :: bdyMaskEdge + real (kind=RKIND), dimension(:,:), pointer :: zz + real (kind=RKIND), dimension(:,:,:), pointer :: zb_cell + real (kind=RKIND), dimension(:,:,:), pointer :: zb3_cell + real (kind=RKIND), dimension(:), pointer :: fzm + real (kind=RKIND), dimension(:), pointer :: fzp #endif @@ -378,6 +404,21 @@ subroutine mpas_atm_dynamics_finalize(domain) call mpas_pool_get_array(mesh, 'bdyMaskEdge', bdyMaskEdge) !$acc exit data delete(bdyMaskEdge) + + call mpas_pool_get_array(mesh, 'zz', zz) + !$acc exit data delete(zz) + + call mpas_pool_get_array(mesh, 'zb_cell', zb_cell) + !$acc exit data delete(zb_cell) + + call mpas_pool_get_array(mesh, 'zb3_cell', zb3_cell) + !$acc exit data delete(zb3_cell) + + call mpas_pool_get_array(mesh, 'fzm', fzm) + !$acc exit data delete(fzm) + + call mpas_pool_get_array(mesh, 'fzp', fzp) + !$acc exit data delete(fzp) #endif end subroutine mpas_atm_dynamics_finalize From 1667ae13f63ca16cf85376a2e0f8e263709be34d Mon Sep 17 00:00:00 2001 From: "G. Dylan Dickerson" Date: Mon, 5 Aug 2024 14:28:19 -0600 Subject: [PATCH 3/4] Add parallel and loop directives to atm_set_smlstep_pert_variables_work Add directives for an initial port of this routine using OpenACC. --- src/core_atmosphere/dynamics/mpas_atm_time_integration.F | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F index 3b408aad96..27c7b92b23 100644 --- a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F +++ b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F @@ -2107,12 +2107,16 @@ subroutine atm_set_smlstep_pert_variables_work(nCells, nEdges, nCellsSolve, & ! this requires us to use the same flux divergence as is used in the theta eqn - see Klemp et al MWR 2003. !! do iCell=cellStart,cellEnd + !$acc parallel + !$acc loop gang worker do iCell=cellSolveStart,cellSolveEnd if (bdyMaskCell(iCell) <= nRelaxZone) then ! no conversion in specified zone, regional_MPAS + !$acc loop seq do i=1,nEdgesOnCell(iCell) iEdge = edgesOnCell(i,iCell) !DIR$ IVDEP + !$acc loop vector do k = 2, nVertLevels flux = edgesOnCell_sign(i,iCell) * (fzm(k) * u_tend(k,iEdge) + fzp(k) * u_tend(k-1,iEdge)) w_tend(k,iCell) = w_tend(k,iCell) & @@ -2120,11 +2124,13 @@ subroutine atm_set_smlstep_pert_variables_work(nCells, nEdges, nCellsSolve, & end do end do !DIR$ IVDEP + !$acc loop vector do k = 2, nVertLevels w_tend(k,iCell) = ( fzm(k) * zz(k,iCell) + fzp(k) * zz(k-1,iCell) ) * w_tend(k,iCell) end do end if ! no conversion in specified zone end do + !$acc end parallel end subroutine atm_set_smlstep_pert_variables_work From d122e6fb274d9dced7db9d64c7f7591bb1259b7e Mon Sep 17 00:00:00 2001 From: "G. Dylan Dickerson" Date: Mon, 5 Aug 2024 14:31:30 -0600 Subject: [PATCH 4/4] Add acc data management to atm_set_smlstep_pert_variables_work Ensure that other, non-invariant, fields are available on the device during this routine and that modified variables are copied out at the end. Timing for these transfers are reported in the output log file in the new timer: `atm_recover_smlstep_pert_variables [ACC_data_xfer]`. The 'default(present)' clause is added to the parallel directive so that any missing data causes a runtime error. --- .../dynamics/mpas_atm_time_integration.F | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F index 27c7b92b23..8effe8480c 100644 --- a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F +++ b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F @@ -2100,6 +2100,10 @@ subroutine atm_set_smlstep_pert_variables_work(nCells, nEdges, nCellsSolve, & integer :: iCell, iEdge, i, k real (kind=RKIND) :: flux + MPAS_ACC_TIMER_START('atm_set_smlstep_pert_variables [ACC_data_xfer]') + !$acc enter data copyin(u_tend, w_tend) + MPAS_ACC_TIMER_STOP('atm_set_smlstep_pert_variables [ACC_data_xfer]') + ! we solve for omega instead of w (see Klemp et al MWR 2007), ! so here we change the w_p tendency to an omega_p tendency @@ -2107,7 +2111,7 @@ subroutine atm_set_smlstep_pert_variables_work(nCells, nEdges, nCellsSolve, & ! this requires us to use the same flux divergence as is used in the theta eqn - see Klemp et al MWR 2003. !! do iCell=cellStart,cellEnd - !$acc parallel + !$acc parallel default(present) !$acc loop gang worker do iCell=cellSolveStart,cellSolveEnd @@ -2132,6 +2136,11 @@ subroutine atm_set_smlstep_pert_variables_work(nCells, nEdges, nCellsSolve, & end do !$acc end parallel + MPAS_ACC_TIMER_START('atm_set_smlstep_pert_variables [ACC_data_xfer]') + !$acc exit data delete(u_tend) + !$acc exit data copyout(w_tend) + MPAS_ACC_TIMER_STOP('atm_set_smlstep_pert_variables [ACC_data_xfer]') + end subroutine atm_set_smlstep_pert_variables_work