diff --git a/src/gen_support.F90 b/src/gen_support.F90 index a7a62dfde..0b89fc796 100644 --- a/src/gen_support.F90 +++ b/src/gen_support.F90 @@ -85,11 +85,7 @@ subroutine smooth_nod2D(arr, N, partit, mesh) arr(node)=work_array(node) ENDDO !$OMP END PARALLEL DO -!$OMP BARRIER -!$OMP MASTER call exchange_nod(arr, partit) -!$OMP END MASTER -!$OMP BARRIER END DO deallocate(work_array) end subroutine smooth_nod2D diff --git a/src/ice_EVP.F90 b/src/ice_EVP.F90 index b646bc6da..9430af743 100755 --- a/src/ice_EVP.F90 +++ b/src/ice_EVP.F90 @@ -819,9 +819,6 @@ subroutine EVPdynamics(ice, partit, mesh) !_______________________________________________________________________ call exchange_nod(U_ice,V_ice,partit, luse_g2g = .true.) -!ifndef ENABLE_OPENACC -!$OMP BARRIER -!endif END DO !--> do shortstep=1, ice%evp_rheol_steps end subroutine EVPdynamics diff --git a/src/ice_fct.F90 b/src/ice_fct.F90 index d60bf352d..7715163ab 100755 --- a/src/ice_fct.F90 +++ b/src/ice_fct.F90 @@ -337,9 +337,6 @@ subroutine ice_solve_low_order(ice, partit, mesh) call exchange_nod(m_templ, partit, luse_g2g = .true.) #endif -#ifndef ENABLE_OPENACC -!$OMP BARRIER -#endif end subroutine ice_solve_low_order ! ! @@ -417,9 +414,6 @@ subroutine ice_solve_high_order(ice, partit, mesh) #if defined (__oifs) || defined (__ifsinterface) call exchange_nod(dm_temp, partit, luse_g2g = .true.) #endif /* (__oifs) */ -#ifndef ENABLE_OPENACC -!$OMP BARRIER -#endif !___________________________________________________________________________ !iterate @@ -482,9 +476,6 @@ subroutine ice_solve_high_order(ice, partit, mesh) #if defined (__oifs) || defined (__ifsinterface) call exchange_nod(dm_temp, partit, luse_g2g = .true.) #endif /* (__oifs) */ -#ifndef ENABLE_OPENACC -!$OMP BARRIER -#endif end do end subroutine ice_solve_high_order ! @@ -838,6 +829,7 @@ subroutine ice_fem_fct(tr_array_id, ice, partit, mesh) !$ACC wait #if defined(_OPENMP) +!$OMP BARRIER !$OMP MASTER #endif call exchange_nod(icepminus, icepplus, partit, luse_g2g = .true.) @@ -1124,7 +1116,6 @@ subroutine ice_fem_fct(tr_array_id, ice, partit, mesh) !$ACC END DATA -!$OMP BARRIER end subroutine ice_fem_fct ! ! @@ -1517,9 +1508,6 @@ subroutine ice_update_for_div(ice, partit, mesh) #if defined (__oifs) || defined (__ifsinterface) call exchange_nod(dm_temp, partit, luse_g2g = .true.) #endif /* (__oifs) */ -#ifndef ENABLE_OPENACC -!$OMP BARRIER -#endif !___________________________________________________________________________ !iterate do n=1,num_iter_solve-1 @@ -1586,9 +1574,6 @@ subroutine ice_update_for_div(ice, partit, mesh) #if defined (__oifs) || defined (__ifsinterface) call exchange_nod(dm_temp, partit, luse_g2g = .true.) #endif /* (__oifs) */ -#ifndef ENABLE_OPENACC -!$OMP BARRIER -#endif end do #ifndef ENABLE_OPENACC diff --git a/src/ice_maEVP.F90 b/src/ice_maEVP.F90 index 62fa9c0a0..37c205d73 100644 --- a/src/ice_maEVP.F90 +++ b/src/ice_maEVP.F90 @@ -856,16 +856,20 @@ subroutine EVPdynamics_m(ice, partit, mesh) end do ! --> do ed=1,myDim_edge2D !$OMP END DO !_______________________________________________________________________ +!$OMP BARRIER !$OMP MASTER call exchange_nod_begin(u_ice_aux, v_ice_aux, partit) !$OMP END MASTER !$OMP BARRIER +! Some thing wierd about why init this during communication +! can't it be moved after exchange_end? !$OMP DO do row=1, myDim_nod2d u_rhs_ice(row)=0.0_WP v_rhs_ice(row)=0.0_WP end do !$OMP END DO +!$OMP BARRIER !$OMP MASTER call exchange_nod_end(partit) !$OMP END MASTER diff --git a/src/ice_oce_coupling.F90 b/src/ice_oce_coupling.F90 index 55ea54eca..6be6af2b3 100755 --- a/src/ice_oce_coupling.F90 +++ b/src/ice_oce_coupling.F90 @@ -350,7 +350,6 @@ subroutine oce_fluxes(ice, dynamics, tracers, partit, mesh) if (use_cavity) then call cavity_heat_water_fluxes_3eq(ice, dynamics, tracers, partit, mesh) call exchange_nod(heat_flux, water_flux, partit) -!$OMP BARRIER end if !___________________________________________________________________________ diff --git a/src/ice_thermo_oce.F90 b/src/ice_thermo_oce.F90 index fd5f35f3b..83932828d 100755 --- a/src/ice_thermo_oce.F90 +++ b/src/ice_thermo_oce.F90 @@ -226,6 +226,7 @@ subroutine thermodynamics(ice, partit, mesh) ustar_aux(i)=sqrt(ustar*ice%cd_oce_ice) end do !$OMP END DO +!$OMP BARRIER !$OMP MASTER call exchange_nod(ustar_aux, partit) !$OMP END MASTER diff --git a/src/oce_adv_tra_driver.F90 b/src/oce_adv_tra_driver.F90 index 52125f40d..b435a0229 100644 --- a/src/oce_adv_tra_driver.F90 +++ b/src/oce_adv_tra_driver.F90 @@ -246,7 +246,6 @@ subroutine do_oce_adv_tra(dt, vel, w, wi, we, tr_num, dynamics, tracers, partit, call adv_tra_ver_upw1(w, ttf, partit, mesh, adv_flux_ver, o_init_zero=.true.) end if call exchange_nod(fct_LO, partit, luse_g2g = .true.) -!$OMP BARRIER end if do_zero_flux=.true. if (trim(tracers%data(tr_num)%tra_adv_lim)=='FCT') do_zero_flux=.false. diff --git a/src/oce_adv_tra_fct.F90 b/src/oce_adv_tra_fct.F90 index f9a382787..e802eafc2 100644 --- a/src/oce_adv_tra_fct.F90 +++ b/src/oce_adv_tra_fct.F90 @@ -397,6 +397,7 @@ subroutine oce_tra_adv_fct(dt, ttf, lo, adf_h, adf_v, fct_ttf_min, fct_ttf_max, !$ACC END PARALLEL LOOP #endif ! fct_minus and fct_plus must be known to neighbouring PE +!$OMP BARRIER !$OMP MASTER call exchange_nod(fct_plus, fct_minus, partit, luse_g2g = .true.) !$OMP END MASTER diff --git a/src/oce_ale.F90 b/src/oce_ale.F90 index 973f81161..cd6f13922 100644 --- a/src/oce_ale.F90 +++ b/src/oce_ale.F90 @@ -1803,7 +1803,6 @@ subroutine compute_ssh_rhs_ale(dynamics, partit, mesh) end if !$OMP END PARALLEL call exchange_nod(ssh_rhs, partit) -!$OMP BARRIER end subroutine compute_ssh_rhs_ale ! ! @@ -1926,7 +1925,6 @@ subroutine compute_hbar_ale(dynamics, partit, mesh) end do !$OMP END PARALLEL DO call exchange_nod(ssh_rhs_old, partit) -!$OMP BARRIER end if !___________________________________________________________________________ ! update the thickness @@ -1943,7 +1941,6 @@ subroutine compute_hbar_ale(dynamics, partit, mesh) end do !$OMP END PARALLEL DO call exchange_nod(hbar, partit) -!$OMP BARRIER !___________________________________________________________________________ ! fill the array for updating the stiffness matrix !$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(elem, elnodes) @@ -2486,7 +2483,6 @@ subroutine vert_vel_ale(dynamics, partit, mesh) call exchange_nod(Wvel, partit) call exchange_nod(hnode_new, partit) ! Or extend cycles above if (Fer_GM) call exchange_nod(fer_Wvel, partit) -!$OMP BARRIER !___________________________________________________________________________ ! calc vertical CFL criteria for debugging purpose and vertical Wvel splitting !$OMP PARALLEL DO diff --git a/src/oce_ale_mixing_kpp.F90 b/src/oce_ale_mixing_kpp.F90 index 22f93f4ca..31bbf526c 100755 --- a/src/oce_ale_mixing_kpp.F90 +++ b/src/oce_ale_mixing_kpp.F90 @@ -374,7 +374,6 @@ SUBROUTINE oce_mixing_KPP(viscAE, diffK, dynamics, tracers, partit, mesh) call smooth_nod(blmc(:,:,j), 3, partit, mesh) end do end if -!$OMP BARRIER ! then combine blmc and viscA/diffK !$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(node, nz, nzmin, nzmax) DO node=1, myDim_nod2D @@ -400,7 +399,6 @@ SUBROUTINE oce_mixing_KPP(viscAE, diffK, dynamics, tracers, partit, mesh) ! OVER ELEMENTS call exchange_nod(viscA, partit) !Warning: don't forget to communicate before averaging on elements!!! -!$OMP BARRIER minmix=3.0e-3_WP !$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(elem, elnodes, nz, nzmin, nzmax) DO elem = 1, myDim_elem2D @@ -597,7 +595,6 @@ SUBROUTINE bldepth(partit, mesh) call exchange_nod(hbl, partit) call smooth_nod(hbl, 3, partit, mesh) end if -!$OMP BARRIER !$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(node, nz, nzmin, nzmax, dzup, coeff_sw) DO node=1, myDim_nod2D @@ -784,7 +781,6 @@ subroutine ri_iwmix(viscA, diffK, dynamics, tracers, partit, mesh) IF (smooth_Ri_hor) then call smooth_nod(diffK(:,:,1), 3, partit, mesh) END IF -!$OMP BARRIER !___________________________________________________________________________ ! compute viscA and diffK !$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(node, nz, nzmin, nzmax, Rigg, ratio, frit, Kv0_b) diff --git a/src/oce_ale_pressure_bv.F90 b/src/oce_ale_pressure_bv.F90 index 939cb1745..2cac272af 100644 --- a/src/oce_ale_pressure_bv.F90 +++ b/src/oce_ale_pressure_bv.F90 @@ -491,7 +491,6 @@ subroutine pressure_bv(tracers, partit, mesh) end if end do !$OMP END DO -!$OMP BARRIER !$OMP END PARALLEL call smooth_nod (bvfreq, 1, partit, mesh) end subroutine pressure_bv @@ -2835,7 +2834,6 @@ subroutine sw_alpha_beta(TF1,SF1, partit, mesh) end do end do !$OMP END DO -!$OMP BARRIER !$OMP END PARALLEL call exchange_nod(sw_alpha, partit) call exchange_nod(sw_beta, partit) @@ -2925,7 +2923,6 @@ subroutine compute_sigma_xy(TF1,SF1, partit, mesh) sigma_xy(2,uln:nln,n) = (-sw_alpha(uln:nln,n)*ty(uln:nln)+sw_beta(uln:nln,n)*sy(uln:nln))/vol(uln:nln)*density_0 END DO !$OMP END DO -!$OMP BARRIER !$OMP END PARALLEL ! call exchange_nod(sigma_xy, partit) CALL MPI_BARRIER(MPI_COMM_FESOM,MPIerr) @@ -2985,7 +2982,6 @@ subroutine compute_neutral_slope(partit, mesh) enddo enddo !$OMP END DO -!$OMP BARRIER !$OMP END PARALLEL call exchange_nod(neutral_slope, partit) call exchange_nod(slope_tapered, partit) diff --git a/src/oce_ale_tracer.F90 b/src/oce_ale_tracer.F90 index febfdadeb..f826ca342 100644 --- a/src/oce_ale_tracer.F90 +++ b/src/oce_ale_tracer.F90 @@ -243,7 +243,6 @@ subroutine solve_tracers_ale(ice, dynamics, tracers, partit, mesh) call relax_to_clim(tr_num, tracers, partit, mesh) end if call exchange_nod(tracers%data(tr_num)%values(:,:), partit) -!$OMP BARRIER end do !!! !$ACC UPDATE HOST (tracers%work%fct_ttf_min, tracers%work%fct_ttf_max, tracers%work%fct_plus, tracers%work%fct_minus) & !!! !$ACC HOST (tracers%work%edge_up_dn_grad) @@ -1236,6 +1235,7 @@ SUBROUTINE diff_part_bh(tr_num, dynamics, tracers, partit, mesh) #endif END DO !$OMP END DO +!$OMP BARRIER !$OMP MASTER call exchange_nod(temporary_ttf, partit) !$OMP END MASTER @@ -1284,7 +1284,6 @@ SUBROUTINE diff_part_bh(tr_num, dynamics, tracers, partit, mesh) !$OMP END DO !$OMP END PARALLEL call exchange_nod(ttf, partit) -!$OMP BARRIER end subroutine diff_part_bh ! ! diff --git a/src/oce_ale_vel_rhs.F90 b/src/oce_ale_vel_rhs.F90 index b951767cb..c43c38cc4 100644 --- a/src/oce_ale_vel_rhs.F90 +++ b/src/oce_ale_vel_rhs.F90 @@ -442,6 +442,7 @@ subroutine momentum_adv_scalar(dynamics, partit, mesh) end do !-->do n=1,myDim_nod2d !$OMP END DO !___________________________________________________________________________ +!$OMP BARRIER !$OMP MASTER call exchange_nod(UVnode_rhs, partit) !$OMP END MASTER diff --git a/src/oce_dyn.F90 b/src/oce_dyn.F90 index d7d5e6f09..c1bdf8974 100755 --- a/src/oce_dyn.F90 +++ b/src/oce_dyn.F90 @@ -145,7 +145,6 @@ SUBROUTINE update_vel(dynamics, partit, mesh) END DO END DO !$OMP END PARALLEL DO -!$OMP BARRIER !!PS Why we do this here eta_n is anyway overwriten through ... !!PS do node=1, myDim_nod2D+eDim_nod2D !!PS if (ulevels_nod2D(node)==1) eta_n(node)=alpha*hbar(node)+(1.0_WP-alpha)*hbar_old(node) @@ -207,7 +206,6 @@ subroutine compute_vel_nodes(dynamics, partit, mesh) END DO END DO !$OMP END PARALLEL DO -!$OMP BARRIER call exchange_nod(UVnode, partit) end subroutine compute_vel_nodes ! @@ -345,6 +343,7 @@ SUBROUTINE visc_filt_bcksct(dynamics, partit, mesh) #endif END DO !$OMP END DO +!$OMP BARRIER !$OMP MASTER call exchange_elem(U_b, partit) call exchange_elem(V_b, partit) @@ -372,6 +371,7 @@ SUBROUTINE visc_filt_bcksct(dynamics, partit, mesh) END DO END DO !$OMP END DO +!$OMP BARRIER !$OMP MASTER call exchange_nod(U_c, V_c, partit) !$OMP END MASTER @@ -485,6 +485,7 @@ SUBROUTINE visc_filt_bilapl(dynamics, partit, mesh) END DO END DO !$OMP END DO +!$OMP BARRIER !$OMP MASTER call exchange_elem(U_c, partit) call exchange_elem(V_c, partit) @@ -608,6 +609,7 @@ SUBROUTINE visc_filt_bidiff(dynamics, partit, mesh) END DO !$OMP END DO +!$OMP BARRIER !$OMP MASTER call exchange_elem(U_c, partit) call exchange_elem(V_c, partit) @@ -859,7 +861,6 @@ subroutine check_validviscopt_5(partit, mesh) end do !$OMP END DO !$OMP END PARALLEL -!$OMP BARRIER !___________________________________________________________________________ ! compute global mean ratio --> core2 Ratio=4.26 (eddy parameterizted), diff --git a/src/oce_fer_gm.F90 b/src/oce_fer_gm.F90 index dbf7d08fb..88cc08c9a 100644 --- a/src/oce_fer_gm.F90 +++ b/src/oce_fer_gm.F90 @@ -158,7 +158,6 @@ subroutine fer_solve_Gamma(partit, mesh) END DO !!! cycle over nodes !$OMP END DO !$OMP END PARALLEL -!$OMP BARRIER call exchange_nod(fer_gamma, partit) END subroutine fer_solve_Gamma ! @@ -206,7 +205,6 @@ subroutine fer_gamma2vel(dynamics, partit, mesh) END DO !$OMP END PARALLEL DO call exchange_elem(fer_uv, partit) -!$OMP BARRIER end subroutine fer_gamma2vel ! ! @@ -412,7 +410,6 @@ subroutine init_Redi_GM(partit, mesh) !fer_compute_C_K_Redi end do !$OMP END DO !$OMP END PARALLEL -!$OMP BARRIER if (Fer_GM) call exchange_nod(fer_c, partit) if (Fer_GM) call exchange_nod(fer_k, partit) if (Redi) call exchange_nod(Ki, partit) diff --git a/src/oce_tracer_mod.F90 b/src/oce_tracer_mod.F90 index 721f3eaf3..2376be131 100755 --- a/src/oce_tracer_mod.F90 +++ b/src/oce_tracer_mod.F90 @@ -49,7 +49,6 @@ SUBROUTINE init_tracers_AB(tr_num, tracers, partit, mesh) if (flag_debug .and. partit%mype==0) print *, achar(27)//'[38m'//' --> call tracer_gradient_z'//achar(27)//'[0m' call tracer_gradient_z(tracers%data(tr_num)%values, partit, mesh) !WHY NOT AB HERE? DSIDOREN! call exchange_elem_end(partit) ! tr_xy used in fill_up_dn_grad -!$OMP BARRIER call exchange_nod_begin(tr_z, partit) ! not used in fill_up_dn_grad diff --git a/src/solver.F90 b/src/solver.F90 index 7b1c7d503..0b06cbdff 100644 --- a/src/solver.F90 +++ b/src/solver.F90 @@ -162,7 +162,6 @@ subroutine ssh_solve_cg(x, rhs, solverinfo, partit, mesh) END DO !$OMP END PARALLEL DO call exchange_nod(rr, partit) -!$OMP BARRIER ! ============= ! z_0=M^{-1} r_0 (M^{-1} is the precondit. matrix) ! pp is the search direction @@ -233,7 +232,6 @@ subroutine ssh_solve_cg(x, rhs, solverinfo, partit, mesh) ! New z ! =========== call exchange_nod(rr, partit) ! Update before matrix-vector multiplications -!$OMP BARRIER !$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(row) DO row=1, myDim_nod2D zz(row)= sum(pr_values(rptr(row):rptr(row+1)-1)*rr(cind(rptr(row):rptr(row+1)-1))) @@ -257,7 +255,6 @@ subroutine ssh_solve_cg(x, rhs, solverinfo, partit, mesh) call MPI_Allreduce(MPI_IN_PLACE, sprod, 2, MPI_DOUBLE, MPI_SUM, partit%MPI_COMM_FESOM, MPIerr) -!$OMP BARRIER ! =========== ! Exit if tolerance is achieved ! =========== @@ -277,7 +274,6 @@ subroutine ssh_solve_cg(x, rhs, solverinfo, partit, mesh) END DO ! At the end: The result is in X, but it needs a halo exchange. call exchange_nod(x, partit) -!$OMP BARRIER end subroutine ssh_solve_cg ! ===================================================================