From 4044e1648a71ab86f19d6114aaff0ef802676f6c Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 10:09:36 +0200 Subject: [PATCH 01/79] new bottom height --- docs/src/grids.md | 4 +- examples/internal_tide.jl | 2 +- src/Grids/grid_utils.jl | 9 ++ .../abstract_grid_fitted_boundary.jl | 18 --- .../distributed_immersed_boundaries.jl | 10 +- src/ImmersedBoundaries/grid_fitted_bottom.jl | 105 +++++++++--------- src/ImmersedBoundaries/partial_cell_bottom.jl | 56 ++++++---- src/MultiRegion/multi_region_grid.jl | 4 +- .../immersed_couette_flow.jl | 4 +- 9 files changed, 105 insertions(+), 107 deletions(-) diff --git a/docs/src/grids.md b/docs/src/grids.md index 71ec02adc0..5e25849b05 100644 --- a/docs/src/grids.md +++ b/docs/src/grids.md @@ -207,11 +207,11 @@ mountain_grid = ImmersedBoundaryGrid(grid, GridFittedBottom(mountain)) ```@example grids using CairoMakie -h = mountain_grid.immersed_boundary.bottom_height +h = mountain_grid.immersed_boundary.z_bottom fig = Figure(size=(600, 600)) ax = Axis(fig[2, 1], xlabel="x (m)", ylabel="y (m)", aspect=1) -hm = heatmap!(ax, h) +hm = heatmap!(ax, - h) Colorbar(fig[1, 1], hm, vertical=false, label="Bottom height (m)") current_figure() diff --git a/examples/internal_tide.jl b/examples/internal_tide.jl index 5b31f18418..a08dedd65a 100644 --- a/examples/internal_tide.jl +++ b/examples/internal_tide.jl @@ -52,7 +52,7 @@ grid = ImmersedBoundaryGrid(underlying_grid, PartialCellBottom(bottom)) # Let's see how the domain with the bathymetry is. x = xnodes(grid, Center()) -bottom_boundary = interior(grid.immersed_boundary.bottom_height, :, 1, 1) +bottom_boundary = interior(grid.immersed_boundary.z_bottom, :, 1, 1) top_boundary = 0 * x using CairoMakie diff --git a/src/Grids/grid_utils.jl b/src/Grids/grid_utils.jl index c881ebeb86..afd199abe0 100644 --- a/src/Grids/grid_utils.jl +++ b/src/Grids/grid_utils.jl @@ -314,6 +314,15 @@ coordinate_summary(topo, Δ::Union{AbstractVector, AbstractMatrix}, name) = name, prettysummary(minimum(parent(Δ))), name, prettysummary(maximum(parent(Δ)))) +##### +##### Bottom height +##### + +@inline bottom_heightᶜᶜᵃ(i, j, k, grid) = grid.Lz +@inline bottom_heightᶜᶠᵃ(i, j, k, grid) = grid.Lz +@inline bottom_heightᶠᶜᵃ(i, j, k, grid) = grid.Lz +@inline bottom_heightᶠᶠᵃ(i, j, k, grid) = grid.Lz + ##### ##### Spherical geometry ##### diff --git a/src/ImmersedBoundaries/abstract_grid_fitted_boundary.jl b/src/ImmersedBoundaries/abstract_grid_fitted_boundary.jl index dcc442b6ac..67d9de83d3 100644 --- a/src/ImmersedBoundaries/abstract_grid_fitted_boundary.jl +++ b/src/ImmersedBoundaries/abstract_grid_fitted_boundary.jl @@ -52,21 +52,3 @@ const AGFB = AbstractGridFittedBoundary @inline immersed_cell(i, j, k, grid::AbstractGrid{<:Any, <:Any, Flat, Flat}, ib::AGFB) = _immersed_cell(i, 1, 1, grid, ib) @inline immersed_cell(i, j, k, grid::AbstractGrid{<:Any, Flat, Flat, Flat}, ib::AGFB) = _immersed_cell(1, 1, 1, grid, ib) end - -function clamp_bottom_height!(bottom_field, grid) - launch!(architecture(grid), grid, :xy, _clamp_bottom_height!, bottom_field, grid) - return nothing -end - -const c = Center() -const f = Face() - -@kernel function _clamp_bottom_height!(z, grid) - i, j = @index(Global, NTuple) - Nz = size(grid, 3) - zmin = znode(i, j, 1, grid, c, c, f) - zmax = znode(i, j, Nz+1, grid, c, c, f) - @inbounds z[i, j, 1] = clamp(z[i, j, 1], zmin, zmax) -end - - diff --git a/src/ImmersedBoundaries/distributed_immersed_boundaries.jl b/src/ImmersedBoundaries/distributed_immersed_boundaries.jl index 4b19202b92..10473ba015 100644 --- a/src/ImmersedBoundaries/distributed_immersed_boundaries.jl +++ b/src/ImmersedBoundaries/distributed_immersed_boundaries.jl @@ -14,7 +14,7 @@ function reconstruct_global_grid(grid::ImmersedBoundaryGrid) arch = grid.architecture local_ib = grid.immersed_boundary global_ug = reconstruct_global_grid(grid.underlying_grid) - global_ib = getnamewrapper(local_ib)(construct_global_array(arch, local_ib.bottom_height, size(grid))) + global_ib = getnamewrapper(local_ib)(construct_global_array(arch, local_ib.z_bottom, size(grid))) return ImmersedBoundaryGrid(global_ug, global_ib) end @@ -33,9 +33,9 @@ function scatter_local_grids(global_grid::ImmersedBoundaryGrid, arch::Distribute local_ug = scatter_local_grids(ug, arch, local_size) # Kinda hacky - local_bottom_height = partition(ib.bottom_height, arch, local_size) + local_z_bottom = partition(ib.z_bottom, arch, local_size) ImmersedBoundaryConstructor = getnamewrapper(ib) - local_ib = ImmersedBoundaryConstructor(local_bottom_height) + local_ib = ImmersedBoundaryConstructor(local_z_bottom) return ImmersedBoundaryGrid(local_ug, local_ib) end @@ -76,10 +76,10 @@ function resize_immersed_boundary(ib::AbstractGridFittedBottom{<:OffsetArray}, g # Check that the size of a bottom field are # consistent with the size of the grid - if any(size(ib.bottom_height) .!= bottom_heigth_size) + if any(size(ib.z_bottom) .!= bottom_heigth_size) @warn "Resizing the bottom field to match the grids' halos" bottom_field = Field((Center, Center, Nothing), grid) - cpu_bottom = on_architecture(CPU(), ib.bottom_height)[1:Nx, 1:Ny] + cpu_bottom = on_architecture(CPU(), ib.z_bottom)[1:Nx, 1:Ny] set!(bottom_field, cpu_bottom) fill_halo_regions!(bottom_field) offset_bottom_array = dropdims(bottom_field.data, dims=3) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index e007f461a1..b21609dd2b 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -15,48 +15,29 @@ import Oceananigans.TurbulenceClosures: z_bottom abstract type AbstractGridFittedBottom{H} <: AbstractGridFittedBoundary end -# To enable comparison with PartialCellBottom in the limiting case that -# fractional cell height is 1.0. -struct CenterImmersedCondition end -struct InterfaceImmersedCondition end - -Base.summary(::CenterImmersedCondition) = "CenterImmersedCondition" -Base.summary(::InterfaceImmersedCondition) = "InterfaceImmersedCondition" - -struct GridFittedBottom{H, I} <: AbstractGridFittedBottom{H} - bottom_height :: H - immersed_condition :: I +struct GridFittedBottom{H} <: AbstractGridFittedBottom{H} + z_bottom :: H end const GFBIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:GridFittedBottom} """ - GridFittedBottom(bottom_height, [immersed_condition=CenterImmersedCondition()]) + GridFittedBottom(z_bottom) Return a bottom immersed boundary. Keyword Arguments ================= -* `bottom_height`: an array or function that gives the height of the - bottom in absolute ``z`` coordinates. - -* `immersed_condition`: Determine whether the part of the domain that is - immersed are all the cell centers that lie below - `bottom_height` (`CenterImmersedCondition()`; default) - or all the cell faces that lie below `bottom_height` - (`InterfaceImmersedCondition()`). The only purpose of - `immersed_condition` to allow `GridFittedBottom` and - `PartialCellBottom` to have the same behavior when the - minimum fractional cell height for partial cells is set - to 0. +* `z_bottom`: an array or function that gives the height of the + bottom in absolute ``z`` coordinates. This is the height of + the bottom interface of the last ``fluid`` cell. """ -GridFittedBottom(bottom_height) = GridFittedBottom(bottom_height, CenterImmersedCondition()) function Base.summary(ib::GridFittedBottom) - zmax = maximum(ib.bottom_height) - zmin = minimum(ib.bottom_height) - zmean = mean(ib.bottom_height) + zmax = maximum(ib.z_bottom) + zmin = minimum(ib.z_bottom) + zmean = mean(ib.z_bottom) summary1 = "GridFittedBottom(" @@ -69,55 +50,69 @@ function Base.summary(ib::GridFittedBottom) return summary1 * summary2 * summary3 end -Base.summary(ib::GridFittedBottom{<:Function}) = @sprintf("GridFittedBottom(%s)", ib.bottom_height) +Base.summary(ib::GridFittedBottom{<:Function}) = @sprintf("GridFittedBottom(%s)", ib.z_bottom) function Base.show(io::IO, ib::GridFittedBottom) print(io, summary(ib), '\n') - print(io, "├── bottom_height: ", prettysummary(ib.bottom_height), '\n') - print(io, "└── immersed_condition: ", summary(ib.immersed_condition)) + print(io, "├── z_bottom: ", prettysummary(ib.z_bottom), '\n') +end + +on_architecture(arch, ib::GridFittedBottom) = GridFittedBottom(on_architecture(ib.bottom_height)) + +function on_architecture(arch, ib::GridFittedBottom{<:Field}) + architecture(ib.bottom_height) == arch && return ib + arch_grid = on_architecture(arch, ib.bottom_height.grid) + new_bottom_height = Field{Center, Center, Nothing}(arch_grid) + set!(new_bottom_height, ib.bottom_height) + fill_halo_regions!(new_bottom_height) + return GridFittedBottom(new_bottom_height, ib.immersed_condition) end +Adapt.adapt_structure(to, ib::GridFittedBottom) = GridFittedBottom(adapt(to, ib.z_bottom)) + """ ImmersedBoundaryGrid(grid, ib::GridFittedBottom) Return a grid with `GridFittedBottom` immersed boundary (`ib`). -Computes `ib.bottom_height` and wraps it in a Field. +Computes `ib.z_bottom` and wraps it in a Field. """ function ImmersedBoundaryGrid(grid, ib::GridFittedBottom) bottom_field = Field{Center, Center, Nothing}(grid) - set!(bottom_field, ib.bottom_height) - @apply_regionally clamp_bottom_height!(bottom_field, grid) + set!(bottom_field, ib.z_bottom) + @apply_regionally correct_z_bottom!(bottom_field, grid) fill_halo_regions!(bottom_field) - new_ib = GridFittedBottom(bottom_field, ib.immersed_condition) + new_ib = GridFittedBottom(bottom_field) TX, TY, TZ = topology(grid) return ImmersedBoundaryGrid{TX, TY, TZ}(grid, new_ib) end -@inline function _immersed_cell(i, j, k, underlying_grid, ib::GridFittedBottom{<:Any, <:InterfaceImmersedCondition}) - z = znode(i, j, k+1, underlying_grid, c, c, f) - h = @inbounds ib.bottom_height[i, j, 1] - return z ≤ h +correct_z_bottom!(bottom_field, grid, ib) = + launch!(architecture(grid), grid, :xy, _correct_z_bottom!, bottom_field, grid, ib) + +@kernel function _correct_z_bottom!(bottom_field, grid, ::GridFittedBottom) + i, j = @index(grid) + zb = @inbounds bottom_field[i, j, 1] + for k in 1:grid.Nz + z⁺ = znode(i, j, k+1, grid, c, c, f) + bottom_cell = zb < z⁺ + @inbounds bottom_field[i, j, 1] = ifelse(bottom_cell, z⁺, zb) + end end -@inline function _immersed_cell(i, j, k, underlying_grid, ib::GridFittedBottom{<:Any, <:CenterImmersedCondition}) - z = znode(i, j, k, underlying_grid, c, c, c) - h = @inbounds ib.bottom_height[i, j, 1] +@inline function _immersed_cell(i, j, k, underlying_grid, ib::GridFittedBottom) + z = znode(i, j, k+1, underlying_grid, c, c, f) + h = @inbounds ib.z_bottom[i, j, 1] return z ≤ h end -@inline z_bottom(i, j, ibg::GFBIBG) = @inbounds ibg.immersed_boundary.bottom_height[i, j, 1] +@inline z_bottom(i, j, ibg::GFBIBG) = @inbounds ibg.immersed_boundary.z_bottom[i, j, 1] -on_architecture(arch, ib::GridFittedBottom) = GridFittedBottom(ib.bottom_height, ib.immersed_condition) - -function on_architecture(arch, ib::GridFittedBottom{<:Field}) - architecture(ib.bottom_height) == arch && return ib - arch_grid = on_architecture(arch, ib.bottom_height.grid) - new_bottom_height = Field{Center, Center, Nothing}(arch_grid) - set!(new_bottom_height, ib.bottom_height) - fill_halo_regions!(new_bottom_height) - return GridFittedBottom(new_bottom_height, ib.immersed_condition) -end +##### +##### Bottom height +##### -Adapt.adapt_structure(to, ib::GridFittedBottom) = GridFittedBottom(adapt(to, ib.bottom_height), - ib.immersed_condition) +@inline bottom_heightᶜᶜᵃ(i, j, k, ibg::GFBIBG) = ibg.immersed_boundary.bottom_height[i, j, 1] +@inline bottom_heightᶜᶠᵃ(i, j, k, ibg::GFBIBG) = min(bottom_heightᶜᶜᵃ(i, j-1, k, ibg), bottom_heightᶜᶜᵃ(i, j, k, ibg)) +@inline bottom_heightᶠᶜᵃ(i, j, k, ibg::GFBIBG) = min(bottom_heightᶜᶜᵃ(i-1, j, k, ibg), bottom_heightᶜᶜᵃ(i, j, k, ibg)) +@inline bottom_heightᶠᶠᵃ(i, j, k, ibg::GFBIBG) = min(bottom_heightᶠᶜᵃ(i, j-1, k, ibg), bottom_heightᶠᶜᵃ(i, j, k, ibg)) diff --git a/src/ImmersedBoundaries/partial_cell_bottom.jl b/src/ImmersedBoundaries/partial_cell_bottom.jl index d08f6db815..d3581b1780 100644 --- a/src/ImmersedBoundaries/partial_cell_bottom.jl +++ b/src/ImmersedBoundaries/partial_cell_bottom.jl @@ -7,16 +7,16 @@ using Printf ##### struct PartialCellBottom{H, E} <: AbstractGridFittedBottom{H} - bottom_height :: H + z_bottom :: H minimum_fractional_cell_height :: E end const PCBIBG{FT, TX, TY, TZ} = ImmersedBoundaryGrid{FT, TX, TY, TZ, <:Any, <:PartialCellBottom} where {FT, TX, TY, TZ} function Base.summary(ib::PartialCellBottom) - zmax = maximum(parent(ib.bottom_height)) - zmin = minimum(parent(ib.bottom_height)) - zmean = mean(parent(ib.bottom_height)) + zmax = maximum(parent(ib.z_bottom)) + zmin = minimum(parent(ib.z_bottom)) + zmean = mean(parent(ib.z_bottom)) summary1 = "PartialCellBottom(" @@ -31,21 +31,21 @@ function Base.summary(ib::PartialCellBottom) end Base.summary(ib::PartialCellBottom{<:Function}) = @sprintf("PartialCellBottom(%s, ϵ=%.1f)", - prettysummary(ib.bottom_height, false), + prettysummary(ib.z_bottom, false), prettysummary(ib.minimum_fractional_cell_height)) function Base.show(io::IO, ib::PartialCellBottom) print(io, summary(ib), '\n') - print(io, "├── bottom_height: ", prettysummary(ib.bottom_height), '\n') + print(io, "├── z_bottom: ", prettysummary(ib.z_bottom), '\n') print(io, "└── minimum_fractional_cell_height: ", prettysummary(ib.minimum_fractional_cell_height)) end """ - PartialCellBottom(bottom_height; minimum_fractional_cell_height=0.2) + PartialCellBottom(z_bottom; minimum_fractional_cell_height=0.2) Return `PartialCellBottom` representing an immersed boundary with "partial" bottom cells. That is, the height of the bottommost cell in each column is reduced -to fit the provided `bottom_height`, which may be a `Field`, `Array`, or function +to fit the provided `z_bottom`, which may be a `Field`, `Array`, or function of `(x, y)`. The height of partial bottom cells is greater than @@ -56,32 +56,44 @@ minimum_fractional_cell_height * Δz, where `Δz` is the original height of the bottom cell underlying grid. """ -function PartialCellBottom(bottom_height; minimum_fractional_cell_height=0.2) - return PartialCellBottom(bottom_height, minimum_fractional_cell_height) +function PartialCellBottom(z_bottom; minimum_fractional_cell_height=0.2) + return PartialCellBottom(z_bottom, minimum_fractional_cell_height) end function ImmersedBoundaryGrid(grid, ib::PartialCellBottom) bottom_field = Field{Center, Center, Nothing}(grid) - set!(bottom_field, ib.bottom_height) - @apply_regionally clamp_bottom_height!(bottom_field, grid) + set!(bottom_field, ib.z_bottom) + @apply_regionally correct_z_bottom!(bottom_field, grid, ib.minimum_fractional_cell_height) fill_halo_regions!(bottom_field) new_ib = PartialCellBottom(bottom_field, ib.minimum_fractional_cell_height) TX, TY, TZ = topology(grid) return ImmersedBoundaryGrid{TX, TY, TZ}(grid, new_ib) end +@kernel function _correct_z_bottom!(bottom_field, grid, ib::PartialCellBottom) + i, j = @index(grid) + zb = @inbounds bottom_field[i, j, 1] + ϵ = ib.minimum_fractional_cell_height + for k in 1:grid.Nz + z⁻ = znode(i, j, k, grid, c, c, f) + Δz = Δzᶜᶜᶜ(i, j, k, underlying_grid) + bottom_cell = zb < z⁻ + Δz * (1 - ϵ) + @inbounds bottom_field[i, j, 1] = ifelse(bottom_cell, z⁻ + Δz * (1 - ϵ), zb) + end +end + function on_architecture(arch, ib::PartialCellBottom{<:Field}) - architecture(ib.bottom_height) == arch && return ib - arch_grid = on_architecture(arch, ib.bottom_height.grid) - new_bottom_height = Field{Center, Center, Nothing}(arch_grid) - copyto!(parent(new_bottom_height), parent(ib.bottom_height)) - return PartialCellBottom(new_bottom_height, ib.minimum_fractional_cell_height) + architecture(ib.z_bottom) == arch && return ib + arch_grid = on_architecture(arch, ib.z_bottom.grid) + new_z_bottom = Field{Center, Center, Nothing}(arch_grid) + copyto!(parent(new_z_bottom), parent(ib.z_bottom)) + return PartialCellBottom(new_z_bottom, ib.minimum_fractional_cell_height) end -Adapt.adapt_structure(to, ib::PartialCellBottom) = PartialCellBottom(adapt(to, ib.bottom_height), +Adapt.adapt_structure(to, ib::PartialCellBottom) = PartialCellBottom(adapt(to, ib.z_bottom), ib.minimum_fractional_cell_height) -on_architecture(to, ib::PartialCellBottom) = PartialCellBottom(on_architecture(to, ib.bottom_height), +on_architecture(to, ib::PartialCellBottom) = PartialCellBottom(on_architecture(to, ib.z_bottom), on_architecture(to, ib.minimum_fractional_cell_height)) """ @@ -108,7 +120,7 @@ Criterion is zb ≥ z - ϵ Δz @inline function _immersed_cell(i, j, k, underlying_grid, ib::PartialCellBottom) # Face node below current cell z = znode(i, j, k, underlying_grid, c, c, f) - zb = @inbounds ib.bottom_height[i, j, 1] + zb = @inbounds ib.z_bottom[i, j, 1] ϵ = ib.minimum_fractional_cell_height # z + Δz is equal to the face above the current cell Δz = Δzᶜᶜᶜ(i, j, k, underlying_grid) @@ -130,7 +142,7 @@ end z = znode(i, j, k+1, underlying_grid, c, c, f) # Get bottom height and fractional Δz parameter - h = @inbounds ib.bottom_height[i, j, 1] + h = @inbounds ib.z_bottom[i, j, 1] ϵ = ibg.immersed_boundary.minimum_fractional_cell_height # Are we in a bottom cell? @@ -176,5 +188,5 @@ YFlatPCBIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Flat, <:Any, <:Any, <:Partial @inline Δzᶠᶠᶜ(i, j, k, ibg::XFlatPCBIBG) = Δzᶜᶠᶜ(i, j, k, ibg) @inline Δzᶠᶠᶜ(i, j, k, ibg::YFlatPCBIBG) = Δzᶠᶜᶜ(i, j, k, ibg) -@inline z_bottom(i, j, ibg::PCBIBG) = @inbounds ibg.immersed_boundary.bottom_height[i, j, 1] +@inline z_bottom(i, j, ibg::PCBIBG) = @inbounds ibg.immersed_boundary.z_bottom[i, j, 1] diff --git a/src/MultiRegion/multi_region_grid.jl b/src/MultiRegion/multi_region_grid.jl index 0c67d75a49..a65a8b5dd8 100644 --- a/src/MultiRegion/multi_region_grid.jl +++ b/src/MultiRegion/multi_region_grid.jl @@ -188,8 +188,8 @@ function reconstruct_global_grid(mrg::ImmersedMultiRegionGrid) return ImmersedBoundaryGrid(global_grid, global_boundary) end -reconstruct_global_boundary(g::GridFittedBottom{<:Field}) = GridFittedBottom(reconstruct_global_field(g.bottom_height), g.immersed_condition) -reconstruct_global_boundary(g::PartialCellBottom{<:Field}) = PartialCellBottom(reconstruct_global_field(g.bottom_height), g.minimum_fractional_cell_height) +reconstruct_global_boundary(g::GridFittedBottom{<:Field}) = GridFittedBottom(reconstruct_global_field(g.z_bottom), g.immersed_condition) +reconstruct_global_boundary(g::PartialCellBottom{<:Field}) = PartialCellBottom(reconstruct_global_field(g.z_bottom), g.minimum_fractional_cell_height) reconstruct_global_boundary(g::GridFittedBoundary{<:Field}) = GridFittedBoundary(reconstruct_global_field(g.mask)) @inline getregion(mrg::ImmersedMultiRegionGrid{FT, TX, TY, TZ}, r) where {FT, TX, TY, TZ} = ImmersedBoundaryGrid{TX, TY, TZ}(_getregion(mrg.underlying_grid, r), _getregion(mrg.immersed_boundary, r)) diff --git a/validation/immersed_boundaries/immersed_couette_flow.jl b/validation/immersed_boundaries/immersed_couette_flow.jl index 90098d6ee4..cd1671f6e7 100644 --- a/validation/immersed_boundaries/immersed_couette_flow.jl +++ b/validation/immersed_boundaries/immersed_couette_flow.jl @@ -10,8 +10,8 @@ Lz = 1 underlying_grid = RectilinearGrid(size=Nz, z=(0, Lz), topology = (Flat, Flat, Bounded)) -@inline bottom_height(x, y) = 0.1 -grid = ImmersedBoundaryGrid(underlying_grid, GridFittedBottom(bottom_height)) +@inline z_bottom(x, y) = 0.1 +grid = ImmersedBoundaryGrid(underlying_grid, GridFittedBottom(z_bottom)) c_immersed_bc = ValueBoundaryCondition(1) c_top_bc = ValueBoundaryCondition(-1) From b83a6ae5098520d28f19268082d9801c6705ce87 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 10:18:09 +0200 Subject: [PATCH 02/79] now it should work --- src/Grids/Grids.jl | 1 + src/ImmersedBoundaries/ImmersedBoundaries.jl | 5 +++- src/ImmersedBoundaries/grid_fitted_bottom.jl | 10 ++++---- src/ImmersedBoundaries/partial_cell_bottom.jl | 2 +- ...distributed_split_explicit_free_surface.jl | 16 +----------- .../split_explicit_free_surface.jl | 17 +------------ .../split_explicit_free_surface_kernels.jl | 25 +++++++++++-------- ...ulti_region_split_explicit_free_surface.jl | 16 +----------- 8 files changed, 29 insertions(+), 63 deletions(-) diff --git a/src/Grids/Grids.jl b/src/Grids/Grids.jl index fd8aebbf21..bbc40581ce 100644 --- a/src/Grids/Grids.jl +++ b/src/Grids/Grids.jl @@ -18,6 +18,7 @@ export xnodes, ynodes, znodes, λnodes, φnodes export spacings export xspacings, yspacings, zspacings, xspacing, yspacing, zspacing export minimum_xspacing, minimum_yspacing, minimum_zspacing +export bottom_heightᶜᶜᵃ, bottom_heightᶠᶜᵃ, bottom_heightᶜᶠᵃ, bottom_heightᶠᶠᵃ export offset_data, new_data export on_architecture diff --git a/src/ImmersedBoundaries/ImmersedBoundaries.jl b/src/ImmersedBoundaries/ImmersedBoundaries.jl index e9281eac32..8cf5e90b08 100644 --- a/src/ImmersedBoundaries/ImmersedBoundaries.jl +++ b/src/ImmersedBoundaries/ImmersedBoundaries.jl @@ -47,16 +47,19 @@ import Oceananigans.Advection: cell_advection_timescale import Oceananigans.Grids: cpu_face_constructor_x, cpu_face_constructor_y, cpu_face_constructor_z, x_domain, y_domain, z_domain -import Oceananigans.Grids: architecture, on_architecture, with_halo, inflate_halo_size_one_dimension, +import Oceananigans.Grids: architecture, with_halo, inflate_halo_size_one_dimension, xnode, ynode, znode, λnode, φnode, node, ξnode, ηnode, rnode, ξname, ηname, rname, node_names, xnodes, ynodes, znodes, λnodes, φnodes, nodes, ξnodes, ηnodes, rnodes, + bottom_heightᶜᶜᵃ, bottom_heightᶠᶜᵃ, bottom_heightᶜᶠᵃ, bottom_heightᶠᶠᵃ, inactive_cell import Oceananigans.Coriolis: φᶠᶠᵃ +import Oceananigans.Architectures: on_architecture + import Oceananigans.Advection: _advective_momentum_flux_Uu, _advective_momentum_flux_Uv, diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index b21609dd2b..1851e39d95 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -91,7 +91,7 @@ correct_z_bottom!(bottom_field, grid, ib) = launch!(architecture(grid), grid, :xy, _correct_z_bottom!, bottom_field, grid, ib) @kernel function _correct_z_bottom!(bottom_field, grid, ::GridFittedBottom) - i, j = @index(grid) + i, j = @index(Global, NTuple) zb = @inbounds bottom_field[i, j, 1] for k in 1:grid.Nz z⁺ = znode(i, j, k+1, grid, c, c, f) @@ -112,7 +112,7 @@ end ##### Bottom height ##### -@inline bottom_heightᶜᶜᵃ(i, j, k, ibg::GFBIBG) = ibg.immersed_boundary.bottom_height[i, j, 1] -@inline bottom_heightᶜᶠᵃ(i, j, k, ibg::GFBIBG) = min(bottom_heightᶜᶜᵃ(i, j-1, k, ibg), bottom_heightᶜᶜᵃ(i, j, k, ibg)) -@inline bottom_heightᶠᶜᵃ(i, j, k, ibg::GFBIBG) = min(bottom_heightᶜᶜᵃ(i-1, j, k, ibg), bottom_heightᶜᶜᵃ(i, j, k, ibg)) -@inline bottom_heightᶠᶠᵃ(i, j, k, ibg::GFBIBG) = min(bottom_heightᶠᶜᵃ(i, j-1, k, ibg), bottom_heightᶠᶜᵃ(i, j, k, ibg)) +@inline bottom_heightᶜᶜᵃ(i, j, k, ibg::AbstractGridFittedBottom) = ibg.immersed_boundary.bottom_height[i, j, 1] +@inline bottom_heightᶜᶠᵃ(i, j, k, ibg::AbstractGridFittedBottom) = min(bottom_heightᶜᶜᵃ(i, j-1, k, ibg), bottom_heightᶜᶜᵃ(i, j, k, ibg)) +@inline bottom_heightᶠᶜᵃ(i, j, k, ibg::AbstractGridFittedBottom) = min(bottom_heightᶜᶜᵃ(i-1, j, k, ibg), bottom_heightᶜᶜᵃ(i, j, k, ibg)) +@inline bottom_heightᶠᶠᵃ(i, j, k, ibg::AbstractGridFittedBottom) = min(bottom_heightᶠᶜᵃ(i, j-1, k, ibg), bottom_heightᶠᶜᵃ(i, j, k, ibg)) diff --git a/src/ImmersedBoundaries/partial_cell_bottom.jl b/src/ImmersedBoundaries/partial_cell_bottom.jl index d3581b1780..00088a1c74 100644 --- a/src/ImmersedBoundaries/partial_cell_bottom.jl +++ b/src/ImmersedBoundaries/partial_cell_bottom.jl @@ -71,7 +71,7 @@ function ImmersedBoundaryGrid(grid, ib::PartialCellBottom) end @kernel function _correct_z_bottom!(bottom_field, grid, ib::PartialCellBottom) - i, j = @index(grid) + i, j = @index(Global, NTuple) zb = @inbounds bottom_field[i, j, 1] ϵ = ib.minimum_fractional_cell_height for k in 1:grid.Nz diff --git a/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl index 6f0afa5a26..d8b472fec8 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl @@ -10,27 +10,13 @@ function SplitExplicitAuxiliaryFields(grid::DistributedGrid) Gᵁ = Field((Face, Center, Nothing), grid) Gⱽ = Field((Center, Face, Nothing), grid) - Hᶠᶜ = Field((Face, Center, Nothing), grid) - Hᶜᶠ = Field((Center, Face, Nothing), grid) - - calculate_column_height!(Hᶠᶜ, (Face, Center, Center)) - calculate_column_height!(Hᶜᶠ, (Center, Face, Center)) - - fill_halo_regions!((Hᶠᶜ, Hᶜᶠ)) - # In a non-parallel grid we calculate only the interior kernel_size = augmented_kernel_size(grid) kernel_offsets = augmented_kernel_offsets(grid) kernel_parameters = KernelParameters(kernel_size, kernel_offsets) - return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, kernel_parameters) -end - -"""Integrate z at locations `location` and set! `height`` with the result""" -@inline function calculate_column_height!(height, location) - dz = GridMetricOperation(location, Δz, height.grid) - return sum!(height, dz) + return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, kernel_parameters) end @inline function augmented_kernel_size(grid::DistributedGrid) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl index 5d9a00f2ed..073fbfc4ea 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl @@ -209,10 +209,6 @@ Base.@kwdef struct SplitExplicitAuxiliaryFields{𝒞ℱ, ℱ𝒞, 𝒦} Gᵁ :: ℱ𝒞 "Vertically-integrated slow barotropic forcing function for `V` (`ReducedField` over ``z``)" Gⱽ :: 𝒞ℱ - "Depth at `(Face, Center)` (`ReducedField` over ``z``)" - Hᶠᶜ :: ℱ𝒞 - "Depth at `(Center, Face)` (`ReducedField` over ``z``)" - Hᶜᶠ :: 𝒞ℱ "kernel size for barotropic time stepping" kernel_parameters :: 𝒦 end @@ -227,20 +223,9 @@ function SplitExplicitAuxiliaryFields(grid::AbstractGrid) Gᵁ = Field((Face, Center, Nothing), grid) Gⱽ = Field((Center, Face, Nothing), grid) - Hᶠᶜ = Field((Face, Center, Nothing), grid) - Hᶜᶠ = Field((Center, Face, Nothing), grid) - - dz = GridMetricOperation((Face, Center, Center), Δz, grid) - sum!(Hᶠᶜ, dz) - - dz = GridMetricOperation((Center, Face, Center), Δz, grid) - sum!(Hᶜᶠ, dz) - - fill_halo_regions!((Hᶠᶜ, Hᶜᶠ)) - kernel_parameters = :xy - return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, kernel_parameters) + return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, kernel_parameters) end """ diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index baca09bbfc..9f793d9168 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -1,3 +1,4 @@ +using Oceananigans.Grids using Oceananigans.Grids: topology using Oceananigans.Utils using Oceananigans.AbstractOperations: Δz @@ -159,20 +160,20 @@ end @kernel function _split_explicit_barotropic_velocity!(averaging_weight, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², - η̅, U̅, V̅, Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, g, + η̅, U̅, V̅, Gᵁ, Gⱽ, g, timestepper) i, j = @index(Global, NTuple) velocity_evolution!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², η̅, U̅, V̅, averaging_weight, - Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, g, + Gᵁ, Gⱽ, g, timestepper) end @inline function velocity_evolution!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², η̅, U̅, V̅, averaging_weight, - Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, g, + Gᵁ, Gⱽ, g, timestepper) k_top = grid.Nz+1 @@ -182,9 +183,12 @@ end advance_previous_velocity!(i, j, k_top-1, timestepper, U, Uᵐ⁻¹, Uᵐ⁻²) advance_previous_velocity!(i, j, k_top-1, timestepper, V, Vᵐ⁻¹, Vᵐ⁻²) + Hᶠᶜ = bottom_heightᶠᶜᵃ(i, j, k_top-1, grid) + Hᶜᶠ = bottom_heightᶜᶠᵃ(i, j, k_top-1, grid) + # ∂τ(U) = - ∇η + G - U[i, j, k_top-1] += Δτ * (- g * Hᶠᶜ[i, j, 1] * ∂xᶠᶜᶠ_η(i, j, k_top, grid, TX, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gᵁ[i, j, k_top-1]) - V[i, j, k_top-1] += Δτ * (- g * Hᶜᶠ[i, j, 1] * ∂yᶜᶠᶠ_η(i, j, k_top, grid, TY, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gⱽ[i, j, k_top-1]) + U[i, j, k_top-1] += Δτ * (- g * Hᶠᶜ * ∂xᶠᶜᶠ_η(i, j, k_top, grid, TX, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gᵁ[i, j, k_top-1]) + V[i, j, k_top-1] += Δτ * (- g * Hᶜᶠ * ∂yᶜᶠᶠ_η(i, j, k_top, grid, TY, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gⱽ[i, j, k_top-1]) # time-averaging η̅[i, j, k_top] += averaging_weight * η[i, j, k_top] @@ -267,24 +271,25 @@ end k_top = grid.Nz+1 @inbounds begin - u[i, j, k] = u[i, j, k] + (U̅[i, j, k_top-1] - U[i, j, k_top-1]) / Hᶠᶜ[i, j, 1] - v[i, j, k] = v[i, j, k] + (V̅[i, j, k_top-1] - V[i, j, k_top-1]) / Hᶜᶠ[i, j, 1] + Hᶠᶜ = bottom_heightᶠᶜᵃ(i, j, k_top-1, grid) + Hᶜᶠ = bottom_heightᶜᶠᵃ(i, j, k_top-1, grid) + + u[i, j, k] = u[i, j, k] + (U̅[i, j, k_top-1] - U[i, j, k_top-1]) / Hᶠᶜ + v[i, j, k] = v[i, j, k] + (V̅[i, j, k_top-1] - V[i, j, k_top-1]) / Hᶜᶠ end end function barotropic_split_explicit_corrector!(u, v, free_surface, grid) sefs = free_surface.state U, V, U̅, V̅ = sefs.U, sefs.V, sefs.U̅, sefs.V̅ - Hᶠᶜ, Hᶜᶠ = free_surface.auxiliary.Hᶠᶜ, free_surface.auxiliary.Hᶜᶠ arch = architecture(grid) - # take out "bad" barotropic mode, # !!!! reusing U and V for this storage since last timestep doesn't matter compute_barotropic_mode!(U, V, grid, u, v) # add in "good" barotropic mode launch!(arch, grid, :xyz, _barotropic_split_explicit_corrector!, - u, v, U̅, V̅, U, V, Hᶠᶜ, Hᶜᶠ, grid) + u, v, U̅, V̅, U, V, grid) return nothing end diff --git a/src/MultiRegion/multi_region_split_explicit_free_surface.jl b/src/MultiRegion/multi_region_split_explicit_free_surface.jl index fb36463b39..10ba130b33 100644 --- a/src/MultiRegion/multi_region_split_explicit_free_surface.jl +++ b/src/MultiRegion/multi_region_split_explicit_free_surface.jl @@ -13,27 +13,13 @@ function SplitExplicitAuxiliaryFields(grid::MultiRegionGrids) Gᵁ = Field((Face, Center, Nothing), grid) Gⱽ = Field((Center, Face, Nothing), grid) - Hᶠᶜ = Field((Face, Center, Nothing), grid) - Hᶜᶠ = Field((Center, Face, Nothing), grid) - - @apply_regionally calculate_column_height!(Hᶠᶜ, (Face, Center, Center)) - @apply_regionally calculate_column_height!(Hᶜᶠ, (Center, Face, Center)) - - fill_halo_regions!((Hᶠᶜ, Hᶜᶠ)) - # In a non-parallel grid we calculate only the interior @apply_regionally kernel_size = augmented_kernel_size(grid, grid.partition) @apply_regionally kernel_offsets = augmented_kernel_offsets(grid, grid.partition) @apply_regionally kernel_parameters = KernelParameters(kernel_size, kernel_offsets) - return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, kernel_parameters) -end - -@inline function calculate_column_height!(height, location) - dz = GridMetricOperation(location, Δz, height.grid) - sum!(height, dz) - return nothing + return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, kernel_parameters) end @inline augmented_kernel_size(grid, ::XPartition) = (size(grid, 1) + 2halo_size(grid)[1]-2, size(grid, 2)) From 5ebaa84481e6dd8e144b651a3dfa42dd08b34890 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 10:19:54 +0200 Subject: [PATCH 03/79] comment --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index 1851e39d95..1824d3b56f 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -30,8 +30,7 @@ Keyword Arguments ================= * `z_bottom`: an array or function that gives the height of the - bottom in absolute ``z`` coordinates. This is the height of - the bottom interface of the last ``fluid`` cell. + bottom in absolute ``z`` coordinates. """ function Base.summary(ib::GridFittedBottom) From f9ad2279e5f720c3a27e6a051321d4128bf87445 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 10:24:55 +0200 Subject: [PATCH 04/79] comment --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index 1824d3b56f..8dc208e790 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -74,7 +74,8 @@ Adapt.adapt_structure(to, ib::GridFittedBottom) = GridFittedBottom(adapt(to, ib. Return a grid with `GridFittedBottom` immersed boundary (`ib`). -Computes `ib.z_bottom` and wraps it in a Field. +Computes `ib.z_bottom` and wraps it in a Field. `ib.z_bottom` is the z-coordinate of top-most interface +of the last ``immersed`` cell in the column. """ function ImmersedBoundaryGrid(grid, ib::GridFittedBottom) bottom_field = Field{Center, Center, Nothing}(grid) From cbdb713a649b59313b903f643c78dc19e76a483e Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 10:54:06 +0200 Subject: [PATCH 05/79] remove circular dependency for now --- Project.toml | 3 +-- test/test_cubed_spheres.jl | 6 +----- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/Project.toml b/Project.toml index 13f1fb6317..06a8553f22 100644 --- a/Project.toml +++ b/Project.toml @@ -79,10 +79,9 @@ julia = "1.9" [extras] DataDeps = "124859b0-ceae-595e-8997-d05f6a7a8dfe" Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" -OrthogonalSphericalShellGrids = "c2be9673-fb75-4747-82dc-aa2bb9f4aed0" SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" TimesDates = "bdfc003b-8df8-5c39-adcd-3a9087f5df4a" [targets] -test = ["DataDeps", "Enzyme", "OrthogonalSphericalShellGrids", "SafeTestsets", "Test", "TimesDates"] +test = ["DataDeps", "Enzyme", "SafeTestsets", "Test", "TimesDates"] diff --git a/test/test_cubed_spheres.jl b/test/test_cubed_spheres.jl index d8984cb0a1..e907dedf68 100644 --- a/test/test_cubed_spheres.jl +++ b/test/test_cubed_spheres.jl @@ -7,8 +7,6 @@ using Oceananigans.CubedSpheres using Oceananigans.Models.HydrostaticFreeSurfaceModels using Oceananigans.Models.HydrostaticFreeSurfaceModels: VerticalVorticityField -using OrthogonalSphericalShellGrids - # To be used in the test below as `KernelFunctionOperation`s @inline intrinsic_vector_x_component(i, j, k, grid, uₑ, vₑ) = @inbounds intrinsic_vector(i, j, k, grid, uₑ, vₑ)[1] @@ -172,12 +170,10 @@ end @testset "Conversion from Intrinsic to Extrinsic reference frame [$(typeof(arch))]" begin @info " Testing the conversion of a vector between the Intrinsic and Extrinsic reference frame" - trg_grid = TripolarGrid(arch, size = (20, 20, 1), z = (0, 1)) - test_vector_rotation(grid) - test_vector_rotation(trg_grid) end + @testset "CubedSphereData and CubedSphereFields [$(typeof(arch))]" begin @info " Testing CubedSphereData and CubedSphereFields [$(typeof(arch))]..." c = model.tracers.c From 7f087680d561eeccb5c959820bb55e951490d820 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 11:20:56 +0200 Subject: [PATCH 06/79] some bugfixes --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 2 +- src/ImmersedBoundaries/partial_cell_bottom.jl | 2 +- .../split_explicit_free_surface_kernels.jl | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index 8dc208e790..fdae0c6a02 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -80,7 +80,7 @@ of the last ``immersed`` cell in the column. function ImmersedBoundaryGrid(grid, ib::GridFittedBottom) bottom_field = Field{Center, Center, Nothing}(grid) set!(bottom_field, ib.z_bottom) - @apply_regionally correct_z_bottom!(bottom_field, grid) + @apply_regionally correct_z_bottom!(bottom_field, grid, ib) fill_halo_regions!(bottom_field) new_ib = GridFittedBottom(bottom_field) TX, TY, TZ = topology(grid) diff --git a/src/ImmersedBoundaries/partial_cell_bottom.jl b/src/ImmersedBoundaries/partial_cell_bottom.jl index 00088a1c74..5117adddf7 100644 --- a/src/ImmersedBoundaries/partial_cell_bottom.jl +++ b/src/ImmersedBoundaries/partial_cell_bottom.jl @@ -63,7 +63,7 @@ end function ImmersedBoundaryGrid(grid, ib::PartialCellBottom) bottom_field = Field{Center, Center, Nothing}(grid) set!(bottom_field, ib.z_bottom) - @apply_regionally correct_z_bottom!(bottom_field, grid, ib.minimum_fractional_cell_height) + @apply_regionally correct_z_bottom!(bottom_field, grid, ib) fill_halo_regions!(bottom_field) new_ib = PartialCellBottom(bottom_field, ib.minimum_fractional_cell_height) TX, TY, TZ = topology(grid) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 9f793d9168..6d31c95973 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -381,7 +381,7 @@ function iterate_split_explicit!(free_surface, grid, Δτᴮ, weights, ::Val{Nsu Vᵐ⁻¹, Vᵐ⁻² = state.Vᵐ⁻¹, state.Vᵐ⁻² ηᵐ, ηᵐ⁻¹, ηᵐ⁻² = state.ηᵐ, state.ηᵐ⁻¹, state.ηᵐ⁻² η̅, U̅, V̅ = state.η̅, state.U̅, state.V̅ - Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ = auxiliary.Gᵁ, auxiliary.Gⱽ, auxiliary.Hᶠᶜ, auxiliary.Hᶜᶠ + Gᵁ, Gⱽ = auxiliary.Gᵁ, auxiliary.Gⱽ timestepper = settings.timestepper @@ -396,7 +396,7 @@ function iterate_split_explicit!(free_surface, grid, Δτᴮ, weights, ::Val{Nsu U_args = (grid, Δτᴮ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², - η̅, U̅, V̅, Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, g, + η̅, U̅, V̅, Gᵁ, Gⱽ, g, timestepper) GC.@preserve η_args U_args begin From 4b7926fd74822f87630ed80eda921b59c102a94f Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 11:26:30 +0200 Subject: [PATCH 07/79] change name to column_height --- src/Grids/Grids.jl | 2 +- src/Grids/grid_utils.jl | 8 +++---- src/ImmersedBoundaries/ImmersedBoundaries.jl | 2 +- src/ImmersedBoundaries/grid_fitted_bottom.jl | 22 +++++++++---------- .../split_explicit_free_surface_kernels.jl | 8 +++---- 5 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/Grids/Grids.jl b/src/Grids/Grids.jl index bbc40581ce..fcc6270001 100644 --- a/src/Grids/Grids.jl +++ b/src/Grids/Grids.jl @@ -18,7 +18,7 @@ export xnodes, ynodes, znodes, λnodes, φnodes export spacings export xspacings, yspacings, zspacings, xspacing, yspacing, zspacing export minimum_xspacing, minimum_yspacing, minimum_zspacing -export bottom_heightᶜᶜᵃ, bottom_heightᶠᶜᵃ, bottom_heightᶜᶠᵃ, bottom_heightᶠᶠᵃ +export column_heightᶜᶜᵃ, column_heightᶠᶜᵃ, column_heightᶜᶠᵃ, column_heightᶠᶠᵃ export offset_data, new_data export on_architecture diff --git a/src/Grids/grid_utils.jl b/src/Grids/grid_utils.jl index afd199abe0..33d60205d1 100644 --- a/src/Grids/grid_utils.jl +++ b/src/Grids/grid_utils.jl @@ -318,10 +318,10 @@ coordinate_summary(topo, Δ::Union{AbstractVector, AbstractMatrix}, name) = ##### Bottom height ##### -@inline bottom_heightᶜᶜᵃ(i, j, k, grid) = grid.Lz -@inline bottom_heightᶜᶠᵃ(i, j, k, grid) = grid.Lz -@inline bottom_heightᶠᶜᵃ(i, j, k, grid) = grid.Lz -@inline bottom_heightᶠᶠᵃ(i, j, k, grid) = grid.Lz +@inline column_heightᶜᶜᵃ(i, j, k, grid) = grid.Lz +@inline column_heightᶜᶠᵃ(i, j, k, grid) = grid.Lz +@inline column_heightᶠᶜᵃ(i, j, k, grid) = grid.Lz +@inline column_heightᶠᶠᵃ(i, j, k, grid) = grid.Lz ##### ##### Spherical geometry diff --git a/src/ImmersedBoundaries/ImmersedBoundaries.jl b/src/ImmersedBoundaries/ImmersedBoundaries.jl index 8cf5e90b08..4a011d1beb 100644 --- a/src/ImmersedBoundaries/ImmersedBoundaries.jl +++ b/src/ImmersedBoundaries/ImmersedBoundaries.jl @@ -53,7 +53,7 @@ import Oceananigans.Grids: architecture, with_halo, inflate_halo_size_one_dimens ξname, ηname, rname, node_names, xnodes, ynodes, znodes, λnodes, φnodes, nodes, ξnodes, ηnodes, rnodes, - bottom_heightᶜᶜᵃ, bottom_heightᶠᶜᵃ, bottom_heightᶜᶠᵃ, bottom_heightᶠᶠᵃ, + column_heightᶜᶜᵃ, column_heightᶠᶜᵃ, column_heightᶜᶠᵃ, column_heightᶠᶠᵃ, inactive_cell import Oceananigans.Coriolis: φᶠᶠᵃ diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index fdae0c6a02..ca9722a073 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -56,15 +56,15 @@ function Base.show(io::IO, ib::GridFittedBottom) print(io, "├── z_bottom: ", prettysummary(ib.z_bottom), '\n') end -on_architecture(arch, ib::GridFittedBottom) = GridFittedBottom(on_architecture(ib.bottom_height)) +on_architecture(arch, ib::GridFittedBottom) = GridFittedBottom(on_architecture(ib.z_bottom)) function on_architecture(arch, ib::GridFittedBottom{<:Field}) - architecture(ib.bottom_height) == arch && return ib - arch_grid = on_architecture(arch, ib.bottom_height.grid) - new_bottom_height = Field{Center, Center, Nothing}(arch_grid) - set!(new_bottom_height, ib.bottom_height) - fill_halo_regions!(new_bottom_height) - return GridFittedBottom(new_bottom_height, ib.immersed_condition) + architecture(ib.z_bottom) == arch && return ib + arch_grid = on_architecture(arch, ib.z_bottom.grid) + new_z_bottom = Field{Center, Center, Nothing}(arch_grid) + set!(new_z_bottom, ib.z_bottom) + fill_halo_regions!(new_z_bottom) + return GridFittedBottom(new_z_bottom) end Adapt.adapt_structure(to, ib::GridFittedBottom) = GridFittedBottom(adapt(to, ib.z_bottom)) @@ -112,7 +112,7 @@ end ##### Bottom height ##### -@inline bottom_heightᶜᶜᵃ(i, j, k, ibg::AbstractGridFittedBottom) = ibg.immersed_boundary.bottom_height[i, j, 1] -@inline bottom_heightᶜᶠᵃ(i, j, k, ibg::AbstractGridFittedBottom) = min(bottom_heightᶜᶜᵃ(i, j-1, k, ibg), bottom_heightᶜᶜᵃ(i, j, k, ibg)) -@inline bottom_heightᶠᶜᵃ(i, j, k, ibg::AbstractGridFittedBottom) = min(bottom_heightᶜᶜᵃ(i-1, j, k, ibg), bottom_heightᶜᶜᵃ(i, j, k, ibg)) -@inline bottom_heightᶠᶠᵃ(i, j, k, ibg::AbstractGridFittedBottom) = min(bottom_heightᶠᶜᵃ(i, j-1, k, ibg), bottom_heightᶠᶜᵃ(i, j, k, ibg)) +@inline column_heightᶜᶜᵃ(i, j, k, ibg::AbstractGridFittedBottom) = ibg.immersed_boundary.column_height[i, j, 1] +@inline column_heightᶜᶠᵃ(i, j, k, ibg::AbstractGridFittedBottom) = min(column_heightᶜᶜᵃ(i, j-1, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) +@inline column_heightᶠᶜᵃ(i, j, k, ibg::AbstractGridFittedBottom) = min(column_heightᶜᶜᵃ(i-1, j, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) +@inline column_heightᶠᶠᵃ(i, j, k, ibg::AbstractGridFittedBottom) = min(column_heightᶠᶜᵃ(i, j-1, k, ibg), column_heightᶠᶜᵃ(i, j, k, ibg)) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 6d31c95973..58c0231f00 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -183,8 +183,8 @@ end advance_previous_velocity!(i, j, k_top-1, timestepper, U, Uᵐ⁻¹, Uᵐ⁻²) advance_previous_velocity!(i, j, k_top-1, timestepper, V, Vᵐ⁻¹, Vᵐ⁻²) - Hᶠᶜ = bottom_heightᶠᶜᵃ(i, j, k_top-1, grid) - Hᶜᶠ = bottom_heightᶜᶠᵃ(i, j, k_top-1, grid) + Hᶠᶜ = column_heightᶠᶜᵃ(i, j, k_top-1, grid) + Hᶜᶠ = column_heightᶜᶠᵃ(i, j, k_top-1, grid) # ∂τ(U) = - ∇η + G U[i, j, k_top-1] += Δτ * (- g * Hᶠᶜ * ∂xᶠᶜᶠ_η(i, j, k_top, grid, TX, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gᵁ[i, j, k_top-1]) @@ -271,8 +271,8 @@ end k_top = grid.Nz+1 @inbounds begin - Hᶠᶜ = bottom_heightᶠᶜᵃ(i, j, k_top-1, grid) - Hᶜᶠ = bottom_heightᶜᶠᵃ(i, j, k_top-1, grid) + Hᶠᶜ = column_heightᶠᶜᵃ(i, j, k_top-1, grid) + Hᶜᶠ = column_heightᶜᶠᵃ(i, j, k_top-1, grid) u[i, j, k] = u[i, j, k] + (U̅[i, j, k_top-1] - U[i, j, k_top-1]) / Hᶠᶜ v[i, j, k] = v[i, j, k] + (V̅[i, j, k_top-1] - V[i, j, k_top-1]) / Hᶜᶠ From 6b7b27ca46ff6c0222ec0a105db401bdafb9a5d6 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 11:30:22 +0200 Subject: [PATCH 08/79] correct column height --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index ca9722a073..8cbe5e998d 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -112,7 +112,8 @@ end ##### Bottom height ##### -@inline column_heightᶜᶜᵃ(i, j, k, ibg::AbstractGridFittedBottom) = ibg.immersed_boundary.column_height[i, j, 1] +@inline column_heightᶜᶜᵃ(i, j, k, ibg::AbstractGridFittedBottom) = znode(i, j, grid.Nz, c, c, f) - ibg.immersed_boundary.column_height[i, j, 1] @inline column_heightᶜᶠᵃ(i, j, k, ibg::AbstractGridFittedBottom) = min(column_heightᶜᶜᵃ(i, j-1, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) @inline column_heightᶠᶜᵃ(i, j, k, ibg::AbstractGridFittedBottom) = min(column_heightᶜᶜᵃ(i-1, j, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) @inline column_heightᶠᶠᵃ(i, j, k, ibg::AbstractGridFittedBottom) = min(column_heightᶠᶜᵃ(i, j-1, k, ibg), column_heightᶠᶜᵃ(i, j, k, ibg)) + From fdee36641a1b786a974627babe06fa3503f54432 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 11:31:11 +0200 Subject: [PATCH 09/79] whoops --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index 8cbe5e998d..9f743195e9 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -112,7 +112,7 @@ end ##### Bottom height ##### -@inline column_heightᶜᶜᵃ(i, j, k, ibg::AbstractGridFittedBottom) = znode(i, j, grid.Nz, c, c, f) - ibg.immersed_boundary.column_height[i, j, 1] +@inline column_heightᶜᶜᵃ(i, j, k, ibg::AbstractGridFittedBottom) = znode(i, j, grid.Nz, c, c, f) - ibg.immersed_boundary.z_bottom[i, j, 1] @inline column_heightᶜᶠᵃ(i, j, k, ibg::AbstractGridFittedBottom) = min(column_heightᶜᶜᵃ(i, j-1, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) @inline column_heightᶠᶜᵃ(i, j, k, ibg::AbstractGridFittedBottom) = min(column_heightᶜᶜᵃ(i-1, j, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) @inline column_heightᶠᶠᵃ(i, j, k, ibg::AbstractGridFittedBottom) = min(column_heightᶠᶜᵃ(i, j-1, k, ibg), column_heightᶠᶜᵃ(i, j, k, ibg)) From d1c145c60af4a6c754a822c632e417e636de2c2d Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 11:31:32 +0200 Subject: [PATCH 10/79] another correction --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index 9f743195e9..e46cb0fe43 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -112,7 +112,7 @@ end ##### Bottom height ##### -@inline column_heightᶜᶜᵃ(i, j, k, ibg::AbstractGridFittedBottom) = znode(i, j, grid.Nz, c, c, f) - ibg.immersed_boundary.z_bottom[i, j, 1] +@inline column_heightᶜᶜᵃ(i, j, k, ibg::AbstractGridFittedBottom) = znode(i, j, grid.Nz, ibg, c, c, f) - ibg.immersed_boundary.z_bottom[i, j, 1] @inline column_heightᶜᶠᵃ(i, j, k, ibg::AbstractGridFittedBottom) = min(column_heightᶜᶜᵃ(i, j-1, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) @inline column_heightᶠᶜᵃ(i, j, k, ibg::AbstractGridFittedBottom) = min(column_heightᶜᶜᵃ(i-1, j, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) @inline column_heightᶠᶠᵃ(i, j, k, ibg::AbstractGridFittedBottom) = min(column_heightᶠᶜᵃ(i, j-1, k, ibg), column_heightᶠᶜᵃ(i, j, k, ibg)) From dbb411ad78a19c4e08bea88f162c8213262a4bf8 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 11:34:25 +0200 Subject: [PATCH 11/79] some more changes --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 12 +++++++----- src/ImmersedBoundaries/partial_cell_bottom.jl | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index e46cb0fe43..e518714eaa 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -95,7 +95,7 @@ correct_z_bottom!(bottom_field, grid, ib) = zb = @inbounds bottom_field[i, j, 1] for k in 1:grid.Nz z⁺ = znode(i, j, k+1, grid, c, c, f) - bottom_cell = zb < z⁺ + bottom_cell = zb ≤ z⁺ @inbounds bottom_field[i, j, 1] = ifelse(bottom_cell, z⁺, zb) end end @@ -112,8 +112,10 @@ end ##### Bottom height ##### -@inline column_heightᶜᶜᵃ(i, j, k, ibg::AbstractGridFittedBottom) = znode(i, j, grid.Nz, ibg, c, c, f) - ibg.immersed_boundary.z_bottom[i, j, 1] -@inline column_heightᶜᶠᵃ(i, j, k, ibg::AbstractGridFittedBottom) = min(column_heightᶜᶜᵃ(i, j-1, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) -@inline column_heightᶠᶜᵃ(i, j, k, ibg::AbstractGridFittedBottom) = min(column_heightᶜᶜᵃ(i-1, j, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) -@inline column_heightᶠᶠᵃ(i, j, k, ibg::AbstractGridFittedBottom) = min(column_heightᶠᶜᵃ(i, j-1, k, ibg), column_heightᶠᶜᵃ(i, j, k, ibg)) +const AGFBIB = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractGridFittedBottom} + +@inline column_heightᶜᶜᵃ(i, j, k, ibg::AGFBIB) = @inbounds znode(i, j, grid.Nz, ibg, c, c, f) - ibg.immersed_boundary.z_bottom[i, j, 1] +@inline column_heightᶜᶠᵃ(i, j, k, ibg::AGFBIB) = min(column_heightᶜᶜᵃ(i, j-1, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) +@inline column_heightᶠᶜᵃ(i, j, k, ibg::AGFBIB) = min(column_heightᶜᶜᵃ(i-1, j, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) +@inline column_heightᶠᶠᵃ(i, j, k, ibg::AGFBIB) = min(column_heightᶠᶜᵃ(i, j-1, k, ibg), column_heightᶠᶜᵃ(i, j, k, ibg)) diff --git a/src/ImmersedBoundaries/partial_cell_bottom.jl b/src/ImmersedBoundaries/partial_cell_bottom.jl index 5117adddf7..d23fe199a7 100644 --- a/src/ImmersedBoundaries/partial_cell_bottom.jl +++ b/src/ImmersedBoundaries/partial_cell_bottom.jl @@ -77,7 +77,7 @@ end for k in 1:grid.Nz z⁻ = znode(i, j, k, grid, c, c, f) Δz = Δzᶜᶜᶜ(i, j, k, underlying_grid) - bottom_cell = zb < z⁻ + Δz * (1 - ϵ) + bottom_cell = zb ≤ z⁻ + Δz * (1 - ϵ) @inbounds bottom_field[i, j, 1] = ifelse(bottom_cell, z⁻ + Δz * (1 - ϵ), zb) end end From 3ec05dff7ac595441aa7055d24684764db5eb46e Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 11:44:44 +0200 Subject: [PATCH 12/79] another correction --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index e518714eaa..83c23b2df4 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -114,7 +114,7 @@ end const AGFBIB = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractGridFittedBottom} -@inline column_heightᶜᶜᵃ(i, j, k, ibg::AGFBIB) = @inbounds znode(i, j, grid.Nz, ibg, c, c, f) - ibg.immersed_boundary.z_bottom[i, j, 1] +@inline column_heightᶜᶜᵃ(i, j, k, ibg::AGFBIB) = @inbounds znode(i, j, grid.Nz+1, ibg, c, c, f) - ibg.immersed_boundary.z_bottom[i, j, 1] @inline column_heightᶜᶠᵃ(i, j, k, ibg::AGFBIB) = min(column_heightᶜᶜᵃ(i, j-1, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) @inline column_heightᶠᶜᵃ(i, j, k, ibg::AGFBIB) = min(column_heightᶜᶜᵃ(i-1, j, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) @inline column_heightᶠᶠᵃ(i, j, k, ibg::AGFBIB) = min(column_heightᶠᶜᵃ(i, j-1, k, ibg), column_heightᶠᶜᵃ(i, j, k, ibg)) From 4e4d40b91052f6cb3352990b906210700a607f49 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 12:28:15 +0200 Subject: [PATCH 13/79] couple of more bugfixes --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 2 +- src/ImmersedBoundaries/partial_cell_bottom.jl | 2 +- .../split_explicit_free_surface_kernels.jl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index 83c23b2df4..e75df50573 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -95,7 +95,7 @@ correct_z_bottom!(bottom_field, grid, ib) = zb = @inbounds bottom_field[i, j, 1] for k in 1:grid.Nz z⁺ = znode(i, j, k+1, grid, c, c, f) - bottom_cell = zb ≤ z⁺ + bottom_cell = z⁺ ≤ zb @inbounds bottom_field[i, j, 1] = ifelse(bottom_cell, z⁺, zb) end end diff --git a/src/ImmersedBoundaries/partial_cell_bottom.jl b/src/ImmersedBoundaries/partial_cell_bottom.jl index d23fe199a7..926b026972 100644 --- a/src/ImmersedBoundaries/partial_cell_bottom.jl +++ b/src/ImmersedBoundaries/partial_cell_bottom.jl @@ -77,7 +77,7 @@ end for k in 1:grid.Nz z⁻ = znode(i, j, k, grid, c, c, f) Δz = Δzᶜᶜᶜ(i, j, k, underlying_grid) - bottom_cell = zb ≤ z⁻ + Δz * (1 - ϵ) + bottom_cell = z⁻ + Δz * (1 - ϵ) ≤ zb @inbounds bottom_field[i, j, 1] = ifelse(bottom_cell, z⁻ + Δz * (1 - ϵ), zb) end end diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 58c0231f00..a0226762f8 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -266,7 +266,7 @@ function initialize_auxiliary_state!(state, η, timestepper) return nothing end -@kernel function _barotropic_split_explicit_corrector!(u, v, U̅, V̅, U, V, Hᶠᶜ, Hᶜᶠ, grid) +@kernel function _barotropic_split_explicit_corrector!(u, v, U̅, V̅, U, V, grid) i, j, k = @index(Global, NTuple) k_top = grid.Nz+1 From a6362eeae2e1b32b6567a91eddc1f5f43bc21d81 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 13:24:03 +0200 Subject: [PATCH 14/79] more bugfixes --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 2 +- src/MultiRegion/multi_region_grid.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index e75df50573..806a4fb22f 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -114,7 +114,7 @@ end const AGFBIB = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractGridFittedBottom} -@inline column_heightᶜᶜᵃ(i, j, k, ibg::AGFBIB) = @inbounds znode(i, j, grid.Nz+1, ibg, c, c, f) - ibg.immersed_boundary.z_bottom[i, j, 1] +@inline column_heightᶜᶜᵃ(i, j, k, ibg::AGFBIB) = @inbounds znode(i, j, ibg.Nz+1, ibg, c, c, f) - ibg.immersed_boundary.z_bottom[i, j, 1] @inline column_heightᶜᶠᵃ(i, j, k, ibg::AGFBIB) = min(column_heightᶜᶜᵃ(i, j-1, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) @inline column_heightᶠᶜᵃ(i, j, k, ibg::AGFBIB) = min(column_heightᶜᶜᵃ(i-1, j, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) @inline column_heightᶠᶠᵃ(i, j, k, ibg::AGFBIB) = min(column_heightᶠᶜᵃ(i, j-1, k, ibg), column_heightᶠᶜᵃ(i, j, k, ibg)) diff --git a/src/MultiRegion/multi_region_grid.jl b/src/MultiRegion/multi_region_grid.jl index a65a8b5dd8..7d28ee6100 100644 --- a/src/MultiRegion/multi_region_grid.jl +++ b/src/MultiRegion/multi_region_grid.jl @@ -188,7 +188,7 @@ function reconstruct_global_grid(mrg::ImmersedMultiRegionGrid) return ImmersedBoundaryGrid(global_grid, global_boundary) end -reconstruct_global_boundary(g::GridFittedBottom{<:Field}) = GridFittedBottom(reconstruct_global_field(g.z_bottom), g.immersed_condition) +reconstruct_global_boundary(g::GridFittedBottom{<:Field}) = GridFittedBottom(reconstruct_global_field(g.z_bottom)) reconstruct_global_boundary(g::PartialCellBottom{<:Field}) = PartialCellBottom(reconstruct_global_field(g.z_bottom), g.minimum_fractional_cell_height) reconstruct_global_boundary(g::GridFittedBoundary{<:Field}) = GridFittedBoundary(reconstruct_global_field(g.mask)) From 3bce6fc823ee375ba42ac05835bb8f557c522569 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 14:14:40 +0200 Subject: [PATCH 15/79] this should make it work --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index 806a4fb22f..a67cecf4b0 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -101,7 +101,7 @@ correct_z_bottom!(bottom_field, grid, ib) = end @inline function _immersed_cell(i, j, k, underlying_grid, ib::GridFittedBottom) - z = znode(i, j, k+1, underlying_grid, c, c, f) + z = znode(i, j, k, underlying_grid, c, c, c) h = @inbounds ib.z_bottom[i, j, 1] return z ≤ h end From 0fa1a67e29d3333ad7086c1b5a2510e5be37e7c8 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 14:16:24 +0200 Subject: [PATCH 16/79] unify the formulations --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 13 +++++++------ src/ImmersedBoundaries/partial_cell_bottom.jl | 9 --------- 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index a67cecf4b0..ae9d0b525a 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -100,20 +100,21 @@ correct_z_bottom!(bottom_field, grid, ib) = end end -@inline function _immersed_cell(i, j, k, underlying_grid, ib::GridFittedBottom) - z = znode(i, j, k, underlying_grid, c, c, c) - h = @inbounds ib.z_bottom[i, j, 1] - return z ≤ h -end @inline z_bottom(i, j, ibg::GFBIBG) = @inbounds ibg.immersed_boundary.z_bottom[i, j, 1] ##### -##### Bottom height +##### _immersed_cell and column height ##### const AGFBIB = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractGridFittedBottom} +@inline function _immersed_cell(i, j, k, underlying_grid, ib::AGFBIB) + z = znode(i, j, k, underlying_grid, c, c, c) + h = @inbounds ib.z_bottom[i, j, 1] + return z ≤ h +end + @inline column_heightᶜᶜᵃ(i, j, k, ibg::AGFBIB) = @inbounds znode(i, j, ibg.Nz+1, ibg, c, c, f) - ibg.immersed_boundary.z_bottom[i, j, 1] @inline column_heightᶜᶠᵃ(i, j, k, ibg::AGFBIB) = min(column_heightᶜᶜᵃ(i, j-1, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) @inline column_heightᶠᶜᵃ(i, j, k, ibg::AGFBIB) = min(column_heightᶜᶜᵃ(i-1, j, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) diff --git a/src/ImmersedBoundaries/partial_cell_bottom.jl b/src/ImmersedBoundaries/partial_cell_bottom.jl index 926b026972..5d1ddf1210 100644 --- a/src/ImmersedBoundaries/partial_cell_bottom.jl +++ b/src/ImmersedBoundaries/partial_cell_bottom.jl @@ -117,15 +117,6 @@ on_architecture(to, ib::PartialCellBottom) = PartialCellBottom(on_architecture(t Criterion is zb ≥ z - ϵ Δz """ -@inline function _immersed_cell(i, j, k, underlying_grid, ib::PartialCellBottom) - # Face node below current cell - z = znode(i, j, k, underlying_grid, c, c, f) - zb = @inbounds ib.z_bottom[i, j, 1] - ϵ = ib.minimum_fractional_cell_height - # z + Δz is equal to the face above the current cell - Δz = Δzᶜᶜᶜ(i, j, k, underlying_grid) - return (z + Δz * (1 - ϵ)) ≤ zb -end @inline function bottom_cell(i, j, k, ibg::PCBIBG) grid = ibg.underlying_grid From 135cac345b32d6e9931c318564e1cabe1ea91030 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 14:18:08 +0200 Subject: [PATCH 17/79] correct implementation --- src/ImmersedBoundaries/partial_cell_bottom.jl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/ImmersedBoundaries/partial_cell_bottom.jl b/src/ImmersedBoundaries/partial_cell_bottom.jl index 5d1ddf1210..966ae98ac4 100644 --- a/src/ImmersedBoundaries/partial_cell_bottom.jl +++ b/src/ImmersedBoundaries/partial_cell_bottom.jl @@ -117,6 +117,11 @@ on_architecture(to, ib::PartialCellBottom) = PartialCellBottom(on_architecture(t Criterion is zb ≥ z - ϵ Δz """ +@inline function _immersed_cell(i, j, k, underlying_grid, ib::PartialCellBottom) + z = znode(i, j, k+1, underlying_grid, c, c, f) + zb = @inbounds ib.z_bottom[i, j, 1] + return z ≤ zb +end @inline function bottom_cell(i, j, k, ibg::PCBIBG) grid = ibg.underlying_grid From 156dada1aa5da8f193c171d43dca7b24e3d0aff6 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 14:18:16 +0200 Subject: [PATCH 18/79] correct implementation --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index ae9d0b525a..a67cecf4b0 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -100,21 +100,20 @@ correct_z_bottom!(bottom_field, grid, ib) = end end +@inline function _immersed_cell(i, j, k, underlying_grid, ib::GridFittedBottom) + z = znode(i, j, k, underlying_grid, c, c, c) + h = @inbounds ib.z_bottom[i, j, 1] + return z ≤ h +end @inline z_bottom(i, j, ibg::GFBIBG) = @inbounds ibg.immersed_boundary.z_bottom[i, j, 1] ##### -##### _immersed_cell and column height +##### Bottom height ##### const AGFBIB = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractGridFittedBottom} -@inline function _immersed_cell(i, j, k, underlying_grid, ib::AGFBIB) - z = znode(i, j, k, underlying_grid, c, c, c) - h = @inbounds ib.z_bottom[i, j, 1] - return z ≤ h -end - @inline column_heightᶜᶜᵃ(i, j, k, ibg::AGFBIB) = @inbounds znode(i, j, ibg.Nz+1, ibg, c, c, f) - ibg.immersed_boundary.z_bottom[i, j, 1] @inline column_heightᶜᶠᵃ(i, j, k, ibg::AGFBIB) = min(column_heightᶜᶜᵃ(i, j-1, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) @inline column_heightᶠᶜᵃ(i, j, k, ibg::AGFBIB) = min(column_heightᶜᶜᵃ(i-1, j, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) From 04e717070cec3c218ca7eec2a590ed118c8fb0b8 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 14:32:21 +0200 Subject: [PATCH 19/79] correct partial cell bottom --- src/ImmersedBoundaries/partial_cell_bottom.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ImmersedBoundaries/partial_cell_bottom.jl b/src/ImmersedBoundaries/partial_cell_bottom.jl index 966ae98ac4..3ece188427 100644 --- a/src/ImmersedBoundaries/partial_cell_bottom.jl +++ b/src/ImmersedBoundaries/partial_cell_bottom.jl @@ -76,7 +76,7 @@ end ϵ = ib.minimum_fractional_cell_height for k in 1:grid.Nz z⁻ = znode(i, j, k, grid, c, c, f) - Δz = Δzᶜᶜᶜ(i, j, k, underlying_grid) + Δz = Δzᶜᶜᶜ(i, j, k, grid) bottom_cell = z⁻ + Δz * (1 - ϵ) ≤ zb @inbounds bottom_field[i, j, 1] = ifelse(bottom_cell, z⁻ + Δz * (1 - ϵ), zb) end From 9258572332bf6acfc51469e601bbd707ec4f1aef Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 14:33:49 +0200 Subject: [PATCH 20/79] use center immersed condition for grid fitted boundary --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index a67cecf4b0..d76e5b6444 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -95,7 +95,8 @@ correct_z_bottom!(bottom_field, grid, ib) = zb = @inbounds bottom_field[i, j, 1] for k in 1:grid.Nz z⁺ = znode(i, j, k+1, grid, c, c, f) - bottom_cell = z⁺ ≤ zb + z = znode(i, j, k+1, grid, c, c, c) + bottom_cell = z ≤ zb @inbounds bottom_field[i, j, 1] = ifelse(bottom_cell, z⁺, zb) end end From 9df333a62383c607efbdf01500901f792d7ae13f Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 14:34:24 +0200 Subject: [PATCH 21/79] use the *correct* center node --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index d76e5b6444..95a1d98efb 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -95,7 +95,7 @@ correct_z_bottom!(bottom_field, grid, ib) = zb = @inbounds bottom_field[i, j, 1] for k in 1:grid.Nz z⁺ = znode(i, j, k+1, grid, c, c, f) - z = znode(i, j, k+1, grid, c, c, c) + z = znode(i, j, k, grid, c, c, c) bottom_cell = z ≤ zb @inbounds bottom_field[i, j, 1] = ifelse(bottom_cell, z⁺, zb) end From 053b26faf7b4c74f912b33d36c15f94dabdfdaa4 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 14:35:33 +0200 Subject: [PATCH 22/79] no h for z-values! --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 6 +++--- src/ImmersedBoundaries/partial_cell_bottom.jl | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index 95a1d98efb..5ddfe41ddd 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -102,9 +102,9 @@ correct_z_bottom!(bottom_field, grid, ib) = end @inline function _immersed_cell(i, j, k, underlying_grid, ib::GridFittedBottom) - z = znode(i, j, k, underlying_grid, c, c, c) - h = @inbounds ib.z_bottom[i, j, 1] - return z ≤ h + z = znode(i, j, k, underlying_grid, c, c, c) + zb = @inbounds ib.z_bottom[i, j, 1] + return z ≤ zb end @inline z_bottom(i, j, ibg::GFBIBG) = @inbounds ibg.immersed_boundary.z_bottom[i, j, 1] diff --git a/src/ImmersedBoundaries/partial_cell_bottom.jl b/src/ImmersedBoundaries/partial_cell_bottom.jl index 3ece188427..51f01ffb08 100644 --- a/src/ImmersedBoundaries/partial_cell_bottom.jl +++ b/src/ImmersedBoundaries/partial_cell_bottom.jl @@ -137,9 +137,9 @@ end # Get node at face above and defining nodes on c,c,f z = znode(i, j, k+1, underlying_grid, c, c, f) - # Get bottom height and fractional Δz parameter - h = @inbounds ib.z_bottom[i, j, 1] - ϵ = ibg.immersed_boundary.minimum_fractional_cell_height + # Get bottom z-coordinate and fractional Δz parameter + zb = @inbounds ib.z_bottom[i, j, 1] + ϵ = ibg.immersed_boundary.minimum_fractional_cell_height # Are we in a bottom cell? at_the_bottom = bottom_cell(i, j, k, ibg) From 61b7e7f0a7729badd5519d4e71e3fa69cb7b722a Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 14:37:10 +0200 Subject: [PATCH 23/79] simplify partial cells --- src/ImmersedBoundaries/partial_cell_bottom.jl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/ImmersedBoundaries/partial_cell_bottom.jl b/src/ImmersedBoundaries/partial_cell_bottom.jl index 51f01ffb08..2e6e9ba50b 100644 --- a/src/ImmersedBoundaries/partial_cell_bottom.jl +++ b/src/ImmersedBoundaries/partial_cell_bottom.jl @@ -139,13 +139,12 @@ end # Get bottom z-coordinate and fractional Δz parameter zb = @inbounds ib.z_bottom[i, j, 1] - ϵ = ibg.immersed_boundary.minimum_fractional_cell_height # Are we in a bottom cell? at_the_bottom = bottom_cell(i, j, k, ibg) - full_Δz = Δzᶜᶜᶜ(i, j, k, ibg.underlying_grid) - partial_Δz = max(ϵ * full_Δz, z - h) + full_Δz = Δzᶜᶜᶜ(i, j, k, ibg.underlying_grid) + partial_Δz = z - zb return ifelse(at_the_bottom, partial_Δz, full_Δz) end From 1098c58cc850dbe8c07e636565053aa76eca2d36 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 14:55:41 +0200 Subject: [PATCH 24/79] make sure we don't go out of bounds --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index 5ddfe41ddd..4fd1efd05a 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -120,3 +120,12 @@ const AGFBIB = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Abstrac @inline column_heightᶠᶜᵃ(i, j, k, ibg::AGFBIB) = min(column_heightᶜᶜᵃ(i-1, j, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) @inline column_heightᶠᶠᵃ(i, j, k, ibg::AGFBIB) = min(column_heightᶠᶜᵃ(i, j-1, k, ibg), column_heightᶠᶜᵃ(i, j, k, ibg)) +# Make sure Δz works for horizontally-Flat topologies. +# (There's no point in using z-Flat with PartialCellBottom). +XFlatAGFIBG = ImmersedBoundaryGrid{<:Any, <:Flat, <:Any, <:Any, <:Any, <:AbstractGridFittedBottom} +YFlatAGFIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Flat, <:Any, <:Any, <:AbstractGridFittedBottom} + +@inline column_heightᶠᶜᵃ(i, j, k, ibg::XFlatAGFIBG) = column_heightᶜᶜᵃ(i, j, k, ibg) +@inline column_heightᶜᶠᵃ(i, j, k, ibg::YFlatAGFIBG) = column_heightᶜᶜᵃ(i, j, k, ibg) +@inline column_heightᶠᶠᵃ(i, j, k, ibg::XFlatAGFIBG) = column_heightᶜᶠᵃ(i, j, k, ibg) +@inline column_heightᶠᶠᵃ(i, j, k, ibg::YFlatAGFIBG) = column_heightᶠᶜᵃ(i, j, k, ibg) From 9e35af6a62c66462de51fcce3b47aa6b4b3a693f Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 15:11:19 +0200 Subject: [PATCH 25/79] back to immersed condition --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 33 +++++++++++++++----- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index 4fd1efd05a..1a2cc89e57 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -15,10 +15,22 @@ import Oceananigans.TurbulenceClosures: z_bottom abstract type AbstractGridFittedBottom{H} <: AbstractGridFittedBoundary end -struct GridFittedBottom{H} <: AbstractGridFittedBottom{H} +# To enable comparison with PartialCellBottom in the limiting case that +# fractional cell height is 1.0. +struct InterfaceImmersedCondition end +struct CenterImmersedCondition end + +struct GridFittedBottom{H, I} <: AbstractGridFittedBottom{H} z_bottom :: H + immersed_condition :: I end + +GridFittedBottom(z_bottom) = GridFittedBottom(z_bottom, CenterImmersedCondition()) + +Base.summary(::CenterImmersedCondition) = "CenterImmersedCondition" +Base.summary(::InterfaceImmersedCondition) = "InterfaceImmersedCondition" + const GFBIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:GridFittedBottom} """ @@ -31,6 +43,11 @@ Keyword Arguments * `z_bottom`: an array or function that gives the height of the bottom in absolute ``z`` coordinates. + +* `immersed_condition`: Determine whether the part of the domain that is immersed are all the cell centers that lie below + `z_bottom` (`CenterImmersedCondition()`; default) or all the cell faces that lie below `bottom_height` (`InterfaceImmersedCondition()`). + The only purpose of `immersed_condition` to allow `GridFittedBottom` and `PartialCellBottom` to have the same behavior when the + minimum fractional cell height for partial cells is set to 0. """ function Base.summary(ib::GridFittedBottom) @@ -56,7 +73,7 @@ function Base.show(io::IO, ib::GridFittedBottom) print(io, "├── z_bottom: ", prettysummary(ib.z_bottom), '\n') end -on_architecture(arch, ib::GridFittedBottom) = GridFittedBottom(on_architecture(ib.z_bottom)) +on_architecture(arch, ib::GridFittedBottom) = GridFittedBottom(on_architecture(ib.z_bottom), ib.immersed_condition) function on_architecture(arch, ib::GridFittedBottom{<:Field}) architecture(ib.z_bottom) == arch && return ib @@ -64,10 +81,10 @@ function on_architecture(arch, ib::GridFittedBottom{<:Field}) new_z_bottom = Field{Center, Center, Nothing}(arch_grid) set!(new_z_bottom, ib.z_bottom) fill_halo_regions!(new_z_bottom) - return GridFittedBottom(new_z_bottom) + return GridFittedBottom(new_z_bottom, ib.immersed_condition) end -Adapt.adapt_structure(to, ib::GridFittedBottom) = GridFittedBottom(adapt(to, ib.z_bottom)) +Adapt.adapt_structure(to, ib::GridFittedBottom) = GridFittedBottom(adapt(to, ib.z_bottom), ib.immersed_condition) """ ImmersedBoundaryGrid(grid, ib::GridFittedBottom) @@ -90,13 +107,14 @@ end correct_z_bottom!(bottom_field, grid, ib) = launch!(architecture(grid), grid, :xy, _correct_z_bottom!, bottom_field, grid, ib) -@kernel function _correct_z_bottom!(bottom_field, grid, ::GridFittedBottom) +@kernel function _correct_z_bottom!(bottom_field, grid, ib::GridFittedBottom) i, j = @index(Global, NTuple) zb = @inbounds bottom_field[i, j, 1] + condition = ib.immersed_condition for k in 1:grid.Nz z⁺ = znode(i, j, k+1, grid, c, c, f) z = znode(i, j, k, grid, c, c, c) - bottom_cell = z ≤ zb + bottom_cell = ifelse(condition isa CenterImmersedCondition, z ≤ zb, z⁺ ≤ zb) @inbounds bottom_field[i, j, 1] = ifelse(bottom_cell, z⁺, zb) end end @@ -120,8 +138,7 @@ const AGFBIB = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Abstrac @inline column_heightᶠᶜᵃ(i, j, k, ibg::AGFBIB) = min(column_heightᶜᶜᵃ(i-1, j, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) @inline column_heightᶠᶠᵃ(i, j, k, ibg::AGFBIB) = min(column_heightᶠᶜᵃ(i, j-1, k, ibg), column_heightᶠᶜᵃ(i, j, k, ibg)) -# Make sure Δz works for horizontally-Flat topologies. -# (There's no point in using z-Flat with PartialCellBottom). +# Make sure column_height works for horizontally-Flat topologies. XFlatAGFIBG = ImmersedBoundaryGrid{<:Any, <:Flat, <:Any, <:Any, <:Any, <:AbstractGridFittedBottom} YFlatAGFIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Flat, <:Any, <:Any, <:AbstractGridFittedBottom} From ac7c96d02bf8e159d4080153d33ced04866db2d3 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 10 Oct 2024 09:49:43 +0200 Subject: [PATCH 26/79] name changes --- docs/src/grids.md | 2 +- examples/internal_tide.jl | 2 +- src/Advection/adapt_advection_order.jl | 98 +++++++++++++++++++ src/BoundaryConditions/apply_flux_bcs.jl | 6 +- src/ImmersedBoundaries/ImmersedBoundaries.jl | 4 +- .../distributed_immersed_boundaries.jl | 10 +- src/ImmersedBoundaries/grid_fitted_bottom.jl | 56 +++++------ src/ImmersedBoundaries/partial_cell_bottom.jl | 46 ++++----- src/MultiRegion/multi_region_grid.jl | 4 +- src/TurbulenceClosures/TurbulenceClosures.jl | 8 +- .../immersed_couette_flow.jl | 4 +- ...multi_region_near_global_quarter_degree.jl | 2 +- .../heterogeneous_windy_convection.jl | 4 +- 13 files changed, 172 insertions(+), 74 deletions(-) create mode 100644 src/Advection/adapt_advection_order.jl diff --git a/docs/src/grids.md b/docs/src/grids.md index 5e25849b05..a6686e5767 100644 --- a/docs/src/grids.md +++ b/docs/src/grids.md @@ -207,7 +207,7 @@ mountain_grid = ImmersedBoundaryGrid(grid, GridFittedBottom(mountain)) ```@example grids using CairoMakie -h = mountain_grid.immersed_boundary.z_bottom +h = mountain_grid.immersed_boundary.bottom_height fig = Figure(size=(600, 600)) ax = Axis(fig[2, 1], xlabel="x (m)", ylabel="y (m)", aspect=1) diff --git a/examples/internal_tide.jl b/examples/internal_tide.jl index a08dedd65a..5b31f18418 100644 --- a/examples/internal_tide.jl +++ b/examples/internal_tide.jl @@ -52,7 +52,7 @@ grid = ImmersedBoundaryGrid(underlying_grid, PartialCellBottom(bottom)) # Let's see how the domain with the bathymetry is. x = xnodes(grid, Center()) -bottom_boundary = interior(grid.immersed_boundary.z_bottom, :, 1, 1) +bottom_boundary = interior(grid.immersed_boundary.bottom_height, :, 1, 1) top_boundary = 0 * x using CairoMakie diff --git a/src/Advection/adapt_advection_order.jl b/src/Advection/adapt_advection_order.jl new file mode 100644 index 0000000000..41f3f60c9a --- /dev/null +++ b/src/Advection/adapt_advection_order.jl @@ -0,0 +1,98 @@ +using Oceananigans.Grids: topology + +""" + adapt_advection_order(advection, grid::AbstractGrid) + +Adapts the advection operator `advection` based on the grid `grid` by adjusting the order of advection in each direction. +For example, if the grid has only one point in the x-direction, the advection operator in the x-direction is set to first order +upwind or 2nd order centered scheme, depending on the original user-specified advection scheme. A high order advection sheme +is reduced to a lower order advection scheme if the grid has fewer points in that direction. + +# Arguments +- `advection`: The original advection scheme. +- `grid::AbstractGrid`: The grid on which the advection scheme is applied. + +If the order of advection is changed in at least one direction, the adapted advection scheme with adjusted advection order returned +by this function is a `FluxFormAdvection`. +""" +function adapt_advection_order(advection, grid::AbstractGrid) + advection_x = x_advection(advection) + advection_y = y_advection(advection) + advection_z = z_advection(advection) + + new_advection_x = adapt_advection_order(advection_x, size(grid, 1), grid) + new_advection_y = adapt_advection_order(advection_y, size(grid, 2), grid) + new_advection_z = adapt_advection_order(advection_z, size(grid, 3), grid) + + # Check that we indeed changed the advection operator + changed_x = new_advection_x != advection_x + changed_y = new_advection_y != advection_y + changed_z = new_advection_z != advection_z + + new_advection = FluxFormAdvection(new_advection_x, new_advection_y, new_advection_z) + changed_advection = any((changed_x, changed_y, changed_z)) + + if changed_x + @info "Using the advection scheme $(summary(new_advection.x)) in the x-direction because size(grid, 1) = $(size(grid, 1))" + end + if changed_y + @info "Using the advection scheme $(summary(new_advection.y)) in the y-direction because size(grid, 2) = $(size(grid, 2))" + end + if changed_z + @info "Using the advection scheme $(summary(new_advection.z)) in the x-direction because size(grid, 3) = $(size(grid, 3))" + end + + return ifelse(changed_advection, new_advection, advection) +end + + +x_advection(flux_form::FluxFormAdvection) = flux_form.x +y_advection(flux_form::FluxFormAdvection) = flux_form.y +z_advection(flux_form::FluxFormAdvection) = flux_form.z + +x_advection(advection) = advection +y_advection(advection) = advection +z_advection(advection) = advection + +# For the moment, we do not adapt the advection order for the VectorInvariant advection scheme +adapt_advection_order(advection::VectorInvariant, grid::AbstractGrid) = advection +adapt_advection_order(advection::Nothing, grid::AbstractGrid) = nothing +adapt_advection_order(advection::Nothing, N::Int, grid::AbstractGrid) = nothing + +##### +##### Directional adapt advection order +##### + +function adapt_advection_order(advection::Centered{B}, N::Int, grid::AbstractGrid) where B + if N >= B + return advection + else + return Centered(; order = 2N) + end +end + +function adapt_advection_order(advection::UpwindBiased{B}, N::Int, grid::AbstractGrid) where B + if N >= B + return advection + else + return UpwindBiased(; order = 2N - 1) + end +end + +""" + new_weno_scheme(grid, order, bounds, XT, YT, ZT) + +Constructs a new WENO scheme based on the given parameters. `XT`, `YT`, and `ZT` is the type of the precomputed weno coefficients in the +x-direction, y-direction and z-direction. A _non-stretched_ WENO scheme has `T` equal to `Nothing` everywhere. In case of a non-stretched WENO scheme, +we rebuild the advection without passing the grid information, otherwise we use the grid to account for stretched directions. +""" +new_weno_scheme(::WENO, grid, order, bounds, ::Type{Nothing}, ::Type{Nothing}, ::Type{Nothing},) = WENO(; order, bounds) +new_weno_scheme(::WENO, grid, order, bounds, XT, YT, ZT) = WENO(grid; order, bounds) + +function adapt_advection_order(advection::WENO{B, FT, XT, YT, ZT}, N::Int, grid::AbstractGrid) where {B, FT, XT, YT, ZT} + if N >= B + return advection + else + return new_weno_scheme(advection, grid, 2N - 1, advection.bounds, XT, YT, ZT) + end +end \ No newline at end of file diff --git a/src/BoundaryConditions/apply_flux_bcs.jl b/src/BoundaryConditions/apply_flux_bcs.jl index c1866fd84e..1ed09e8b43 100644 --- a/src/BoundaryConditions/apply_flux_bcs.jl +++ b/src/BoundaryConditions/apply_flux_bcs.jl @@ -78,7 +78,7 @@ Apply a top and/or bottom boundary condition to variable `c`. """ @kernel function _apply_z_bcs!(Gc, loc, grid, bottom_bc, top_bc, args) i, j = @index(Global, NTuple) - apply_z_bottom_bc!(Gc, loc, bottom_bc, i, j, grid, args...) + apply_bottom_height_bc!(Gc, loc, bottom_bc, i, j, grid, args...) apply_z_top_bc!(Gc, loc, top_bc, i, j, grid, args...) end @@ -88,7 +88,7 @@ end @inline apply_y_north_bc!(Gc, loc, ::NotFluxBC, args...) = nothing @inline apply_y_south_bc!(Gc, loc, ::NotFluxBC, args...) = nothing @inline apply_z_top_bc!(Gc, loc, ::NotFluxBC, args...) = nothing -@inline apply_z_bottom_bc!(Gc, loc, ::NotFluxBC, args...) = nothing +@inline apply_bottom_height_bc!(Gc, loc, ::NotFluxBC, args...) = nothing @inline flip(::Center) = Face() @inline flip(::Face) = Center() @@ -120,7 +120,7 @@ end return nothing end -@inline function apply_z_bottom_bc!(Gc, loc, bottom_flux::BC{<:Flux}, i, j, grid, args...) +@inline function apply_bottom_height_bc!(Gc, loc, bottom_flux::BC{<:Flux}, i, j, grid, args...) LX, LY, LZ = loc @inbounds Gc[i, j, 1] += getbc(bottom_flux, i, j, grid, args...) * Az(i, j, 1, grid, LX, LY, flip(LZ)) / volume(i, j, 1, grid, LX, LY, LZ) return nothing diff --git a/src/ImmersedBoundaries/ImmersedBoundaries.jl b/src/ImmersedBoundaries/ImmersedBoundaries.jl index 4a011d1beb..869b44dd8b 100644 --- a/src/ImmersedBoundaries/ImmersedBoundaries.jl +++ b/src/ImmersedBoundaries/ImmersedBoundaries.jl @@ -94,7 +94,7 @@ import Oceananigans.TurbulenceClosures: νᶠᶠᶜ, νᶜᶠᶠ, νᶠᶜᶠ, - z_bottom + bottom_height import Oceananigans.Fields: fractional_x_index, fractional_y_index, fractional_z_index @@ -162,7 +162,7 @@ with_halo(halo, ibg::ImmersedBoundaryGrid) = inflate_halo_size_one_dimension(req_H, old_H, _, ::IBG) = max(req_H + 1, old_H) inflate_halo_size_one_dimension(req_H, old_H, ::Type{Flat}, ::IBG) = 0 -@inline z_bottom(i, j, ibg::IBG) = error("The function `bottom` has not been defined for $(summary(ibg))!") +@inline bottom_height(i, j, ibg::IBG) = error("The function `bottom` has not been defined for $(summary(ibg))!") function Base.summary(grid::ImmersedBoundaryGrid) FT = eltype(grid) diff --git a/src/ImmersedBoundaries/distributed_immersed_boundaries.jl b/src/ImmersedBoundaries/distributed_immersed_boundaries.jl index 10473ba015..4b19202b92 100644 --- a/src/ImmersedBoundaries/distributed_immersed_boundaries.jl +++ b/src/ImmersedBoundaries/distributed_immersed_boundaries.jl @@ -14,7 +14,7 @@ function reconstruct_global_grid(grid::ImmersedBoundaryGrid) arch = grid.architecture local_ib = grid.immersed_boundary global_ug = reconstruct_global_grid(grid.underlying_grid) - global_ib = getnamewrapper(local_ib)(construct_global_array(arch, local_ib.z_bottom, size(grid))) + global_ib = getnamewrapper(local_ib)(construct_global_array(arch, local_ib.bottom_height, size(grid))) return ImmersedBoundaryGrid(global_ug, global_ib) end @@ -33,9 +33,9 @@ function scatter_local_grids(global_grid::ImmersedBoundaryGrid, arch::Distribute local_ug = scatter_local_grids(ug, arch, local_size) # Kinda hacky - local_z_bottom = partition(ib.z_bottom, arch, local_size) + local_bottom_height = partition(ib.bottom_height, arch, local_size) ImmersedBoundaryConstructor = getnamewrapper(ib) - local_ib = ImmersedBoundaryConstructor(local_z_bottom) + local_ib = ImmersedBoundaryConstructor(local_bottom_height) return ImmersedBoundaryGrid(local_ug, local_ib) end @@ -76,10 +76,10 @@ function resize_immersed_boundary(ib::AbstractGridFittedBottom{<:OffsetArray}, g # Check that the size of a bottom field are # consistent with the size of the grid - if any(size(ib.z_bottom) .!= bottom_heigth_size) + if any(size(ib.bottom_height) .!= bottom_heigth_size) @warn "Resizing the bottom field to match the grids' halos" bottom_field = Field((Center, Center, Nothing), grid) - cpu_bottom = on_architecture(CPU(), ib.z_bottom)[1:Nx, 1:Ny] + cpu_bottom = on_architecture(CPU(), ib.bottom_height)[1:Nx, 1:Ny] set!(bottom_field, cpu_bottom) fill_halo_regions!(bottom_field) offset_bottom_array = dropdims(bottom_field.data, dims=3) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index 1a2cc89e57..b3ac5f35fc 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -7,7 +7,7 @@ using Oceananigans.Fields: fill_halo_regions! using Oceananigans.BoundaryConditions: FBC using Printf -import Oceananigans.TurbulenceClosures: z_bottom +import Oceananigans.TurbulenceClosures: bottom_height ##### ##### GridFittedBottom (2.5D immersed boundary with modified bottom height) @@ -21,12 +21,12 @@ struct InterfaceImmersedCondition end struct CenterImmersedCondition end struct GridFittedBottom{H, I} <: AbstractGridFittedBottom{H} - z_bottom :: H + bottom_height :: H immersed_condition :: I end -GridFittedBottom(z_bottom) = GridFittedBottom(z_bottom, CenterImmersedCondition()) +GridFittedBottom(bottom_height) = GridFittedBottom(bottom_height, CenterImmersedCondition()) Base.summary(::CenterImmersedCondition) = "CenterImmersedCondition" Base.summary(::InterfaceImmersedCondition) = "InterfaceImmersedCondition" @@ -34,26 +34,26 @@ Base.summary(::InterfaceImmersedCondition) = "InterfaceImmersedCondition" const GFBIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:GridFittedBottom} """ - GridFittedBottom(z_bottom) + GridFittedBottom(bottom_height) Return a bottom immersed boundary. Keyword Arguments ================= -* `z_bottom`: an array or function that gives the height of the +* `bottom_height`: an array or function that gives the height of the bottom in absolute ``z`` coordinates. * `immersed_condition`: Determine whether the part of the domain that is immersed are all the cell centers that lie below - `z_bottom` (`CenterImmersedCondition()`; default) or all the cell faces that lie below `bottom_height` (`InterfaceImmersedCondition()`). + `bottom_height` (`CenterImmersedCondition()`; default) or all the cell faces that lie below `bottom_height` (`InterfaceImmersedCondition()`). The only purpose of `immersed_condition` to allow `GridFittedBottom` and `PartialCellBottom` to have the same behavior when the minimum fractional cell height for partial cells is set to 0. """ function Base.summary(ib::GridFittedBottom) - zmax = maximum(ib.z_bottom) - zmin = minimum(ib.z_bottom) - zmean = mean(ib.z_bottom) + zmax = maximum(ib.bottom_height) + zmin = minimum(ib.bottom_height) + zmean = mean(ib.bottom_height) summary1 = "GridFittedBottom(" @@ -66,48 +66,48 @@ function Base.summary(ib::GridFittedBottom) return summary1 * summary2 * summary3 end -Base.summary(ib::GridFittedBottom{<:Function}) = @sprintf("GridFittedBottom(%s)", ib.z_bottom) +Base.summary(ib::GridFittedBottom{<:Function}) = @sprintf("GridFittedBottom(%s)", ib.bottom_height) function Base.show(io::IO, ib::GridFittedBottom) print(io, summary(ib), '\n') - print(io, "├── z_bottom: ", prettysummary(ib.z_bottom), '\n') + print(io, "├── bottom_height: ", prettysummary(ib.bottom_height), '\n') end -on_architecture(arch, ib::GridFittedBottom) = GridFittedBottom(on_architecture(ib.z_bottom), ib.immersed_condition) +on_architecture(arch, ib::GridFittedBottom) = GridFittedBottom(on_architecture(ib.bottom_height), ib.immersed_condition) function on_architecture(arch, ib::GridFittedBottom{<:Field}) - architecture(ib.z_bottom) == arch && return ib - arch_grid = on_architecture(arch, ib.z_bottom.grid) - new_z_bottom = Field{Center, Center, Nothing}(arch_grid) - set!(new_z_bottom, ib.z_bottom) - fill_halo_regions!(new_z_bottom) - return GridFittedBottom(new_z_bottom, ib.immersed_condition) + architecture(ib.bottom_height) == arch && return ib + arch_grid = on_architecture(arch, ib.bottom_height.grid) + new_bottom_height = Field{Center, Center, Nothing}(arch_grid) + set!(new_bottom_height, ib.bottom_height) + fill_halo_regions!(new_bottom_height) + return GridFittedBottom(new_bottom_height, ib.immersed_condition) end -Adapt.adapt_structure(to, ib::GridFittedBottom) = GridFittedBottom(adapt(to, ib.z_bottom), ib.immersed_condition) +Adapt.adapt_structure(to, ib::GridFittedBottom) = GridFittedBottom(adapt(to, ib.bottom_height), ib.immersed_condition) """ ImmersedBoundaryGrid(grid, ib::GridFittedBottom) Return a grid with `GridFittedBottom` immersed boundary (`ib`). -Computes `ib.z_bottom` and wraps it in a Field. `ib.z_bottom` is the z-coordinate of top-most interface +Computes `ib.bottom_height` and wraps it in a Field. `ib.bottom_height` is the z-coordinate of top-most interface of the last ``immersed`` cell in the column. """ function ImmersedBoundaryGrid(grid, ib::GridFittedBottom) bottom_field = Field{Center, Center, Nothing}(grid) - set!(bottom_field, ib.z_bottom) - @apply_regionally correct_z_bottom!(bottom_field, grid, ib) + set!(bottom_field, ib.bottom_height) + @apply_regionally correct_bottom_height!(bottom_field, grid, ib) fill_halo_regions!(bottom_field) new_ib = GridFittedBottom(bottom_field) TX, TY, TZ = topology(grid) return ImmersedBoundaryGrid{TX, TY, TZ}(grid, new_ib) end -correct_z_bottom!(bottom_field, grid, ib) = - launch!(architecture(grid), grid, :xy, _correct_z_bottom!, bottom_field, grid, ib) +correct_bottom_height!(bottom_field, grid, ib) = + launch!(architecture(grid), grid, :xy, _correct_bottom_height!, bottom_field, grid, ib) -@kernel function _correct_z_bottom!(bottom_field, grid, ib::GridFittedBottom) +@kernel function _correct_bottom_height!(bottom_field, grid, ib::GridFittedBottom) i, j = @index(Global, NTuple) zb = @inbounds bottom_field[i, j, 1] condition = ib.immersed_condition @@ -121,11 +121,11 @@ end @inline function _immersed_cell(i, j, k, underlying_grid, ib::GridFittedBottom) z = znode(i, j, k, underlying_grid, c, c, c) - zb = @inbounds ib.z_bottom[i, j, 1] + zb = @inbounds ib.bottom_height[i, j, 1] return z ≤ zb end -@inline z_bottom(i, j, ibg::GFBIBG) = @inbounds ibg.immersed_boundary.z_bottom[i, j, 1] +@inline bottom_height(i, j, ibg::GFBIBG) = @inbounds ibg.immersed_boundary.bottom_height[i, j, 1] ##### ##### Bottom height @@ -133,7 +133,7 @@ end const AGFBIB = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractGridFittedBottom} -@inline column_heightᶜᶜᵃ(i, j, k, ibg::AGFBIB) = @inbounds znode(i, j, ibg.Nz+1, ibg, c, c, f) - ibg.immersed_boundary.z_bottom[i, j, 1] +@inline column_heightᶜᶜᵃ(i, j, k, ibg::AGFBIB) = @inbounds znode(i, j, ibg.Nz+1, ibg, c, c, f) - ibg.immersed_boundary.bottom_height[i, j, 1] @inline column_heightᶜᶠᵃ(i, j, k, ibg::AGFBIB) = min(column_heightᶜᶜᵃ(i, j-1, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) @inline column_heightᶠᶜᵃ(i, j, k, ibg::AGFBIB) = min(column_heightᶜᶜᵃ(i-1, j, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) @inline column_heightᶠᶠᵃ(i, j, k, ibg::AGFBIB) = min(column_heightᶠᶜᵃ(i, j-1, k, ibg), column_heightᶠᶜᵃ(i, j, k, ibg)) diff --git a/src/ImmersedBoundaries/partial_cell_bottom.jl b/src/ImmersedBoundaries/partial_cell_bottom.jl index 2e6e9ba50b..7988fa9ec9 100644 --- a/src/ImmersedBoundaries/partial_cell_bottom.jl +++ b/src/ImmersedBoundaries/partial_cell_bottom.jl @@ -7,16 +7,16 @@ using Printf ##### struct PartialCellBottom{H, E} <: AbstractGridFittedBottom{H} - z_bottom :: H + bottom_height :: H minimum_fractional_cell_height :: E end const PCBIBG{FT, TX, TY, TZ} = ImmersedBoundaryGrid{FT, TX, TY, TZ, <:Any, <:PartialCellBottom} where {FT, TX, TY, TZ} function Base.summary(ib::PartialCellBottom) - zmax = maximum(parent(ib.z_bottom)) - zmin = minimum(parent(ib.z_bottom)) - zmean = mean(parent(ib.z_bottom)) + zmax = maximum(parent(ib.bottom_height)) + zmin = minimum(parent(ib.bottom_height)) + zmean = mean(parent(ib.bottom_height)) summary1 = "PartialCellBottom(" @@ -31,21 +31,21 @@ function Base.summary(ib::PartialCellBottom) end Base.summary(ib::PartialCellBottom{<:Function}) = @sprintf("PartialCellBottom(%s, ϵ=%.1f)", - prettysummary(ib.z_bottom, false), + prettysummary(ib.bottom_height, false), prettysummary(ib.minimum_fractional_cell_height)) function Base.show(io::IO, ib::PartialCellBottom) print(io, summary(ib), '\n') - print(io, "├── z_bottom: ", prettysummary(ib.z_bottom), '\n') + print(io, "├── bottom_height: ", prettysummary(ib.bottom_height), '\n') print(io, "└── minimum_fractional_cell_height: ", prettysummary(ib.minimum_fractional_cell_height)) end """ - PartialCellBottom(z_bottom; minimum_fractional_cell_height=0.2) + PartialCellBottom(bottom_height; minimum_fractional_cell_height=0.2) Return `PartialCellBottom` representing an immersed boundary with "partial" bottom cells. That is, the height of the bottommost cell in each column is reduced -to fit the provided `z_bottom`, which may be a `Field`, `Array`, or function +to fit the provided `bottom_height`, which may be a `Field`, `Array`, or function of `(x, y)`. The height of partial bottom cells is greater than @@ -56,21 +56,21 @@ minimum_fractional_cell_height * Δz, where `Δz` is the original height of the bottom cell underlying grid. """ -function PartialCellBottom(z_bottom; minimum_fractional_cell_height=0.2) - return PartialCellBottom(z_bottom, minimum_fractional_cell_height) +function PartialCellBottom(bottom_height; minimum_fractional_cell_height=0.2) + return PartialCellBottom(bottom_height, minimum_fractional_cell_height) end function ImmersedBoundaryGrid(grid, ib::PartialCellBottom) bottom_field = Field{Center, Center, Nothing}(grid) - set!(bottom_field, ib.z_bottom) - @apply_regionally correct_z_bottom!(bottom_field, grid, ib) + set!(bottom_field, ib.bottom_height) + @apply_regionally correct_bottom_height!(bottom_field, grid, ib) fill_halo_regions!(bottom_field) new_ib = PartialCellBottom(bottom_field, ib.minimum_fractional_cell_height) TX, TY, TZ = topology(grid) return ImmersedBoundaryGrid{TX, TY, TZ}(grid, new_ib) end -@kernel function _correct_z_bottom!(bottom_field, grid, ib::PartialCellBottom) +@kernel function _correct_bottom_height!(bottom_field, grid, ib::PartialCellBottom) i, j = @index(Global, NTuple) zb = @inbounds bottom_field[i, j, 1] ϵ = ib.minimum_fractional_cell_height @@ -83,17 +83,17 @@ end end function on_architecture(arch, ib::PartialCellBottom{<:Field}) - architecture(ib.z_bottom) == arch && return ib - arch_grid = on_architecture(arch, ib.z_bottom.grid) - new_z_bottom = Field{Center, Center, Nothing}(arch_grid) - copyto!(parent(new_z_bottom), parent(ib.z_bottom)) - return PartialCellBottom(new_z_bottom, ib.minimum_fractional_cell_height) + architecture(ib.bottom_height) == arch && return ib + arch_grid = on_architecture(arch, ib.bottom_height.grid) + new_bottom_height = Field{Center, Center, Nothing}(arch_grid) + copyto!(parent(new_bottom_height), parent(ib.bottom_height)) + return PartialCellBottom(new_bottom_height, ib.minimum_fractional_cell_height) end -Adapt.adapt_structure(to, ib::PartialCellBottom) = PartialCellBottom(adapt(to, ib.z_bottom), +Adapt.adapt_structure(to, ib::PartialCellBottom) = PartialCellBottom(adapt(to, ib.bottom_height), ib.minimum_fractional_cell_height) -on_architecture(to, ib::PartialCellBottom) = PartialCellBottom(on_architecture(to, ib.z_bottom), +on_architecture(to, ib::PartialCellBottom) = PartialCellBottom(on_architecture(to, ib.bottom_height), on_architecture(to, ib.minimum_fractional_cell_height)) """ @@ -119,7 +119,7 @@ Criterion is zb ≥ z - ϵ Δz """ @inline function _immersed_cell(i, j, k, underlying_grid, ib::PartialCellBottom) z = znode(i, j, k+1, underlying_grid, c, c, f) - zb = @inbounds ib.z_bottom[i, j, 1] + zb = @inbounds ib.bottom_height[i, j, 1] return z ≤ zb end @@ -138,7 +138,7 @@ end z = znode(i, j, k+1, underlying_grid, c, c, f) # Get bottom z-coordinate and fractional Δz parameter - zb = @inbounds ib.z_bottom[i, j, 1] + zb = @inbounds ib.bottom_height[i, j, 1] # Are we in a bottom cell? at_the_bottom = bottom_cell(i, j, k, ibg) @@ -183,5 +183,5 @@ YFlatPCBIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Flat, <:Any, <:Any, <:Partial @inline Δzᶠᶠᶜ(i, j, k, ibg::XFlatPCBIBG) = Δzᶜᶠᶜ(i, j, k, ibg) @inline Δzᶠᶠᶜ(i, j, k, ibg::YFlatPCBIBG) = Δzᶠᶜᶜ(i, j, k, ibg) -@inline z_bottom(i, j, ibg::PCBIBG) = @inbounds ibg.immersed_boundary.z_bottom[i, j, 1] +@inline bottom_height(i, j, ibg::PCBIBG) = @inbounds ibg.immersed_boundary.bottom_height[i, j, 1] diff --git a/src/MultiRegion/multi_region_grid.jl b/src/MultiRegion/multi_region_grid.jl index 7d28ee6100..f338750b24 100644 --- a/src/MultiRegion/multi_region_grid.jl +++ b/src/MultiRegion/multi_region_grid.jl @@ -188,8 +188,8 @@ function reconstruct_global_grid(mrg::ImmersedMultiRegionGrid) return ImmersedBoundaryGrid(global_grid, global_boundary) end -reconstruct_global_boundary(g::GridFittedBottom{<:Field}) = GridFittedBottom(reconstruct_global_field(g.z_bottom)) -reconstruct_global_boundary(g::PartialCellBottom{<:Field}) = PartialCellBottom(reconstruct_global_field(g.z_bottom), g.minimum_fractional_cell_height) +reconstruct_global_boundary(g::GridFittedBottom{<:Field}) = GridFittedBottom(reconstruct_global_field(g.bottom_height)) +reconstruct_global_boundary(g::PartialCellBottom{<:Field}) = PartialCellBottom(reconstruct_global_field(g.bottom_height), g.minimum_fractional_cell_height) reconstruct_global_boundary(g::GridFittedBoundary{<:Field}) = GridFittedBoundary(reconstruct_global_field(g.mask)) @inline getregion(mrg::ImmersedMultiRegionGrid{FT, TX, TY, TZ}, r) where {FT, TX, TY, TZ} = ImmersedBoundaryGrid{TX, TY, TZ}(_getregion(mrg.underlying_grid, r), _getregion(mrg.immersed_boundary, r)) diff --git a/src/TurbulenceClosures/TurbulenceClosures.jl b/src/TurbulenceClosures/TurbulenceClosures.jl index f810ce5e2f..7046a2f1cc 100644 --- a/src/TurbulenceClosures/TurbulenceClosures.jl +++ b/src/TurbulenceClosures/TurbulenceClosures.jl @@ -120,14 +120,14 @@ const c = Center() const f = Face() @inline z_top(i, j, grid) = znode(i, j, grid.Nz+1, grid, c, c, f) -@inline z_bottom(i, j, grid) = znode(i, j, 1, grid, c, c, f) +@inline bottom_height(i, j, grid) = znode(i, j, 1, grid, c, c, f) @inline depthᶜᶜᶠ(i, j, k, grid) = clip(z_top(i, j, grid) - znode(i, j, k, grid, c, c, f)) @inline depthᶜᶜᶜ(i, j, k, grid) = clip(z_top(i, j, grid) - znode(i, j, k, grid, c, c, c)) -@inline total_depthᶜᶜᵃ(i, j, grid) = clip(z_top(i, j, grid) - z_bottom(i, j, grid)) +@inline total_depthᶜᶜᵃ(i, j, grid) = clip(z_top(i, j, grid) - bottom_height(i, j, grid)) @inline function height_above_bottomᶜᶜᶠ(i, j, k, grid) - h = znode(i, j, k, grid, c, c, f) - z_bottom(i, j, grid) + h = znode(i, j, k, grid, c, c, f) - bottom_height(i, j, grid) # Limit by thickness of cell below Δz = Δzᶜᶜᶜ(i, j, k-1, grid) @@ -136,7 +136,7 @@ end @inline function height_above_bottomᶜᶜᶜ(i, j, k, grid) Δz = Δzᶜᶜᶜ(i, j, k, grid) - h = znode(i, j, k, grid, c, c, c) - z_bottom(i, j, grid) + h = znode(i, j, k, grid, c, c, c) - bottom_height(i, j, grid) return max(Δz/2, h) end diff --git a/validation/immersed_boundaries/immersed_couette_flow.jl b/validation/immersed_boundaries/immersed_couette_flow.jl index cd1671f6e7..90098d6ee4 100644 --- a/validation/immersed_boundaries/immersed_couette_flow.jl +++ b/validation/immersed_boundaries/immersed_couette_flow.jl @@ -10,8 +10,8 @@ Lz = 1 underlying_grid = RectilinearGrid(size=Nz, z=(0, Lz), topology = (Flat, Flat, Bounded)) -@inline z_bottom(x, y) = 0.1 -grid = ImmersedBoundaryGrid(underlying_grid, GridFittedBottom(z_bottom)) +@inline bottom_height(x, y) = 0.1 +grid = ImmersedBoundaryGrid(underlying_grid, GridFittedBottom(bottom_height)) c_immersed_bc = ValueBoundaryCondition(1) c_top_bc = ValueBoundaryCondition(-1) diff --git a/validation/multi_region/multi_region_near_global_quarter_degree.jl b/validation/multi_region/multi_region_near_global_quarter_degree.jl index 6bb671cb8c..ce62c65f82 100644 --- a/validation/multi_region/multi_region_near_global_quarter_degree.jl +++ b/validation/multi_region/multi_region_near_global_quarter_degree.jl @@ -147,7 +147,7 @@ biharmonic_viscosity = HorizontalScalarBiharmonicDiffusivity(ν=νhb, discrete @inline cyclic_interpolate(u₁::Number, u₂, time) = u₁ + mod(time / thirty_days, 1) * (u₂ - u₁) Δz_top = @allowscalar Δzᵃᵃᶜ(1, 1, grid.Nz, grid.underlying_grid) -Δz_bottom = @allowscalar Δzᵃᵃᶜ(1, 1, 1, grid.underlying_grid) +Δbottom_height = @allowscalar Δzᵃᵃᶜ(1, 1, 1, grid.underlying_grid) @inline function surface_wind_stress(i, j, grid, clock, fields, τ) time = clock.time diff --git a/validation/vertical_mixing_closures/heterogeneous_windy_convection.jl b/validation/vertical_mixing_closures/heterogeneous_windy_convection.jl index 6faf2950d0..ce55c42dee 100644 --- a/validation/vertical_mixing_closures/heterogeneous_windy_convection.jl +++ b/validation/vertical_mixing_closures/heterogeneous_windy_convection.jl @@ -34,8 +34,8 @@ grid = RectilinearGrid(size = (Nx, Ny, Nz), z = z, topology=(Periodic, Bounded, Bounded)) -z_bottom(x, y) = - Lz * (1 - (2y / Ly)^2) -grid = ImmersedBoundaryGrid(grid, PartialCellBottom(z_bottom, minimum_fractional_cell_height=0.1)) +bottom_height(x, y) = - Lz * (1 - (2y / Ly)^2) +grid = ImmersedBoundaryGrid(grid, PartialCellBottom(bottom_height, minimum_fractional_cell_height=0.1)) @show grid @inline Qᵇ(x, y, t) = 1e-7 From f8d5c1857f054f687b3cb91f3bea21bf37525aee Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 10 Oct 2024 14:22:26 +0200 Subject: [PATCH 27/79] bugfix --- ext/OceananigansMakieExt.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/OceananigansMakieExt.jl b/ext/OceananigansMakieExt.jl index 9c8fe12bad..398f847fec 100644 --- a/ext/OceananigansMakieExt.jl +++ b/ext/OceananigansMakieExt.jl @@ -37,7 +37,7 @@ end convert_arguments(pl::Type{<:AbstractPlot}, f::Field) = convert_arguments(pl, convert_field_argument(f)...) -function convert_arguments(pl::Type{<:AbstractPlot}, fop::AbstractOperation) +function convert_arguments(pl::Type{<:AbstractPlot}, op::AbstractOperation) f = Field(op) compute!(f) return convert_arguments(pl, f) From fd8a0287b34df94397b3e9eb5f6ef336c381198a Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 10 Oct 2024 14:26:02 +0200 Subject: [PATCH 28/79] another bugfix --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index b3ac5f35fc..455d35666c 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -25,7 +25,6 @@ struct GridFittedBottom{H, I} <: AbstractGridFittedBottom{H} immersed_condition :: I end - GridFittedBottom(bottom_height) = GridFittedBottom(bottom_height, CenterImmersedCondition()) Base.summary(::CenterImmersedCondition) = "CenterImmersedCondition" @@ -73,7 +72,7 @@ function Base.show(io::IO, ib::GridFittedBottom) print(io, "├── bottom_height: ", prettysummary(ib.bottom_height), '\n') end -on_architecture(arch, ib::GridFittedBottom) = GridFittedBottom(on_architecture(ib.bottom_height), ib.immersed_condition) +on_architecture(arch, ib::GridFittedBottom) = GridFittedBottom(on_architecture(arch, ib.bottom_height), ib.immersed_condition) function on_architecture(arch, ib::GridFittedBottom{<:Field}) architecture(ib.bottom_height) == arch && return ib @@ -84,7 +83,7 @@ function on_architecture(arch, ib::GridFittedBottom{<:Field}) return GridFittedBottom(new_bottom_height, ib.immersed_condition) end -Adapt.adapt_structure(to, ib::GridFittedBottom) = GridFittedBottom(adapt(to, ib.bottom_height), ib.immersed_condition) +Adapt.adapt_structure(to, ib::GridFittedBottom) = GridFittedBottom(adapt(to, ib.bottom_height), adapt(to, ib.immersed_condition)) """ ImmersedBoundaryGrid(grid, ib::GridFittedBottom) From f518d414b1d843ff7f58a72a797770fffeeeb8e9 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 11 Oct 2024 09:29:58 +0200 Subject: [PATCH 29/79] fix bugs --- src/MultiRegion/multi_region_field.jl | 2 ++ src/MultiRegion/multi_region_grid.jl | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/MultiRegion/multi_region_field.jl b/src/MultiRegion/multi_region_field.jl index cb6d06cda8..6abbee7ce2 100644 --- a/src/MultiRegion/multi_region_field.jl +++ b/src/MultiRegion/multi_region_field.jl @@ -70,6 +70,8 @@ Base.size(f::GriddedMultiRegionField) = size(getregion(f.grid, 1)) Reconstruct a global field from `mrf::MultiRegionField` on the `CPU`. """ function reconstruct_global_field(mrf::MultiRegionField) + + # TODO: Is this correct? Shall we reconstruct a global field on the architecture of the grid? global_grid = on_architecture(CPU(), reconstruct_global_grid(mrf.grid)) indices = reconstruct_global_indices(mrf.indices, mrf.grid.partition, size(global_grid)) global_field = Field(location(mrf), global_grid; indices) diff --git a/src/MultiRegion/multi_region_grid.jl b/src/MultiRegion/multi_region_grid.jl index f338750b24..61e419c16a 100644 --- a/src/MultiRegion/multi_region_grid.jl +++ b/src/MultiRegion/multi_region_grid.jl @@ -184,11 +184,12 @@ end function reconstruct_global_grid(mrg::ImmersedMultiRegionGrid) global_grid = reconstruct_global_grid(mrg.underlying_grid) global_boundary = reconstruct_global_boundary(mrg.immersed_boundary) + global_boundary = on_architecture(architecture(mrg), global_boundary) return ImmersedBoundaryGrid(global_grid, global_boundary) end -reconstruct_global_boundary(g::GridFittedBottom{<:Field}) = GridFittedBottom(reconstruct_global_field(g.bottom_height)) +reconstruct_global_boundary(g::GridFittedBottom{<:Field}) = GridFittedBottom(reconstruct_global_field(g.bottom_height), g.immersed_condition) reconstruct_global_boundary(g::PartialCellBottom{<:Field}) = PartialCellBottom(reconstruct_global_field(g.bottom_height), g.minimum_fractional_cell_height) reconstruct_global_boundary(g::GridFittedBoundary{<:Field}) = GridFittedBoundary(reconstruct_global_field(g.mask)) From 983b9560c0334c7f12de8797482f24b011902aab Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 11 Oct 2024 09:33:35 +0200 Subject: [PATCH 30/79] some corrections --- docs/src/grids.md | 2 +- src/Advection/adapt_advection_order.jl | 98 ------------------------ src/BoundaryConditions/apply_flux_bcs.jl | 4 +- 3 files changed, 3 insertions(+), 101 deletions(-) delete mode 100644 src/Advection/adapt_advection_order.jl diff --git a/docs/src/grids.md b/docs/src/grids.md index a6686e5767..71ec02adc0 100644 --- a/docs/src/grids.md +++ b/docs/src/grids.md @@ -211,7 +211,7 @@ h = mountain_grid.immersed_boundary.bottom_height fig = Figure(size=(600, 600)) ax = Axis(fig[2, 1], xlabel="x (m)", ylabel="y (m)", aspect=1) -hm = heatmap!(ax, - h) +hm = heatmap!(ax, h) Colorbar(fig[1, 1], hm, vertical=false, label="Bottom height (m)") current_figure() diff --git a/src/Advection/adapt_advection_order.jl b/src/Advection/adapt_advection_order.jl deleted file mode 100644 index 41f3f60c9a..0000000000 --- a/src/Advection/adapt_advection_order.jl +++ /dev/null @@ -1,98 +0,0 @@ -using Oceananigans.Grids: topology - -""" - adapt_advection_order(advection, grid::AbstractGrid) - -Adapts the advection operator `advection` based on the grid `grid` by adjusting the order of advection in each direction. -For example, if the grid has only one point in the x-direction, the advection operator in the x-direction is set to first order -upwind or 2nd order centered scheme, depending on the original user-specified advection scheme. A high order advection sheme -is reduced to a lower order advection scheme if the grid has fewer points in that direction. - -# Arguments -- `advection`: The original advection scheme. -- `grid::AbstractGrid`: The grid on which the advection scheme is applied. - -If the order of advection is changed in at least one direction, the adapted advection scheme with adjusted advection order returned -by this function is a `FluxFormAdvection`. -""" -function adapt_advection_order(advection, grid::AbstractGrid) - advection_x = x_advection(advection) - advection_y = y_advection(advection) - advection_z = z_advection(advection) - - new_advection_x = adapt_advection_order(advection_x, size(grid, 1), grid) - new_advection_y = adapt_advection_order(advection_y, size(grid, 2), grid) - new_advection_z = adapt_advection_order(advection_z, size(grid, 3), grid) - - # Check that we indeed changed the advection operator - changed_x = new_advection_x != advection_x - changed_y = new_advection_y != advection_y - changed_z = new_advection_z != advection_z - - new_advection = FluxFormAdvection(new_advection_x, new_advection_y, new_advection_z) - changed_advection = any((changed_x, changed_y, changed_z)) - - if changed_x - @info "Using the advection scheme $(summary(new_advection.x)) in the x-direction because size(grid, 1) = $(size(grid, 1))" - end - if changed_y - @info "Using the advection scheme $(summary(new_advection.y)) in the y-direction because size(grid, 2) = $(size(grid, 2))" - end - if changed_z - @info "Using the advection scheme $(summary(new_advection.z)) in the x-direction because size(grid, 3) = $(size(grid, 3))" - end - - return ifelse(changed_advection, new_advection, advection) -end - - -x_advection(flux_form::FluxFormAdvection) = flux_form.x -y_advection(flux_form::FluxFormAdvection) = flux_form.y -z_advection(flux_form::FluxFormAdvection) = flux_form.z - -x_advection(advection) = advection -y_advection(advection) = advection -z_advection(advection) = advection - -# For the moment, we do not adapt the advection order for the VectorInvariant advection scheme -adapt_advection_order(advection::VectorInvariant, grid::AbstractGrid) = advection -adapt_advection_order(advection::Nothing, grid::AbstractGrid) = nothing -adapt_advection_order(advection::Nothing, N::Int, grid::AbstractGrid) = nothing - -##### -##### Directional adapt advection order -##### - -function adapt_advection_order(advection::Centered{B}, N::Int, grid::AbstractGrid) where B - if N >= B - return advection - else - return Centered(; order = 2N) - end -end - -function adapt_advection_order(advection::UpwindBiased{B}, N::Int, grid::AbstractGrid) where B - if N >= B - return advection - else - return UpwindBiased(; order = 2N - 1) - end -end - -""" - new_weno_scheme(grid, order, bounds, XT, YT, ZT) - -Constructs a new WENO scheme based on the given parameters. `XT`, `YT`, and `ZT` is the type of the precomputed weno coefficients in the -x-direction, y-direction and z-direction. A _non-stretched_ WENO scheme has `T` equal to `Nothing` everywhere. In case of a non-stretched WENO scheme, -we rebuild the advection without passing the grid information, otherwise we use the grid to account for stretched directions. -""" -new_weno_scheme(::WENO, grid, order, bounds, ::Type{Nothing}, ::Type{Nothing}, ::Type{Nothing},) = WENO(; order, bounds) -new_weno_scheme(::WENO, grid, order, bounds, XT, YT, ZT) = WENO(grid; order, bounds) - -function adapt_advection_order(advection::WENO{B, FT, XT, YT, ZT}, N::Int, grid::AbstractGrid) where {B, FT, XT, YT, ZT} - if N >= B - return advection - else - return new_weno_scheme(advection, grid, 2N - 1, advection.bounds, XT, YT, ZT) - end -end \ No newline at end of file diff --git a/src/BoundaryConditions/apply_flux_bcs.jl b/src/BoundaryConditions/apply_flux_bcs.jl index 1ed09e8b43..2a69fad07c 100644 --- a/src/BoundaryConditions/apply_flux_bcs.jl +++ b/src/BoundaryConditions/apply_flux_bcs.jl @@ -78,7 +78,7 @@ Apply a top and/or bottom boundary condition to variable `c`. """ @kernel function _apply_z_bcs!(Gc, loc, grid, bottom_bc, top_bc, args) i, j = @index(Global, NTuple) - apply_bottom_height_bc!(Gc, loc, bottom_bc, i, j, grid, args...) + apply_z_bottom_bc!(Gc, loc, bottom_bc, i, j, grid, args...) apply_z_top_bc!(Gc, loc, top_bc, i, j, grid, args...) end @@ -88,7 +88,7 @@ end @inline apply_y_north_bc!(Gc, loc, ::NotFluxBC, args...) = nothing @inline apply_y_south_bc!(Gc, loc, ::NotFluxBC, args...) = nothing @inline apply_z_top_bc!(Gc, loc, ::NotFluxBC, args...) = nothing -@inline apply_bottom_height_bc!(Gc, loc, ::NotFluxBC, args...) = nothing +@inline apply_z_bottom_bc!(Gc, loc, ::NotFluxBC, args...) = nothing @inline flip(::Center) = Face() @inline flip(::Face) = Center() From dfada799cb91a1945a95b99100288e04bcc2d4cb Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 11 Oct 2024 12:19:34 +0200 Subject: [PATCH 31/79] another bugfix --- src/BoundaryConditions/apply_flux_bcs.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BoundaryConditions/apply_flux_bcs.jl b/src/BoundaryConditions/apply_flux_bcs.jl index 2a69fad07c..c1866fd84e 100644 --- a/src/BoundaryConditions/apply_flux_bcs.jl +++ b/src/BoundaryConditions/apply_flux_bcs.jl @@ -120,7 +120,7 @@ end return nothing end -@inline function apply_bottom_height_bc!(Gc, loc, bottom_flux::BC{<:Flux}, i, j, grid, args...) +@inline function apply_z_bottom_bc!(Gc, loc, bottom_flux::BC{<:Flux}, i, j, grid, args...) LX, LY, LZ = loc @inbounds Gc[i, j, 1] += getbc(bottom_flux, i, j, grid, args...) * Az(i, j, 1, grid, LX, LY, flip(LZ)) / volume(i, j, 1, grid, LX, LY, LZ) return nothing From 2705b03818c459ab23d7cc61e3e3f6c5f241109b Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 11 Oct 2024 16:46:53 +0200 Subject: [PATCH 32/79] domain_depth --- src/Grids/Grids.jl | 2 +- src/Grids/grid_utils.jl | 8 ++++---- src/ImmersedBoundaries/ImmersedBoundaries.jl | 2 +- src/ImmersedBoundaries/grid_fitted_bottom.jl | 18 +++++++++--------- .../split_explicit_free_surface_kernels.jl | 8 ++++---- src/TurbulenceClosures/TurbulenceClosures.jl | 13 ++++++------- .../catke_equation.jl | 2 +- .../catke_mixing_length.jl | 8 ++++---- 8 files changed, 30 insertions(+), 31 deletions(-) diff --git a/src/Grids/Grids.jl b/src/Grids/Grids.jl index fcc6270001..e88ca07980 100644 --- a/src/Grids/Grids.jl +++ b/src/Grids/Grids.jl @@ -18,7 +18,7 @@ export xnodes, ynodes, znodes, λnodes, φnodes export spacings export xspacings, yspacings, zspacings, xspacing, yspacing, zspacing export minimum_xspacing, minimum_yspacing, minimum_zspacing -export column_heightᶜᶜᵃ, column_heightᶠᶜᵃ, column_heightᶜᶠᵃ, column_heightᶠᶠᵃ +export domain_depthᶜᶜᵃ, domain_depthᶠᶜᵃ, domain_depthᶜᶠᵃ, domain_depthᶠᶠᵃ export offset_data, new_data export on_architecture diff --git a/src/Grids/grid_utils.jl b/src/Grids/grid_utils.jl index 33d60205d1..efcefd2297 100644 --- a/src/Grids/grid_utils.jl +++ b/src/Grids/grid_utils.jl @@ -318,10 +318,10 @@ coordinate_summary(topo, Δ::Union{AbstractVector, AbstractMatrix}, name) = ##### Bottom height ##### -@inline column_heightᶜᶜᵃ(i, j, k, grid) = grid.Lz -@inline column_heightᶜᶠᵃ(i, j, k, grid) = grid.Lz -@inline column_heightᶠᶜᵃ(i, j, k, grid) = grid.Lz -@inline column_heightᶠᶠᵃ(i, j, k, grid) = grid.Lz +@inline domain_depthᶜᶜᵃ(i, j, grid) = grid.Lz +@inline domain_depthᶜᶠᵃ(i, j, grid) = grid.Lz +@inline domain_depthᶠᶜᵃ(i, j, grid) = grid.Lz +@inline domain_depthᶠᶠᵃ(i, j, grid) = grid.Lz ##### ##### Spherical geometry diff --git a/src/ImmersedBoundaries/ImmersedBoundaries.jl b/src/ImmersedBoundaries/ImmersedBoundaries.jl index 869b44dd8b..5805e04e05 100644 --- a/src/ImmersedBoundaries/ImmersedBoundaries.jl +++ b/src/ImmersedBoundaries/ImmersedBoundaries.jl @@ -53,7 +53,7 @@ import Oceananigans.Grids: architecture, with_halo, inflate_halo_size_one_dimens ξname, ηname, rname, node_names, xnodes, ynodes, znodes, λnodes, φnodes, nodes, ξnodes, ηnodes, rnodes, - column_heightᶜᶜᵃ, column_heightᶠᶜᵃ, column_heightᶜᶠᵃ, column_heightᶠᶠᵃ, + domain_depthᶜᶜᵃ, domain_depthᶠᶜᵃ, domain_depthᶜᶠᵃ, domain_depthᶠᶠᵃ, inactive_cell import Oceananigans.Coriolis: φᶠᶠᵃ diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index 455d35666c..d70ac77cff 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -124,7 +124,7 @@ end return z ≤ zb end -@inline bottom_height(i, j, ibg::GFBIBG) = @inbounds ibg.immersed_boundary.bottom_height[i, j, 1] +@inline z_bottom(i, j, ibg::GFBIBG) = @inbounds ibg.immersed_boundary.bottom_height[i, j, 1] ##### ##### Bottom height @@ -132,16 +132,16 @@ end const AGFBIB = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractGridFittedBottom} -@inline column_heightᶜᶜᵃ(i, j, k, ibg::AGFBIB) = @inbounds znode(i, j, ibg.Nz+1, ibg, c, c, f) - ibg.immersed_boundary.bottom_height[i, j, 1] -@inline column_heightᶜᶠᵃ(i, j, k, ibg::AGFBIB) = min(column_heightᶜᶜᵃ(i, j-1, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) -@inline column_heightᶠᶜᵃ(i, j, k, ibg::AGFBIB) = min(column_heightᶜᶜᵃ(i-1, j, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) -@inline column_heightᶠᶠᵃ(i, j, k, ibg::AGFBIB) = min(column_heightᶠᶜᵃ(i, j-1, k, ibg), column_heightᶠᶜᵃ(i, j, k, ibg)) +@inline domain_depthᶜᶜᵃ(i, j, ibg::AGFBIB) = @inbounds znode(i, j, ibg.Nz+1, ibg, c, c, f) - ibg.immersed_boundary.bottom_height[i, j, 1] +@inline domain_depthᶜᶠᵃ(i, j, ibg::AGFBIB) = min(domain_depthᶜᶜᵃ(i, j-1, ibg), domain_depthᶜᶜᵃ(i, j, ibg)) +@inline domain_depthᶠᶜᵃ(i, j, ibg::AGFBIB) = min(domain_depthᶜᶜᵃ(i-1, j, ibg), domain_depthᶜᶜᵃ(i, j, ibg)) +@inline domain_depthᶠᶠᵃ(i, j, ibg::AGFBIB) = min(domain_depthᶠᶜᵃ(i, j-1, ibg), domain_depthᶠᶜᵃ(i, j, ibg)) # Make sure column_height works for horizontally-Flat topologies. XFlatAGFIBG = ImmersedBoundaryGrid{<:Any, <:Flat, <:Any, <:Any, <:Any, <:AbstractGridFittedBottom} YFlatAGFIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Flat, <:Any, <:Any, <:AbstractGridFittedBottom} -@inline column_heightᶠᶜᵃ(i, j, k, ibg::XFlatAGFIBG) = column_heightᶜᶜᵃ(i, j, k, ibg) -@inline column_heightᶜᶠᵃ(i, j, k, ibg::YFlatAGFIBG) = column_heightᶜᶜᵃ(i, j, k, ibg) -@inline column_heightᶠᶠᵃ(i, j, k, ibg::XFlatAGFIBG) = column_heightᶜᶠᵃ(i, j, k, ibg) -@inline column_heightᶠᶠᵃ(i, j, k, ibg::YFlatAGFIBG) = column_heightᶠᶜᵃ(i, j, k, ibg) +@inline domain_depthᶠᶜᵃ(i, j, k, ibg::XFlatAGFIBG) = domain_depthᶜᶜᵃ(i, j, ibg) +@inline domain_depthᶜᶠᵃ(i, j, k, ibg::YFlatAGFIBG) = domain_depthᶜᶜᵃ(i, j, ibg) +@inline domain_depthᶠᶠᵃ(i, j, k, ibg::XFlatAGFIBG) = domain_depthᶜᶠᵃ(i, j, ibg) +@inline domain_depthᶠᶠᵃ(i, j, k, ibg::YFlatAGFIBG) = domain_depthᶠᶜᵃ(i, j, ibg) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index a0226762f8..6b118d0868 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -183,8 +183,8 @@ end advance_previous_velocity!(i, j, k_top-1, timestepper, U, Uᵐ⁻¹, Uᵐ⁻²) advance_previous_velocity!(i, j, k_top-1, timestepper, V, Vᵐ⁻¹, Vᵐ⁻²) - Hᶠᶜ = column_heightᶠᶜᵃ(i, j, k_top-1, grid) - Hᶜᶠ = column_heightᶜᶠᵃ(i, j, k_top-1, grid) + Hᶠᶜ = domain_depthᶠᶜᵃ(i, j, grid) + Hᶜᶠ = domain_depthᶜᶠᵃ(i, j, grid) # ∂τ(U) = - ∇η + G U[i, j, k_top-1] += Δτ * (- g * Hᶠᶜ * ∂xᶠᶜᶠ_η(i, j, k_top, grid, TX, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gᵁ[i, j, k_top-1]) @@ -271,8 +271,8 @@ end k_top = grid.Nz+1 @inbounds begin - Hᶠᶜ = column_heightᶠᶜᵃ(i, j, k_top-1, grid) - Hᶜᶠ = column_heightᶜᶠᵃ(i, j, k_top-1, grid) + Hᶠᶜ = domain_depthᶠᶜᵃ(i, j, grid) + Hᶜᶠ = domain_depthᶜᶠᵃ(i, j, grid) u[i, j, k] = u[i, j, k] + (U̅[i, j, k_top-1] - U[i, j, k_top-1]) / Hᶠᶜ v[i, j, k] = v[i, j, k] + (V̅[i, j, k_top-1] - V[i, j, k_top-1]) / Hᶜᶠ diff --git a/src/TurbulenceClosures/TurbulenceClosures.jl b/src/TurbulenceClosures/TurbulenceClosures.jl index 7046a2f1cc..c32faf9bae 100644 --- a/src/TurbulenceClosures/TurbulenceClosures.jl +++ b/src/TurbulenceClosures/TurbulenceClosures.jl @@ -119,15 +119,14 @@ end const c = Center() const f = Face() -@inline z_top(i, j, grid) = znode(i, j, grid.Nz+1, grid, c, c, f) -@inline bottom_height(i, j, grid) = znode(i, j, 1, grid, c, c, f) +@inline z_top(i, j, grid) = znode(i, j, grid.Nz+1, grid, c, c, f) +@inline z_bottom(i, j, grid) = znode(i, j, 1, grid, c, c, f) -@inline depthᶜᶜᶠ(i, j, k, grid) = clip(z_top(i, j, grid) - znode(i, j, k, grid, c, c, f)) -@inline depthᶜᶜᶜ(i, j, k, grid) = clip(z_top(i, j, grid) - znode(i, j, k, grid, c, c, c)) -@inline total_depthᶜᶜᵃ(i, j, grid) = clip(z_top(i, j, grid) - bottom_height(i, j, grid)) +@inline depthᶜᶜᶠ(i, j, k, grid) = clip(z_top(i, j, grid) - znode(i, j, k, grid, c, c, f)) +@inline depthᶜᶜᶜ(i, j, k, grid) = clip(z_top(i, j, grid) - znode(i, j, k, grid, c, c, c)) @inline function height_above_bottomᶜᶜᶠ(i, j, k, grid) - h = znode(i, j, k, grid, c, c, f) - bottom_height(i, j, grid) + h = znode(i, j, k, grid, c, c, f) - z_bottom(i, j, grid) # Limit by thickness of cell below Δz = Δzᶜᶜᶜ(i, j, k-1, grid) @@ -136,7 +135,7 @@ end @inline function height_above_bottomᶜᶜᶜ(i, j, k, grid) Δz = Δzᶜᶜᶜ(i, j, k, grid) - h = znode(i, j, k, grid, c, c, c) - bottom_height(i, j, grid) + h = znode(i, j, k, grid, c, c, c) - z_bottom(i, j, grid) return max(Δz/2, h) end diff --git a/src/TurbulenceClosures/turbulence_closure_implementations/TKEBasedVerticalDiffusivities/catke_equation.jl b/src/TurbulenceClosures/turbulence_closure_implementations/TKEBasedVerticalDiffusivities/catke_equation.jl index a08fb4b969..73ddb5310c 100644 --- a/src/TurbulenceClosures/turbulence_closure_implementations/TKEBasedVerticalDiffusivities/catke_equation.jl +++ b/src/TurbulenceClosures/turbulence_closure_implementations/TKEBasedVerticalDiffusivities/catke_equation.jl @@ -58,7 +58,7 @@ end ℓ★ = ifelse(isnan(ℓ★), zero(grid), ℓ★) ℓᴰ = max(ℓ★, ℓʰ) - H = total_depthᶜᶜᵃ(i, j, grid) + H = domain_depthᶜᶜᵃ(i, j, grid) return min(H, ℓᴰ) end diff --git a/src/TurbulenceClosures/turbulence_closure_implementations/TKEBasedVerticalDiffusivities/catke_mixing_length.jl b/src/TurbulenceClosures/turbulence_closure_implementations/TKEBasedVerticalDiffusivities/catke_mixing_length.jl index 8b2620f6b0..19e81270fb 100644 --- a/src/TurbulenceClosures/turbulence_closure_implementations/TKEBasedVerticalDiffusivities/catke_mixing_length.jl +++ b/src/TurbulenceClosures/turbulence_closure_implementations/TKEBasedVerticalDiffusivities/catke_mixing_length.jl @@ -5,7 +5,7 @@ using ..TurbulenceClosures: height_above_bottomᶜᶜᶠ, depthᶜᶜᶜ, height_above_bottomᶜᶜᶜ, - total_depthᶜᶜᵃ + domain_depthᶜᶜᵃ """ struct CATKEMixingLength{FT} @@ -232,7 +232,7 @@ end ℓ★ = ifelse(isnan(ℓ★), zero(grid), ℓ★) ℓu = max(ℓ★, ℓʰ) - H = total_depthᶜᶜᵃ(i, j, grid) + H = domain_depthᶜᶜᵃ(i, j, grid) return min(H, ℓu) end @@ -252,7 +252,7 @@ end ℓ★ = ifelse(isnan(ℓ★), zero(grid), ℓ★) ℓc = max(ℓ★, ℓʰ) - H = total_depthᶜᶜᵃ(i, j, grid) + H = domain_depthᶜᶜᵃ(i, j, grid) return min(H, ℓc) end @@ -272,7 +272,7 @@ end ℓ★ = ifelse(isnan(ℓ★), zero(grid), ℓ★) ℓe = max(ℓ★, ℓʰ) - H = total_depthᶜᶜᵃ(i, j, grid) + H = domain_depthᶜᶜᵃ(i, j, grid) return min(H, ℓe) end From cc9cd4ff0697019dcfee2be7c17cfcc02f970479 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 11 Oct 2024 17:13:23 +0200 Subject: [PATCH 33/79] some remaining `z_bottom`s --- src/ImmersedBoundaries/ImmersedBoundaries.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ImmersedBoundaries/ImmersedBoundaries.jl b/src/ImmersedBoundaries/ImmersedBoundaries.jl index 5805e04e05..f823f54dfd 100644 --- a/src/ImmersedBoundaries/ImmersedBoundaries.jl +++ b/src/ImmersedBoundaries/ImmersedBoundaries.jl @@ -94,7 +94,7 @@ import Oceananigans.TurbulenceClosures: νᶠᶠᶜ, νᶜᶠᶠ, νᶠᶜᶠ, - bottom_height + z_bottom import Oceananigans.Fields: fractional_x_index, fractional_y_index, fractional_z_index @@ -162,7 +162,7 @@ with_halo(halo, ibg::ImmersedBoundaryGrid) = inflate_halo_size_one_dimension(req_H, old_H, _, ::IBG) = max(req_H + 1, old_H) inflate_halo_size_one_dimension(req_H, old_H, ::Type{Flat}, ::IBG) = 0 -@inline bottom_height(i, j, ibg::IBG) = error("The function `bottom` has not been defined for $(summary(ibg))!") +@inline z_bottom(i, j, ibg::IBG) = error("The function `bottom` has not been defined for $(summary(ibg))!") function Base.summary(grid::ImmersedBoundaryGrid) FT = eltype(grid) From 45b5cd4296e9c646df15e2bffaa4573682632ad1 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 11 Oct 2024 17:14:23 +0200 Subject: [PATCH 34/79] back as it was --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index d70ac77cff..b54e75921e 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -33,7 +33,7 @@ Base.summary(::InterfaceImmersedCondition) = "InterfaceImmersedCondition" const GFBIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:GridFittedBottom} """ - GridFittedBottom(bottom_height) + GridFittedBottom(bottom_height, [immersed_condition=CenterImmersedCondition()]) Return a bottom immersed boundary. From d381037662c46649caead41b9e3ff8a0d2294a4b Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 16 Oct 2024 17:18:36 +0200 Subject: [PATCH 35/79] bugfix --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index b54e75921e..e6d75bc7fa 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -141,7 +141,7 @@ const AGFBIB = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Abstrac XFlatAGFIBG = ImmersedBoundaryGrid{<:Any, <:Flat, <:Any, <:Any, <:Any, <:AbstractGridFittedBottom} YFlatAGFIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Flat, <:Any, <:Any, <:AbstractGridFittedBottom} -@inline domain_depthᶠᶜᵃ(i, j, k, ibg::XFlatAGFIBG) = domain_depthᶜᶜᵃ(i, j, ibg) -@inline domain_depthᶜᶠᵃ(i, j, k, ibg::YFlatAGFIBG) = domain_depthᶜᶜᵃ(i, j, ibg) -@inline domain_depthᶠᶠᵃ(i, j, k, ibg::XFlatAGFIBG) = domain_depthᶜᶠᵃ(i, j, ibg) -@inline domain_depthᶠᶠᵃ(i, j, k, ibg::YFlatAGFIBG) = domain_depthᶠᶜᵃ(i, j, ibg) +@inline domain_depthᶠᶜᵃ(i, j, ibg::XFlatAGFIBG) = domain_depthᶜᶜᵃ(i, j, ibg) +@inline domain_depthᶜᶠᵃ(i, j, ibg::YFlatAGFIBG) = domain_depthᶜᶜᵃ(i, j, ibg) +@inline domain_depthᶠᶠᵃ(i, j, ibg::XFlatAGFIBG) = domain_depthᶜᶠᵃ(i, j, ibg) +@inline domain_depthᶠᶠᵃ(i, j, ibg::YFlatAGFIBG) = domain_depthᶠᶜᵃ(i, j, ibg) From b8401cc0cc87d31241fe9c67948ecf35576ff387 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 16 Oct 2024 20:14:03 +0200 Subject: [PATCH 36/79] correct correction --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 1 + src/ImmersedBoundaries/partial_cell_bottom.jl | 1 + 2 files changed, 2 insertions(+) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index e6d75bc7fa..29452164c0 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -109,6 +109,7 @@ correct_bottom_height!(bottom_field, grid, ib) = @kernel function _correct_bottom_height!(bottom_field, grid, ib::GridFittedBottom) i, j = @index(Global, NTuple) zb = @inbounds bottom_field[i, j, 1] + @inbounds bottom_field[i, j, 1] = znode(i, j, 1, grid, c, c, f) condition = ib.immersed_condition for k in 1:grid.Nz z⁺ = znode(i, j, k+1, grid, c, c, f) diff --git a/src/ImmersedBoundaries/partial_cell_bottom.jl b/src/ImmersedBoundaries/partial_cell_bottom.jl index 7988fa9ec9..7034c01dc6 100644 --- a/src/ImmersedBoundaries/partial_cell_bottom.jl +++ b/src/ImmersedBoundaries/partial_cell_bottom.jl @@ -73,6 +73,7 @@ end @kernel function _correct_bottom_height!(bottom_field, grid, ib::PartialCellBottom) i, j = @index(Global, NTuple) zb = @inbounds bottom_field[i, j, 1] + @inbounds bottom_field[i, j, 1] = znode(i, j, 1, grid, c, c, f) ϵ = ib.minimum_fractional_cell_height for k in 1:grid.Nz z⁻ = znode(i, j, k, grid, c, c, f) From b967754bd00596f44aeadc480509e55414508953 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 21 Oct 2024 09:28:19 +0200 Subject: [PATCH 37/79] static_column_depth --- src/Grids/Grids.jl | 2 +- src/Grids/grid_utils.jl | 8 +- src/ImmersedBoundaries/ImmersedBoundaries.jl | 2 +- src/ImmersedBoundaries/grid_fitted_bottom.jl | 16 ++-- ...ompute_hydrostatic_free_surface_buffers.jl | 73 +++++++++++++++++++ .../split_explicit_free_surface_kernels.jl | 8 +- .../catke_equation.jl | 2 +- .../catke_mixing_length.jl | 8 +- 8 files changed, 96 insertions(+), 23 deletions(-) create mode 100644 src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_buffers.jl diff --git a/src/Grids/Grids.jl b/src/Grids/Grids.jl index e88ca07980..27bd8f3a4a 100644 --- a/src/Grids/Grids.jl +++ b/src/Grids/Grids.jl @@ -18,7 +18,7 @@ export xnodes, ynodes, znodes, λnodes, φnodes export spacings export xspacings, yspacings, zspacings, xspacing, yspacing, zspacing export minimum_xspacing, minimum_yspacing, minimum_zspacing -export domain_depthᶜᶜᵃ, domain_depthᶠᶜᵃ, domain_depthᶜᶠᵃ, domain_depthᶠᶠᵃ +export static_column_depthᶜᶜᵃ, static_column_depthᶠᶜᵃ, static_column_depthᶜᶠᵃ, static_column_depthᶠᶠᵃ export offset_data, new_data export on_architecture diff --git a/src/Grids/grid_utils.jl b/src/Grids/grid_utils.jl index efcefd2297..4bee71f311 100644 --- a/src/Grids/grid_utils.jl +++ b/src/Grids/grid_utils.jl @@ -318,10 +318,10 @@ coordinate_summary(topo, Δ::Union{AbstractVector, AbstractMatrix}, name) = ##### Bottom height ##### -@inline domain_depthᶜᶜᵃ(i, j, grid) = grid.Lz -@inline domain_depthᶜᶠᵃ(i, j, grid) = grid.Lz -@inline domain_depthᶠᶜᵃ(i, j, grid) = grid.Lz -@inline domain_depthᶠᶠᵃ(i, j, grid) = grid.Lz +@inline static_column_depthᶜᶜᵃ(i, j, grid) = grid.Lz +@inline static_column_depthᶜᶠᵃ(i, j, grid) = grid.Lz +@inline static_column_depthᶠᶜᵃ(i, j, grid) = grid.Lz +@inline static_column_depthᶠᶠᵃ(i, j, grid) = grid.Lz ##### ##### Spherical geometry diff --git a/src/ImmersedBoundaries/ImmersedBoundaries.jl b/src/ImmersedBoundaries/ImmersedBoundaries.jl index f823f54dfd..afad4b152a 100644 --- a/src/ImmersedBoundaries/ImmersedBoundaries.jl +++ b/src/ImmersedBoundaries/ImmersedBoundaries.jl @@ -53,7 +53,7 @@ import Oceananigans.Grids: architecture, with_halo, inflate_halo_size_one_dimens ξname, ηname, rname, node_names, xnodes, ynodes, znodes, λnodes, φnodes, nodes, ξnodes, ηnodes, rnodes, - domain_depthᶜᶜᵃ, domain_depthᶠᶜᵃ, domain_depthᶜᶠᵃ, domain_depthᶠᶠᵃ, + static_column_depthᶜᶜᵃ, static_column_depthᶠᶜᵃ, static_column_depthᶜᶠᵃ, static_column_depthᶠᶠᵃ, inactive_cell import Oceananigans.Coriolis: φᶠᶠᵃ diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index 29452164c0..b44d253bb6 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -133,16 +133,16 @@ end const AGFBIB = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractGridFittedBottom} -@inline domain_depthᶜᶜᵃ(i, j, ibg::AGFBIB) = @inbounds znode(i, j, ibg.Nz+1, ibg, c, c, f) - ibg.immersed_boundary.bottom_height[i, j, 1] -@inline domain_depthᶜᶠᵃ(i, j, ibg::AGFBIB) = min(domain_depthᶜᶜᵃ(i, j-1, ibg), domain_depthᶜᶜᵃ(i, j, ibg)) -@inline domain_depthᶠᶜᵃ(i, j, ibg::AGFBIB) = min(domain_depthᶜᶜᵃ(i-1, j, ibg), domain_depthᶜᶜᵃ(i, j, ibg)) -@inline domain_depthᶠᶠᵃ(i, j, ibg::AGFBIB) = min(domain_depthᶠᶜᵃ(i, j-1, ibg), domain_depthᶠᶜᵃ(i, j, ibg)) +@inline static_column_depthᶜᶜᵃ(i, j, ibg::AGFBIB) = @inbounds znode(i, j, ibg.Nz+1, ibg, c, c, f) - ibg.immersed_boundary.bottom_height[i, j, 1] +@inline static_column_depthᶜᶠᵃ(i, j, ibg::AGFBIB) = min(static_column_depthᶜᶜᵃ(i, j-1, ibg), static_column_depthᶜᶜᵃ(i, j, ibg)) +@inline static_column_depthᶠᶜᵃ(i, j, ibg::AGFBIB) = min(static_column_depthᶜᶜᵃ(i-1, j, ibg), static_column_depthᶜᶜᵃ(i, j, ibg)) +@inline static_column_depthᶠᶠᵃ(i, j, ibg::AGFBIB) = min(static_column_depthᶠᶜᵃ(i, j-1, ibg), static_column_depthᶠᶜᵃ(i, j, ibg)) # Make sure column_height works for horizontally-Flat topologies. XFlatAGFIBG = ImmersedBoundaryGrid{<:Any, <:Flat, <:Any, <:Any, <:Any, <:AbstractGridFittedBottom} YFlatAGFIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Flat, <:Any, <:Any, <:AbstractGridFittedBottom} -@inline domain_depthᶠᶜᵃ(i, j, ibg::XFlatAGFIBG) = domain_depthᶜᶜᵃ(i, j, ibg) -@inline domain_depthᶜᶠᵃ(i, j, ibg::YFlatAGFIBG) = domain_depthᶜᶜᵃ(i, j, ibg) -@inline domain_depthᶠᶠᵃ(i, j, ibg::XFlatAGFIBG) = domain_depthᶜᶠᵃ(i, j, ibg) -@inline domain_depthᶠᶠᵃ(i, j, ibg::YFlatAGFIBG) = domain_depthᶠᶜᵃ(i, j, ibg) +@inline static_column_depthᶠᶜᵃ(i, j, ibg::XFlatAGFIBG) = static_column_depthᶜᶜᵃ(i, j, ibg) +@inline static_column_depthᶜᶠᵃ(i, j, ibg::YFlatAGFIBG) = static_column_depthᶜᶜᵃ(i, j, ibg) +@inline static_column_depthᶠᶠᵃ(i, j, ibg::XFlatAGFIBG) = static_column_depthᶜᶠᵃ(i, j, ibg) +@inline static_column_depthᶠᶠᵃ(i, j, ibg::YFlatAGFIBG) = static_column_depthᶠᶜᵃ(i, j, ibg) diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_buffers.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_buffers.jl new file mode 100644 index 0000000000..7dc110d662 --- /dev/null +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_buffers.jl @@ -0,0 +1,73 @@ +import Oceananigans.Models: compute_buffer_tendencies! + +using Oceananigans.Grids: halo_size +using Oceananigans.DistributedComputations: DistributedActiveCellsIBG +using Oceananigans.ImmersedBoundaries: retrieve_interior_active_cells_map +using Oceananigans.Models.NonhydrostaticModels: buffer_tendency_kernel_parameters, + buffer_p_kernel_parameters, + buffer_κ_kernel_parameters, + buffer_parameters + +# We assume here that top/bottom BC are always synchronized (no partitioning in z) +function compute_buffer_tendencies!(model::HydrostaticFreeSurfaceModel) + grid = model.grid + arch = architecture(grid) + + w_parameters = buffer_w_kernel_parameters(grid, arch) + p_parameters = buffer_p_kernel_parameters(grid, arch) + κ_parameters = buffer_κ_kernel_parameters(grid, model.closure, arch) + + # We need new values for `w`, `p` and `κ` + compute_auxiliaries!(model; w_parameters, p_parameters, κ_parameters) + + # parameters for communicating North / South / East / West side + compute_buffer_tendency_contributions!(grid, arch, model) + + return nothing +end + +function compute_buffer_tendency_contributions!(grid, arch, model) + kernel_parameters = buffer_tendency_kernel_parameters(grid, arch) + compute_hydrostatic_free_surface_tendency_contributions!(model, kernel_parameters) + return nothing +end + +function compute_buffer_tendency_contributions!(grid::DistributedActiveCellsIBG, arch, model) + maps = grid.interior_active_cells + + for (name, map) in zip(keys(maps), maps) + + # If there exists a buffer map, then we compute the buffer contributions. If not, the + # buffer contributions have already been calculated. We exclude the interior because it has + # already been calculated + compute_buffer = (name != :interior) && !isnothing(map) + + if compute_buffer + active_cells_map = retrieve_interior_active_cells_map(grid, Val(name)) + compute_hydrostatic_free_surface_tendency_contributions!(model, :xyz; active_cells_map) + end + end + + return nothing +end + +# w needs computing in the range - H + 1 : 0 and N - 1 : N + H - 1 +function buffer_w_kernel_parameters(grid, arch) + Nx, Ny, _ = size(grid) + Hx, Hy, _ = halo_size(grid) + + Sx = (Hx, Ny+2) + Sy = (Nx+2, Hy) + + # Offsets in tangential direction are == -1 to + # cover the required corners + param_west = (-Hx+2:1, 0:Ny+1) + param_east = (Nx:Nx+Hx-1, 0:Ny+1) + param_south = (0:Nx+1, -Hy+2:1) + param_north = (0:Nx+1, Ny:Ny+Hy-1) + + params = (param_west, param_east, param_south, param_north) + + return buffer_parameters(params, grid, arch) +end + diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 6b118d0868..10a41cc4c8 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -183,8 +183,8 @@ end advance_previous_velocity!(i, j, k_top-1, timestepper, U, Uᵐ⁻¹, Uᵐ⁻²) advance_previous_velocity!(i, j, k_top-1, timestepper, V, Vᵐ⁻¹, Vᵐ⁻²) - Hᶠᶜ = domain_depthᶠᶜᵃ(i, j, grid) - Hᶜᶠ = domain_depthᶜᶠᵃ(i, j, grid) + Hᶠᶜ = static_column_depthᶠᶜᵃ(i, j, grid) + Hᶜᶠ = static_column_depthᶜᶠᵃ(i, j, grid) # ∂τ(U) = - ∇η + G U[i, j, k_top-1] += Δτ * (- g * Hᶠᶜ * ∂xᶠᶜᶠ_η(i, j, k_top, grid, TX, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gᵁ[i, j, k_top-1]) @@ -271,8 +271,8 @@ end k_top = grid.Nz+1 @inbounds begin - Hᶠᶜ = domain_depthᶠᶜᵃ(i, j, grid) - Hᶜᶠ = domain_depthᶜᶠᵃ(i, j, grid) + Hᶠᶜ = static_column_depthᶠᶜᵃ(i, j, grid) + Hᶜᶠ = static_column_depthᶜᶠᵃ(i, j, grid) u[i, j, k] = u[i, j, k] + (U̅[i, j, k_top-1] - U[i, j, k_top-1]) / Hᶠᶜ v[i, j, k] = v[i, j, k] + (V̅[i, j, k_top-1] - V[i, j, k_top-1]) / Hᶜᶠ diff --git a/src/TurbulenceClosures/turbulence_closure_implementations/TKEBasedVerticalDiffusivities/catke_equation.jl b/src/TurbulenceClosures/turbulence_closure_implementations/TKEBasedVerticalDiffusivities/catke_equation.jl index 73ddb5310c..bf2aaa7bfd 100644 --- a/src/TurbulenceClosures/turbulence_closure_implementations/TKEBasedVerticalDiffusivities/catke_equation.jl +++ b/src/TurbulenceClosures/turbulence_closure_implementations/TKEBasedVerticalDiffusivities/catke_equation.jl @@ -58,7 +58,7 @@ end ℓ★ = ifelse(isnan(ℓ★), zero(grid), ℓ★) ℓᴰ = max(ℓ★, ℓʰ) - H = domain_depthᶜᶜᵃ(i, j, grid) + H = static_column_depthᶜᶜᵃ(i, j, grid) return min(H, ℓᴰ) end diff --git a/src/TurbulenceClosures/turbulence_closure_implementations/TKEBasedVerticalDiffusivities/catke_mixing_length.jl b/src/TurbulenceClosures/turbulence_closure_implementations/TKEBasedVerticalDiffusivities/catke_mixing_length.jl index 19e81270fb..16d0366d37 100644 --- a/src/TurbulenceClosures/turbulence_closure_implementations/TKEBasedVerticalDiffusivities/catke_mixing_length.jl +++ b/src/TurbulenceClosures/turbulence_closure_implementations/TKEBasedVerticalDiffusivities/catke_mixing_length.jl @@ -5,7 +5,7 @@ using ..TurbulenceClosures: height_above_bottomᶜᶜᶠ, depthᶜᶜᶜ, height_above_bottomᶜᶜᶜ, - domain_depthᶜᶜᵃ + static_column_depthᶜᶜᵃ """ struct CATKEMixingLength{FT} @@ -232,7 +232,7 @@ end ℓ★ = ifelse(isnan(ℓ★), zero(grid), ℓ★) ℓu = max(ℓ★, ℓʰ) - H = domain_depthᶜᶜᵃ(i, j, grid) + H = static_column_depthᶜᶜᵃ(i, j, grid) return min(H, ℓu) end @@ -252,7 +252,7 @@ end ℓ★ = ifelse(isnan(ℓ★), zero(grid), ℓ★) ℓc = max(ℓ★, ℓʰ) - H = domain_depthᶜᶜᵃ(i, j, grid) + H = static_column_depthᶜᶜᵃ(i, j, grid) return min(H, ℓc) end @@ -272,7 +272,7 @@ end ℓ★ = ifelse(isnan(ℓ★), zero(grid), ℓ★) ℓe = max(ℓ★, ℓʰ) - H = domain_depthᶜᶜᵃ(i, j, grid) + H = static_column_depthᶜᶜᵃ(i, j, grid) return min(H, ℓe) end From 6046d2d084c0fe2d053d0a7f2dead31aa9f234b5 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 21 Oct 2024 16:22:51 +0200 Subject: [PATCH 38/79] Update src/Grids/grid_utils.jl Co-authored-by: Gregory L. Wagner --- src/Grids/grid_utils.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Grids/grid_utils.jl b/src/Grids/grid_utils.jl index 4bee71f311..178d7ab87c 100644 --- a/src/Grids/grid_utils.jl +++ b/src/Grids/grid_utils.jl @@ -315,7 +315,7 @@ coordinate_summary(topo, Δ::Union{AbstractVector, AbstractMatrix}, name) = name, prettysummary(maximum(parent(Δ)))) ##### -##### Bottom height +##### Static column depth ##### @inline static_column_depthᶜᶜᵃ(i, j, grid) = grid.Lz From a10df1613cc5a888ff4b75f67725ad3215cb426e Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Oct 2024 10:49:54 +0200 Subject: [PATCH 39/79] address comments --- Project.toml | 3 ++- src/ImmersedBoundaries/grid_fitted_bottom.jl | 25 +++++++++++-------- src/ImmersedBoundaries/partial_cell_bottom.jl | 7 ++---- src/MultiRegion/multi_region_grid.jl | 12 ++++----- ...multi_region_near_global_quarter_degree.jl | 2 +- 5 files changed, 26 insertions(+), 23 deletions(-) diff --git a/Project.toml b/Project.toml index 06a8553f22..13f1fb6317 100644 --- a/Project.toml +++ b/Project.toml @@ -79,9 +79,10 @@ julia = "1.9" [extras] DataDeps = "124859b0-ceae-595e-8997-d05f6a7a8dfe" Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" +OrthogonalSphericalShellGrids = "c2be9673-fb75-4747-82dc-aa2bb9f4aed0" SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" TimesDates = "bdfc003b-8df8-5c39-adcd-3a9087f5df4a" [targets] -test = ["DataDeps", "Enzyme", "SafeTestsets", "Test", "TimesDates"] +test = ["DataDeps", "Enzyme", "OrthogonalSphericalShellGrids", "SafeTestsets", "Test", "TimesDates"] diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index 9679d1bc5a..c21106c175 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -23,8 +23,6 @@ struct GridFittedBottom{H, I} <: AbstractGridFittedBottom{H} immersed_condition :: I end -GridFittedBottom(bottom_height) = GridFittedBottom(bottom_height, CenterImmersedCondition()) - Base.summary(::CenterImmersedCondition) = "CenterImmersedCondition" Base.summary(::InterfaceImmersedCondition) = "InterfaceImmersedCondition" @@ -38,14 +36,21 @@ Return a bottom immersed boundary. Keyword Arguments ================= -* `bottom_height`: an array or function that gives the height of the - bottom in absolute ``z`` coordinates. -* `immersed_condition`: Determine whether the part of the domain that is immersed are all the cell centers that lie below - `bottom_height` (`CenterImmersedCondition()`; default) or all the cell faces that lie below `bottom_height` (`InterfaceImmersedCondition()`). - The only purpose of `immersed_condition` to allow `GridFittedBottom` and `PartialCellBottom` to have the same behavior when the - minimum fractional cell height for partial cells is set to 0. +* `bottom_height`: an array or function that gives the height of the + bottom in absolute ``z`` coordinates. + +* `immersed_condition`: Determine whether the part of the domain that is + immersed are all the cell centers that lie below + `bottom_height` (`CenterImmersedCondition()`; default) + or all the cell faces that lie below `bottom_height` + (`InterfaceImmersedCondition()`). The only purpose of + `immersed_condition` to allow `GridFittedBottom` and + `PartialCellBottom` to have the same behavior when the + minimum fractional cell height for partial cells is set + to 0. """ +GridFittedBottom(bottom_height) = GridFittedBottom(bottom_height, CenterImmersedCondition()) function Base.summary(ib::GridFittedBottom) zmax = maximum(ib.bottom_height) @@ -123,14 +128,14 @@ end return z ≤ zb end -@inline z_bottom(i, j, ibg::GFBIBG) = @inbounds ibg.immersed_boundary.bottom_height[i, j, 1] - ##### ##### Bottom height ##### const AGFBIB = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractGridFittedBottom} +@inline z_bottom(i, j, ibg::AGFBIBG) = @inbounds ibg.immersed_boundary.bottom_height[i, j, 1] + @inline static_column_depthᶜᶜᵃ(i, j, ibg::AGFBIB) = @inbounds znode(i, j, ibg.Nz+1, ibg, c, c, f) - ibg.immersed_boundary.bottom_height[i, j, 1] @inline static_column_depthᶜᶠᵃ(i, j, ibg::AGFBIB) = min(static_column_depthᶜᶜᵃ(i, j-1, ibg), static_column_depthᶜᶜᵃ(i, j, ibg)) @inline static_column_depthᶠᶜᵃ(i, j, ibg::AGFBIB) = min(static_column_depthᶜᶜᵃ(i-1, j, ibg), static_column_depthᶜᶜᵃ(i, j, ibg)) diff --git a/src/ImmersedBoundaries/partial_cell_bottom.jl b/src/ImmersedBoundaries/partial_cell_bottom.jl index 7034c01dc6..cd8a261ec3 100644 --- a/src/ImmersedBoundaries/partial_cell_bottom.jl +++ b/src/ImmersedBoundaries/partial_cell_bottom.jl @@ -63,14 +63,14 @@ end function ImmersedBoundaryGrid(grid, ib::PartialCellBottom) bottom_field = Field{Center, Center, Nothing}(grid) set!(bottom_field, ib.bottom_height) - @apply_regionally correct_bottom_height!(bottom_field, grid, ib) + @apply_regionally compute_numerical_bottom_height!(bottom_field, grid, ib) fill_halo_regions!(bottom_field) new_ib = PartialCellBottom(bottom_field, ib.minimum_fractional_cell_height) TX, TY, TZ = topology(grid) return ImmersedBoundaryGrid{TX, TY, TZ}(grid, new_ib) end -@kernel function _correct_bottom_height!(bottom_field, grid, ib::PartialCellBottom) +@kernel function _compute_numerical_bottom_height!(bottom_field, grid, ib::PartialCellBottom) i, j = @index(Global, NTuple) zb = @inbounds bottom_field[i, j, 1] @inbounds bottom_field[i, j, 1] = znode(i, j, 1, grid, c, c, f) @@ -183,6 +183,3 @@ YFlatPCBIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Flat, <:Any, <:Any, <:Partial @inline Δzᶜᶠᶠ(i, j, k, ibg::YFlatPCBIBG) = Δzᶜᶜᶠ(i, j, k, ibg) @inline Δzᶠᶠᶜ(i, j, k, ibg::XFlatPCBIBG) = Δzᶜᶠᶜ(i, j, k, ibg) @inline Δzᶠᶠᶜ(i, j, k, ibg::YFlatPCBIBG) = Δzᶠᶜᶜ(i, j, k, ibg) - -@inline bottom_height(i, j, ibg::PCBIBG) = @inbounds ibg.immersed_boundary.bottom_height[i, j, 1] - diff --git a/src/MultiRegion/multi_region_grid.jl b/src/MultiRegion/multi_region_grid.jl index 61e419c16a..0702b20d43 100644 --- a/src/MultiRegion/multi_region_grid.jl +++ b/src/MultiRegion/multi_region_grid.jl @@ -183,15 +183,15 @@ end function reconstruct_global_grid(mrg::ImmersedMultiRegionGrid) global_grid = reconstruct_global_grid(mrg.underlying_grid) - global_boundary = reconstruct_global_boundary(mrg.immersed_boundary) - global_boundary = on_architecture(architecture(mrg), global_boundary) + global_immersed_boundary = reconstruct_global_immersed_boundary(mrg.immersed_boundary) + global_immersed_boundary = on_architecture(architecture(mrg), global_immersed_boundary) - return ImmersedBoundaryGrid(global_grid, global_boundary) + return ImmersedBoundaryGrid(global_grid, global_immersed_boundary) end -reconstruct_global_boundary(g::GridFittedBottom{<:Field}) = GridFittedBottom(reconstruct_global_field(g.bottom_height), g.immersed_condition) -reconstruct_global_boundary(g::PartialCellBottom{<:Field}) = PartialCellBottom(reconstruct_global_field(g.bottom_height), g.minimum_fractional_cell_height) -reconstruct_global_boundary(g::GridFittedBoundary{<:Field}) = GridFittedBoundary(reconstruct_global_field(g.mask)) +reconstruct_global_immersed_boundary(g::GridFittedBottom{<:Field}) = GridFittedBottom(reconstruct_global_field(g.bottom_height), g.immersed_condition) +reconstruct_global_immersed_boundary(g::PartialCellBottom{<:Field}) = PartialCellBottom(reconstruct_global_field(g.bottom_height), g.minimum_fractional_cell_height) +reconstruct_global_immersed_boundary(g::GridFittedBoundary{<:Field}) = GridFittedBoundary(reconstruct_global_field(g.mask)) @inline getregion(mrg::ImmersedMultiRegionGrid{FT, TX, TY, TZ}, r) where {FT, TX, TY, TZ} = ImmersedBoundaryGrid{TX, TY, TZ}(_getregion(mrg.underlying_grid, r), _getregion(mrg.immersed_boundary, r)) @inline _getregion(mrg::ImmersedMultiRegionGrid{FT, TX, TY, TZ}, r) where {FT, TX, TY, TZ} = ImmersedBoundaryGrid{TX, TY, TZ}( getregion(mrg.underlying_grid, r), getregion(mrg.immersed_boundary, r)) diff --git a/validation/multi_region/multi_region_near_global_quarter_degree.jl b/validation/multi_region/multi_region_near_global_quarter_degree.jl index ce62c65f82..6bb671cb8c 100644 --- a/validation/multi_region/multi_region_near_global_quarter_degree.jl +++ b/validation/multi_region/multi_region_near_global_quarter_degree.jl @@ -147,7 +147,7 @@ biharmonic_viscosity = HorizontalScalarBiharmonicDiffusivity(ν=νhb, discrete @inline cyclic_interpolate(u₁::Number, u₂, time) = u₁ + mod(time / thirty_days, 1) * (u₂ - u₁) Δz_top = @allowscalar Δzᵃᵃᶜ(1, 1, grid.Nz, grid.underlying_grid) -Δbottom_height = @allowscalar Δzᵃᵃᶜ(1, 1, 1, grid.underlying_grid) +Δz_bottom = @allowscalar Δzᵃᵃᶜ(1, 1, 1, grid.underlying_grid) @inline function surface_wind_stress(i, j, grid, clock, fields, τ) time = clock.time From 5ee7623516c00d0132c1d87fcbd2e400deef7684 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Oct 2024 10:55:40 +0200 Subject: [PATCH 40/79] new comment --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index c21106c175..8ef8ab25df 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -128,14 +128,18 @@ end return z ≤ zb end -##### -##### Bottom height +##### +##### Utilities for `AbstractGridFittedBottom` ImmersedBoundaryGrids ##### const AGFBIB = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractGridFittedBottom} @inline z_bottom(i, j, ibg::AGFBIBG) = @inbounds ibg.immersed_boundary.bottom_height[i, j, 1] +##### +##### Static column depth +##### + @inline static_column_depthᶜᶜᵃ(i, j, ibg::AGFBIB) = @inbounds znode(i, j, ibg.Nz+1, ibg, c, c, f) - ibg.immersed_boundary.bottom_height[i, j, 1] @inline static_column_depthᶜᶠᵃ(i, j, ibg::AGFBIB) = min(static_column_depthᶜᶜᵃ(i, j-1, ibg), static_column_depthᶜᶜᵃ(i, j, ibg)) @inline static_column_depthᶠᶜᵃ(i, j, ibg::AGFBIB) = min(static_column_depthᶜᶜᵃ(i-1, j, ibg), static_column_depthᶜᶜᵃ(i, j, ibg)) From 9777ee6460dc15f944e2cafd4dc8aba6db0f9cc1 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Oct 2024 10:58:20 +0200 Subject: [PATCH 41/79] another name change --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index 8ef8ab25df..1644e91fab 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -99,17 +99,17 @@ of the last ``immersed`` cell in the column. function ImmersedBoundaryGrid(grid, ib::GridFittedBottom) bottom_field = Field{Center, Center, Nothing}(grid) set!(bottom_field, ib.bottom_height) - @apply_regionally correct_bottom_height!(bottom_field, grid, ib) + @apply_regionally compute_numerical_bottom_height!(bottom_field, grid, ib) fill_halo_regions!(bottom_field) new_ib = GridFittedBottom(bottom_field) TX, TY, TZ = topology(grid) return ImmersedBoundaryGrid{TX, TY, TZ}(grid, new_ib) end -correct_bottom_height!(bottom_field, grid, ib) = - launch!(architecture(grid), grid, :xy, _correct_bottom_height!, bottom_field, grid, ib) +compute_numerical_bottom_height!(bottom_field, grid, ib) = + launch!(architecture(grid), grid, :xy, _compute_numerical_bottom_height!, bottom_field, grid, ib) -@kernel function _correct_bottom_height!(bottom_field, grid, ib::GridFittedBottom) +@kernel function _compute_numerical_bottom_height!(bottom_field, grid, ib::GridFittedBottom) i, j = @index(Global, NTuple) zb = @inbounds bottom_field[i, j, 1] @inbounds bottom_field[i, j, 1] = znode(i, j, 1, grid, c, c, f) From 1d219af3159ebc31802bc102bce4d2f876b29da6 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Oct 2024 11:07:42 +0200 Subject: [PATCH 42/79] AGFBIBG istead of AGFBIB and z_bottom only in TurbulenceClosures --- src/ImmersedBoundaries/ImmersedBoundaries.jl | 4 ---- src/ImmersedBoundaries/grid_fitted_bottom.jl | 18 ++++++------------ src/TurbulenceClosures/TurbulenceClosures.jl | 5 +++-- 3 files changed, 9 insertions(+), 18 deletions(-) diff --git a/src/ImmersedBoundaries/ImmersedBoundaries.jl b/src/ImmersedBoundaries/ImmersedBoundaries.jl index 3976b79453..7047948e31 100644 --- a/src/ImmersedBoundaries/ImmersedBoundaries.jl +++ b/src/ImmersedBoundaries/ImmersedBoundaries.jl @@ -95,10 +95,6 @@ with_halo(halo, ibg::ImmersedBoundaryGrid) = inflate_halo_size_one_dimension(req_H, old_H, _, ::IBG) = max(req_H + 1, old_H) inflate_halo_size_one_dimension(req_H, old_H, ::Type{Flat}, ::IBG) = 0 -# Defining the bottom -@inline z_bottom(i, j, grid) = znode(i, j, 1, grid, c, c, f) -@inline z_bottom(i, j, ibg::IBG) = error("The function `bottom` has not been defined for $(summary(ibg))!") - function Base.summary(grid::ImmersedBoundaryGrid) FT = eltype(grid) TX, TY, TZ = topology(grid) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index 1644e91fab..af62726ab3 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -128,22 +128,16 @@ end return z ≤ zb end -##### -##### Utilities for `AbstractGridFittedBottom` ImmersedBoundaryGrids -##### - -const AGFBIB = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractGridFittedBottom} - -@inline z_bottom(i, j, ibg::AGFBIBG) = @inbounds ibg.immersed_boundary.bottom_height[i, j, 1] - ##### ##### Static column depth ##### -@inline static_column_depthᶜᶜᵃ(i, j, ibg::AGFBIB) = @inbounds znode(i, j, ibg.Nz+1, ibg, c, c, f) - ibg.immersed_boundary.bottom_height[i, j, 1] -@inline static_column_depthᶜᶠᵃ(i, j, ibg::AGFBIB) = min(static_column_depthᶜᶜᵃ(i, j-1, ibg), static_column_depthᶜᶜᵃ(i, j, ibg)) -@inline static_column_depthᶠᶜᵃ(i, j, ibg::AGFBIB) = min(static_column_depthᶜᶜᵃ(i-1, j, ibg), static_column_depthᶜᶜᵃ(i, j, ibg)) -@inline static_column_depthᶠᶠᵃ(i, j, ibg::AGFBIB) = min(static_column_depthᶠᶜᵃ(i, j-1, ibg), static_column_depthᶠᶜᵃ(i, j, ibg)) +const AGFBIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractGridFittedBottom} + +@inline static_column_depthᶜᶜᵃ(i, j, ibg::AGFBIBG) = @inbounds znode(i, j, ibg.Nz+1, ibg, c, c, f) - ibg.immersed_boundary.bottom_height[i, j, 1] +@inline static_column_depthᶜᶠᵃ(i, j, ibg::AGFBIBG) = min(static_column_depthᶜᶜᵃ(i, j-1, ibg), static_column_depthᶜᶜᵃ(i, j, ibg)) +@inline static_column_depthᶠᶜᵃ(i, j, ibg::AGFBIBG) = min(static_column_depthᶜᶜᵃ(i-1, j, ibg), static_column_depthᶜᶜᵃ(i, j, ibg)) +@inline static_column_depthᶠᶠᵃ(i, j, ibg::AGFBIBG) = min(static_column_depthᶠᶜᵃ(i, j-1, ibg), static_column_depthᶠᶜᵃ(i, j, ibg)) # Make sure column_height works for horizontally-Flat topologies. XFlatAGFIBG = ImmersedBoundaryGrid{<:Any, <:Flat, <:Any, <:Any, <:Any, <:AbstractGridFittedBottom} diff --git a/src/TurbulenceClosures/TurbulenceClosures.jl b/src/TurbulenceClosures/TurbulenceClosures.jl index eb683c11d7..e451b7d455 100644 --- a/src/TurbulenceClosures/TurbulenceClosures.jl +++ b/src/TurbulenceClosures/TurbulenceClosures.jl @@ -50,12 +50,10 @@ using Oceananigans.Utils using Oceananigans.Architectures: AbstractArchitecture, device using Oceananigans.Fields: FunctionField -using Oceananigans.ImmersedBoundaries: z_bottom import Oceananigans.Grids: required_halo_size_x, required_halo_size_y, required_halo_size_z import Oceananigans.Architectures: on_architecture -import Oceananigans.ImmersedBoundaries: z_bottom const VerticallyBoundedGrid{FT} = AbstractGrid{FT, <:Any, <:Any, <:Bounded} ##### @@ -124,8 +122,11 @@ end const c = Center() const f = Face() +const AGFBIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractGridFittedBottom} + @inline z_top(i, j, grid) = znode(i, j, grid.Nz+1, grid, c, c, f) @inline z_bottom(i, j, grid) = znode(i, j, 1, grid, c, c, f) +@inline z_bottom(i, j, ibg::ACFBIGB) = @inbounds ibg.immersed_boundary.bottom_height[i, j, 1] @inline depthᶜᶜᶠ(i, j, k, grid) = clip(z_top(i, j, grid) - znode(i, j, k, grid, c, c, f)) @inline depthᶜᶜᶜ(i, j, k, grid) = clip(z_top(i, j, grid) - znode(i, j, k, grid, c, c, c)) From 238fa741339582200c1a31613a2af452ff257399 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Oct 2024 11:13:19 +0200 Subject: [PATCH 43/79] some bugfixes --- src/TurbulenceClosures/TurbulenceClosures.jl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/TurbulenceClosures/TurbulenceClosures.jl b/src/TurbulenceClosures/TurbulenceClosures.jl index e451b7d455..6effcffc72 100644 --- a/src/TurbulenceClosures/TurbulenceClosures.jl +++ b/src/TurbulenceClosures/TurbulenceClosures.jl @@ -50,6 +50,8 @@ using Oceananigans.Utils using Oceananigans.Architectures: AbstractArchitecture, device using Oceananigans.Fields: FunctionField +using Oceananigans.ImmersedBoundaries +using Oceananigans.ImmersedBoundaries: AbstractGridFittedBottom import Oceananigans.Grids: required_halo_size_x, required_halo_size_y, required_halo_size_z import Oceananigans.Architectures: on_architecture @@ -119,6 +121,10 @@ end @inline clip(x) = max(zero(x), x) +##### +##### Height, Depth and Bottom interfaces +##### + const c = Center() const f = Face() @@ -126,7 +132,7 @@ const AGFBIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Abstra @inline z_top(i, j, grid) = znode(i, j, grid.Nz+1, grid, c, c, f) @inline z_bottom(i, j, grid) = znode(i, j, 1, grid, c, c, f) -@inline z_bottom(i, j, ibg::ACFBIGB) = @inbounds ibg.immersed_boundary.bottom_height[i, j, 1] +@inline z_bottom(i, j, ibg::AGFBIBG) = @inbounds ibg.immersed_boundary.bottom_height[i, j, 1] @inline depthᶜᶜᶠ(i, j, k, grid) = clip(z_top(i, j, grid) - znode(i, j, k, grid, c, c, f)) @inline depthᶜᶜᶜ(i, j, k, grid) = clip(z_top(i, j, grid) - znode(i, j, k, grid, c, c, c)) From 90dfa7af1d1cf0c4f2de33e2db969ab913905ad5 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Oct 2024 13:58:26 +0200 Subject: [PATCH 44/79] better definition of bottom height --- src/ImmersedBoundaries/partial_cell_bottom.jl | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src/ImmersedBoundaries/partial_cell_bottom.jl b/src/ImmersedBoundaries/partial_cell_bottom.jl index cd8a261ec3..92fd2ac305 100644 --- a/src/ImmersedBoundaries/partial_cell_bottom.jl +++ b/src/ImmersedBoundaries/partial_cell_bottom.jl @@ -72,14 +72,28 @@ end @kernel function _compute_numerical_bottom_height!(bottom_field, grid, ib::PartialCellBottom) i, j = @index(Global, NTuple) + + # Save analytical bottom height zb = @inbounds bottom_field[i, j, 1] - @inbounds bottom_field[i, j, 1] = znode(i, j, 1, grid, c, c, f) + + # Cap bottom height at Lz and at rnode(i, j, grid.Nz+1, grid, c, c, f) + domain_bottom = znode(i, j, 1, grid, c, c, f) + domain_top = znode(i, j, grid.Nz+1, grid, c, c, f) + @inbounds bottom_field[i, j, 1] = clamp(zb, domain_bottom, domain_top) + ϵ = ib.minimum_fractional_cell_height for k in 1:grid.Nz - z⁻ = znode(i, j, k, grid, c, c, f) + z⁻ = znode(i, j, k, grid, c, c, f) + z⁺ = znode(i, j, k+1, grid, c, c, f) Δz = Δzᶜᶜᶜ(i, j, k, grid) - bottom_cell = z⁻ + Δz * (1 - ϵ) ≤ zb - @inbounds bottom_field[i, j, 1] = ifelse(bottom_cell, z⁻ + Δz * (1 - ϵ), zb) + bottom_cell = (z⁻ ≤ zb) & (z⁺ ≥ zb) + + # If the size of the bottom cell is less than ϵ Δz, use + # a simple `GridFittedBottom` approach where the bottom + # height is the top interface of cell k. + capped_zb = ifelse(zb < z⁻ + Δz * (1 - ϵ), zb, z⁺) + + @inbounds bottom_field[i, j, 1] = ifelse(bottom_cell, capped_zb, bottom_field[i, j, 1]) end end From f2402f0b4505e097101c1396e90ae91fbfd3896f Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Oct 2024 14:49:41 +0200 Subject: [PATCH 45/79] fixed partial cell --- src/ImmersedBoundaries/partial_cell_bottom.jl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/ImmersedBoundaries/partial_cell_bottom.jl b/src/ImmersedBoundaries/partial_cell_bottom.jl index 92fd2ac305..40c6bed202 100644 --- a/src/ImmersedBoundaries/partial_cell_bottom.jl +++ b/src/ImmersedBoundaries/partial_cell_bottom.jl @@ -133,9 +133,12 @@ Criterion is zb ≥ z - ϵ Δz """ @inline function _immersed_cell(i, j, k, underlying_grid, ib::PartialCellBottom) - z = znode(i, j, k+1, underlying_grid, c, c, f) + z⁻ = znode(i, j, k, underlying_grid, c, c, f) + ϵ = ib.minimum_fractional_cell_height + Δrz= Δzᶜᶜᶜ(i, j, k, underlying_grid) + z★ = z⁻ + Δz * (1 - ϵ) zb = @inbounds ib.bottom_height[i, j, 1] - return z ≤ zb + return z★ < zb end @inline function bottom_cell(i, j, k, ibg::PCBIBG) From 55d7432ecb42bd4db63350845a01aaf12b5d596f Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Oct 2024 14:50:13 +0200 Subject: [PATCH 46/79] fixed partial cells --- src/ImmersedBoundaries/partial_cell_bottom.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ImmersedBoundaries/partial_cell_bottom.jl b/src/ImmersedBoundaries/partial_cell_bottom.jl index 40c6bed202..decaa17763 100644 --- a/src/ImmersedBoundaries/partial_cell_bottom.jl +++ b/src/ImmersedBoundaries/partial_cell_bottom.jl @@ -135,7 +135,7 @@ Criterion is zb ≥ z - ϵ Δz @inline function _immersed_cell(i, j, k, underlying_grid, ib::PartialCellBottom) z⁻ = znode(i, j, k, underlying_grid, c, c, f) ϵ = ib.minimum_fractional_cell_height - Δrz= Δzᶜᶜᶜ(i, j, k, underlying_grid) + Δz = Δzᶜᶜᶜ(i, j, k, underlying_grid) z★ = z⁻ + Δz * (1 - ϵ) zb = @inbounds ib.bottom_height[i, j, 1] return z★ < zb From 849c253d1e2e2638140c857b578fb2bccb987d18 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Oct 2024 15:16:52 +0200 Subject: [PATCH 47/79] remove OrthogonalSphericalShellGrids while we decide what to do --- Project.toml | 3 +- difference | 302 +++++++++++++++++++++++++++++++++++++ difference_se | 126 ++++++++++++++++ mynew_model.jl | 139 +++++++++++++++++ test/test_cubed_spheres.jl | 1 - 5 files changed, 568 insertions(+), 3 deletions(-) create mode 100644 difference create mode 100644 difference_se create mode 100644 mynew_model.jl diff --git a/Project.toml b/Project.toml index 13f1fb6317..06a8553f22 100644 --- a/Project.toml +++ b/Project.toml @@ -79,10 +79,9 @@ julia = "1.9" [extras] DataDeps = "124859b0-ceae-595e-8997-d05f6a7a8dfe" Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" -OrthogonalSphericalShellGrids = "c2be9673-fb75-4747-82dc-aa2bb9f4aed0" SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" TimesDates = "bdfc003b-8df8-5c39-adcd-3a9087f5df4a" [targets] -test = ["DataDeps", "Enzyme", "OrthogonalSphericalShellGrids", "SafeTestsets", "Test", "TimesDates"] +test = ["DataDeps", "Enzyme", "SafeTestsets", "Test", "TimesDates"] diff --git a/difference b/difference new file mode 100644 index 0000000000..5ef0c89b63 --- /dev/null +++ b/difference @@ -0,0 +1,302 @@ + ++const PGX = AbstractGrid{<:Any, <:Periodic} ++const BGX = AbstractGrid{<:Any, <:Bounded} ++const RGX = AbstractGrid{<:Any, <:RightConnected} ++ ++const PGY = AbstractGrid{<:Any, <:Periodic} ++const BGY = AbstractGrid{<:Any, <:Bounded} ++const RGY = AbstractGrid{<:Any, <:RightConnected} ++ ++@inline ℑxᶠᵃᵃ_η(i, j, k, grid, η) = ℑxᶠᵃᵃ(i, j, k, grid, η) ++@inline ℑyᵃᶠᵃ_η(i, j, k, grid, η) = ℑyᵃᶠᵃ(i, j, k, grid, η) ++@inline ℑxᶠᵃᵃ_η(i, j, k, grid::PGX, η) = ifelse(i == 1, (η[1, j, k] + η[grid.Nx, j, k]) / 2, ℑxᶠᵃᵃ(i, j, k, grid, η)) ++@inline ℑyᵃᶠᵃ_η(i, j, k, grid::PGY, η) = ifelse(j == 1, (η[i, 1, k] + η[i, grid.Ny, k]) / 2, ℑyᵃᶠᵃ(i, j, k, grid, η)) ++@inline ℑxᶠᵃᵃ_η(i, j, k, grid::BGX, η) = ifelse(i == 1, η[1, j, k], ℑxᶠᵃᵃ(i, j, k, grid, η)) ++@inline ℑyᵃᶠᵃ_η(i, j, k, grid::BGY, η) = ifelse(j == 1, η[i, 1, k], ℑyᵃᶠᵃ(i, j, k, grid, η)) ++@inline ℑxᶠᵃᵃ_η(i, j, k, grid::RGX, η) = ifelse(i == 1, η[1, j, k], ℑxᶠᵃᵃ(i, j, k, grid, η)) ++@inline ℑyᵃᶠᵃ_η(i, j, k, grid::RGY, η) = ifelse(j == 1, η[i, 1, k], ℑyᵃᶠᵃ(i, j, k, grid, η)) ++ + # Time stepping extrapolation U★, and η★ + + # AB3 step +@@ -158,6 +178,14 @@ end + return nothing + end + ++# Linear free surface implementation ++@inline dynamic_column_depthᶠᶜᵃ(i, j, k, grid, η) = static_column_depthᶠᶜᵃ(i, j, grid) ++@inline dynamic_column_depthᶜᶠᵃ(i, j, k, grid, η) = static_column_depthᶜᶠᵃ(i, j, grid) ++ ++# Non-linear free surface implementation ++@inline dynamic_column_depthᶠᶜᵃ(i, j, k, grid::ZStarSpacingGrid, η) = static_column_depthᶠᶜᵃ(i, j, grid) + ℑxᶠᵃᵃ_η(i, j, k, grid, η) ++@inline dynamic_column_depthᶜᶠᵃ(i, j, k, grid::ZStarSpacingGrid, η) = static_column_depthᶜᶠᵃ(i, j, grid) + ℑyᵃᶠᵃ_η(i, j, k, grid, η) ++ + @kernel function _split_explicit_barotropic_velocity!(averaging_weight, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², + U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², + η̅, U̅, V̅, Gᵁ, Gⱽ, g, +@@ -183,9 +211,9 @@ end + advance_previous_velocity!(i, j, k_top-1, timestepper, U, Uᵐ⁻¹, Uᵐ⁻²) + advance_previous_velocity!(i, j, k_top-1, timestepper, V, Vᵐ⁻¹, Vᵐ⁻²) + +- Hᶠᶜ = static_column_depthᶠᶜᵃ(i, j, grid) +- Hᶜᶠ = static_column_depthᶜᶠᵃ(i, j, grid) +- ++ Hᶠᶜ = dynamic_column_depthᶠᶜᵃ(i, j, k_top, grid, η) ++ Hᶜᶠ = dynamic_column_depthᶠᶜᵃ(i, j, k_top, grid, η) ++ + # ∂τ(U) = - ∇η + G + U[i, j, k_top-1] += Δτ * (- g * Hᶠᶜ * ∂xᶠᶜᶠ_η(i, j, k_top, grid, TX, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gᵁ[i, j, k_top-1]) + V[i, j, k_top-1] += Δτ * (- g * Hᶜᶠ * ∂yᶜᶠᶠ_η(i, j, k_top, grid, TY, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gⱽ[i, j, k_top-1]) +@@ -199,16 +227,26 @@ end + + # Barotropic Model Kernels + # u_Δz = u * Δz ++# For Zstar vertical spacing the vertical integral includes the dynamic height ++# Remember, the vertical coordinate has not yet been updated! ++# For this reason the integration has to be performed manually + @kernel function _barotropic_mode_kernel!(U, V, grid, ::Nothing, u, v) + i, j = @index(Global, NTuple) + k_top = grid.Nz+1 + +- @inbounds U[i, j, k_top-1] = Δzᶠᶜᶜ(i, j, 1, grid) * u[i, j, 1] +- @inbounds V[i, j, k_top-1] = Δzᶜᶠᶜ(i, j, 1, grid) * v[i, j, 1] ++ hᶠᶜ = static_column_depthᶠᶜᵃ(i, j, grid) ++ hᶜᶠ = static_column_depthᶜᶠᵃ(i, j, grid) ++ ++ sᶠᶜ = ifelse(hᶠᶜ == 0, one(grid), dynamic_column_depthᶠᶜᵃ(i, j, k_top, grid, η) / hᶠᶜ) ++ sᶜᶠ = ifelse(hᶜᶠ == 0, one(grid), dynamic_column_depthᶜᶠᵃ(i, j, k_top, grid, η) / hᶜᶠ) + +- for k in 2:grid.Nz +- @inbounds U[i, j, k_top-1] += Δzᶠᶜᶜ(i, j, k, grid) * u[i, j, k] +- @inbounds V[i, j, k_top-1] += Δzᶜᶠᶜ(i, j, k, grid) * v[i, j, k] ++ # hand unroll first loop ++ @inbounds U[i, j, k_top-1] = u[i, j, 1] * Δrᶠᶜᶜ(i, j, 1, grid) * sᶠᶜ ++ @inbounds V[i, j, k_top-1] = v[i, j, 1] * Δrᶜᶠᶜ(i, j, 1, grid) * sᶜᶠ ++ ++ @unroll for k in 2:grid.Nz ++ @inbounds U[i, j, k_top-1] += u[i, j, k] * Δrᶠᶜᶜ(i, j, k, grid) * sᶠᶜ ++ @inbounds V[i, j, k_top-1] += v[i, j, k] * Δrᶜᶠᶜ(i, j, k, grid) * sᶜᶠ + end + end + +@@ -219,19 +257,55 @@ end + i, j = active_linear_index_to_tuple(idx, active_cells_map) + k_top = grid.Nz+1 + +- @inbounds U[i, j, k_top-1] = Δzᶠᶜᶜ(i, j, 1, grid) * u[i, j, 1] +- @inbounds V[i, j, k_top-1] = Δzᶜᶠᶜ(i, j, 1, grid) * v[i, j, 1] ++ hᶠᶜ = static_column_depthᶠᶜᵃ(i, j, grid) ++ hᶜᶠ = static_column_depthᶜᶠᵃ(i, j, grid) ++ ++ sᶠᶜ = ifelse(hᶠᶜ == 0, one(grid), dynamic_column_depthᶠᶜᵃ(i, j, k_top, grid, η) / hᶠᶜ) ++ sᶜᶠ = ifelse(hᶜᶠ == 0, one(grid), dynamic_column_depthᶜᶠᵃ(i, j, k_top, grid, η) / hᶜᶠ) + +- for k in 2:grid.Nz +- @inbounds U[i, j, k_top-1] += Δzᶠᶜᶜ(i, j, k, grid) * u[i, j, k] +- @inbounds V[i, j, k_top-1] += Δzᶜᶠᶜ(i, j, k, grid) * v[i, j, k] ++ # hand unroll first loop ++ @inbounds U[i, j, k_top-1] = u[i, j, 1] * Δrᶠᶜᶜ(i, j, 1, grid) * sᶠᶜ ++ @inbounds V[i, j, k_top-1] = v[i, j, 1] * Δrᶜᶠᶜ(i, j, 1, grid) * sᶜᶠ ++ ++ @unroll for k in 2:grid.Nz ++ @inbounds U[i, j, k_top-1] += u[i, j, k] * Δrᶠᶜᶜ(i, j, k, grid) * sᶠᶜ ++ @inbounds V[i, j, k_top-1] += v[i, j, k] * Δrᶜᶠᶜ(i, j, k, grid) * sᶜᶠ + end + end + +-@inline function compute_barotropic_mode!(U, V, grid, u, v) ++@inline function compute_barotropic_mode!(U, V, grid, u, v, η) + active_cells_map = retrieve_surface_active_cells_map(grid) + +- launch!(architecture(grid), grid, :xy, _barotropic_mode_kernel!, U, V, grid, active_cells_map, u, v; active_cells_map) ++ launch!(architecture(grid), grid, :xy, _barotropic_mode_kernel!, U, V, grid, active_cells_map, u, v, η; active_cells_map) ++ ++ return nothing ++end ++ ++@kernel function _barotropic_split_explicit_corrector!(u, v, grid, U̅, V̅, U, V, η) ++ i, j, k = @index(Global, NTuple) ++ k_top = grid.Nz + 1 ++ ++ Hᶠᶜ = dynamic_column_depthᶠᶜᵃ(i, j, k_top, grid, η) ++ Hᶜᶠ = dynamic_column_depthᶜᶠᵃ(i, j, k_top, grid, η) ++ ++ @inbounds begin ++ u[i, j, k] = u[i, j, k] + (U̅[i, j, k_top-1] - U[i, j, k_top-1]) / Hᶠᶜ ++ v[i, j, k] = v[i, j, k] + (V̅[i, j, k_top-1] - V[i, j, k_top-1]) / Hᶜᶠ ++ end ++end ++ ++function barotropic_split_explicit_corrector!(u, v, free_surface, grid) ++ sefs = free_surface.state ++ U, V, U̅, V̅ = sefs.U, sefs.V, sefs.U̅, sefs.V̅ ++ arch = architecture(grid) ++ ++ # take out "bad" barotropic mode, ++ # !!!! reusing U and V for this storage since last timestep doesn't matter ++ compute_barotropic_mode!(U, V, grid, u, v, free_surface.η) ++ # add in "good" barotropic mode ++ ++ launch!(arch, grid, :xyz, _barotropic_split_explicit_corrector!, ++ u, v, grid, U̅, V̅, U, V, free_surface.η) + + return nothing + end +@@ -266,34 +340,6 @@ function initialize_auxiliary_state!(state, η, timestepper) + return nothing + end + +-@kernel function _barotropic_split_explicit_corrector!(u, v, U̅, V̅, U, V, grid) +- i, j, k = @index(Global, NTuple) +- k_top = grid.Nz+1 +- +- @inbounds begin +- Hᶠᶜ = static_column_depthᶠᶜᵃ(i, j, grid) +- Hᶜᶠ = static_column_depthᶜᶠᵃ(i, j, grid) +- +- u[i, j, k] = u[i, j, k] + (U̅[i, j, k_top-1] - U[i, j, k_top-1]) / Hᶠᶜ +- v[i, j, k] = v[i, j, k] + (V̅[i, j, k_top-1] - V[i, j, k_top-1]) / Hᶜᶠ +- end +-end +- +-function barotropic_split_explicit_corrector!(u, v, free_surface, grid) +- sefs = free_surface.state +- U, V, U̅, V̅ = sefs.U, sefs.V, sefs.U̅, sefs.V̅ +- arch = architecture(grid) +- +- # take out "bad" barotropic mode, +- # !!!! reusing U and V for this storage since last timestep doesn't matter +- compute_barotropic_mode!(U, V, grid, u, v) +- # add in "good" barotropic mode +- launch!(arch, grid, :xyz, _barotropic_split_explicit_corrector!, +- u, v, U̅, V̅, U, V, grid) +- +- return nothing +-end +- + """ + Explicitly step forward η in substeps. + """ +@@ -301,8 +347,13 @@ ab2_step_free_surface!(free_surface::SplitExplicitFreeSurface, model, Δt, χ) = + split_explicit_free_surface_step!(free_surface, model, Δt, χ) + + function initialize_free_surface!(sefs::SplitExplicitFreeSurface, grid, velocities) +- @apply_regionally compute_barotropic_mode!(sefs.state.U̅, sefs.state.V̅, grid, velocities.u, velocities.v) ++ U̅, V̅ = sefs.state.U̅, sefs.state.V̅ ++ u, v, _ = velocities ++ ++ @apply_regionally compute_barotropic_mode!(U̅, V̅, grid, u, v, sefs.η) + fill_halo_regions!((sefs.state.U̅, sefs.state.V̅, sefs.η)) ++ ++ return nothing + end + + function split_explicit_free_surface_step!(free_surface::SplitExplicitFreeSurface, model, Δt, χ) +@@ -336,6 +387,10 @@ function split_explicit_free_surface_step!(free_surface::SplitExplicitFreeSurfac + set!(free_surface.η, free_surface.state.η̅) + end + ++ # This is needed for the barotropic mode calculations, so it cannot be done asynchronously ++ fill_halo_regions!(free_surface.η) ++ ++ # Velocities can be passed asynchronously + fields_to_fill = (free_surface.state.U̅, free_surface.state.V̅) + fill_halo_regions!(fields_to_fill; async = true) + +@@ -363,9 +418,6 @@ const MINIMUM_SUBSTEPS = 5 + @inline calculate_adaptive_settings(substepping::FTS, substeps) = weights_from_substeps(eltype(substepping.Δt_barotropic), + substeps, substepping.averaging_kernel) + +-const FixedSubstepsSetting{N} = SplitExplicitSettings{<:FixedSubstepNumber{<:Any, <:NTuple{N, <:Any}}} where N +-const FixedSubstepsSplitExplicit{F} = SplitExplicitFreeSurface{<:Any, <:Any, <:Any, <:Any, <:FixedSubstepsSetting{N}} where N +- + function iterate_split_explicit!(free_surface, grid, Δτᴮ, weights, ::Val{Nsubsteps}) where Nsubsteps + arch = architecture(grid) + +@@ -424,12 +476,12 @@ end + i, j = @index(Global, NTuple) + k_top = grid.Nz + 1 + +- @inbounds Gᵁ[i, j, k_top-1] = Δzᶠᶜᶜ(i, j, 1, grid) * ab2_step_Gu(i, j, 1, grid, Gu⁻, Guⁿ, χ) +- @inbounds Gⱽ[i, j, k_top-1] = Δzᶜᶠᶜ(i, j, 1, grid) * ab2_step_Gv(i, j, 1, grid, Gv⁻, Gvⁿ, χ) ++ @inbounds Gᵁ[i, j, k_top-1] = Δrᶠᶜᶜ(i, j, 1, grid) * ab2_step_Gu(i, j, 1, grid, Gu⁻, Guⁿ, χ) ++ @inbounds Gⱽ[i, j, k_top-1] = Δrᶜᶠᶜ(i, j, 1, grid) * ab2_step_Gv(i, j, 1, grid, Gv⁻, Gvⁿ, χ) + + for k in 2:grid.Nz +- @inbounds Gᵁ[i, j, k_top-1] += Δzᶠᶜᶜ(i, j, k, grid) * ab2_step_Gu(i, j, k, grid, Gu⁻, Guⁿ, χ) +- @inbounds Gⱽ[i, j, k_top-1] += Δzᶜᶠᶜ(i, j, k, grid) * ab2_step_Gv(i, j, k, grid, Gv⁻, Gvⁿ, χ) ++ @inbounds Gᵁ[i, j, k_top-1] += Δrᶠᶜᶜ(i, j, k, grid) * ab2_step_Gu(i, j, k, grid, Gu⁻, Guⁿ, χ) ++ @inbounds Gⱽ[i, j, k_top-1] += Δrᶜᶠᶜ(i, j, k, grid) * ab2_step_Gv(i, j, k, grid, Gv⁻, Gvⁿ, χ) + end + end + +@@ -439,24 +491,44 @@ end + i, j = active_linear_index_to_tuple(idx, active_cells_map) + k_top = grid.Nz+1 + +- @inbounds Gᵁ[i, j, k_top-1] = Δzᶠᶜᶜ(i, j, 1, grid) * ab2_step_Gu(i, j, 1, grid, Gu⁻, Guⁿ, χ) +- @inbounds Gⱽ[i, j, k_top-1] = Δzᶜᶠᶜ(i, j, 1, grid) * ab2_step_Gv(i, j, 1, grid, Gv⁻, Gvⁿ, χ) ++ @inbounds Gᵁ[i, j, k_top-1] = Δrᶠᶜᶜ(i, j, 1, grid) * ab2_step_Gu(i, j, 1, grid, Gu⁻, Guⁿ, χ) ++ @inbounds Gⱽ[i, j, k_top-1] = Δrᶜᶠᶜ(i, j, 1, grid) * ab2_step_Gv(i, j, 1, grid, Gv⁻, Gvⁿ, χ) + + for k in 2:grid.Nz +- @inbounds Gᵁ[i, j, k_top-1] += Δzᶠᶜᶜ(i, j, k, grid) * ab2_step_Gu(i, j, k, grid, Gu⁻, Guⁿ, χ) +- @inbounds Gⱽ[i, j, k_top-1] += Δzᶜᶠᶜ(i, j, k, grid) * ab2_step_Gv(i, j, k, grid, Gv⁻, Gvⁿ, χ) ++ @inbounds Gᵁ[i, j, k_top-1] += Δrᶠᶜᶜ(i, j, k, grid) * ab2_step_Gu(i, j, k, grid, Gu⁻, Guⁿ, χ) ++ @inbounds Gⱽ[i, j, k_top-1] += Δrᶜᶠᶜ(i, j, k, grid) * ab2_step_Gv(i, j, k, grid, Gv⁻, Gvⁿ, χ) + end + end + +-@inline ab2_step_Gu(i, j, k, grid, G⁻, Gⁿ, χ::FT) where FT = +- @inbounds ifelse(peripheral_node(i, j, k, grid, f, c, c), zero(grid), (convert(FT, 1.5) + χ) * Gⁿ[i, j, k] - G⁻[i, j, k] * (convert(FT, 0.5) + χ)) ++@inline function ab2_step_Gu(i, j, k, grid, G⁻, Gⁿ, χ::FT) where FT ++ C¹ = convert(FT, 3/2) + χ ++ C² = convert(FT, 1/2) + χ ++ ++ Fⁿ = @inbounds C¹ * Gⁿ[i, j, k] * vertical_scaling(i, j, k, grid, Face(), Center(), Center()) ++ F⁻ = @inbounds C² * G⁻[i, j, k] * previous_vertical_scaling(i, j, k, grid, Face(), Center(), Center()) ++ ++ Gi = Fⁿ - F⁻ ++ ++ return ifelse(peripheral_node(i, j, k, grid, f, c, c), zero(grid), Gi) ++end ++ ++@inline function ab2_step_Gv(i, j, k, grid, G⁻, Gⁿ, χ::FT) where FT ++ C¹ = convert(FT, 3/2) + χ ++ C² = convert(FT, 1/2) + χ ++ ++ Fⁿ = @inbounds C¹ * Gⁿ[i, j, k] * vertical_scaling(i, j, k, grid, Center(), Face(), Center()) ++ F⁻ = @inbounds C² * G⁻[i, j, k] * previous_vertical_scaling(i, j, k, grid, Center(), Face(), Center()) + +-@inline ab2_step_Gv(i, j, k, grid, G⁻, Gⁿ, χ::FT) where FT = +- @inbounds ifelse(peripheral_node(i, j, k, grid, c, f, c), zero(grid), (convert(FT, 1.5) + χ) * Gⁿ[i, j, k] - G⁻[i, j, k] * (convert(FT, 0.5) + χ)) ++ Gi = Fⁿ - F⁻ ++ ++ return ifelse(peripheral_node(i, j, k, grid, f, c, c), zero(grid), Gi) ++end + + # Setting up the RHS for the barotropic step (tendencies of the barotropic velocity components) + # This function is called after `calculate_tendency` and before `ab2_step_velocities!` + function setup_free_surface!(model, free_surface::SplitExplicitFreeSurface, χ) ++ ++ grid = model.grid + + # we start the time integration of η from the average ηⁿ + Gu⁻ = model.timestepper.G⁻.u +@@ -466,7 +538,7 @@ function setup_free_surface!(model, free_surface::SplitExplicitFreeSurface, χ) + + auxiliary = free_surface.auxiliary + +- @apply_regionally setup_split_explicit_tendency!(auxiliary, model.grid, Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ) ++ @apply_regionally setup_split_explicit_tendency!(auxiliary, grid, Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ) + + fields_to_fill = (auxiliary.Gᵁ, auxiliary.Gⱽ) + fill_halo_regions!(fields_to_fill; async = true) +@@ -483,5 +555,4 @@ end + return nothing + end + +-wait_free_surface_communication!(free_surface, arch) = nothing +- ++wait_free_surface_communication!(free_surface, arch) = nothing +\ No newline at end of file diff --git a/difference_se b/difference_se new file mode 100644 index 0000000000..75b7e8da6d --- /dev/null +++ b/difference_se @@ -0,0 +1,126 @@ +diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +index 073fbfc4e..6992a410d 100644 +--- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl ++++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +@@ -3,8 +3,9 @@ using Oceananigans.Architectures + using Oceananigans.Fields + using Oceananigans.Grids + using Oceananigans.Grids: AbstractGrid ++using Oceananigans.BoundaryConditions: default_prognostic_bc + using Oceananigans.AbstractOperations: Δz, GridMetricOperation +- ++using Oceananigans.ImmersedBoundaries: immersed_peripheral_node, c, f + using Adapt + using Base + using KernelAbstractions: @index, @kernel +@@ -158,6 +159,10 @@ Base.@kwdef struct SplitExplicitState{CC, ACC, FC, AFC, CF, ACF} + U̅ :: FC + "The time-filtered barotropic meridional velocity. (`ReducedField` over ``z``)" + V̅ :: CF ++ # "The time-filtered barotropic zonal velocity. (`ReducedField` over ``z``)" ++ # Ũ :: FC ++ # "The time-filtered barotropic meridional velocity. (`ReducedField` over ``z``)" ++ # Ṽ :: CF + end + + """ +@@ -179,18 +184,20 @@ function SplitExplicitState(grid::AbstractGrid, timestepper) + ηᵐ⁻¹ = auxiliary_free_surface_field(grid, timestepper) + ηᵐ⁻² = auxiliary_free_surface_field(grid, timestepper) + +- U = XFaceField(grid, indices = (:, :, Nz)) +- V = YFaceField(grid, indices = (:, :, Nz)) +- +- Uᵐ⁻¹ = auxiliary_barotropic_U_field(grid, timestepper) +- Vᵐ⁻¹ = auxiliary_barotropic_V_field(grid, timestepper) +- Uᵐ⁻² = auxiliary_barotropic_U_field(grid, timestepper) +- Vᵐ⁻² = auxiliary_barotropic_V_field(grid, timestepper) ++ 𝒰 = VelocityFields(grid) ++ ++ U = Field(𝒰.u, indices = (:, :, Nz)) ++ V = Field(𝒰.v, indices = (:, :, Nz)) + +- U̅ = XFaceField(grid, indices = (:, :, Nz)) +- V̅ = YFaceField(grid, indices = (:, :, Nz)) ++ Uᵐ⁻¹ = auxiliary_barotropic_velocity_field(U, timestepper) ++ Vᵐ⁻¹ = auxiliary_barotropic_velocity_field(V, timestepper) ++ Uᵐ⁻² = auxiliary_barotropic_velocity_field(U, timestepper) ++ Vᵐ⁻² = auxiliary_barotropic_velocity_field(V, timestepper) ++ ++ U̅ = deepcopy(U) ++ V̅ = deepcopy(V) + +- return SplitExplicitState(; ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², η̅, U̅, V̅) ++ return SplitExplicitState(; ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², η̅, U̅, V̅) + end + + """ +@@ -204,7 +211,7 @@ large (or `:xy` in case of a serial computation), and start computing from + + $(FIELDS) + """ +-Base.@kwdef struct SplitExplicitAuxiliaryFields{𝒞ℱ, ℱ𝒞, 𝒦} ++Base.@kwdef struct SplitExplicitAuxiliaryFields{ℱ𝒞, 𝒞ℱ, 𝒦} + "Vertically-integrated slow barotropic forcing function for `U` (`ReducedField` over ``z``)" + Gᵁ :: ℱ𝒞 + "Vertically-integrated slow barotropic forcing function for `V` (`ReducedField` over ``z``)" +@@ -220,8 +227,8 @@ Return the `SplitExplicitAuxiliaryFields` for `grid`. + """ + function SplitExplicitAuxiliaryFields(grid::AbstractGrid) + +- Gᵁ = Field((Face, Center, Nothing), grid) +- Gⱽ = Field((Center, Face, Nothing), grid) ++ Gᵁ = Field{Face, Center, Nothing}(grid) ++ Gⱽ = Field{Center, Face, Nothing}(grid) + + kernel_parameters = :xy + +@@ -244,14 +251,11 @@ end + struct AdamsBashforth3Scheme end + struct ForwardBackwardScheme end + +- + auxiliary_free_surface_field(grid, ::AdamsBashforth3Scheme) = ZFaceField(grid, indices = (:, :, size(grid, 3)+1)) + auxiliary_free_surface_field(grid, ::ForwardBackwardScheme) = nothing + +-auxiliary_barotropic_U_field(grid, ::AdamsBashforth3Scheme) = XFaceField(grid, indices = (:, :, size(grid, 3))) +-auxiliary_barotropic_U_field(grid, ::ForwardBackwardScheme) = nothing +-auxiliary_barotropic_V_field(grid, ::AdamsBashforth3Scheme) = YFaceField(grid, indices = (:, :, size(grid, 3))) +-auxiliary_barotropic_V_field(grid, ::ForwardBackwardScheme) = nothing ++auxiliary_barotropic_velocity_field(U, ::AdamsBashforth3Scheme) = deecopy(u) ++auxiliary_barotropic_velocity_field(U, ::ForwardBackwardScheme) = nothing + + # (p = 2, q = 4, r = 0.18927) minimize dispersion error from Shchepetkin and McWilliams (2005): https://doi.org/10.1016/j.ocemod.2004.08.002 + @inline function averaging_shape_function(τ::FT; p = 2, q = 4, r = FT(0.18927)) where FT +@@ -306,7 +310,7 @@ end + + averaging_weights = averaging_weights[1:idx] + averaging_weights ./= sum(averaging_weights) +- ++ + return Δτ, tuple(averaging_weights...) + end + +@@ -390,11 +394,19 @@ end + + # Adapt + Adapt.adapt_structure(to, free_surface::SplitExplicitFreeSurface) = +- SplitExplicitFreeSurface(Adapt.adapt(to, free_surface.η), nothing, nothing, ++ SplitExplicitFreeSurface(Adapt.adapt(to, free_surface.η), ++ nothing, ++ Adapt.adapt(to, free_surface.auxiliary), + free_surface.gravitational_acceleration, nothing) + +-for Type in (:SplitExplicitFreeSurface, +- :SplitExplicitSettings, ++# Adapt ++Adapt.adapt_structure(to, auxiliary::SplitExplicitAuxiliaryFields) = ++ SplitExplicitAuxiliaryFields(Adapt.adapt(to, auxiliary.Gᵁ), ++ Adapt.adapt(to, auxiliary.Gⱽ), ++ nothing) ++ ++for Type in (:SplitExplicitFreeSurface, ++ :SplitExplicitSettings, + :SplitExplicitState, + :SplitExplicitAuxiliaryFields, + :FixedTimeStepSize, diff --git a/mynew_model.jl b/mynew_model.jl new file mode 100644 index 0000000000..a9467b4cb0 --- /dev/null +++ b/mynew_model.jl @@ -0,0 +1,139 @@ +using Oceananigans +using Oceananigans: tupleit +using Oceananigans.Grids +using Oceananigans.Architectures: architecture +using Oceananigans.Advection: div_Uc +using Oceananigans.BoundaryConditions +using Oceananigans.Utils +using Oceananigans.Fields: TracerFields +using Oceananigans.Models.HydrostaticFreeSurfaceModels: PrescribedVelocityFields, tracernames, validate_tracer_advection, with_tracers, tracernames +using KernelAbstractions: @index, @kernel + +import Oceananigans.Models: prognostic_fields + +using Oceananigans.TimeSteppers: RungeKutta3TimeStepper, store_tendencies!, rk3_substep_field! +import Oceananigans.TimeSteppers: update_state!, time_step!, compute_tendencies!, rk3_substep! + +mutable struct AdvectiveModel{G, A, V, T, B} + grid :: G + advection :: A + velocities :: V + tracers :: T + timestepper :: B +end + +struct DirectSpaceTimeAder end + +function AdvectiveModel(; grid, + velocities = PrescribedVelocityFields(; u = 1), + tracers = :b, + timestepper = nothing, + advection = WENO(; order = 5)) + + # Next, we form a list of default boundary conditions: + tracers = tupleit(tracers) # supports tracers=:c keyword argument (for example) + + tracers = TracerFields(tracers, grid, NamedTuple()) + Gⁿ = deepcopy(tracers) + G⁻ = deepcopy(tracers) + + if timestepper isa Nothing + timestepper = DirectSpaceTime() + else + timestepper = RungeKutta3TimeStepper(grid, tracernames(tracers); Gⁿ = Gⁿ, G⁻ = G⁻) + end + + model = AdvectiveModel(grid, advection, velocities, tracers, timestepper) + update_state!(model) + + return model +end + +prognostic_fields(model::AdvectiveModel) = model.tracers + +update_state!(model::AdvectiveModel) = + fill_halo_regions!(model.tracers) + +function time_step!(model::AdvectiveModel, Δt) + Δt == 0 && @warn "Δt == 0 may cause model blowup!" + + # Be paranoid and update state at iteration 0, in case run! is not used: + update_state!(model) + + γ¹ = model.timestepper.γ¹ + γ² = model.timestepper.γ² + γ³ = model.timestepper.γ³ + + ζ² = model.timestepper.ζ² + ζ³ = model.timestepper.ζ³ + + compute_tendencies!(model) + rk3_substep!(model, Δt, γ¹, nothing) + store_tendencies!(model) + update_state!(model) + + # + # Second stage + # + + compute_tendencies!(model) + rk3_substep!(model, Δt, γ², ζ²) + store_tendencies!(model) + update_state!(model) + + # + # Third stage + # + + compute_tendencies!(model) + rk3_substep!(model, Δt, γ³, ζ³) + update_state!(model) + + return nothing +end + +function rk3_substep!(model::AdvectiveModel, Δt, γ, ζ) + grid = model.grid + arch = architecture(grid) + tracers = model.tracers + velocities = model.velocities + + for (tracer, Gⁿ, G⁻) in zip(tracers, model.timestepper.Gⁿ, model.timestepper.G⁻) + launch!(arch, grid, :xyz, rk3_substep_field!, tracer, Δt, γ, ζ, Gⁿ, G⁻) + end + + return nothing +end + +function compute_tendencies!(model::AdvectiveModel) + grid = model.grid + arch = architecture(grid) + tendencies = model.timestepper.Gⁿ + tracers = model.tracers + velocities = model.velocities + + for (tracer, G) in zip(tracers, tendencies) + launch!(arch, grid, :xyz, _compute_tendency!, G, grid, model.advection, velocities, tracer) + end + + return nothing +end + +@kernel function _compute_tendency!(G, grid, advection, U, c) + i, j, k = @index(Global, NTuple) + @inbounds G[i, j, k] = - div_Uc(i, j, k, grid, advection, U, c) +end + +function time_step!(model::AdvectiveModel{G, A, V, <:DirectSpaceTimeAder, B}, Δt) where {G, A, V, B} + Δt == 0 && @warn "Δt == 0 may cause model blowup!" + + # Be paranoid and update state at iteration 0, in case run! is not used: + update_state!(model) + + compute_tendencies!(model) + rk3_substep!(model, Δt, γ¹, nothing) + store_tendencies!(model) + update_state!(model) + + return nothing +end diff --git a/test/test_cubed_spheres.jl b/test/test_cubed_spheres.jl index e907dedf68..a950dc5cbf 100644 --- a/test/test_cubed_spheres.jl +++ b/test/test_cubed_spheres.jl @@ -173,7 +173,6 @@ end test_vector_rotation(grid) end - @testset "CubedSphereData and CubedSphereFields [$(typeof(arch))]" begin @info " Testing CubedSphereData and CubedSphereFields [$(typeof(arch))]..." c = model.tracers.c From 510a858cc8ff520894ef1a8e0e551cf188e67077 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Oct 2024 15:18:30 +0200 Subject: [PATCH 48/79] these files shouldn't go here --- difference | 302 ------------------------------------------------- difference_se | 126 --------------------- mynew_model.jl | 139 ----------------------- 3 files changed, 567 deletions(-) delete mode 100644 difference delete mode 100644 difference_se delete mode 100644 mynew_model.jl diff --git a/difference b/difference deleted file mode 100644 index 5ef0c89b63..0000000000 --- a/difference +++ /dev/null @@ -1,302 +0,0 @@ - -+const PGX = AbstractGrid{<:Any, <:Periodic} -+const BGX = AbstractGrid{<:Any, <:Bounded} -+const RGX = AbstractGrid{<:Any, <:RightConnected} -+ -+const PGY = AbstractGrid{<:Any, <:Periodic} -+const BGY = AbstractGrid{<:Any, <:Bounded} -+const RGY = AbstractGrid{<:Any, <:RightConnected} -+ -+@inline ℑxᶠᵃᵃ_η(i, j, k, grid, η) = ℑxᶠᵃᵃ(i, j, k, grid, η) -+@inline ℑyᵃᶠᵃ_η(i, j, k, grid, η) = ℑyᵃᶠᵃ(i, j, k, grid, η) -+@inline ℑxᶠᵃᵃ_η(i, j, k, grid::PGX, η) = ifelse(i == 1, (η[1, j, k] + η[grid.Nx, j, k]) / 2, ℑxᶠᵃᵃ(i, j, k, grid, η)) -+@inline ℑyᵃᶠᵃ_η(i, j, k, grid::PGY, η) = ifelse(j == 1, (η[i, 1, k] + η[i, grid.Ny, k]) / 2, ℑyᵃᶠᵃ(i, j, k, grid, η)) -+@inline ℑxᶠᵃᵃ_η(i, j, k, grid::BGX, η) = ifelse(i == 1, η[1, j, k], ℑxᶠᵃᵃ(i, j, k, grid, η)) -+@inline ℑyᵃᶠᵃ_η(i, j, k, grid::BGY, η) = ifelse(j == 1, η[i, 1, k], ℑyᵃᶠᵃ(i, j, k, grid, η)) -+@inline ℑxᶠᵃᵃ_η(i, j, k, grid::RGX, η) = ifelse(i == 1, η[1, j, k], ℑxᶠᵃᵃ(i, j, k, grid, η)) -+@inline ℑyᵃᶠᵃ_η(i, j, k, grid::RGY, η) = ifelse(j == 1, η[i, 1, k], ℑyᵃᶠᵃ(i, j, k, grid, η)) -+ - # Time stepping extrapolation U★, and η★ - - # AB3 step -@@ -158,6 +178,14 @@ end - return nothing - end - -+# Linear free surface implementation -+@inline dynamic_column_depthᶠᶜᵃ(i, j, k, grid, η) = static_column_depthᶠᶜᵃ(i, j, grid) -+@inline dynamic_column_depthᶜᶠᵃ(i, j, k, grid, η) = static_column_depthᶜᶠᵃ(i, j, grid) -+ -+# Non-linear free surface implementation -+@inline dynamic_column_depthᶠᶜᵃ(i, j, k, grid::ZStarSpacingGrid, η) = static_column_depthᶠᶜᵃ(i, j, grid) + ℑxᶠᵃᵃ_η(i, j, k, grid, η) -+@inline dynamic_column_depthᶜᶠᵃ(i, j, k, grid::ZStarSpacingGrid, η) = static_column_depthᶜᶠᵃ(i, j, grid) + ℑyᵃᶠᵃ_η(i, j, k, grid, η) -+ - @kernel function _split_explicit_barotropic_velocity!(averaging_weight, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², - U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², - η̅, U̅, V̅, Gᵁ, Gⱽ, g, -@@ -183,9 +211,9 @@ end - advance_previous_velocity!(i, j, k_top-1, timestepper, U, Uᵐ⁻¹, Uᵐ⁻²) - advance_previous_velocity!(i, j, k_top-1, timestepper, V, Vᵐ⁻¹, Vᵐ⁻²) - -- Hᶠᶜ = static_column_depthᶠᶜᵃ(i, j, grid) -- Hᶜᶠ = static_column_depthᶜᶠᵃ(i, j, grid) -- -+ Hᶠᶜ = dynamic_column_depthᶠᶜᵃ(i, j, k_top, grid, η) -+ Hᶜᶠ = dynamic_column_depthᶠᶜᵃ(i, j, k_top, grid, η) -+ - # ∂τ(U) = - ∇η + G - U[i, j, k_top-1] += Δτ * (- g * Hᶠᶜ * ∂xᶠᶜᶠ_η(i, j, k_top, grid, TX, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gᵁ[i, j, k_top-1]) - V[i, j, k_top-1] += Δτ * (- g * Hᶜᶠ * ∂yᶜᶠᶠ_η(i, j, k_top, grid, TY, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gⱽ[i, j, k_top-1]) -@@ -199,16 +227,26 @@ end - - # Barotropic Model Kernels - # u_Δz = u * Δz -+# For Zstar vertical spacing the vertical integral includes the dynamic height -+# Remember, the vertical coordinate has not yet been updated! -+# For this reason the integration has to be performed manually - @kernel function _barotropic_mode_kernel!(U, V, grid, ::Nothing, u, v) - i, j = @index(Global, NTuple) - k_top = grid.Nz+1 - -- @inbounds U[i, j, k_top-1] = Δzᶠᶜᶜ(i, j, 1, grid) * u[i, j, 1] -- @inbounds V[i, j, k_top-1] = Δzᶜᶠᶜ(i, j, 1, grid) * v[i, j, 1] -+ hᶠᶜ = static_column_depthᶠᶜᵃ(i, j, grid) -+ hᶜᶠ = static_column_depthᶜᶠᵃ(i, j, grid) -+ -+ sᶠᶜ = ifelse(hᶠᶜ == 0, one(grid), dynamic_column_depthᶠᶜᵃ(i, j, k_top, grid, η) / hᶠᶜ) -+ sᶜᶠ = ifelse(hᶜᶠ == 0, one(grid), dynamic_column_depthᶜᶠᵃ(i, j, k_top, grid, η) / hᶜᶠ) - -- for k in 2:grid.Nz -- @inbounds U[i, j, k_top-1] += Δzᶠᶜᶜ(i, j, k, grid) * u[i, j, k] -- @inbounds V[i, j, k_top-1] += Δzᶜᶠᶜ(i, j, k, grid) * v[i, j, k] -+ # hand unroll first loop -+ @inbounds U[i, j, k_top-1] = u[i, j, 1] * Δrᶠᶜᶜ(i, j, 1, grid) * sᶠᶜ -+ @inbounds V[i, j, k_top-1] = v[i, j, 1] * Δrᶜᶠᶜ(i, j, 1, grid) * sᶜᶠ -+ -+ @unroll for k in 2:grid.Nz -+ @inbounds U[i, j, k_top-1] += u[i, j, k] * Δrᶠᶜᶜ(i, j, k, grid) * sᶠᶜ -+ @inbounds V[i, j, k_top-1] += v[i, j, k] * Δrᶜᶠᶜ(i, j, k, grid) * sᶜᶠ - end - end - -@@ -219,19 +257,55 @@ end - i, j = active_linear_index_to_tuple(idx, active_cells_map) - k_top = grid.Nz+1 - -- @inbounds U[i, j, k_top-1] = Δzᶠᶜᶜ(i, j, 1, grid) * u[i, j, 1] -- @inbounds V[i, j, k_top-1] = Δzᶜᶠᶜ(i, j, 1, grid) * v[i, j, 1] -+ hᶠᶜ = static_column_depthᶠᶜᵃ(i, j, grid) -+ hᶜᶠ = static_column_depthᶜᶠᵃ(i, j, grid) -+ -+ sᶠᶜ = ifelse(hᶠᶜ == 0, one(grid), dynamic_column_depthᶠᶜᵃ(i, j, k_top, grid, η) / hᶠᶜ) -+ sᶜᶠ = ifelse(hᶜᶠ == 0, one(grid), dynamic_column_depthᶜᶠᵃ(i, j, k_top, grid, η) / hᶜᶠ) - -- for k in 2:grid.Nz -- @inbounds U[i, j, k_top-1] += Δzᶠᶜᶜ(i, j, k, grid) * u[i, j, k] -- @inbounds V[i, j, k_top-1] += Δzᶜᶠᶜ(i, j, k, grid) * v[i, j, k] -+ # hand unroll first loop -+ @inbounds U[i, j, k_top-1] = u[i, j, 1] * Δrᶠᶜᶜ(i, j, 1, grid) * sᶠᶜ -+ @inbounds V[i, j, k_top-1] = v[i, j, 1] * Δrᶜᶠᶜ(i, j, 1, grid) * sᶜᶠ -+ -+ @unroll for k in 2:grid.Nz -+ @inbounds U[i, j, k_top-1] += u[i, j, k] * Δrᶠᶜᶜ(i, j, k, grid) * sᶠᶜ -+ @inbounds V[i, j, k_top-1] += v[i, j, k] * Δrᶜᶠᶜ(i, j, k, grid) * sᶜᶠ - end - end - --@inline function compute_barotropic_mode!(U, V, grid, u, v) -+@inline function compute_barotropic_mode!(U, V, grid, u, v, η) - active_cells_map = retrieve_surface_active_cells_map(grid) - -- launch!(architecture(grid), grid, :xy, _barotropic_mode_kernel!, U, V, grid, active_cells_map, u, v; active_cells_map) -+ launch!(architecture(grid), grid, :xy, _barotropic_mode_kernel!, U, V, grid, active_cells_map, u, v, η; active_cells_map) -+ -+ return nothing -+end -+ -+@kernel function _barotropic_split_explicit_corrector!(u, v, grid, U̅, V̅, U, V, η) -+ i, j, k = @index(Global, NTuple) -+ k_top = grid.Nz + 1 -+ -+ Hᶠᶜ = dynamic_column_depthᶠᶜᵃ(i, j, k_top, grid, η) -+ Hᶜᶠ = dynamic_column_depthᶜᶠᵃ(i, j, k_top, grid, η) -+ -+ @inbounds begin -+ u[i, j, k] = u[i, j, k] + (U̅[i, j, k_top-1] - U[i, j, k_top-1]) / Hᶠᶜ -+ v[i, j, k] = v[i, j, k] + (V̅[i, j, k_top-1] - V[i, j, k_top-1]) / Hᶜᶠ -+ end -+end -+ -+function barotropic_split_explicit_corrector!(u, v, free_surface, grid) -+ sefs = free_surface.state -+ U, V, U̅, V̅ = sefs.U, sefs.V, sefs.U̅, sefs.V̅ -+ arch = architecture(grid) -+ -+ # take out "bad" barotropic mode, -+ # !!!! reusing U and V for this storage since last timestep doesn't matter -+ compute_barotropic_mode!(U, V, grid, u, v, free_surface.η) -+ # add in "good" barotropic mode -+ -+ launch!(arch, grid, :xyz, _barotropic_split_explicit_corrector!, -+ u, v, grid, U̅, V̅, U, V, free_surface.η) - - return nothing - end -@@ -266,34 +340,6 @@ function initialize_auxiliary_state!(state, η, timestepper) - return nothing - end - --@kernel function _barotropic_split_explicit_corrector!(u, v, U̅, V̅, U, V, grid) -- i, j, k = @index(Global, NTuple) -- k_top = grid.Nz+1 -- -- @inbounds begin -- Hᶠᶜ = static_column_depthᶠᶜᵃ(i, j, grid) -- Hᶜᶠ = static_column_depthᶜᶠᵃ(i, j, grid) -- -- u[i, j, k] = u[i, j, k] + (U̅[i, j, k_top-1] - U[i, j, k_top-1]) / Hᶠᶜ -- v[i, j, k] = v[i, j, k] + (V̅[i, j, k_top-1] - V[i, j, k_top-1]) / Hᶜᶠ -- end --end -- --function barotropic_split_explicit_corrector!(u, v, free_surface, grid) -- sefs = free_surface.state -- U, V, U̅, V̅ = sefs.U, sefs.V, sefs.U̅, sefs.V̅ -- arch = architecture(grid) -- -- # take out "bad" barotropic mode, -- # !!!! reusing U and V for this storage since last timestep doesn't matter -- compute_barotropic_mode!(U, V, grid, u, v) -- # add in "good" barotropic mode -- launch!(arch, grid, :xyz, _barotropic_split_explicit_corrector!, -- u, v, U̅, V̅, U, V, grid) -- -- return nothing --end -- - """ - Explicitly step forward η in substeps. - """ -@@ -301,8 +347,13 @@ ab2_step_free_surface!(free_surface::SplitExplicitFreeSurface, model, Δt, χ) = - split_explicit_free_surface_step!(free_surface, model, Δt, χ) - - function initialize_free_surface!(sefs::SplitExplicitFreeSurface, grid, velocities) -- @apply_regionally compute_barotropic_mode!(sefs.state.U̅, sefs.state.V̅, grid, velocities.u, velocities.v) -+ U̅, V̅ = sefs.state.U̅, sefs.state.V̅ -+ u, v, _ = velocities -+ -+ @apply_regionally compute_barotropic_mode!(U̅, V̅, grid, u, v, sefs.η) - fill_halo_regions!((sefs.state.U̅, sefs.state.V̅, sefs.η)) -+ -+ return nothing - end - - function split_explicit_free_surface_step!(free_surface::SplitExplicitFreeSurface, model, Δt, χ) -@@ -336,6 +387,10 @@ function split_explicit_free_surface_step!(free_surface::SplitExplicitFreeSurfac - set!(free_surface.η, free_surface.state.η̅) - end - -+ # This is needed for the barotropic mode calculations, so it cannot be done asynchronously -+ fill_halo_regions!(free_surface.η) -+ -+ # Velocities can be passed asynchronously - fields_to_fill = (free_surface.state.U̅, free_surface.state.V̅) - fill_halo_regions!(fields_to_fill; async = true) - -@@ -363,9 +418,6 @@ const MINIMUM_SUBSTEPS = 5 - @inline calculate_adaptive_settings(substepping::FTS, substeps) = weights_from_substeps(eltype(substepping.Δt_barotropic), - substeps, substepping.averaging_kernel) - --const FixedSubstepsSetting{N} = SplitExplicitSettings{<:FixedSubstepNumber{<:Any, <:NTuple{N, <:Any}}} where N --const FixedSubstepsSplitExplicit{F} = SplitExplicitFreeSurface{<:Any, <:Any, <:Any, <:Any, <:FixedSubstepsSetting{N}} where N -- - function iterate_split_explicit!(free_surface, grid, Δτᴮ, weights, ::Val{Nsubsteps}) where Nsubsteps - arch = architecture(grid) - -@@ -424,12 +476,12 @@ end - i, j = @index(Global, NTuple) - k_top = grid.Nz + 1 - -- @inbounds Gᵁ[i, j, k_top-1] = Δzᶠᶜᶜ(i, j, 1, grid) * ab2_step_Gu(i, j, 1, grid, Gu⁻, Guⁿ, χ) -- @inbounds Gⱽ[i, j, k_top-1] = Δzᶜᶠᶜ(i, j, 1, grid) * ab2_step_Gv(i, j, 1, grid, Gv⁻, Gvⁿ, χ) -+ @inbounds Gᵁ[i, j, k_top-1] = Δrᶠᶜᶜ(i, j, 1, grid) * ab2_step_Gu(i, j, 1, grid, Gu⁻, Guⁿ, χ) -+ @inbounds Gⱽ[i, j, k_top-1] = Δrᶜᶠᶜ(i, j, 1, grid) * ab2_step_Gv(i, j, 1, grid, Gv⁻, Gvⁿ, χ) - - for k in 2:grid.Nz -- @inbounds Gᵁ[i, j, k_top-1] += Δzᶠᶜᶜ(i, j, k, grid) * ab2_step_Gu(i, j, k, grid, Gu⁻, Guⁿ, χ) -- @inbounds Gⱽ[i, j, k_top-1] += Δzᶜᶠᶜ(i, j, k, grid) * ab2_step_Gv(i, j, k, grid, Gv⁻, Gvⁿ, χ) -+ @inbounds Gᵁ[i, j, k_top-1] += Δrᶠᶜᶜ(i, j, k, grid) * ab2_step_Gu(i, j, k, grid, Gu⁻, Guⁿ, χ) -+ @inbounds Gⱽ[i, j, k_top-1] += Δrᶜᶠᶜ(i, j, k, grid) * ab2_step_Gv(i, j, k, grid, Gv⁻, Gvⁿ, χ) - end - end - -@@ -439,24 +491,44 @@ end - i, j = active_linear_index_to_tuple(idx, active_cells_map) - k_top = grid.Nz+1 - -- @inbounds Gᵁ[i, j, k_top-1] = Δzᶠᶜᶜ(i, j, 1, grid) * ab2_step_Gu(i, j, 1, grid, Gu⁻, Guⁿ, χ) -- @inbounds Gⱽ[i, j, k_top-1] = Δzᶜᶠᶜ(i, j, 1, grid) * ab2_step_Gv(i, j, 1, grid, Gv⁻, Gvⁿ, χ) -+ @inbounds Gᵁ[i, j, k_top-1] = Δrᶠᶜᶜ(i, j, 1, grid) * ab2_step_Gu(i, j, 1, grid, Gu⁻, Guⁿ, χ) -+ @inbounds Gⱽ[i, j, k_top-1] = Δrᶜᶠᶜ(i, j, 1, grid) * ab2_step_Gv(i, j, 1, grid, Gv⁻, Gvⁿ, χ) - - for k in 2:grid.Nz -- @inbounds Gᵁ[i, j, k_top-1] += Δzᶠᶜᶜ(i, j, k, grid) * ab2_step_Gu(i, j, k, grid, Gu⁻, Guⁿ, χ) -- @inbounds Gⱽ[i, j, k_top-1] += Δzᶜᶠᶜ(i, j, k, grid) * ab2_step_Gv(i, j, k, grid, Gv⁻, Gvⁿ, χ) -+ @inbounds Gᵁ[i, j, k_top-1] += Δrᶠᶜᶜ(i, j, k, grid) * ab2_step_Gu(i, j, k, grid, Gu⁻, Guⁿ, χ) -+ @inbounds Gⱽ[i, j, k_top-1] += Δrᶜᶠᶜ(i, j, k, grid) * ab2_step_Gv(i, j, k, grid, Gv⁻, Gvⁿ, χ) - end - end - --@inline ab2_step_Gu(i, j, k, grid, G⁻, Gⁿ, χ::FT) where FT = -- @inbounds ifelse(peripheral_node(i, j, k, grid, f, c, c), zero(grid), (convert(FT, 1.5) + χ) * Gⁿ[i, j, k] - G⁻[i, j, k] * (convert(FT, 0.5) + χ)) -+@inline function ab2_step_Gu(i, j, k, grid, G⁻, Gⁿ, χ::FT) where FT -+ C¹ = convert(FT, 3/2) + χ -+ C² = convert(FT, 1/2) + χ -+ -+ Fⁿ = @inbounds C¹ * Gⁿ[i, j, k] * vertical_scaling(i, j, k, grid, Face(), Center(), Center()) -+ F⁻ = @inbounds C² * G⁻[i, j, k] * previous_vertical_scaling(i, j, k, grid, Face(), Center(), Center()) -+ -+ Gi = Fⁿ - F⁻ -+ -+ return ifelse(peripheral_node(i, j, k, grid, f, c, c), zero(grid), Gi) -+end -+ -+@inline function ab2_step_Gv(i, j, k, grid, G⁻, Gⁿ, χ::FT) where FT -+ C¹ = convert(FT, 3/2) + χ -+ C² = convert(FT, 1/2) + χ -+ -+ Fⁿ = @inbounds C¹ * Gⁿ[i, j, k] * vertical_scaling(i, j, k, grid, Center(), Face(), Center()) -+ F⁻ = @inbounds C² * G⁻[i, j, k] * previous_vertical_scaling(i, j, k, grid, Center(), Face(), Center()) - --@inline ab2_step_Gv(i, j, k, grid, G⁻, Gⁿ, χ::FT) where FT = -- @inbounds ifelse(peripheral_node(i, j, k, grid, c, f, c), zero(grid), (convert(FT, 1.5) + χ) * Gⁿ[i, j, k] - G⁻[i, j, k] * (convert(FT, 0.5) + χ)) -+ Gi = Fⁿ - F⁻ -+ -+ return ifelse(peripheral_node(i, j, k, grid, f, c, c), zero(grid), Gi) -+end - - # Setting up the RHS for the barotropic step (tendencies of the barotropic velocity components) - # This function is called after `calculate_tendency` and before `ab2_step_velocities!` - function setup_free_surface!(model, free_surface::SplitExplicitFreeSurface, χ) -+ -+ grid = model.grid - - # we start the time integration of η from the average ηⁿ - Gu⁻ = model.timestepper.G⁻.u -@@ -466,7 +538,7 @@ function setup_free_surface!(model, free_surface::SplitExplicitFreeSurface, χ) - - auxiliary = free_surface.auxiliary - -- @apply_regionally setup_split_explicit_tendency!(auxiliary, model.grid, Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ) -+ @apply_regionally setup_split_explicit_tendency!(auxiliary, grid, Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ) - - fields_to_fill = (auxiliary.Gᵁ, auxiliary.Gⱽ) - fill_halo_regions!(fields_to_fill; async = true) -@@ -483,5 +555,4 @@ end - return nothing - end - --wait_free_surface_communication!(free_surface, arch) = nothing -- -+wait_free_surface_communication!(free_surface, arch) = nothing -\ No newline at end of file diff --git a/difference_se b/difference_se deleted file mode 100644 index 75b7e8da6d..0000000000 --- a/difference_se +++ /dev/null @@ -1,126 +0,0 @@ -diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl -index 073fbfc4e..6992a410d 100644 ---- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl -+++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl -@@ -3,8 +3,9 @@ using Oceananigans.Architectures - using Oceananigans.Fields - using Oceananigans.Grids - using Oceananigans.Grids: AbstractGrid -+using Oceananigans.BoundaryConditions: default_prognostic_bc - using Oceananigans.AbstractOperations: Δz, GridMetricOperation -- -+using Oceananigans.ImmersedBoundaries: immersed_peripheral_node, c, f - using Adapt - using Base - using KernelAbstractions: @index, @kernel -@@ -158,6 +159,10 @@ Base.@kwdef struct SplitExplicitState{CC, ACC, FC, AFC, CF, ACF} - U̅ :: FC - "The time-filtered barotropic meridional velocity. (`ReducedField` over ``z``)" - V̅ :: CF -+ # "The time-filtered barotropic zonal velocity. (`ReducedField` over ``z``)" -+ # Ũ :: FC -+ # "The time-filtered barotropic meridional velocity. (`ReducedField` over ``z``)" -+ # Ṽ :: CF - end - - """ -@@ -179,18 +184,20 @@ function SplitExplicitState(grid::AbstractGrid, timestepper) - ηᵐ⁻¹ = auxiliary_free_surface_field(grid, timestepper) - ηᵐ⁻² = auxiliary_free_surface_field(grid, timestepper) - -- U = XFaceField(grid, indices = (:, :, Nz)) -- V = YFaceField(grid, indices = (:, :, Nz)) -- -- Uᵐ⁻¹ = auxiliary_barotropic_U_field(grid, timestepper) -- Vᵐ⁻¹ = auxiliary_barotropic_V_field(grid, timestepper) -- Uᵐ⁻² = auxiliary_barotropic_U_field(grid, timestepper) -- Vᵐ⁻² = auxiliary_barotropic_V_field(grid, timestepper) -+ 𝒰 = VelocityFields(grid) -+ -+ U = Field(𝒰.u, indices = (:, :, Nz)) -+ V = Field(𝒰.v, indices = (:, :, Nz)) - -- U̅ = XFaceField(grid, indices = (:, :, Nz)) -- V̅ = YFaceField(grid, indices = (:, :, Nz)) -+ Uᵐ⁻¹ = auxiliary_barotropic_velocity_field(U, timestepper) -+ Vᵐ⁻¹ = auxiliary_barotropic_velocity_field(V, timestepper) -+ Uᵐ⁻² = auxiliary_barotropic_velocity_field(U, timestepper) -+ Vᵐ⁻² = auxiliary_barotropic_velocity_field(V, timestepper) -+ -+ U̅ = deepcopy(U) -+ V̅ = deepcopy(V) - -- return SplitExplicitState(; ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², η̅, U̅, V̅) -+ return SplitExplicitState(; ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², η̅, U̅, V̅) - end - - """ -@@ -204,7 +211,7 @@ large (or `:xy` in case of a serial computation), and start computing from - - $(FIELDS) - """ --Base.@kwdef struct SplitExplicitAuxiliaryFields{𝒞ℱ, ℱ𝒞, 𝒦} -+Base.@kwdef struct SplitExplicitAuxiliaryFields{ℱ𝒞, 𝒞ℱ, 𝒦} - "Vertically-integrated slow barotropic forcing function for `U` (`ReducedField` over ``z``)" - Gᵁ :: ℱ𝒞 - "Vertically-integrated slow barotropic forcing function for `V` (`ReducedField` over ``z``)" -@@ -220,8 +227,8 @@ Return the `SplitExplicitAuxiliaryFields` for `grid`. - """ - function SplitExplicitAuxiliaryFields(grid::AbstractGrid) - -- Gᵁ = Field((Face, Center, Nothing), grid) -- Gⱽ = Field((Center, Face, Nothing), grid) -+ Gᵁ = Field{Face, Center, Nothing}(grid) -+ Gⱽ = Field{Center, Face, Nothing}(grid) - - kernel_parameters = :xy - -@@ -244,14 +251,11 @@ end - struct AdamsBashforth3Scheme end - struct ForwardBackwardScheme end - -- - auxiliary_free_surface_field(grid, ::AdamsBashforth3Scheme) = ZFaceField(grid, indices = (:, :, size(grid, 3)+1)) - auxiliary_free_surface_field(grid, ::ForwardBackwardScheme) = nothing - --auxiliary_barotropic_U_field(grid, ::AdamsBashforth3Scheme) = XFaceField(grid, indices = (:, :, size(grid, 3))) --auxiliary_barotropic_U_field(grid, ::ForwardBackwardScheme) = nothing --auxiliary_barotropic_V_field(grid, ::AdamsBashforth3Scheme) = YFaceField(grid, indices = (:, :, size(grid, 3))) --auxiliary_barotropic_V_field(grid, ::ForwardBackwardScheme) = nothing -+auxiliary_barotropic_velocity_field(U, ::AdamsBashforth3Scheme) = deecopy(u) -+auxiliary_barotropic_velocity_field(U, ::ForwardBackwardScheme) = nothing - - # (p = 2, q = 4, r = 0.18927) minimize dispersion error from Shchepetkin and McWilliams (2005): https://doi.org/10.1016/j.ocemod.2004.08.002 - @inline function averaging_shape_function(τ::FT; p = 2, q = 4, r = FT(0.18927)) where FT -@@ -306,7 +310,7 @@ end - - averaging_weights = averaging_weights[1:idx] - averaging_weights ./= sum(averaging_weights) -- -+ - return Δτ, tuple(averaging_weights...) - end - -@@ -390,11 +394,19 @@ end - - # Adapt - Adapt.adapt_structure(to, free_surface::SplitExplicitFreeSurface) = -- SplitExplicitFreeSurface(Adapt.adapt(to, free_surface.η), nothing, nothing, -+ SplitExplicitFreeSurface(Adapt.adapt(to, free_surface.η), -+ nothing, -+ Adapt.adapt(to, free_surface.auxiliary), - free_surface.gravitational_acceleration, nothing) - --for Type in (:SplitExplicitFreeSurface, -- :SplitExplicitSettings, -+# Adapt -+Adapt.adapt_structure(to, auxiliary::SplitExplicitAuxiliaryFields) = -+ SplitExplicitAuxiliaryFields(Adapt.adapt(to, auxiliary.Gᵁ), -+ Adapt.adapt(to, auxiliary.Gⱽ), -+ nothing) -+ -+for Type in (:SplitExplicitFreeSurface, -+ :SplitExplicitSettings, - :SplitExplicitState, - :SplitExplicitAuxiliaryFields, - :FixedTimeStepSize, diff --git a/mynew_model.jl b/mynew_model.jl deleted file mode 100644 index a9467b4cb0..0000000000 --- a/mynew_model.jl +++ /dev/null @@ -1,139 +0,0 @@ -using Oceananigans -using Oceananigans: tupleit -using Oceananigans.Grids -using Oceananigans.Architectures: architecture -using Oceananigans.Advection: div_Uc -using Oceananigans.BoundaryConditions -using Oceananigans.Utils -using Oceananigans.Fields: TracerFields -using Oceananigans.Models.HydrostaticFreeSurfaceModels: PrescribedVelocityFields, tracernames, validate_tracer_advection, with_tracers, tracernames -using KernelAbstractions: @index, @kernel - -import Oceananigans.Models: prognostic_fields - -using Oceananigans.TimeSteppers: RungeKutta3TimeStepper, store_tendencies!, rk3_substep_field! -import Oceananigans.TimeSteppers: update_state!, time_step!, compute_tendencies!, rk3_substep! - -mutable struct AdvectiveModel{G, A, V, T, B} - grid :: G - advection :: A - velocities :: V - tracers :: T - timestepper :: B -end - -struct DirectSpaceTimeAder end - -function AdvectiveModel(; grid, - velocities = PrescribedVelocityFields(; u = 1), - tracers = :b, - timestepper = nothing, - advection = WENO(; order = 5)) - - # Next, we form a list of default boundary conditions: - tracers = tupleit(tracers) # supports tracers=:c keyword argument (for example) - - tracers = TracerFields(tracers, grid, NamedTuple()) - Gⁿ = deepcopy(tracers) - G⁻ = deepcopy(tracers) - - if timestepper isa Nothing - timestepper = DirectSpaceTime() - else - timestepper = RungeKutta3TimeStepper(grid, tracernames(tracers); Gⁿ = Gⁿ, G⁻ = G⁻) - end - - model = AdvectiveModel(grid, advection, velocities, tracers, timestepper) - update_state!(model) - - return model -end - -prognostic_fields(model::AdvectiveModel) = model.tracers - -update_state!(model::AdvectiveModel) = - fill_halo_regions!(model.tracers) - -function time_step!(model::AdvectiveModel, Δt) - Δt == 0 && @warn "Δt == 0 may cause model blowup!" - - # Be paranoid and update state at iteration 0, in case run! is not used: - update_state!(model) - - γ¹ = model.timestepper.γ¹ - γ² = model.timestepper.γ² - γ³ = model.timestepper.γ³ - - ζ² = model.timestepper.ζ² - ζ³ = model.timestepper.ζ³ - - compute_tendencies!(model) - rk3_substep!(model, Δt, γ¹, nothing) - store_tendencies!(model) - update_state!(model) - - # - # Second stage - # - - compute_tendencies!(model) - rk3_substep!(model, Δt, γ², ζ²) - store_tendencies!(model) - update_state!(model) - - # - # Third stage - # - - compute_tendencies!(model) - rk3_substep!(model, Δt, γ³, ζ³) - update_state!(model) - - return nothing -end - -function rk3_substep!(model::AdvectiveModel, Δt, γ, ζ) - grid = model.grid - arch = architecture(grid) - tracers = model.tracers - velocities = model.velocities - - for (tracer, Gⁿ, G⁻) in zip(tracers, model.timestepper.Gⁿ, model.timestepper.G⁻) - launch!(arch, grid, :xyz, rk3_substep_field!, tracer, Δt, γ, ζ, Gⁿ, G⁻) - end - - return nothing -end - -function compute_tendencies!(model::AdvectiveModel) - grid = model.grid - arch = architecture(grid) - tendencies = model.timestepper.Gⁿ - tracers = model.tracers - velocities = model.velocities - - for (tracer, G) in zip(tracers, tendencies) - launch!(arch, grid, :xyz, _compute_tendency!, G, grid, model.advection, velocities, tracer) - end - - return nothing -end - -@kernel function _compute_tendency!(G, grid, advection, U, c) - i, j, k = @index(Global, NTuple) - @inbounds G[i, j, k] = - div_Uc(i, j, k, grid, advection, U, c) -end - -function time_step!(model::AdvectiveModel{G, A, V, <:DirectSpaceTimeAder, B}, Δt) where {G, A, V, B} - Δt == 0 && @warn "Δt == 0 may cause model blowup!" - - # Be paranoid and update state at iteration 0, in case run! is not used: - update_state!(model) - - compute_tendencies!(model) - rk3_substep!(model, Δt, γ¹, nothing) - store_tendencies!(model) - update_state!(model) - - return nothing -end From fe2d552fc5216f0565c9e0c4b7eeadc25c6be49c Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 28 Oct 2024 18:45:54 +0100 Subject: [PATCH 49/79] new operators --- .../split_explicit_free_surface_kernels.jl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 4bf9428d72..0042755123 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -83,7 +83,6 @@ end @inline function free_surface_evolution!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², timestepper) k_top = grid.Nz+1 - TX, TY, _ = topology(grid) @inbounds begin advance_previous_free_surface!(i, j, k_top, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) @@ -122,8 +121,8 @@ end Hᶜᶠ = static_column_depthᶜᶠᵃ(i, j, grid) # ∂τ(U) = - ∇η + G - U[i, j, k_top-1] += Δτ * (- g * Hᶠᶜ * ∂xᶠᶜᶠ_η(i, j, k_top, grid, TX, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gᵁ[i, j, k_top-1]) - V[i, j, k_top-1] += Δτ * (- g * Hᶜᶠ * ∂yᶜᶠᶠ_η(i, j, k_top, grid, TY, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gⱽ[i, j, k_top-1]) + U[i, j, k_top-1] += Δτ * (- g * Hᶠᶜ * ∂xTᶠᶜᶠ(i, j, k_top, grid, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gᵁ[i, j, k_top-1]) + V[i, j, k_top-1] += Δτ * (- g * Hᶜᶠ * ∂yTᶜᶠᶠ(i, j, k_top, grid, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gⱽ[i, j, k_top-1]) # time-averaging η̅[i, j, k_top] += averaging_weight * η[i, j, k_top] From c57a321c9deed6a187bab5a7cf869d5f116002ea Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 30 Oct 2024 14:29:26 +0100 Subject: [PATCH 50/79] this was removed --- test/test_cubed_spheres.jl | 227 ------------------------------------- 1 file changed, 227 deletions(-) delete mode 100644 test/test_cubed_spheres.jl diff --git a/test/test_cubed_spheres.jl b/test/test_cubed_spheres.jl deleted file mode 100644 index a950dc5cbf..0000000000 --- a/test/test_cubed_spheres.jl +++ /dev/null @@ -1,227 +0,0 @@ -include("dependencies_for_runtests.jl") -include("data_dependencies.jl") - -using Statistics: mean -using Oceananigans.Operators -using Oceananigans.CubedSpheres -using Oceananigans.Models.HydrostaticFreeSurfaceModels -using Oceananigans.Models.HydrostaticFreeSurfaceModels: VerticalVorticityField - -# To be used in the test below as `KernelFunctionOperation`s -@inline intrinsic_vector_x_component(i, j, k, grid, uₑ, vₑ) = - @inbounds intrinsic_vector(i, j, k, grid, uₑ, vₑ)[1] - -@inline intrinsic_vector_y_component(i, j, k, grid, uₑ, vₑ) = - @inbounds intrinsic_vector(i, j, k, grid, uₑ, vₑ)[2] - -@inline extrinsic_vector_x_component(i, j, k, grid, uₑ, vₑ) = - @inbounds intrinsic_vector(i, j, k, grid, uₑ, vₑ)[1] - -@inline extrinsic_vector_y_component(i, j, k, grid, uₑ, vₑ) = - @inbounds intrinsic_vector(i, j, k, grid, uₑ, vₑ)[2] - -function kinetic_energy(u, v) - ke = Field(0.5 * (u * u + v * v)) - return compute!(ke) -end - -function test_vector_rotation(grid) - u = XFaceField(grid) - v = YFaceField(grid) - - # Purely longitudinal flow in the extrinsic coordinate system - set!(u, 1) - set!(v, 0) - - # Convert it to an "Instrinsic" reference frame - uᵢ = KernelFunctionOperation{Face, Center, Center}(intrinsic_vector_x_component, grid, u, v) - vᵢ = KernelFunctionOperation{Center, Face, Center}(intrinsic_vector_y_component, grid, u, v) - - uᵢ = compute!(Field(uᵢ)) - vᵢ = compute!(Field(vᵢ)) - - # The extrema of u and v, as well as their mean value should - # be equivalent on an "intrinsic" frame - @test maximum(uᵢ) ≈ maximum(vᵢ) - @test minimum(uᵢ) ≈ minimum(vᵢ) - @test mean(uᵢ) ≈ mean(vᵢ) - @test mean(uᵢ) > 0 # The mean value should be positive - - # Kinetic energy should remain the same - KE = kinetic_energy(uᵢ, vᵢ) - @test all(on_architecture(CPU(), interior(KE)) .≈ 0.5) - - # Convert it back to a purely zonal velocity (vₑ == 0) - uₑ = KernelFunctionOperation{Face, Center, Center}(extrinsic_vector_x_component, grid, uᵢ, vᵢ) - vₑ = KernelFunctionOperation{Center, Face, Center}(extrinsic_vector_y_component, grid, uᵢ, vᵢ) - - uₑ = compute!(Field(uₑ)) - vₑ = compute!(Field(vₑ)) - - # Make sure that the flow was converted back to a - # purely zonal flow in the extrensic frame (v ≈ 0) - @test all(on_architecture(CPU(), interior(vₑ)) .≈ 0) - @test all(on_architecture(CPU(), interior(uₑ)) .≈ 1) - - # Purely meridional flow in the extrinsic coordinate system - set!(u, 0) - set!(v, 1) - - # Convert it to an "Instrinsic" reference frame - uᵢ = KernelFunctionOperation{Face, Center, Center}(intrinsic_vector_x_component, grid, u, v) - vᵢ = KernelFunctionOperation{Center, Face, Center}(intrinsic_vector_y_component, grid, u, v) - - uᵢ = compute!(Field(uᵢ)) - vᵢ = compute!(Field(vᵢ)) - - # The extrema of u and v, as well as their mean value should - # be equivalent on an "intrinsic" frame - @test maximum(uᵢ) ≈ maximum(vᵢ) - @test minimum(uᵢ) ≈ minimum(vᵢ) - @test mean(uᵢ) ≈ mean(vᵢ) - @test mean(vᵢ) > 0 # The mean value should be positive - - # Kinetic energy should remain the same - KE = kinetic_energy(uᵢ, vᵢ) - @test all(on_architecture(CPU(), interior(KE)) .≈ 0.5) - - # Convert it back to a purely zonal velocity (vₑ == 0) - uₑ = KernelFunctionOperation{Face, Center, Center}(extrinsic_vector_x_component, grid, uᵢ, vᵢ) - vₑ = KernelFunctionOperation{Center, Face, Center}(extrinsic_vector_y_component, grid, uᵢ, vᵢ) - - uₑ = compute!(Field(uₑ)) - vₑ = compute!(Field(vₑ)) - - # Make sure that the flow was converted back to a - # purely zonal flow in the extrensic frame (v ≈ 0) - @test all(on_architecture(CPU(), interior(vₑ)) .≈ 1) - @test all(on_architecture(CPU(), interior(uₑ)) .≈ 0) - - # Mixed zonal and meridional flow. - set!(u, 0.5) - set!(v, 0.5) - - # Convert it to an "Instrinsic" reference frame - uᵢ = KernelFunctionOperation{Face, Center, Center}(intrinsic_vector_x_component, grid, u, v) - vᵢ = KernelFunctionOperation{Center, Face, Center}(intrinsic_vector_y_component, grid, u, v) - - uᵢ = compute!(Field(uᵢ)) - vᵢ = compute!(Field(vᵢ)) - - # The extrema of u and v, as well as their mean value should - # be equivalent on an "intrinsic" frame - @test maximum(uᵢ) ≈ maximum(vᵢ) - @test minimum(uᵢ) ≈ minimum(vᵢ) - @test mean(uᵢ) ≈ mean(vᵢ) - @test mean(vᵢ) > 0 # The mean value should be positive - - # Kinetic energy should remain the same - KE = kinetic_energy(uᵢ, vᵢ) - @test all(on_architecture(CPU(), interior(KE)) .≈ 0.25) - - # Convert it back to a purely zonal velocity (vₑ == 0) - uₑ = KernelFunctionOperation{Face, Center, Center}(extrinsic_vector_x_component, grid, uᵢ, vᵢ) - vₑ = KernelFunctionOperation{Center, Face, Center}(extrinsic_vector_y_component, grid, uᵢ, vᵢ) - - uₑ = compute!(Field(uₑ)) - vₑ = compute!(Field(vₑ)) - - # Make sure that the flow was converted back to a - # purely zonal flow in the extrensic frame (v ≈ 0) - @test all(on_architecture(CPU(), interior(vₑ)) .≈ 0.5) - @test all(on_architecture(CPU(), interior(uₑ)) .≈ 0.5) -end - - -@testset "Cubed spheres" begin - - @testset "Conformal cubed sphere grid" begin - @info " Testing conformal cubed sphere grid..." - - grid = ConformalCubedSphereGrid(panel_size=(10, 10, 1), z=(-1, 0)) - @test try show(grid); println(); true; catch; false; end - end - - for arch in archs - - @info " Constructing a ConformalCubedSphereGrid from file [$(typeof(arch))]..." - - # These tests cause an undefined `Bound Access Error` on GPU's CI with the new CUDA version. - # The error is not reproducible neither on Tartarus nor on Sverdrup. - # These are excised for the moment (PR #2253) as Cubed sphere will be reworked - if !(arch isa GPU) - # Prototype grid and model for subsequent tests - cs32_filepath = datadep"cubed_sphere_32_grid/cubed_sphere_32_grid.jld2" - grid = OldConformalCubedSphereGrid(cs32_filepath, arch, Nz=1, z=(-1, 0)) - - @info " Constructing a HydrostaticFreeSurfaceModel on a ConformalCubedSphereGrid [$(typeof(arch))]..." - - free_surface = ExplicitFreeSurface(gravitational_acceleration=0.1) - model = HydrostaticFreeSurfaceModel(; grid, free_surface, - momentum_advection = VectorInvariant(), - coriolis = nothing, - closure = nothing, - tracers = :c, - buoyancy = nothing) - - @testset "Constructing a grid from file [$(typeof(arch))]" begin - @test grid isa ConformalCubedSphereGrid - end - - @testset "Conversion from Intrinsic to Extrinsic reference frame [$(typeof(arch))]" begin - @info " Testing the conversion of a vector between the Intrinsic and Extrinsic reference frame" - test_vector_rotation(grid) - end - - @testset "CubedSphereData and CubedSphereFields [$(typeof(arch))]" begin - @info " Testing CubedSphereData and CubedSphereFields [$(typeof(arch))]..." - c = model.tracers.c - η = model.free_surface.η - - set!(c, 0) - set!(η, 0) - - CUDA.allowscalar(true) - @test all(all(face_c .== 0) for face_c in faces(c)) - @test all(all(face_η .== 0) for face_η in faces(η)) - CUDA.allowscalar(false) - - @test maximum(abs, c) == 0 - @test minimum(abs, c) == 0 - @test mean(c) == 0 - - @test maximum(abs, η) == 0 - @test minimum(abs, η) == 0 - @test mean(η) == 0 - end - - @testset "Constructing a HydrostaticFreeSurfaceModel [$(typeof(arch))]" begin - @test model isa HydrostaticFreeSurfaceModel - end - - @testset "Time stepping a HydrostaticFreeSurfaceModel [$(typeof(arch))]" begin - @info " Time-stepping HydrostaticFreeSurfaceModel on a ConformalCubedSphereGrid [$(typeof(arch))]..." - time_step!(model, 1) - @test try time_step!(model, 1); true; catch; false; end - end - - @testset "VerticalVorticityField on ConformalCubedSphereGrid [$(typeof(arch))]" begin - @info " Testing VerticalVorticityField on a ConformalCubedSphereGrid [$(typeof(arch))]..." - ζ = VerticalVorticityField(model) - - @test ζ isa Field - - set!(model, u = (x, y, z) -> rand()) - - @test try - compute!(ζ) - true - catch err - println(sprint(showerror, err)) - false - end - @test maximum(abs, ζ) > 0 # fingers crossed - end - end - end -end From d78d6597ce59aa73b58b8a58ea89a79b3ec8464d Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 30 Oct 2024 14:46:06 +0100 Subject: [PATCH 51/79] start refactoring --- .../HydrostaticFreeSurfaceModels.jl | 15 +++++++++++++++ .../hydrostatic_free_surface_field_tuples.jl | 16 ++++++++++++++++ .../split_explicit_free_surface.jl | 19 ++++++++----------- 3 files changed, 39 insertions(+), 11 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl index a602a289f7..6eb0d8c603 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl @@ -85,6 +85,13 @@ Return a flattened `NamedTuple` of the prognostic fields associated with `Hydros η = free_surface.η), tracers) +@inline hydrostatic_prognostic_fields(velocities, free_surface::SplitExplicitFreeSurface, tracers) = merge((u = velocities.u, + v = velocities.v, + η = free_surface.η, + U = free_surface.barotropic_velocities.U, + V = free_surface.barotropic_velocities.V), + tracers) + @inline hydrostatic_prognostic_fields(velocities, ::Nothing, tracers) = merge((u = velocities.u, v = velocities.v), tracers) @@ -95,6 +102,14 @@ Return a flattened `NamedTuple` of the prognostic fields associated with `Hydros tracers, (; η = free_surface.η)) +@inline hydrostatic_fields(velocities, free_surface::SplitExplicitFreeSurface, tracers) = merge((u = velocities.u, + v = velocities.v, + w = velocities.w, + η = free_surface.η, + U = free_surface.barotropic_velocities.U, + V = free_surface.barotropic_velocities.V), + tracers) + @inline hydrostatic_fields(velocities, ::Nothing, tracers) = merge((u = velocities.u, v = velocities.v, w = velocities.w), diff --git a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_field_tuples.jl b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_field_tuples.jl index 92f787bfd2..019a3f3eff 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_field_tuples.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_field_tuples.jl @@ -9,9 +9,25 @@ function HydrostaticFreeSurfaceVelocityFields(::Nothing, grid, clock, bcs=NamedT end function HydrostaticFreeSurfaceTendencyFields(velocities, free_surface, grid, tracer_names) + u = XFaceField(grid) + v = YFaceField(grid) + tracers = TracerFields(tracer_names, grid) + return merge((u=u, v=v), tracers) +end + +function HydrostaticFreeSurfaceTendencyFields(velocities, free_surface::ExplicitFreeSurface, grid, tracer_names) u = XFaceField(grid) v = YFaceField(grid) η = free_surface_displacement_field(velocities, free_surface, grid) tracers = TracerFields(tracer_names, grid) return merge((u=u, v=v, η=η), tracers) end + +function HydrostaticFreeSurfaceTendencyFields(velocities, ::SplitExplicitFreeSurface, grid, tracer_names) + u = XFaceField(grid) + v = YFaceField(grid) + U = Field{Face, Center, Nothing}(grid) + V = Field{Center, Face, Nothing}(grid) + tracers = TracerFields(tracer_names, grid) + return merge((u=u, v=v, U=U, V=V), tracers) +end diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl index 073fbfc4ea..c0983daf82 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl @@ -18,17 +18,14 @@ The split-explicit free surface solver. $(FIELDS) """ -struct SplitExplicitFreeSurface{𝒩, 𝒮, ℱ, 𝒫 ,ℰ} <: AbstractFreeSurface{𝒩, 𝒫} - "The instantaneous free surface (`ReducedField`)" - η :: 𝒩 - "The entire state for the split-explicit solver (`SplitExplicitState`)" - state :: 𝒮 - "Parameters for timestepping split-explicit solver (`NamedTuple`)" - auxiliary :: ℱ - "Gravitational acceleration" - gravitational_acceleration :: 𝒫 - "Settings for the split-explicit scheme" - settings :: ℰ +struct SplitExplicitFreeSurface{H, U, M, FT, K , S, T} <: AbstractFreeSurface{H, FT} + η :: H + barotropic_velocities :: U # A namedtuple with U, V + filtered_state :: M # A namedtuple with η, U, V averaged throughout the substepping + gravitational_acceleration :: FT + kernel_parameters :: K + substepping :: S # Either `FixedSubstepNumber` or `FixedTimeStepSize` + timestepper :: T # redesigned to contain all auxiliary field and settings necessary to the particular timestepping end """ From 86532788f5a46937d51b25cd92cfe3fd4d41ff84 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 30 Oct 2024 15:02:33 +0100 Subject: [PATCH 52/79] start refactoring --- .../HydrostaticFreeSurfaceModels.jl | 2 +- .../split_explicit_free_surface.jl | 141 ++++++------------ 2 files changed, 49 insertions(+), 94 deletions(-) rename src/Models/HydrostaticFreeSurfaceModels/{ => SplitExplicitFreeSurfaces}/split_explicit_free_surface.jl (81%) diff --git a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl index 6eb0d8c603..d9f0ec6980 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl @@ -46,7 +46,7 @@ include("matrix_implicit_free_surface_solver.jl") include("implicit_free_surface.jl") # Split-Explicit free-surface solver functionality -include("split_explicit_free_surface.jl") +include("SplitExplicitFreeSurfaces/split_explicit_free_surface.jl") include("distributed_split_explicit_free_surface.jl") include("split_explicit_free_surface_kernels.jl") diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_free_surface.jl similarity index 81% rename from src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl rename to src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_free_surface.jl index c0983daf82..808402e4ff 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_free_surface.jl @@ -92,21 +92,58 @@ function SplitExplicitFreeSurface(grid = nothing; averaging_kernel = averaging_shape_function, timestepper = ForwardBackwardScheme()) - settings = SplitExplicitSettings(grid; - gravitational_acceleration, - substeps, - cfl, - fixed_Δt, - averaging_kernel, - timestepper) + if !isnothing(grid) + FT = eltype(grid) + else + # this is a fallback and only used via the outer constructor, + # in case no grid is provided; when afterwards the free surfade + # is materialized via materialize_free_surface + # FT becomes eltype(grid) + FT = Float64 + end + + gravitational_acceleration = convert(FT, gravitational_acceleration) + substepping = split_explicit_substepping(cfl, substeps, fixed_Δt, grid, averaging_kernel, gravitational_acceleration) + + kernel_parameters = :xy return SplitExplicitFreeSurface(nothing, nothing, nothing, gravitational_acceleration, + kernel_parameters, + substepping, settings) end +# Simplest case: we have the substeps and the averaging kernel +function split_explicit_substepping(::Nothing, substeps, fixed_Δt, grid, averaging_kernel, gravitational_acceleration) + FT = eltype(gravitational_acceleration) + fractional_step_size, averaging_weights = weights_from_substeps(FT, substeps, averaging_kernel) + return FixedSubstepNumber(fractional_step_size, averaging_weights) +end + +# The substeps are calculated dynamically when a cfl without a fixed_Δt is provided +function split_explicit_substepping(cfl, ::Nothing, ::Nothing, grid, averaging_kernel, gravitational_acceleration) + if isnothing(grid) + throw(ArgumentError(string("Need to provide the grid to calculate the barotropic substeps from the cfl. ", + "For example, SplitExplicitFreeSurface(grid, cfl=0.7, ...)"))) + end + cfl = convert(eltype(grid), cfl) + + return FixedTimeStepSize(grid; cfl, averaging_kernel) +end + +# The number of substeps are calculated based on the cfl and the fixed_Δt +function split_explicit_substepping(cfl, ::Nothing, fixed_Δt, grid, averaging_kernel, gravitational_acceleration) + + substepping = split_explicit_substepping(cfl, nothing, nothing, grid, averaging_kernel, gravitational_acceleration) + substeps = ceil(Int, 2 * fixed_Δt / substepping.Δt_barotropic) + substepping = split_explicit_substepping(nothing, substeps, nothing, grid, averaging_kernel, gravitational_acceleration) + + return substepping +end + # Internal function for HydrostaticFreeSurfaceModel function materialize_free_surface(free_surface::SplitExplicitFreeSurface, velocities, grid) settings = SplitExplicitSettings(grid; free_surface.settings.settings_kwargs...) @@ -115,11 +152,13 @@ function materialize_free_surface(free_surface::SplitExplicitFreeSurface, veloci gravitational_acceleration = convert(eltype(grid), free_surface.gravitational_acceleration) + timestepper = materialize_timestepper(free_surface.timestepper, grid) + return SplitExplicitFreeSurface(η, SplitExplicitState(grid, settings.timestepper), - SplitExplicitAuxiliaryFields(grid), + free_surface.kernel_parameters, gravitational_acceleration, - settings) + timestepper) end @@ -190,41 +229,6 @@ function SplitExplicitState(grid::AbstractGrid, timestepper) return SplitExplicitState(; ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², η̅, U̅, V̅) end -""" - struct SplitExplicitAuxiliaryFields - -A type containing auxiliary fields for the split-explicit free surface. - -The barotropic time stepping is launched on a grid `(kernel_size[1], kernel_size[2])` -large (or `:xy` in case of a serial computation), and start computing from -`(i - kernel_offsets[1], j - kernel_offsets[2])`. - -$(FIELDS) -""" -Base.@kwdef struct SplitExplicitAuxiliaryFields{𝒞ℱ, ℱ𝒞, 𝒦} - "Vertically-integrated slow barotropic forcing function for `U` (`ReducedField` over ``z``)" - Gᵁ :: ℱ𝒞 - "Vertically-integrated slow barotropic forcing function for `V` (`ReducedField` over ``z``)" - Gⱽ :: 𝒞ℱ - "kernel size for barotropic time stepping" - kernel_parameters :: 𝒦 -end - -""" - SplitExplicitAuxiliaryFields(grid) - -Return the `SplitExplicitAuxiliaryFields` for `grid`. -""" -function SplitExplicitAuxiliaryFields(grid::AbstractGrid) - - Gᵁ = Field((Face, Center, Nothing), grid) - Gⱽ = Field((Center, Face, Nothing), grid) - - kernel_parameters = :xy - - return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, kernel_parameters) -end - """ struct SplitExplicitSettings @@ -307,54 +311,6 @@ end return Δτ, tuple(averaging_weights...) end -function SplitExplicitSettings(grid = nothing; - gravitational_acceleration = g_Earth, - substeps = nothing, - cfl = nothing, - fixed_Δt = nothing, - averaging_kernel = averaging_shape_function, - timestepper = ForwardBackwardScheme()) - - settings_kwargs = (; gravitational_acceleration, - substeps, - cfl, - fixed_Δt, - averaging_kernel, - timestepper) - - if !isnothing(grid) - FT = eltype(grid) - else - # this is a fallback and only used via the outer constructor, - # in case no grid is provided; when afterwards the free surfade - # is materialized via materialize_free_surface - # FT becomes eltype(grid) - FT = Float64 - end - - if (!isnothing(substeps) && !isnothing(cfl)) || (isnothing(substeps) && isnothing(cfl)) - throw(ArgumentError("either specify a cfl or a number of substeps")) - end - - if !isnothing(cfl) - if isnothing(grid) - throw(ArgumentError(string("Need to provide the grid to calculate the barotropic substeps from the cfl. ", - "For example, SplitExplicitFreeSurface(grid, cfl=0.7, ...)"))) - end - substepping = FixedTimeStepSize(grid; cfl, gravitational_acceleration, averaging_kernel) - if isnothing(fixed_Δt) - return SplitExplicitSettings(substepping, timestepper, settings_kwargs) - else - substeps = ceil(Int, 2 * fixed_Δt / substepping.Δt_barotropic) - end - end - - fractional_step_size, averaging_weights = weights_from_substeps(FT, substeps, averaging_kernel) - substepping = FixedSubstepNumber(fractional_step_size, averaging_weights) - - return SplitExplicitSettings(substepping, timestepper, settings_kwargs) -end - # Convenience Functions for grabbing free surface free_surface(free_surface::SplitExplicitFreeSurface) = free_surface.η @@ -393,7 +349,6 @@ Adapt.adapt_structure(to, free_surface::SplitExplicitFreeSurface) = for Type in (:SplitExplicitFreeSurface, :SplitExplicitSettings, :SplitExplicitState, - :SplitExplicitAuxiliaryFields, :FixedTimeStepSize, :FixedSubstepNumber) From 4c728b3f264700aca525c15a3200c3b4de059158 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 30 Oct 2024 20:02:42 +0100 Subject: [PATCH 53/79] some more refactoring --- .../barotropic_correction.jl | 73 +++++ .../barotropic_kernels.jl} | 271 ++++-------------- .../setup_split_explicit.jl | 97 +++++++ .../split_explicit_free_surface.jl | 170 +++++------ 4 files changed, 284 insertions(+), 327 deletions(-) create mode 100644 src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_correction.jl rename src/Models/HydrostaticFreeSurfaceModels/{split_explicit_free_surface_kernels.jl => SplitExplicitFreeSurfaces/barotropic_kernels.jl} (54%) create mode 100644 src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/setup_split_explicit.jl diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_correction.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_correction.jl new file mode 100644 index 0000000000..81008c51ca --- /dev/null +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_correction.jl @@ -0,0 +1,73 @@ + +function initialize_free_surface!(sefs::SplitExplicitFreeSurface, grid, velocities) + @apply_regionally compute_barotropic_mode!(sefs.state.U̅, sefs.state.V̅, grid, velocities.u, velocities.v) + fill_halo_regions!((sefs.state.U̅, sefs.state.V̅, sefs.η)) +end + + +# Barotropic Model Kernels +# u_Δz = u * Δz +@kernel function _barotropic_mode_kernel!(U, V, grid, ::Nothing, u, v) + i, j = @index(Global, NTuple) + k_top = grid.Nz+1 + + @inbounds U[i, j, k_top-1] = Δzᶠᶜᶜ(i, j, 1, grid) * u[i, j, 1] + @inbounds V[i, j, k_top-1] = Δzᶜᶠᶜ(i, j, 1, grid) * v[i, j, 1] + + for k in 2:grid.Nz + @inbounds U[i, j, k_top-1] += Δzᶠᶜᶜ(i, j, k, grid) * u[i, j, k] + @inbounds V[i, j, k_top-1] += Δzᶜᶠᶜ(i, j, k, grid) * v[i, j, k] + end +end + +# Barotropic Model Kernels +# u_Δz = u * Δz +@kernel function _barotropic_mode_kernel!(U, V, grid, active_cells_map, u, v) + idx = @index(Global, Linear) + i, j = active_linear_index_to_tuple(idx, active_cells_map) + k_top = grid.Nz+1 + + @inbounds U[i, j, k_top-1] = Δzᶠᶜᶜ(i, j, 1, grid) * u[i, j, 1] + @inbounds V[i, j, k_top-1] = Δzᶜᶠᶜ(i, j, 1, grid) * v[i, j, 1] + + for k in 2:grid.Nz + @inbounds U[i, j, k_top-1] += Δzᶠᶜᶜ(i, j, k, grid) * u[i, j, k] + @inbounds V[i, j, k_top-1] += Δzᶜᶠᶜ(i, j, k, grid) * v[i, j, k] + end +end + +@inline function compute_barotropic_mode!(U, V, grid, u, v) + active_cells_map = retrieve_surface_active_cells_map(grid) + + launch!(architecture(grid), grid, :xy, _barotropic_mode_kernel!, U, V, grid, active_cells_map, u, v; active_cells_map) + + return nothing +end + +@kernel function _barotropic_split_explicit_corrector!(u, v, U̅, V̅, U, V, grid) + i, j, k = @index(Global, NTuple) + k_top = grid.Nz+1 + + @inbounds begin + Hᶠᶜ = static_column_depthᶠᶜᵃ(i, j, grid) + Hᶜᶠ = static_column_depthᶜᶠᵃ(i, j, grid) + + u[i, j, k] = u[i, j, k] + (U̅[i, j, k_top-1] - U[i, j, k_top-1]) / Hᶠᶜ + v[i, j, k] = v[i, j, k] + (V̅[i, j, k_top-1] - V[i, j, k_top-1]) / Hᶜᶠ + end +end + +function barotropic_split_explicit_corrector!(u, v, free_surface, grid) + sefs = free_surface.state + U, V, U̅, V̅ = sefs.U, sefs.V, sefs.U̅, sefs.V̅ + arch = architecture(grid) + + # take out "bad" barotropic mode, + # !!!! reusing U and V for this storage since last timestep doesn't matter + compute_barotropic_mode!(U, V, grid, u, v) + # add in "good" barotropic mode + launch!(arch, grid, :xyz, _barotropic_split_explicit_corrector!, + u, v, U̅, V̅, U, V, grid) + + return nothing +end diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_kernels.jl similarity index 54% rename from src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl rename to src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_kernels.jl index 0042755123..50ad5d05b3 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_kernels.jl @@ -131,157 +131,6 @@ end end end -# Barotropic Model Kernels -# u_Δz = u * Δz -@kernel function _barotropic_mode_kernel!(U, V, grid, ::Nothing, u, v) - i, j = @index(Global, NTuple) - k_top = grid.Nz+1 - - @inbounds U[i, j, k_top-1] = Δzᶠᶜᶜ(i, j, 1, grid) * u[i, j, 1] - @inbounds V[i, j, k_top-1] = Δzᶜᶠᶜ(i, j, 1, grid) * v[i, j, 1] - - for k in 2:grid.Nz - @inbounds U[i, j, k_top-1] += Δzᶠᶜᶜ(i, j, k, grid) * u[i, j, k] - @inbounds V[i, j, k_top-1] += Δzᶜᶠᶜ(i, j, k, grid) * v[i, j, k] - end -end - -# Barotropic Model Kernels -# u_Δz = u * Δz -@kernel function _barotropic_mode_kernel!(U, V, grid, active_cells_map, u, v) - idx = @index(Global, Linear) - i, j = active_linear_index_to_tuple(idx, active_cells_map) - k_top = grid.Nz+1 - - @inbounds U[i, j, k_top-1] = Δzᶠᶜᶜ(i, j, 1, grid) * u[i, j, 1] - @inbounds V[i, j, k_top-1] = Δzᶜᶠᶜ(i, j, 1, grid) * v[i, j, 1] - - for k in 2:grid.Nz - @inbounds U[i, j, k_top-1] += Δzᶠᶜᶜ(i, j, k, grid) * u[i, j, k] - @inbounds V[i, j, k_top-1] += Δzᶜᶠᶜ(i, j, k, grid) * v[i, j, k] - end -end - -@inline function compute_barotropic_mode!(U, V, grid, u, v) - active_cells_map = retrieve_surface_active_cells_map(grid) - - launch!(architecture(grid), grid, :xy, _barotropic_mode_kernel!, U, V, grid, active_cells_map, u, v; active_cells_map) - - return nothing -end - -function initialize_free_surface_state!(state, η, timestepper) - - parent(state.U) .= parent(state.U̅) - parent(state.V) .= parent(state.V̅) - - initialize_auxiliary_state!(state, η, timestepper) - - fill!(state.η̅, 0) - fill!(state.U̅, 0) - fill!(state.V̅, 0) - - return nothing -end - -initialize_auxiliary_state!(state, η, ::ForwardBackwardScheme) = nothing - -function initialize_auxiliary_state!(state, η, timestepper) - parent(state.Uᵐ⁻¹) .= parent(state.U̅) - parent(state.Vᵐ⁻¹) .= parent(state.V̅) - - parent(state.Uᵐ⁻²) .= parent(state.U̅) - parent(state.Vᵐ⁻²) .= parent(state.V̅) - - parent(state.ηᵐ) .= parent(η) - parent(state.ηᵐ⁻¹) .= parent(η) - parent(state.ηᵐ⁻²) .= parent(η) - - return nothing -end - -@kernel function _barotropic_split_explicit_corrector!(u, v, U̅, V̅, U, V, grid) - i, j, k = @index(Global, NTuple) - k_top = grid.Nz+1 - - @inbounds begin - Hᶠᶜ = static_column_depthᶠᶜᵃ(i, j, grid) - Hᶜᶠ = static_column_depthᶜᶠᵃ(i, j, grid) - - u[i, j, k] = u[i, j, k] + (U̅[i, j, k_top-1] - U[i, j, k_top-1]) / Hᶠᶜ - v[i, j, k] = v[i, j, k] + (V̅[i, j, k_top-1] - V[i, j, k_top-1]) / Hᶜᶠ - end -end - -function barotropic_split_explicit_corrector!(u, v, free_surface, grid) - sefs = free_surface.state - U, V, U̅, V̅ = sefs.U, sefs.V, sefs.U̅, sefs.V̅ - arch = architecture(grid) - - # take out "bad" barotropic mode, - # !!!! reusing U and V for this storage since last timestep doesn't matter - compute_barotropic_mode!(U, V, grid, u, v) - # add in "good" barotropic mode - launch!(arch, grid, :xyz, _barotropic_split_explicit_corrector!, - u, v, U̅, V̅, U, V, grid) - - return nothing -end - -""" -Explicitly step forward η in substeps. -""" -ab2_step_free_surface!(free_surface::SplitExplicitFreeSurface, model, Δt, χ) = - split_explicit_free_surface_step!(free_surface, model, Δt, χ) - -function initialize_free_surface!(sefs::SplitExplicitFreeSurface, grid, velocities) - @apply_regionally compute_barotropic_mode!(sefs.state.U̅, sefs.state.V̅, grid, velocities.u, velocities.v) - fill_halo_regions!((sefs.state.U̅, sefs.state.V̅, sefs.η)) -end - -function split_explicit_free_surface_step!(free_surface::SplitExplicitFreeSurface, model, Δt, χ) - - # Note: free_surface.η.grid != model.grid for DistributedSplitExplicitFreeSurface - # since halo_size(free_surface.η.grid) != halo_size(model.grid) - free_surface_grid = free_surface.η.grid - - # Wait for previous set up - wait_free_surface_communication!(free_surface, architecture(free_surface_grid)) - - # Calculate the substepping parameterers - settings = free_surface.settings - Nsubsteps = calculate_substeps(settings.substepping, Δt) - - # barotropic time step as fraction of baroclinic step and averaging weights - fractional_Δt, weights = calculate_adaptive_settings(settings.substepping, Nsubsteps) - Nsubsteps = length(weights) - - # barotropic time step in seconds - Δτᴮ = fractional_Δt * Δt - - # reset free surface averages - @apply_regionally begin - initialize_free_surface_state!(free_surface.state, free_surface.η, settings.timestepper) - - # Solve for the free surface at tⁿ⁺¹ - iterate_split_explicit!(free_surface, free_surface_grid, Δτᴮ, weights, Val(Nsubsteps)) - - # Reset eta for the next timestep - set!(free_surface.η, free_surface.state.η̅) - end - - fields_to_fill = (free_surface.state.U̅, free_surface.state.V̅) - fill_halo_regions!(fields_to_fill; async = true) - - # Preparing velocities for the barotropic correction - @apply_regionally begin - mask_immersed_field!(model.velocities.u) - mask_immersed_field!(model.velocities.v) - end - - return nothing -end - # Change name const FNS = FixedSubstepNumber const FTS = FixedTimeStepSize @@ -300,37 +149,28 @@ const MINIMUM_SUBSTEPS = 5 const FixedSubstepsSetting{N} = SplitExplicitSettings{<:FixedSubstepNumber{<:Any, <:NTuple{N, <:Any}}} where N const FixedSubstepsSplitExplicit{F} = SplitExplicitFreeSurface{<:Any, <:Any, <:Any, <:Any, <:FixedSubstepsSetting{N}} where N -function iterate_split_explicit!(free_surface, grid, Δτᴮ, weights, ::Val{Nsubsteps}) where Nsubsteps +function iterate_split_explicit!(free_surface, grid, GUⁿ, GVⁿ, Δτᴮ, weights, ::Val{Nsubsteps}) where Nsubsteps arch = architecture(grid) - η = free_surface.η - state = free_surface.state - auxiliary = free_surface.auxiliary - settings = free_surface.settings - g = free_surface.gravitational_acceleration + η = free_surface.η + grid = free_surface.η.grid + state = free_surface.filtered_state + timestepper = free_surface.timestepper + g = free_surface.gravitational_acceleration + parameters = free_surface.kernel_parameters # unpack state quantities, parameters and forcing terms - U, V = state.U, state.V - Uᵐ⁻¹, Uᵐ⁻² = state.Uᵐ⁻¹, state.Uᵐ⁻² - Vᵐ⁻¹, Vᵐ⁻² = state.Vᵐ⁻¹, state.Vᵐ⁻² - ηᵐ, ηᵐ⁻¹, ηᵐ⁻² = state.ηᵐ, state.ηᵐ⁻¹, state.ηᵐ⁻² - η̅, U̅, V̅ = state.η̅, state.U̅, state.V̅ - Gᵁ, Gⱽ = auxiliary.Gᵁ, auxiliary.Gⱽ - - timestepper = settings.timestepper - - parameters = auxiliary.kernel_parameters + U, V = free_surface.barotropic_velocities + η̅, U̅, V̅ = state.η̅, state.U̅, state.V̅ free_surface_kernel!, _ = configure_kernel(arch, grid, parameters, _split_explicit_free_surface!) barotropic_velocity_kernel!, _ = configure_kernel(arch, grid, parameters, _split_explicit_barotropic_velocity!) - η_args = (grid, Δτᴮ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², - U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², + η_args = (grid, Δτᴮ, η, U, V, timestepper) - U_args = (grid, Δτᴮ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², - U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², - η̅, U̅, V̅, Gᵁ, Gⱽ, g, + U_args = (grid, Δτᴮ, η, U, V, + η̅, U̅, V̅, GUⁿ, GVⁿ, g, timestepper) GC.@preserve η_args U_args begin @@ -353,68 +193,57 @@ function iterate_split_explicit!(free_surface, grid, Δτᴮ, weights, ::Val{Nsu return nothing end -# Calculate RHS for the barotropic time step. -@kernel function _compute_integrated_ab2_tendencies!(Gᵁ, Gⱽ, grid, ::Nothing, Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ) - i, j = @index(Global, NTuple) - k_top = grid.Nz + 1 +##### +##### SplitExplicitFreeSurface barotropic subcylicing +##### - @inbounds Gᵁ[i, j, k_top-1] = Δzᶠᶜᶜ(i, j, 1, grid) * ab2_step_Gu(i, j, 1, grid, Gu⁻, Guⁿ, χ) - @inbounds Gⱽ[i, j, k_top-1] = Δzᶜᶠᶜ(i, j, 1, grid) * ab2_step_Gv(i, j, 1, grid, Gv⁻, Gvⁿ, χ) - - for k in 2:grid.Nz - @inbounds Gᵁ[i, j, k_top-1] += Δzᶠᶜᶜ(i, j, k, grid) * ab2_step_Gu(i, j, k, grid, Gu⁻, Guⁿ, χ) - @inbounds Gⱽ[i, j, k_top-1] += Δzᶜᶠᶜ(i, j, k, grid) * ab2_step_Gv(i, j, k, grid, Gv⁻, Gvⁿ, χ) - end -end +ab2_step_free_surface!(free_surface::SplitExplicitFreeSurface, model, Δt, χ) = + split_explicit_free_surface_step!(free_surface, model, Δt) -# Calculate RHS for the barotropic time step.q -@kernel function _compute_integrated_ab2_tendencies!(Gᵁ, Gⱽ, grid, active_cells_map, Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ) - idx = @index(Global, Linear) - i, j = active_linear_index_to_tuple(idx, active_cells_map) - k_top = grid.Nz+1 +function split_explicit_free_surface_step!(free_surface::SplitExplicitFreeSurface, model, Δt) - @inbounds Gᵁ[i, j, k_top-1] = Δzᶠᶜᶜ(i, j, 1, grid) * ab2_step_Gu(i, j, 1, grid, Gu⁻, Guⁿ, χ) - @inbounds Gⱽ[i, j, k_top-1] = Δzᶜᶠᶜ(i, j, 1, grid) * ab2_step_Gv(i, j, 1, grid, Gv⁻, Gvⁿ, χ) + # Note: free_surface.η.grid != model.grid for DistributedSplitExplicitFreeSurface + # since halo_size(free_surface.η.grid) != halo_size(model.grid) + free_surface_grid = free_surface.η.grid + filtered_state = free_surface.filtered_state + substepping = free_surface.substepping + timestepper = free_surface.timestepper - for k in 2:grid.Nz - @inbounds Gᵁ[i, j, k_top-1] += Δzᶠᶜᶜ(i, j, k, grid) * ab2_step_Gu(i, j, k, grid, Gu⁻, Guⁿ, χ) - @inbounds Gⱽ[i, j, k_top-1] += Δzᶜᶠᶜ(i, j, k, grid) * ab2_step_Gv(i, j, k, grid, Gv⁻, Gvⁿ, χ) - end -end + # Wait for previous set up + wait_free_surface_communication!(free_surface, architecture(free_surface_grid)) -@inline ab2_step_Gu(i, j, k, grid, G⁻, Gⁿ, χ::FT) where FT = - @inbounds ifelse(peripheral_node(i, j, k, grid, f, c, c), zero(grid), (convert(FT, 1.5) + χ) * Gⁿ[i, j, k] - G⁻[i, j, k] * (convert(FT, 0.5) + χ)) + # Calculate the substepping parameterers + # barotropic time step as fraction of baroclinic step and averaging weights + Nsubsteps = calculate_substeps(substepping, Δt) + fractional_Δt, weights = calculate_adaptive_settings(substepping, Nsubsteps) + Nsubsteps = length(weights) -@inline ab2_step_Gv(i, j, k, grid, G⁻, Gⁿ, χ::FT) where FT = - @inbounds ifelse(peripheral_node(i, j, k, grid, c, f, c), zero(grid), (convert(FT, 1.5) + χ) * Gⁿ[i, j, k] - G⁻[i, j, k] * (convert(FT, 0.5) + χ)) + # barotropic time step in seconds + Δτᴮ = fractional_Δt * Δt -# Setting up the RHS for the barotropic step (tendencies of the barotropic velocity components) -# This function is called after `calculate_tendency` and before `ab2_step_velocities!` -function setup_free_surface!(model, free_surface::SplitExplicitFreeSurface, χ) + # Slow forcing terms + GUⁿ = model.timestepper.Gⁿ.U + GVⁿ = model.timestepper.Gⁿ.V - # we start the time integration of η from the average ηⁿ - Gu⁻ = model.timestepper.G⁻.u - Gv⁻ = model.timestepper.G⁻.v - Guⁿ = model.timestepper.Gⁿ.u - Gvⁿ = model.timestepper.Gⁿ.v + # reset free surface averages + @apply_regionally begin + initialize_free_surface_state!(filtered_state, free_surface.η, free_surface.barotropic_velocities, timestepper) - auxiliary = free_surface.auxiliary + # Solve for the free surface at tⁿ⁺¹ + iterate_split_explicit!(free_surface, free_surface_grid, GUⁿ, GVⁿ, Δτᴮ, weights, Val(Nsubsteps)) - @apply_regionally setup_split_explicit_tendency!(auxiliary, model.grid, Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ) + # Reset eta for the next timestep + set!(free_surface.η, free_surface.filtered_state.η̅) + end - fields_to_fill = (auxiliary.Gᵁ, auxiliary.Gⱽ) + fields_to_fill = (free_surface.filtered_state.U̅, free_surface.filtered_state.V̅) fill_halo_regions!(fields_to_fill; async = true) - return nothing -end - -@inline function setup_split_explicit_tendency!(auxiliary, grid, Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ) - active_cells_map = retrieve_surface_active_cells_map(grid) - - launch!(architecture(grid), grid, :xy, _compute_integrated_ab2_tendencies!, auxiliary.Gᵁ, auxiliary.Gⱽ, grid, - active_cells_map, Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ; active_cells_map) + # Preparing velocities for the barotropic correction + @apply_regionally begin + mask_immersed_field!(model.velocities.u) + mask_immersed_field!(model.velocities.v) + end return nothing end - -wait_free_surface_communication!(free_surface, arch) = nothing diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/setup_split_explicit.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/setup_split_explicit.jl new file mode 100644 index 0000000000..9301b53746 --- /dev/null +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/setup_split_explicit.jl @@ -0,0 +1,97 @@ + +function initialize_free_surface_state!(filtered_state, η, barotropic_velocities, timestepper) + + parent(barotropic_velocities.U) .= parent(filtered_state.U̅) + parent(barotropic_velocities.V) .= parent(filtered_state.V̅) + + initialize_auxiliary_state!(filtered_state, η, timestepper) + + fill!(filtered_state.η̅, 0) + fill!(filtered_state.U̅, 0) + fill!(filtered_state.V̅, 0) + + return nothing +end + +initialize_auxiliary_state!(state, η, ::ForwardBackwardScheme) = nothing + +function initialize_auxiliary_state!(filtered_state, η, timestepper::AdamsBashforth3Scheme) + parent(timestepper.Uᵐ⁻¹) .= parent(filtered_state.U̅) + parent(timestepper.Vᵐ⁻¹) .= parent(filtered_state.V̅) + + parent(timestepper.Uᵐ⁻²) .= parent(filtered_state.U̅) + parent(timestepper.Vᵐ⁻²) .= parent(filtered_state.V̅) + + parent(timestepper.ηᵐ) .= parent(η) + parent(timestepper.ηᵐ⁻¹) .= parent(η) + parent(timestepper.ηᵐ⁻²) .= parent(η) + + return nothing +end + +# Calculate RHS for the barotropic time step. +@kernel function _compute_integrated_ab2_tendencies!(Gᵁ, Gⱽ, grid, ::Nothing, Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ) + i, j = @index(Global, NTuple) + k_top = grid.Nz + 1 + + @inbounds Gᵁ[i, j, k_top-1] = Δzᶠᶜᶜ(i, j, 1, grid) * ab2_step_Gu(i, j, 1, grid, Gu⁻, Guⁿ, χ) + @inbounds Gⱽ[i, j, k_top-1] = Δzᶜᶠᶜ(i, j, 1, grid) * ab2_step_Gv(i, j, 1, grid, Gv⁻, Gvⁿ, χ) + + for k in 2:grid.Nz + @inbounds Gᵁ[i, j, k_top-1] += Δzᶠᶜᶜ(i, j, k, grid) * ab2_step_Gu(i, j, k, grid, Gu⁻, Guⁿ, χ) + @inbounds Gⱽ[i, j, k_top-1] += Δzᶜᶠᶜ(i, j, k, grid) * ab2_step_Gv(i, j, k, grid, Gv⁻, Gvⁿ, χ) + end +end + +# Calculate RHS for the barotropic time step.q +@kernel function _compute_integrated_ab2_tendencies!(Gᵁ, Gⱽ, grid, active_cells_map, Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ) + idx = @index(Global, Linear) + i, j = active_linear_index_to_tuple(idx, active_cells_map) + k_top = grid.Nz+1 + + @inbounds Gᵁ[i, j, k_top-1] = Δzᶠᶜᶜ(i, j, 1, grid) * ab2_step_Gu(i, j, 1, grid, Gu⁻, Guⁿ, χ) + @inbounds Gⱽ[i, j, k_top-1] = Δzᶜᶠᶜ(i, j, 1, grid) * ab2_step_Gv(i, j, 1, grid, Gv⁻, Gvⁿ, χ) + + for k in 2:grid.Nz + @inbounds Gᵁ[i, j, k_top-1] += Δzᶠᶜᶜ(i, j, k, grid) * ab2_step_Gu(i, j, k, grid, Gu⁻, Guⁿ, χ) + @inbounds Gⱽ[i, j, k_top-1] += Δzᶜᶠᶜ(i, j, k, grid) * ab2_step_Gv(i, j, k, grid, Gv⁻, Gvⁿ, χ) + end +end + +@inline ab2_step_Gu(i, j, k, grid, G⁻, Gⁿ, χ::FT) where FT = + @inbounds ifelse(peripheral_node(i, j, k, grid, f, c, c), zero(grid), (convert(FT, 1.5) + χ) * Gⁿ[i, j, k] - G⁻[i, j, k] * (convert(FT, 0.5) + χ)) + +@inline ab2_step_Gv(i, j, k, grid, G⁻, Gⁿ, χ::FT) where FT = + @inbounds ifelse(peripheral_node(i, j, k, grid, c, f, c), zero(grid), (convert(FT, 1.5) + χ) * Gⁿ[i, j, k] - G⁻[i, j, k] * (convert(FT, 0.5) + χ)) + +# Setting up the RHS for the barotropic step (tendencies of the barotropic velocity components) +# This function is called after `calculate_tendency` and before `ab2_step_velocities!` +function setup_free_surface!(model, ::SplitExplicitFreeSurface, χ) + + # we start the time integration of η from the average ηⁿ + Gu⁻ = model.timestepper.G⁻.u + Gv⁻ = model.timestepper.G⁻.v + Guⁿ = model.timestepper.Gⁿ.u + Gvⁿ = model.timestepper.Gⁿ.v + + GUⁿ = model.timestepper.Gⁿ.U + GVⁿ = model.timestepper.Gⁿ.V + + @apply_regionally setup_split_explicit_tendency!(GUⁿ, GVⁿ, model.grid, Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ) + + fields_to_fill = (GUⁿ, GVⁿ) + fill_halo_regions!(fields_to_fill; async = true) + + return nothing +end + +@inline function setup_split_explicit_tendency!(GUⁿ, GVⁿ, grid, Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ) + active_cells_map = retrieve_surface_active_cells_map(grid) + + launch!(architecture(grid), grid, :xy, _compute_integrated_ab2_tendencies!, GUⁿ, GVⁿ, grid, + active_cells_map, Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ; active_cells_map) + + return nothing +end + +wait_free_surface_communication!(free_surface, arch) = nothing diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_free_surface.jl index 808402e4ff..7837cad46a 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_free_surface.jl @@ -134,7 +134,7 @@ function split_explicit_substepping(cfl, ::Nothing, ::Nothing, grid, averaging_k return FixedTimeStepSize(grid; cfl, averaging_kernel) end -# The number of substeps are calculated based on the cfl and the fixed_Δt +# The number of substeps are calculated based on the cfl an2d the fixed_Δt function split_explicit_substepping(cfl, ::Nothing, fixed_Δt, grid, averaging_kernel, gravitational_acceleration) substepping = split_explicit_substepping(cfl, nothing, nothing, grid, averaging_kernel, gravitational_acceleration) @@ -144,115 +144,90 @@ function split_explicit_substepping(cfl, ::Nothing, fixed_Δt, grid, averaging_k return substepping end +# TODO: When open boundary conditions are online +# We need to calculate the barotropic boundary conditions +# from the baroclinic boundary conditions by integrating the BC upwards +@inline west_barotropic_bc(baroclinic_velocity) = baroclinic_velocity.boundary_conditions.west +@inline east_barotropic_bc(baroclinic_velocity) = baroclinic_velocity.boundary_conditions.east +@inline south_barotropic_bc(baroclinic_velocity) = baroclinic_velocity.boundary_conditions.south +@inline north_barotropic_bc(baroclinic_velocity) = baroclinic_velocity.boundary_conditions.north + +@inline barotropic_bc(baroclinic_velocity) = FieldBoundaryConditions( + west = west_barotropic_bc(baroclinic_velocity), + east = east_barotropic_bc(baroclinic_velocity), + south = south_barotropic_bc(baroclinic_velocity), + north = north_barotropic_bc(baroclinic_velocity), + top = nothing, + bottom = nothing +) + # Internal function for HydrostaticFreeSurfaceModel function materialize_free_surface(free_surface::SplitExplicitFreeSurface, velocities, grid) - settings = SplitExplicitSettings(grid; free_surface.settings.settings_kwargs...) η = free_surface_displacement_field(velocities, free_surface, grid) + η̅ = free_surface_displacement_field(velocities, free_surface, grid) - gravitational_acceleration = convert(eltype(grid), free_surface.gravitational_acceleration) + u_baroclinic = velocities.u + v_baroclinic = velocities.v - timestepper = materialize_timestepper(free_surface.timestepper, grid) + u_bc = barotropic_bc(u_baroclinic) + v_bc = barotropic_bc(v_baroclinic) - return SplitExplicitFreeSurface(η, - SplitExplicitState(grid, settings.timestepper), - free_surface.kernel_parameters, - gravitational_acceleration, - timestepper) -end + U = Field{Center, Center, Nothing}(grid, boundary_conditions = u_bc) + V = Field{Center, Center, Nothing}(grid, boundary_conditions = v_bc) + U̅ = Field{Center, Center, Nothing}(grid, boundary_conditions = u_bc) + V̅ = Field{Center, Center, Nothing}(grid, boundary_conditions = v_bc) -""" - struct SplitExplicitState + filtered_state = (η = η̅, U = U̅, V = V̅) + barotropic_velocities = (U = U, V = V) -A type containing the state fields for the split-explicit free surface. + gravitational_acceleration = convert(eltype(grid), free_surface.gravitational_acceleration) + timestepper = materialize_timestepper(free_surface.timestepper, grid, free_surface, velocities, u_bc, v_bc) -$(FIELDS) -""" -Base.@kwdef struct SplitExplicitState{CC, ACC, FC, AFC, CF, ACF} - "The free surface at time `m`. (`ReducedField` over ``z``)" - ηᵐ :: ACC - "The free surface at time `m-1`. (`ReducedField` over ``z``)" - ηᵐ⁻¹ :: ACC - "The free surface at time `m-2`. (`ReducedField` over ``z``)" - ηᵐ⁻² :: ACC - "The barotropic zonal velocity at time `m`. (`ReducedField` over ``z``)" - U :: FC - "The barotropic zonal velocity at time `m-1`. (`ReducedField` over ``z``)" - Uᵐ⁻¹ :: AFC - "The barotropic zonal velocity at time `m-2`. (`ReducedField` over ``z``)" - Uᵐ⁻² :: AFC - "The barotropic meridional velocity at time `m`. (`ReducedField` over ``z``)" - V :: CF - "The barotropic meridional velocity at time `m-1`. (`ReducedField` over ``z``)" - Vᵐ⁻¹ :: ACF - "The barotropic meridional velocity at time `m-2`. (`ReducedField` over ``z``)" - Vᵐ⁻² :: ACF - "The time-filtered free surface. (`ReducedField` over ``z``)" - η̅ :: CC - "The time-filtered barotropic zonal velocity. (`ReducedField` over ``z``)" - U̅ :: FC - "The time-filtered barotropic meridional velocity. (`ReducedField` over ``z``)" - V̅ :: CF + return SplitExplicitFreeSurface(η, + barotropic_velocities, + filtered_state, + gravitational_acceleration, + free_surface.kernel_parameters, + free_surface.substepping, + timestepper) end -""" - SplitExplicitState(grid, timestepper) - -Return the split-explicit state for `grid`. - -Note that `η̅` is solely used for setting the `η` at the next substep iteration -- it essentially -acts as a filter for `η`. Values with superscripts `m-1` and `m-2` correspond to previous stored -time steps to allow using a higher-order time stepping scheme, e.g., `AdamsBashforth3Scheme`. -""" -function SplitExplicitState(grid::AbstractGrid, timestepper) - - Nz = size(grid, 3) - - η̅ = ZFaceField(grid, indices = (:, :, Nz+1)) - - ηᵐ = auxiliary_free_surface_field(grid, timestepper) - ηᵐ⁻¹ = auxiliary_free_surface_field(grid, timestepper) - ηᵐ⁻² = auxiliary_free_surface_field(grid, timestepper) - - U = XFaceField(grid, indices = (:, :, Nz)) - V = YFaceField(grid, indices = (:, :, Nz)) - - Uᵐ⁻¹ = auxiliary_barotropic_U_field(grid, timestepper) - Vᵐ⁻¹ = auxiliary_barotropic_V_field(grid, timestepper) - Uᵐ⁻² = auxiliary_barotropic_U_field(grid, timestepper) - Vᵐ⁻² = auxiliary_barotropic_V_field(grid, timestepper) - - U̅ = XFaceField(grid, indices = (:, :, Nz)) - V̅ = YFaceField(grid, indices = (:, :, Nz)) - - return SplitExplicitState(; ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², η̅, U̅, V̅) +function materialize_timestepper(timestepper::Symbol, args...) + fullname = Symbol(name, :Scheme) + TS = getglobal(@__MODULE__, fullname) + return materialize_timestepper(TS, args...) end -""" - struct SplitExplicitSettings +struct ForwardBackwardScheme end -A type containing settings for the split-explicit free surface. +materialize_timestepper(::ForwardBackwardScheme, grid, args...) = ForwardBackwardScheme() -$(FIELDS) -""" -struct SplitExplicitSettings{𝒩, 𝒮} - substepping :: 𝒩 # Either `FixedSubstepNumber` or `FixedTimeStepSize`" - timestepper :: 𝒮 # time-stepping scheme - settings_kwargs :: NamedTuple # kwargs to reproduce current settings +struct AdamsBashforth3Scheme{CC, FC, CF} + ηᵐ :: CC + ηᵐ⁻¹ :: CC + ηᵐ⁻² :: CC + Uᵐ⁻¹ :: FC + Uᵐ⁻² :: FC + Vᵐ⁻¹ :: CF + Vᵐ⁻² :: CF end -struct AdamsBashforth3Scheme end -struct ForwardBackwardScheme end +AdamsBashforth3Scheme() = AdamsBashforth3Scheme(nothing, nothing, nothing, nothing, nothing, nothing, nothing) +function materialize_timestepper(::AdamsBashforth3Scheme, grid, free_surface, velocities, u_bc, v_bc) + ηᵐ = free_surface_displacement_field(velocities, free_surface, grid) + ηᵐ⁻¹ = free_surface_displacement_field(velocities, free_surface, grid) + ηᵐ⁻² = free_surface_displacement_field(velocities, free_surface, grid) -auxiliary_free_surface_field(grid, ::AdamsBashforth3Scheme) = ZFaceField(grid, indices = (:, :, size(grid, 3)+1)) -auxiliary_free_surface_field(grid, ::ForwardBackwardScheme) = nothing + Uᵐ⁻¹ = Field{Face, Center, Nothing}(grid; boundary_conditions = u_bc) + Uᵐ⁻² = Field{Face, Center, Nothing}(grid; boundary_conditions = u_bc) + Vᵐ⁻¹ = Field{Center, Face, Nothing}(grid; boundary_conditions = v_bc) + Vᵐ⁻² = Field{Center, Face, Nothing}(grid; boundary_conditions = v_bc) -auxiliary_barotropic_U_field(grid, ::AdamsBashforth3Scheme) = XFaceField(grid, indices = (:, :, size(grid, 3))) -auxiliary_barotropic_U_field(grid, ::ForwardBackwardScheme) = nothing -auxiliary_barotropic_V_field(grid, ::AdamsBashforth3Scheme) = YFaceField(grid, indices = (:, :, size(grid, 3))) -auxiliary_barotropic_V_field(grid, ::ForwardBackwardScheme) = nothing + return AdamsBashforth3Scheme(ηᵐ, ηᵐ⁻¹, ηᵐ⁻², Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻²) +end # (p = 2, q = 4, r = 0.18927) minimize dispersion error from Shchepetkin and McWilliams (2005): https://doi.org/10.1016/j.ocemod.2004.08.002 @inline function averaging_shape_function(τ::FT; p = 2, q = 4, r = FT(0.18927)) where FT @@ -318,10 +293,6 @@ free_surface(free_surface::SplitExplicitFreeSurface) = free_surface.η @inline explicit_barotropic_pressure_x_gradient(i, j, k, grid, ::SplitExplicitFreeSurface) = zero(grid) @inline explicit_barotropic_pressure_y_gradient(i, j, k, grid, ::SplitExplicitFreeSurface) = zero(grid) -# convenience functor -(sefs::SplitExplicitFreeSurface)(settings::SplitExplicitSettings) = - SplitExplicitFreeSurface(sefs.η, sefs.state, sefs.auxiliary, sefs.gravitational_acceleration, settings) - Base.summary(s::FixedTimeStepSize) = string("Barotropic time step equal to $(prettytime(s.Δt_barotropic))") Base.summary(s::FixedSubstepNumber) = string("Barotropic fractional step equal to $(s.fractional_step_size) times the baroclinic step") @@ -329,26 +300,13 @@ Base.summary(sefs::SplitExplicitFreeSurface) = string("SplitExplicitFreeSurface Base.show(io::IO, sefs::SplitExplicitFreeSurface) = print(io, "$(summary(sefs))\n") -function reset!(sefs::SplitExplicitFreeSurface) - for name in propertynames(sefs.state) - var = getproperty(sefs.state, name) - fill!(var, 0) - end - - fill!(sefs.auxiliary.Gᵁ, 0) - fill!(sefs.auxiliary.Gⱽ, 0) - - return nothing -end - # Adapt Adapt.adapt_structure(to, free_surface::SplitExplicitFreeSurface) = SplitExplicitFreeSurface(Adapt.adapt(to, free_surface.η), nothing, nothing, free_surface.gravitational_acceleration, nothing) for Type in (:SplitExplicitFreeSurface, - :SplitExplicitSettings, - :SplitExplicitState, + :AdamsBashforth3Scheme, :FixedTimeStepSize, :FixedSubstepNumber) From 04087add6f02c0ae71186b4394178a2dc78907ec Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 30 Oct 2024 20:23:47 +0100 Subject: [PATCH 54/79] big overhaul --- .../barotropic_correction.jl | 55 +++++++------------ .../barotropic_kernels.jl | 15 +++-- .../setup_split_explicit.jl | 33 ++++++----- 3 files changed, 49 insertions(+), 54 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_correction.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_correction.jl index 81008c51ca..2af652b573 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_correction.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_correction.jl @@ -1,50 +1,35 @@ - -function initialize_free_surface!(sefs::SplitExplicitFreeSurface, grid, velocities) - @apply_regionally compute_barotropic_mode!(sefs.state.U̅, sefs.state.V̅, grid, velocities.u, velocities.v) - fill_halo_regions!((sefs.state.U̅, sefs.state.V̅, sefs.η)) -end - - # Barotropic Model Kernels # u_Δz = u * Δz @kernel function _barotropic_mode_kernel!(U, V, grid, ::Nothing, u, v) i, j = @index(Global, NTuple) - k_top = grid.Nz+1 - - @inbounds U[i, j, k_top-1] = Δzᶠᶜᶜ(i, j, 1, grid) * u[i, j, 1] - @inbounds V[i, j, k_top-1] = Δzᶜᶠᶜ(i, j, 1, grid) * v[i, j, 1] - - for k in 2:grid.Nz - @inbounds U[i, j, k_top-1] += Δzᶠᶜᶜ(i, j, k, grid) * u[i, j, k] - @inbounds V[i, j, k_top-1] += Δzᶜᶠᶜ(i, j, k, grid) * v[i, j, k] - end + barotropic_mode_kernel!(U, V, i, j, grid, u, v) end -# Barotropic Model Kernels -# u_Δz = u * Δz @kernel function _barotropic_mode_kernel!(U, V, grid, active_cells_map, u, v) idx = @index(Global, Linear) i, j = active_linear_index_to_tuple(idx, active_cells_map) - k_top = grid.Nz+1 + barotropic_mode_kernel!(U, V, i, j, grid, u, v) +end - @inbounds U[i, j, k_top-1] = Δzᶠᶜᶜ(i, j, 1, grid) * u[i, j, 1] - @inbounds V[i, j, k_top-1] = Δzᶜᶠᶜ(i, j, 1, grid) * v[i, j, 1] +@inline function barotropic_mode_kernel!(U, V, i, j, grid, u, v) + @inbounds U[i, j, 1] = Δzᶠᶜᶜ(i, j, 1, grid) * u[i, j, 1] + @inbounds V[i, j, 1] = Δzᶜᶠᶜ(i, j, 1, grid) * v[i, j, 1] for k in 2:grid.Nz - @inbounds U[i, j, k_top-1] += Δzᶠᶜᶜ(i, j, k, grid) * u[i, j, k] - @inbounds V[i, j, k_top-1] += Δzᶜᶠᶜ(i, j, k, grid) * v[i, j, k] + @inbounds U[i, j, 1] += Δzᶠᶜᶜ(i, j, k, grid) * u[i, j, k] + @inbounds V[i, j, 1] += Δzᶜᶠᶜ(i, j, k, grid) * v[i, j, k] end + + return nothing end @inline function compute_barotropic_mode!(U, V, grid, u, v) active_cells_map = retrieve_surface_active_cells_map(grid) - launch!(architecture(grid), grid, :xy, _barotropic_mode_kernel!, U, V, grid, active_cells_map, u, v; active_cells_map) - return nothing end -@kernel function _barotropic_split_explicit_corrector!(u, v, U̅, V̅, U, V, grid) +@kernel function _barotropic_split_explicit_corrector!(u, v, U, V, U̅, V̅, grid) i, j, k = @index(Global, NTuple) k_top = grid.Nz+1 @@ -52,22 +37,24 @@ end Hᶠᶜ = static_column_depthᶠᶜᵃ(i, j, grid) Hᶜᶠ = static_column_depthᶜᶠᵃ(i, j, grid) - u[i, j, k] = u[i, j, k] + (U̅[i, j, k_top-1] - U[i, j, k_top-1]) / Hᶠᶜ - v[i, j, k] = v[i, j, k] + (V̅[i, j, k_top-1] - V[i, j, k_top-1]) / Hᶜᶠ + u[i, j, k] = u[i, j, k] + (U[i, j, 1] - U̅[i, j, 1]) / Hᶠᶜ + v[i, j, k] = v[i, j, k] + (V[i, j, 1] - V̅[i, j, 1]) / Hᶜᶠ end end function barotropic_split_explicit_corrector!(u, v, free_surface, grid) - sefs = free_surface.state - U, V, U̅, V̅ = sefs.U, sefs.V, sefs.U̅, sefs.V̅ - arch = architecture(grid) + state = free_surface.filtered_state + U, V = free_surface.barotropic_velocities + U̅, V̅ = state.U, state.V + arch = architecture(grid) # take out "bad" barotropic mode, - # !!!! reusing U and V for this storage since last timestep doesn't matter - compute_barotropic_mode!(U, V, grid, u, v) + # !!!! reusing U̅ and V̅ for U and V have already been restarted + compute_barotropic_mode!(U̅, V̅, grid, u, v) + # add in "good" barotropic mode launch!(arch, grid, :xyz, _barotropic_split_explicit_corrector!, - u, v, U̅, V̅, U, V, grid) + u, v, U, V, U̅, V̅, grid) return nothing end diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_kernels.jl index 50ad5d05b3..ff7799dc9b 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_kernels.jl @@ -208,6 +208,7 @@ function split_explicit_free_surface_step!(free_surface::SplitExplicitFreeSurfac filtered_state = free_surface.filtered_state substepping = free_surface.substepping timestepper = free_surface.timestepper + velocities = free_surface.barotropic_velocities # Wait for previous set up wait_free_surface_communication!(free_surface, architecture(free_surface_grid)) @@ -227,17 +228,19 @@ function split_explicit_free_surface_step!(free_surface::SplitExplicitFreeSurfac # reset free surface averages @apply_regionally begin - initialize_free_surface_state!(filtered_state, free_surface.η, free_surface.barotropic_velocities, timestepper) + initialize_free_surface_state!(filtered_state, free_surface.η, velocities, timestepper) # Solve for the free surface at tⁿ⁺¹ iterate_split_explicit!(free_surface, free_surface_grid, GUⁿ, GVⁿ, Δτᴮ, weights, Val(Nsubsteps)) - - # Reset eta for the next timestep - set!(free_surface.η, free_surface.filtered_state.η̅) end - fields_to_fill = (free_surface.filtered_state.U̅, free_surface.filtered_state.V̅) - fill_halo_regions!(fields_to_fill; async = true) + # Reset eta and velocities for the next timestep + set!(free_surface.η, filtered_state.η) + set!(velocities.U, filtered_state.U) + set!(velocities.U, filtered_state.V) + + # fields_to_fill = (velocities.U, velocities.V) TODO: do this? + # fill_halo_regions!(fields_to_fill; async = true) # Preparing velocities for the barotropic correction @apply_regionally begin diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/setup_split_explicit.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/setup_split_explicit.jl index 9301b53746..79413f402a 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/setup_split_explicit.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/setup_split_explicit.jl @@ -1,26 +1,31 @@ +# `initialize_free_surface!` is called at the beginning of the simulation to initialize the free surface state +# from the initial velocity conditions. +function initialize_free_surface!(sefs::SplitExplicitFreeSurface, grid, velocities) + @apply_regionally compute_barotropic_mode!(sefs.state.U̅, sefs.state.V̅, grid, velocities.u, velocities.v) + fill_halo_regions!((sefs.state.U̅, sefs.state.V̅, sefs.η)) +end -function initialize_free_surface_state!(filtered_state, η, barotropic_velocities, timestepper) - - parent(barotropic_velocities.U) .= parent(filtered_state.U̅) - parent(barotropic_velocities.V) .= parent(filtered_state.V̅) +# `initialize_free_surface_state!` is called at the beginning of the substepping to +# reset the filtered state to zero and reinitialize the state from the filtered state. +function initialize_free_surface_state!(filtered_state, η, velocities, timestepper) - initialize_auxiliary_state!(filtered_state, η, timestepper) + initialize_free_surface_timestepper!(timestepper, η, velocities) - fill!(filtered_state.η̅, 0) - fill!(filtered_state.U̅, 0) - fill!(filtered_state.V̅, 0) + fill!(filtered_state.η, 0) + fill!(filtered_state.U, 0) + fill!(filtered_state.V, 0) return nothing end -initialize_auxiliary_state!(state, η, ::ForwardBackwardScheme) = nothing +initialize_auxiliary_state!(::ForwardBackwardScheme, args...) = nothing -function initialize_auxiliary_state!(filtered_state, η, timestepper::AdamsBashforth3Scheme) - parent(timestepper.Uᵐ⁻¹) .= parent(filtered_state.U̅) - parent(timestepper.Vᵐ⁻¹) .= parent(filtered_state.V̅) +function initialize_auxiliary_state!(timestepper::AdamsBashforth3Scheme, η, velocities) + parent(timestepper.Uᵐ⁻¹) .= parent(velocities.U) + parent(timestepper.Vᵐ⁻¹) .= parent(velocities.V) - parent(timestepper.Uᵐ⁻²) .= parent(filtered_state.U̅) - parent(timestepper.Vᵐ⁻²) .= parent(filtered_state.V̅) + parent(timestepper.Uᵐ⁻²) .= parent(velocities.U) + parent(timestepper.Vᵐ⁻²) .= parent(velocities.V) parent(timestepper.ηᵐ) .= parent(η) parent(timestepper.ηᵐ⁻¹) .= parent(η) From b9ec92bfbf462d6c9753a006e00138a2fa688200 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 31 Oct 2024 19:23:20 +0100 Subject: [PATCH 55/79] more refactoring --- .../HydrostaticFreeSurfaceModels.jl | 6 +- .../barotropic_kernels.jl | 2 +- ...distributed_split_explicit_free_surface.jl | 106 ++++++++++-------- .../setup_split_explicit.jl | 2 - .../hydrostatic_free_surface_field_tuples.jl | 6 +- src/MultiRegion/multi_region_models.jl | 2 - ...ulti_region_split_explicit_free_surface.jl | 77 +++++++------ 7 files changed, 111 insertions(+), 90 deletions(-) rename src/Models/HydrostaticFreeSurfaceModels/{ => SplitExplicitFreeSurfaces}/distributed_split_explicit_free_surface.jl (53%) diff --git a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl index d9f0ec6980..4367de9a3e 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl @@ -47,8 +47,10 @@ include("implicit_free_surface.jl") # Split-Explicit free-surface solver functionality include("SplitExplicitFreeSurfaces/split_explicit_free_surface.jl") -include("distributed_split_explicit_free_surface.jl") -include("split_explicit_free_surface_kernels.jl") +include("SplitExplicitFreeSurfaces/setup_split_explicit.jl") +include("SplitExplicitFreeSurfaces/barotropic_kernels.jl") +include("SplitExplicitFreeSurfaces/barotropic_correction.jl") +include("SplitExplicitFreeSurfaces/distributed_split_explicit_free_surface.jl") include("hydrostatic_free_surface_field_tuples.jl") include("hydrostatic_free_surface_model.jl") diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_kernels.jl index ff7799dc9b..0518c26149 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_kernels.jl @@ -211,7 +211,7 @@ function split_explicit_free_surface_step!(free_surface::SplitExplicitFreeSurfac velocities = free_surface.barotropic_velocities # Wait for previous set up - wait_free_surface_communication!(free_surface, architecture(free_surface_grid)) + wait_free_surface_communication!(free_surface, model, architecture(free_surface_grid)) # Calculate the substepping parameterers # barotropic time step as fraction of baroclinic step and averaging weights diff --git a/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/distributed_split_explicit_free_surface.jl similarity index 53% rename from src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl rename to src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/distributed_split_explicit_free_surface.jl index d8b472fec8..37cea8d7e6 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/distributed_split_explicit_free_surface.jl @@ -5,18 +5,61 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: SplitExplicitState, Spli import Oceananigans.Models.HydrostaticFreeSurfaceModels: materialize_free_surface, SplitExplicitAuxiliaryFields -function SplitExplicitAuxiliaryFields(grid::DistributedGrid) +# Internal function for HydrostaticFreeSurfaceModel +function materialize_free_surface(free_surface::SplitExplicitFreeSurface, velocities, grid::DistributedGrid) + + settings = free_surface.settings + + old_halos = halo_size(grid) + Nsubsteps = length(settings.substepping.averaging_weights) + + extended_halos = distributed_split_explicit_halos(old_halos, Nsubsteps+1, grid) + extended_grid = with_halo(extended_halos, grid) + + η = free_surface_displacement_field(velocities, free_surface, extended_grid) + η̅ = free_surface_displacement_field(velocities, free_surface, extended_grid) - Gᵁ = Field((Face, Center, Nothing), grid) - Gⱽ = Field((Center, Face, Nothing), grid) + u_baroclinic = velocities.u + v_baroclinic = velocities.v - # In a non-parallel grid we calculate only the interior - kernel_size = augmented_kernel_size(grid) - kernel_offsets = augmented_kernel_offsets(grid) - - kernel_parameters = KernelParameters(kernel_size, kernel_offsets) + u_bc = barotropic_bc(u_baroclinic) + v_bc = barotropic_bc(v_baroclinic) + + U = Field{Center, Center, Nothing}(extended_grid, boundary_conditions = u_bc) + V = Field{Center, Center, Nothing}(extended_grid, boundary_conditions = v_bc) + + U̅ = Field{Center, Center, Nothing}(extended_grid, boundary_conditions = u_bc) + V̅ = Field{Center, Center, Nothing}(extended_grid, boundary_conditions = v_bc) - return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, kernel_parameters) + filtered_state = (η = η̅, U = U̅, V = V̅) + barotropic_velocities = (U = U, V = V) + + gravitational_acceleration = convert(eltype(extended_grid), free_surface.gravitational_acceleration) + timestepper = materialize_timestepper(free_surface.timestepper, extended_grid, free_surface, velocities, u_bc, v_bc) + + # In a non-parallel grid we calculate only the interior + kernel_size = augmented_kernel_size(extended_grid) + kernel_offsets = augmented_kernel_offsets(extended_grid) + + kernel_parameters = KernelParameters(kernel_size, kernel_offsets) + + return SplitExplicitFreeSurface(η, + barotropic_velocities, + filtered_state, + gravitational_acceleration, + kernel_parameters, + free_surface.substepping, + timestepper) +end + +@inline function distributed_split_explicit_halos(old_halos, step_halo, grid::DistributedGrid) + + Rx, Ry, _ = architecture(grid).ranks + + Ax = Rx == 1 ? old_halos[1] : max(step_halo, old_halos[1]) + Ay = Ry == 1 ? old_halos[2] : max(step_halo, old_halos[2]) + + return (Ax, Ay, old_halos[3]) end @inline function augmented_kernel_size(grid::DistributedGrid) @@ -45,52 +88,21 @@ end return (Ax, Ay) end -# Internal function for HydrostaticFreeSurfaceModel -function materialize_free_surface(free_surface::SplitExplicitFreeSurface, velocities, grid::DistributedGrid) - - settings = free_surface.settings - - old_halos = halo_size(grid) - Nsubsteps = length(settings.substepping.averaging_weights) - - extended_halos = distributed_split_explicit_halos(old_halos, Nsubsteps+1, grid) - extended_grid = with_halo(extended_halos, grid) - - Nze = size(extended_grid, 3) - η = ZFaceField(extended_grid, indices = (:, :, Nze+1)) - - return SplitExplicitFreeSurface(η, - SplitExplicitState(extended_grid, settings.timestepper), - SplitExplicitAuxiliaryFields(extended_grid), - free_surface.gravitational_acceleration, - free_surface.settings) -end - -@inline function distributed_split_explicit_halos(old_halos, step_halo, grid::DistributedGrid) - - Rx, Ry, _ = architecture(grid).ranks - - Ax = Rx == 1 ? old_halos[1] : max(step_halo, old_halos[1]) - Ay = Ry == 1 ? old_halos[2] : max(step_halo, old_halos[2]) - - return (Ax, Ay, old_halos[3]) -end - const DistributedSplitExplicit = SplitExplicitFreeSurface{<:DistributedField} -wait_free_surface_communication!(::DistributedSplitExplicit, ::SynchronizedDistributed) = nothing +wait_free_surface_communication!(free_surface, model, arch) = nothing +wait_free_surface_communication!(::DistributedSplitExplicit, model, ::SynchronizedDistributed) = nothing -function wait_free_surface_communication!(free_surface::DistributedSplitExplicit, arch) +function wait_free_surface_communication!(free_surface::DistributedSplitExplicit, model, arch) - state = free_surface.state - - for field in (state.U̅, state.V̅) + for field in (barotropic_velocities.U, barotropic_velocities.V) synchronize_communication!(field) end - auxiliary = free_surface.auxiliary + Gᵁ = model.timestepper.Gⁿ.U + Gⱽ = model.timestepper.Gⁿ.V - for field in (auxiliary.Gᵁ, auxiliary.Gⱽ) + for field in (Gᵁ, Gⱽ) synchronize_communication!(field) end diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/setup_split_explicit.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/setup_split_explicit.jl index 79413f402a..bf50f54b52 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/setup_split_explicit.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/setup_split_explicit.jl @@ -98,5 +98,3 @@ end return nothing end - -wait_free_surface_communication!(free_surface, arch) = nothing diff --git a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_field_tuples.jl b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_field_tuples.jl index 019a3f3eff..465cadd146 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_field_tuples.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_field_tuples.jl @@ -23,11 +23,11 @@ function HydrostaticFreeSurfaceTendencyFields(velocities, free_surface::Explicit return merge((u=u, v=v, η=η), tracers) end -function HydrostaticFreeSurfaceTendencyFields(velocities, ::SplitExplicitFreeSurface, grid, tracer_names) +function HydrostaticFreeSurfaceTendencyFields(velocities, free_surface::SplitExplicitFreeSurface, grid, tracer_names) u = XFaceField(grid) v = YFaceField(grid) - U = Field{Face, Center, Nothing}(grid) - V = Field{Center, Face, Nothing}(grid) + U = deepcopy(free_surface.barotropic_velocities.U) + V = deepcopy(free_surface.barotropic_velocities.V) tracers = TracerFields(tracer_names, grid) return merge((u=u, v=v, U=U, V=V), tracers) end diff --git a/src/MultiRegion/multi_region_models.jl b/src/MultiRegion/multi_region_models.jl index 95f677e434..21f22f70da 100644 --- a/src/MultiRegion/multi_region_models.jl +++ b/src/MultiRegion/multi_region_models.jl @@ -35,8 +35,6 @@ Types = (:HydrostaticFreeSurfaceModel, :ImplicitFreeSurface, :ExplicitFreeSurface, :QuasiAdamsBashforth2TimeStepper, - :SplitExplicitAuxiliaryFields, - :SplitExplicitState, :SplitExplicitFreeSurface, :PrescribedVelocityFields, :ConjugateGradientSolver, diff --git a/src/MultiRegion/multi_region_split_explicit_free_surface.jl b/src/MultiRegion/multi_region_split_explicit_free_surface.jl index 10ba130b33..5c2b05ee71 100644 --- a/src/MultiRegion/multi_region_split_explicit_free_surface.jl +++ b/src/MultiRegion/multi_region_split_explicit_free_surface.jl @@ -1,58 +1,69 @@ using Oceananigans.Utils using Oceananigans.AbstractOperations: GridMetricOperation, Δz using Oceananigans.Models.HydrostaticFreeSurfaceModels: SplitExplicitFreeSurface, - SplitExplicitSettings, - SplitExplicitState, FixedSubstepNumber, FixedTimeStepSize, calculate_substeps -import Oceananigans.Models.HydrostaticFreeSurfaceModels: materialize_free_surface, SplitExplicitAuxiliaryFields +import Oceananigans.Models.HydrostaticFreeSurfaceModels: materialize_free_surface -function SplitExplicitAuxiliaryFields(grid::MultiRegionGrids) +# Internal function for HydrostaticFreeSurfaceModel +function materialize_free_surface(free_surface::SplitExplicitFreeSurface, velocities, grid::MultiRegionGrids) - Gᵁ = Field((Face, Center, Nothing), grid) - Gⱽ = Field((Center, Face, Nothing), grid) + free_surface.substepping isa FixedTimeStepSize && + throw(ArgumentError("SplitExplicitFreeSurface on MultiRegionGrids only suports FixedSubstepNumber; re-initialize SplitExplicitFreeSurface using substeps kwarg")) - # In a non-parallel grid we calculate only the interior - @apply_regionally kernel_size = augmented_kernel_size(grid, grid.partition) - @apply_regionally kernel_offsets = augmented_kernel_offsets(grid, grid.partition) + switch_device!(grid.devices[1]) - @apply_regionally kernel_parameters = KernelParameters(kernel_size, kernel_offsets) + old_halos = halo_size(getregion(grid, 1)) + Nsubsteps = calculate_substeps(settings.substepping) - return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, kernel_parameters) -end + extended_halos = multiregion_split_explicit_halos(old_halos, Nsubsteps+1, grid.partition) + extended_grid = with_halo(new_halos, grid) -@inline augmented_kernel_size(grid, ::XPartition) = (size(grid, 1) + 2halo_size(grid)[1]-2, size(grid, 2)) -@inline augmented_kernel_size(grid, ::YPartition) = (size(grid, 1), size(grid, 2) + 2halo_size(grid)[2]-2) -@inline augmented_kernel_size(grid, ::CubedSpherePartition) = (size(grid, 1) + 2halo_size(grid)[1]-2, size(grid, 2) + 2halo_size(grid)[2]-2) + η = free_surface_displacement_field(velocities, free_surface, extended_grid) + η̅ = free_surface_displacement_field(velocities, free_surface, extended_grid) -@inline augmented_kernel_offsets(grid, ::XPartition) = (- halo_size(grid)[1] + 1, 0) -@inline augmented_kernel_offsets(grid, ::YPartition) = (0, - halo_size(grid)[2] + 1) -@inline augmented_kernel_offsets(grid, ::CubedSpherePartition) = (- halo_size(grid)[2] + 1, - halo_size(grid)[2] + 1) + u_baroclinic = velocities.u + v_baroclinic = velocities.v -# Internal function for HydrostaticFreeSurfaceModel -function materialize_free_surface(free_surface::SplitExplicitFreeSurface, velocities, grid::MultiRegionGrids) - settings = SplitExplicitSettings(grid; free_surface.settings.settings_kwargs...) + u_bc = barotropic_bc(u_baroclinic) + v_bc = barotropic_bc(v_baroclinic) - settings.substepping isa FixedTimeStepSize && - throw(ArgumentError("SplitExplicitFreeSurface on MultiRegionGrids only suports FixedSubstepNumber; re-initialize SplitExplicitFreeSurface using substeps kwarg")) + U = Field{Center, Center, Nothing}(extended_grid, boundary_conditions = u_bc) + V = Field{Center, Center, Nothing}(extended_grid, boundary_conditions = v_bc) - switch_device!(grid.devices[1]) + U̅ = Field{Center, Center, Nothing}(extended_grid, boundary_conditions = u_bc) + V̅ = Field{Center, Center, Nothing}(extended_grid, boundary_conditions = v_bc) - old_halos = halo_size(getregion(grid, 1)) - Nsubsteps = calculate_substeps(settings.substepping) + filtered_state = (η = η̅, U = U̅, V = V̅) + barotropic_velocities = (U = U, V = V) - new_halos = multiregion_split_explicit_halos(old_halos, Nsubsteps+1, grid.partition) - new_grid = with_halo(new_halos, grid) + gravitational_acceleration = convert(eltype(extended_grid), free_surface.gravitational_acceleration) + timestepper = materialize_timestepper(free_surface.timestepper, extended_grid, free_surface, velocities, u_bc, v_bc) - η = ZFaceField(new_grid, indices = (:, :, size(new_grid, 3)+1)) + # In a non-parallel grid we calculate only the interior + @apply_regionally kernel_size = augmented_kernel_size(grid, grid.partition) + @apply_regionally kernel_offsets = augmented_kernel_offsets(grid, grid.partition) + + @apply_regionally kernel_parameters = KernelParameters(kernel_size, kernel_offsets) return SplitExplicitFreeSurface(η, - SplitExplicitState(new_grid, free_surface.settings.timestepper), - SplitExplicitAuxiliaryFields(new_grid), - free_surface.gravitational_acceleration, - free_surface.settings) + barotropic_velocities, + filtered_state, + gravitational_acceleration, + kernel_parameters, + free_surface.substepping, + timestepper) end @inline multiregion_split_explicit_halos(old_halos, step_halo, ::XPartition) = (max(step_halo, old_halos[1]), old_halos[2], old_halos[3]) @inline multiregion_split_explicit_halos(old_halos, step_halo, ::YPartition) = (old_halos[1], max(step_halo, old_halo[2]), old_halos[3]) + +@inline augmented_kernel_size(grid, ::XPartition) = (size(grid, 1) + 2halo_size(grid)[1]-2, size(grid, 2)) +@inline augmented_kernel_size(grid, ::YPartition) = (size(grid, 1), size(grid, 2) + 2halo_size(grid)[2]-2) +@inline augmented_kernel_size(grid, ::CubedSpherePartition) = (size(grid, 1) + 2halo_size(grid)[1]-2, size(grid, 2) + 2halo_size(grid)[2]-2) + +@inline augmented_kernel_offsets(grid, ::XPartition) = (- halo_size(grid)[1] + 1, 0) +@inline augmented_kernel_offsets(grid, ::YPartition) = (0, - halo_size(grid)[2] + 1) +@inline augmented_kernel_offsets(grid, ::CubedSpherePartition) = (- halo_size(grid)[2] + 1, - halo_size(grid)[2] + 1) + From 57be084abb846d58b3999b40f147f2601884fbb9 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 31 Oct 2024 19:49:08 +0100 Subject: [PATCH 56/79] compiles --- .../barotropic_kernels.jl | 76 +++++++++---------- ...distributed_split_explicit_free_surface.jl | 4 +- .../setup_split_explicit.jl | 9 ++- .../split_explicit_free_surface.jl | 2 +- .../barotropic_pressure_correction.jl | 4 +- ...ute_hydrostatic_free_surface_tendencies.jl | 4 +- test/test_hydrostatic_regression.jl | 3 +- ...test_split_explicit_free_surface_solver.jl | 3 - .../test_split_explicit_vertical_integrals.jl | 3 - 9 files changed, 53 insertions(+), 55 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_kernels.jl index 0518c26149..2f2c709029 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_kernels.jl @@ -42,87 +42,90 @@ const μ = 1 - δ - γ - ϵ # Time stepping extrapolation U★, and η★ # AB3 step -@inline function U★(i, j, k, grid, ::AdamsBashforth3Scheme, Uᵐ, Uᵐ⁻¹, Uᵐ⁻²) +@inline function U★(i, j, k, grid, ::AdamsBashforth3Scheme, Uᵐ) FT = eltype(grid) - return @inbounds FT(α) * Uᵐ[i, j, k] + FT(θ) * Uᵐ⁻¹[i, j, k] + FT(β) * Uᵐ⁻²[i, j, k] + return @inbounds FT(α) * Uᵐ[i, j, k] + FT(θ) * t.Uᵐ⁻¹[i, j, k] + FT(β) * t.Uᵐ⁻²[i, j, k] end -@inline function η★(i, j, k, grid, ::AdamsBashforth3Scheme, ηᵐ⁺¹, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) +@inline function η★(i, j, k, grid, t::AdamsBashforth3Scheme, ηᵐ⁺¹) FT = eltype(grid) - return @inbounds FT(δ) * ηᵐ⁺¹[i, j, k] + FT(μ) * ηᵐ[i, j, k] + FT(γ) * ηᵐ⁻¹[i, j, k] + FT(ϵ) * ηᵐ⁻²[i, j, k] + return @inbounds FT(δ) * ηᵐ⁺¹[i, j, k] + FT(μ) * t.ηᵐ[i, j, k] + FT(γ) * t.ηᵐ⁻¹[i, j, k] + FT(ϵ) * t.ηᵐ⁻²[i, j, k] end # Forward Backward Step -@inline U★(i, j, k, grid, ::ForwardBackwardScheme, U, args...) = @inbounds U[i, j, k] -@inline η★(i, j, k, grid, ::ForwardBackwardScheme, η, args...) = @inbounds η[i, j, k] +@inline U★(i, j, k, grid, ::ForwardBackwardScheme, U) = @inbounds U[i, j, k] +@inline η★(i, j, k, grid, ::ForwardBackwardScheme, η) = @inbounds η[i, j, k] -@inline advance_previous_velocity!(i, j, k, ::ForwardBackwardScheme, U, Uᵐ⁻¹, Uᵐ⁻²) = nothing +@inline advance_previous_velocities!(::ForwardBackwardScheme, i, j, k, U) = nothing -@inline function advance_previous_velocity!(i, j, k, ::AdamsBashforth3Scheme, U, Uᵐ⁻¹, Uᵐ⁻²) - @inbounds Uᵐ⁻²[i, j, k] = Uᵐ⁻¹[i, j, k] - @inbounds Uᵐ⁻¹[i, j, k] = U[i, j, k] +@inline function advance_previous_velocities!(t::AdamsBashforth3Scheme, i, j, k, U) + @inbounds t.Uᵐ⁻²[i, j, k] = t.Uᵐ⁻¹[i, j, k] + @inbounds t.Uᵐ⁻¹[i, j, k] = U[i, j, k] return nothing end -@inline advance_previous_free_surface!(i, j, k, ::ForwardBackwardScheme, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) = nothing +@inline advance_previous_free_surface!(::ForwardBackwardScheme, i, j, k, η) = nothing -@inline function advance_previous_free_surface!(i, j, k, ::AdamsBashforth3Scheme, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) - @inbounds ηᵐ⁻²[i, j, k] = ηᵐ⁻¹[i, j, k] - @inbounds ηᵐ⁻¹[i, j, k] = ηᵐ[i, j, k] - @inbounds ηᵐ[i, j, k] = η[i, j, k] +@inline function advance_previous_free_surface!(t::AdamsBashforth3Scheme, i, j, k, η) + @inbounds t.ηᵐ⁻²[i, j, k] = t.ηᵐ⁻¹[i, j, k] + @inbounds t.ηᵐ⁻¹[i, j, k] = t.ηᵐ[i, j, k] + @inbounds t.ηᵐ[i, j, k] = η[i, j, k] return nothing end -@kernel function _split_explicit_free_surface!(grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², timestepper) +@kernel function _split_explicit_free_surface!(grid, Δτ, η, U, V, timestepper) i, j = @index(Global, NTuple) - free_surface_evolution!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², timestepper) + free_surface_evolution!(η, i, j, grid, Δτ, U, V, timestepper) end -@inline function free_surface_evolution!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², timestepper) +@inline function free_surface_evolution!(η, i, j, grid, Δτ, U, V, timestepper) k_top = grid.Nz+1 @inbounds begin - advance_previous_free_surface!(i, j, k_top, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + advance_previous_free_surface!(timestepper, i, j, k_top, η) - η[i, j, k_top] -= Δτ * (div_Txᶜᶜᶠ(i, j, k_top-1, grid, U★, timestepper, U, Uᵐ⁻¹, Uᵐ⁻²) + - div_Tyᶜᶜᶠ(i, j, k_top-1, grid, U★, timestepper, V, Vᵐ⁻¹, Vᵐ⁻²)) + η[i, j, k_top] -= Δτ * (div_Txᶜᶜᶠ(i, j, k_top-1, grid, U★, timestepper, U) + + div_Tyᶜᶜᶠ(i, j, k_top-1, grid, U★, timestepper, V)) end return nothing end -@kernel function _split_explicit_barotropic_velocity!(averaging_weight, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², - U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², - η̅, U̅, V̅, Gᵁ, Gⱽ, g, +@kernel function _split_explicit_barotropic_velocity!(averaging_weight, grid, Δτ, + η, U, V, + η̅, U̅, V̅, + Gᵁ, Gⱽ, g, timestepper) i, j = @index(Global, NTuple) - velocity_evolution!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², - U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², - η̅, U̅, V̅, averaging_weight, + velocity_evolution!(η, U, V, + η̅, U̅, V̅, + i, j, grid, Δτ, + averaging_weight, Gᵁ, Gⱽ, g, timestepper) end -@inline function velocity_evolution!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², - U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², - η̅, U̅, V̅, averaging_weight, +@inline function velocity_evolution!(η, U, V, + η̅, U̅, V̅, + i, j, grid, Δτ, + averaging_weight, Gᵁ, Gⱽ, g, timestepper) k_top = grid.Nz+1 @inbounds begin - advance_previous_velocity!(i, j, k_top-1, timestepper, U, Uᵐ⁻¹, Uᵐ⁻²) - advance_previous_velocity!(i, j, k_top-1, timestepper, V, Vᵐ⁻¹, Vᵐ⁻²) + advance_previous_velocities!(timestepper, i, j, k_top-1, U) + advance_previous_velocities!(timestepper, i, j, k_top-1, V) Hᶠᶜ = static_column_depthᶠᶜᵃ(i, j, grid) Hᶜᶠ = static_column_depthᶜᶠᵃ(i, j, grid) # ∂τ(U) = - ∇η + G - U[i, j, k_top-1] += Δτ * (- g * Hᶠᶜ * ∂xTᶠᶜᶠ(i, j, k_top, grid, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gᵁ[i, j, k_top-1]) - V[i, j, k_top-1] += Δτ * (- g * Hᶜᶠ * ∂yTᶜᶠᶠ(i, j, k_top, grid, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gⱽ[i, j, k_top-1]) + U[i, j, k_top-1] += Δτ * (- g * Hᶠᶜ * ∂xTᶠᶜᶠ(i, j, k_top, grid, η★, timestepper, η) + Gᵁ[i, j, k_top-1]) + V[i, j, k_top-1] += Δτ * (- g * Hᶜᶠ * ∂yTᶜᶠᶠ(i, j, k_top, grid, η★, timestepper, η) + Gⱽ[i, j, k_top-1]) # time-averaging η̅[i, j, k_top] += averaging_weight * η[i, j, k_top] @@ -146,9 +149,6 @@ const MINIMUM_SUBSTEPS = 5 @inline calculate_adaptive_settings(substepping::FTS, substeps) = weights_from_substeps(eltype(substepping.Δt_barotropic), substeps, substepping.averaging_kernel) -const FixedSubstepsSetting{N} = SplitExplicitSettings{<:FixedSubstepNumber{<:Any, <:NTuple{N, <:Any}}} where N -const FixedSubstepsSplitExplicit{F} = SplitExplicitFreeSurface{<:Any, <:Any, <:Any, <:Any, <:FixedSubstepsSetting{N}} where N - function iterate_split_explicit!(free_surface, grid, GUⁿ, GVⁿ, Δτᴮ, weights, ::Val{Nsubsteps}) where Nsubsteps arch = architecture(grid) @@ -161,7 +161,7 @@ function iterate_split_explicit!(free_surface, grid, GUⁿ, GVⁿ, Δτᴮ, weig # unpack state quantities, parameters and forcing terms U, V = free_surface.barotropic_velocities - η̅, U̅, V̅ = state.η̅, state.U̅, state.V̅ + η̅, U̅, V̅ = state.η, state.U, state.V free_surface_kernel!, _ = configure_kernel(arch, grid, parameters, _split_explicit_free_surface!) barotropic_velocity_kernel!, _ = configure_kernel(arch, grid, parameters, _split_explicit_barotropic_velocity!) diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/distributed_split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/distributed_split_explicit_free_surface.jl index 37cea8d7e6..eeab4b8c61 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/distributed_split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/distributed_split_explicit_free_surface.jl @@ -1,9 +1,9 @@ using Oceananigans.AbstractOperations: GridMetricOperation, Δz using Oceananigans.DistributedComputations: DistributedGrid, DistributedField using Oceananigans.DistributedComputations: SynchronizedDistributed, synchronize_communication! -using Oceananigans.Models.HydrostaticFreeSurfaceModels: SplitExplicitState, SplitExplicitFreeSurface +using Oceananigans.Models.HydrostaticFreeSurfaceModels: SplitExplicitFreeSurface -import Oceananigans.Models.HydrostaticFreeSurfaceModels: materialize_free_surface, SplitExplicitAuxiliaryFields +import Oceananigans.Models.HydrostaticFreeSurfaceModels: materialize_free_surface # Internal function for HydrostaticFreeSurfaceModel function materialize_free_surface(free_surface::SplitExplicitFreeSurface, velocities, grid::DistributedGrid) diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/setup_split_explicit.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/setup_split_explicit.jl index bf50f54b52..26a6d0b7d4 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/setup_split_explicit.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/setup_split_explicit.jl @@ -1,8 +1,9 @@ # `initialize_free_surface!` is called at the beginning of the simulation to initialize the free surface state # from the initial velocity conditions. function initialize_free_surface!(sefs::SplitExplicitFreeSurface, grid, velocities) - @apply_regionally compute_barotropic_mode!(sefs.state.U̅, sefs.state.V̅, grid, velocities.u, velocities.v) - fill_halo_regions!((sefs.state.U̅, sefs.state.V̅, sefs.η)) + barotropic_velocities = sefs.barotropic_velocities + @apply_regionally compute_barotropic_mode!(barotropic_velocities.U, barotropic_velocities.V, grid, velocities.u, velocities.v) + fill_halo_regions!((barotropic_velocities.U, barotropic_velocities.V, sefs.η)) end # `initialize_free_surface_state!` is called at the beginning of the substepping to @@ -18,9 +19,9 @@ function initialize_free_surface_state!(filtered_state, η, velocities, timestep return nothing end -initialize_auxiliary_state!(::ForwardBackwardScheme, args...) = nothing +initialize_free_surface_timestepper!(::ForwardBackwardScheme, args...) = nothing -function initialize_auxiliary_state!(timestepper::AdamsBashforth3Scheme, η, velocities) +function initialize_free_surface_timestepper!(timestepper::AdamsBashforth3Scheme, η, velocities) parent(timestepper.Uᵐ⁻¹) .= parent(velocities.U) parent(timestepper.Vᵐ⁻¹) .= parent(velocities.V) diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_free_surface.jl index 7837cad46a..aa6606dcd1 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_free_surface.jl @@ -113,7 +113,7 @@ function SplitExplicitFreeSurface(grid = nothing; gravitational_acceleration, kernel_parameters, substepping, - settings) + timestepper) end # Simplest case: we have the substeps and the averaging kernel diff --git a/src/Models/HydrostaticFreeSurfaceModels/barotropic_pressure_correction.jl b/src/Models/HydrostaticFreeSurfaceModels/barotropic_pressure_correction.jl index 382236feec..a0536107aa 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/barotropic_pressure_correction.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/barotropic_pressure_correction.jl @@ -30,8 +30,8 @@ function pressure_correct_velocities!(model::ImplicitFreeSurfaceHFSM, Δt) return nothing end -calculate_free_surface_tendency!(grid, ::ImplicitFreeSurfaceHFSM , args...) = nothing -calculate_free_surface_tendency!(grid, ::SplitExplicitFreeSurfaceHFSM, args...) = nothing +compute_free_surface_tendency!(grid, ::ImplicitFreeSurfaceHFSM , args...) = nothing +compute_free_surface_tendency!(grid, ::SplitExplicitFreeSurfaceHFSM, args...) = nothing function pressure_correct_velocities!(model::SplitExplicitFreeSurfaceHFSM, Δt) u, v, _ = model.velocities diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl index 82c71b05dd..449b263a16 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl @@ -185,7 +185,9 @@ function compute_hydrostatic_boundary_tendency_contributions!(Gⁿ, arch, veloci end # Free surface - apply_flux_bcs!(Gⁿ.η, displacement(free_surface), arch, args) + if free_surface isa ExplicitFreeSurface + apply_flux_bcs!(Gⁿ.η, displacement(free_surface), arch, args) + end # Tracer fields for i in propertynames(tracers) diff --git a/test/test_hydrostatic_regression.jl b/test/test_hydrostatic_regression.jl index 62de3c8ffe..a6696be3b7 100644 --- a/test/test_hydrostatic_regression.jl +++ b/test/test_hydrostatic_regression.jl @@ -79,7 +79,8 @@ include("regression_tests/hydrostatic_free_turbulence_regression_test.jl") gravitational_acceleration = 1.0, substeps = 5) - for free_surface in [explicit_free_surface, implicit_free_surface, split_explicit_free_surface] + # for free_surface in [explicit_free_surface, implicit_free_surface, split_explicit_free_surface] + for free_surface in [split_explicit_free_surface] # GPU + ImplicitFreeSurface + precompute metrics cannot be tested on sverdrup at the moment # because "uses too much parameter space (maximum 0x1100 bytes)" error diff --git a/test/test_split_explicit_free_surface_solver.jl b/test/test_split_explicit_free_surface_solver.jl index 2786bf70c9..6f4af2ed4d 100644 --- a/test/test_split_explicit_free_surface_solver.jl +++ b/test/test_split_explicit_free_surface_solver.jl @@ -6,9 +6,6 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: calculate_substeps, constant_averaging_kernel, materialize_free_surface, SplitExplicitFreeSurface, - SplitExplicitState, - SplitExplicitAuxiliaryFields, - SplitExplicitSettings, iterate_split_explicit! @testset "Split-Explicit Dynamics" begin diff --git a/test/test_split_explicit_vertical_integrals.jl b/test/test_split_explicit_vertical_integrals.jl index ca7713ad2b..d7912e2619 100644 --- a/test/test_split_explicit_vertical_integrals.jl +++ b/test/test_split_explicit_vertical_integrals.jl @@ -2,9 +2,6 @@ include("dependencies_for_runtests.jl") using Oceananigans.Models.HydrostaticFreeSurfaceModels using Oceananigans.Models.HydrostaticFreeSurfaceModels: SplitExplicitFreeSurface -using Oceananigans.Models.HydrostaticFreeSurfaceModels: SplitExplicitState, - SplitExplicitAuxiliaryFields, - SplitExplicitSettings using Oceananigans.Models.HydrostaticFreeSurfaceModels: compute_barotropic_mode!, barotropic_split_explicit_corrector!, initialize_free_surface_state! From c522ad34b85d20cff08cd09b8d5e66af85fb51d3 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 31 Oct 2024 20:28:17 +0100 Subject: [PATCH 57/79] more refactoring... --- .../HydrostaticFreeSurfaceModels.jl | 1 + .../barotropic_kernels.jl | 52 +---------- .../split_explicit_free_surface.jl | 48 ---------- .../split_explicit_timesteppers.jl | 87 +++++++++++++++++++ 4 files changed, 90 insertions(+), 98 deletions(-) create mode 100644 src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_timesteppers.jl diff --git a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl index 4367de9a3e..9e83c7fbd4 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl @@ -46,6 +46,7 @@ include("matrix_implicit_free_surface_solver.jl") include("implicit_free_surface.jl") # Split-Explicit free-surface solver functionality +include("SplitExplicitFreeSurfaces/split_explicit_timesteppers.jl") include("SplitExplicitFreeSurfaces/split_explicit_free_surface.jl") include("SplitExplicitFreeSurfaces/setup_split_explicit.jl") include("SplitExplicitFreeSurfaces/barotropic_kernels.jl") diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_kernels.jl index 2f2c709029..d5e242d131 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_kernels.jl @@ -16,15 +16,6 @@ using Printf using KernelAbstractions: @index, @kernel using KernelAbstractions.Extras.LoopInfo: @unroll -# constants for AB3 time stepping scheme (from https://doi.org/10.1016/j.ocemod.2004.08.002) -const β = 0.281105 -const α = 1.5 + β -const θ = - 0.5 - 2β -const γ = 0.088 -const δ = 0.614 -const ϵ = 0.013 -const μ = 1 - δ - γ - ϵ - # Evolution Kernels # # ∂t(η) = -∇⋅U @@ -37,50 +28,11 @@ const μ = 1 - δ - γ - ϵ @inline div_Txᶜᶜᶠ(i, j, k, grid, U★::Function, args...) = 1 / Azᶜᶜᶠ(i, j, k, grid) * δxTᶜᵃᵃ(i, j, k, grid, Δy_qᶠᶜᶠ, U★, args...) @inline div_Tyᶜᶜᶠ(i, j, k, grid, V★::Function, args...) = 1 / Azᶜᶜᶠ(i, j, k, grid) * δyTᵃᶜᵃ(i, j, k, grid, Δx_qᶜᶠᶠ, V★, args...) -# The functions `η★` `U★` and `V★` represent the value of free surface, barotropic zonal and meridional velocity at time step m+1/2 - -# Time stepping extrapolation U★, and η★ - -# AB3 step -@inline function U★(i, j, k, grid, ::AdamsBashforth3Scheme, Uᵐ) - FT = eltype(grid) - return @inbounds FT(α) * Uᵐ[i, j, k] + FT(θ) * t.Uᵐ⁻¹[i, j, k] + FT(β) * t.Uᵐ⁻²[i, j, k] -end - -@inline function η★(i, j, k, grid, t::AdamsBashforth3Scheme, ηᵐ⁺¹) - FT = eltype(grid) - return @inbounds FT(δ) * ηᵐ⁺¹[i, j, k] + FT(μ) * t.ηᵐ[i, j, k] + FT(γ) * t.ηᵐ⁻¹[i, j, k] + FT(ϵ) * t.ηᵐ⁻²[i, j, k] -end - -# Forward Backward Step -@inline U★(i, j, k, grid, ::ForwardBackwardScheme, U) = @inbounds U[i, j, k] -@inline η★(i, j, k, grid, ::ForwardBackwardScheme, η) = @inbounds η[i, j, k] - -@inline advance_previous_velocities!(::ForwardBackwardScheme, i, j, k, U) = nothing - -@inline function advance_previous_velocities!(t::AdamsBashforth3Scheme, i, j, k, U) - @inbounds t.Uᵐ⁻²[i, j, k] = t.Uᵐ⁻¹[i, j, k] - @inbounds t.Uᵐ⁻¹[i, j, k] = U[i, j, k] - - return nothing -end - -@inline advance_previous_free_surface!(::ForwardBackwardScheme, i, j, k, η) = nothing - -@inline function advance_previous_free_surface!(t::AdamsBashforth3Scheme, i, j, k, η) - @inbounds t.ηᵐ⁻²[i, j, k] = t.ηᵐ⁻¹[i, j, k] - @inbounds t.ηᵐ⁻¹[i, j, k] = t.ηᵐ[i, j, k] - @inbounds t.ηᵐ[i, j, k] = η[i, j, k] - - return nothing -end - @kernel function _split_explicit_free_surface!(grid, Δτ, η, U, V, timestepper) i, j = @index(Global, NTuple) free_surface_evolution!(η, i, j, grid, Δτ, U, V, timestepper) end - @inline function free_surface_evolution!(η, i, j, grid, Δτ, U, V, timestepper) k_top = grid.Nz+1 @@ -239,8 +191,8 @@ function split_explicit_free_surface_step!(free_surface::SplitExplicitFreeSurfac set!(velocities.U, filtered_state.U) set!(velocities.U, filtered_state.V) - # fields_to_fill = (velocities.U, velocities.V) TODO: do this? - # fill_halo_regions!(fields_to_fill; async = true) + fields_to_fill = (velocities.U, velocities.V, free_surface.η) # TODO: do this? + fill_halo_regions!(fields_to_fill; async = true) # Preparing velocities for the barotropic correction @apply_regionally begin diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_free_surface.jl index aa6606dcd1..f8e10bbea4 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_free_surface.jl @@ -1,16 +1,3 @@ -using Oceananigans -using Oceananigans.Architectures -using Oceananigans.Fields -using Oceananigans.Grids -using Oceananigans.Grids: AbstractGrid -using Oceananigans.AbstractOperations: Δz, GridMetricOperation - -using Adapt -using Base -using KernelAbstractions: @index, @kernel - -import Oceananigans.TimeSteppers: reset! - """ struct SplitExplicitFreeSurface @@ -194,41 +181,6 @@ function materialize_free_surface(free_surface::SplitExplicitFreeSurface, veloci timestepper) end -function materialize_timestepper(timestepper::Symbol, args...) - fullname = Symbol(name, :Scheme) - TS = getglobal(@__MODULE__, fullname) - return materialize_timestepper(TS, args...) -end - -struct ForwardBackwardScheme end - -materialize_timestepper(::ForwardBackwardScheme, grid, args...) = ForwardBackwardScheme() - -struct AdamsBashforth3Scheme{CC, FC, CF} - ηᵐ :: CC - ηᵐ⁻¹ :: CC - ηᵐ⁻² :: CC - Uᵐ⁻¹ :: FC - Uᵐ⁻² :: FC - Vᵐ⁻¹ :: CF - Vᵐ⁻² :: CF -end - -AdamsBashforth3Scheme() = AdamsBashforth3Scheme(nothing, nothing, nothing, nothing, nothing, nothing, nothing) - -function materialize_timestepper(::AdamsBashforth3Scheme, grid, free_surface, velocities, u_bc, v_bc) - ηᵐ = free_surface_displacement_field(velocities, free_surface, grid) - ηᵐ⁻¹ = free_surface_displacement_field(velocities, free_surface, grid) - ηᵐ⁻² = free_surface_displacement_field(velocities, free_surface, grid) - - Uᵐ⁻¹ = Field{Face, Center, Nothing}(grid; boundary_conditions = u_bc) - Uᵐ⁻² = Field{Face, Center, Nothing}(grid; boundary_conditions = u_bc) - Vᵐ⁻¹ = Field{Center, Face, Nothing}(grid; boundary_conditions = v_bc) - Vᵐ⁻² = Field{Center, Face, Nothing}(grid; boundary_conditions = v_bc) - - return AdamsBashforth3Scheme(ηᵐ, ηᵐ⁻¹, ηᵐ⁻², Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻²) -end - # (p = 2, q = 4, r = 0.18927) minimize dispersion error from Shchepetkin and McWilliams (2005): https://doi.org/10.1016/j.ocemod.2004.08.002 @inline function averaging_shape_function(τ::FT; p = 2, q = 4, r = FT(0.18927)) where FT τ₀ = (p + 2) * (p + q + 2) / (p + 1) / (p + q + 1) diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_timesteppers.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_timesteppers.jl new file mode 100644 index 0000000000..bc5db14714 --- /dev/null +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_timesteppers.jl @@ -0,0 +1,87 @@ +using Oceananigans +using Oceananigans.Architectures +using Oceananigans.Fields +using Oceananigans.Grids +using Oceananigans.Grids: AbstractGrid + +using Adapt +using Base +using KernelAbstractions: @index, @kernel + +function materialize_timestepper(name::Symbol, args...) + fullname = Symbol(name, :Scheme) + TS = getglobal(@__MODULE__, fullname) + return materialize_timestepper(TS, args...) +end + +struct ForwardBackwardScheme end + +materialize_timestepper(::ForwardBackwardScheme, grid, args...) = ForwardBackwardScheme() + +struct AdamsBashforth3Scheme{CC, FC, CF, FT} + ηᵐ :: CC + ηᵐ⁻¹ :: CC + ηᵐ⁻² :: CC + Uᵐ⁻¹ :: FC + Uᵐ⁻² :: FC + Vᵐ⁻¹ :: CF + Vᵐ⁻² :: CF + β :: FT + α :: FT + θ :: FT + γ :: FT + δ :: FT + ϵ :: FT + μ :: FT +end + +AdamsBashforth3Scheme(; β = 0.281105, α = 1.5 + β, θ = - 0.5 - 2β, γ = 0.088, δ = 0.614, ϵ = 0.013, μ = 1 - δ - γ - ϵ) = + AdamsBashforth3Scheme(nothing, nothing, nothing, nothing, nothing, nothing, nothing, β, α, θ, γ, δ, ϵ, μ) + +function materialize_timestepper(t::AdamsBashforth3Scheme, grid, free_surface, velocities, u_bc, v_bc) + ηᵐ = free_surface_displacement_field(velocities, free_surface, grid) + ηᵐ⁻¹ = free_surface_displacement_field(velocities, free_surface, grid) + ηᵐ⁻² = free_surface_displacement_field(velocities, free_surface, grid) + + Uᵐ⁻¹ = Field{Face, Center, Nothing}(grid; boundary_conditions = u_bc) + Uᵐ⁻² = Field{Face, Center, Nothing}(grid; boundary_conditions = u_bc) + Vᵐ⁻¹ = Field{Center, Face, Nothing}(grid; boundary_conditions = v_bc) + Vᵐ⁻² = Field{Center, Face, Nothing}(grid; boundary_conditions = v_bc) + + FT = eltype(grid) + + β = convert(FT, t.β) + α = convert(FT, t.α) + θ = convert(FT, t.θ) + γ = convert(FT, t.γ) + δ = convert(FT, t.δ) + ϵ = convert(FT, t.ϵ) + μ = convert(FT, t.μ) + return AdamsBashforth3Scheme(ηᵐ, ηᵐ⁻¹, ηᵐ⁻², Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², β, α, θ, γ, δ, ϵ, μ) +end + +# The functions `η★` `U★` and `V★` represent the value of free surface, barotropic zonal and meridional velocity at time step m+1/2 +@inline U★(i, j, k, grid, t::ForwardBackwardScheme, Uᵐ) = @inbounds Uᵐ[i, j, k] +@inline U★(i, j, k, grid, t::AdamsBashforth3Scheme, Uᵐ) = @inbounds t.α * Uᵐ[i, j, k] + t.θ * t.Uᵐ⁻¹[i, j, k] + t.β * t.Uᵐ⁻²[i, j, k] + +@inline η★(i, j, k, grid, t::ForwardBackwardScheme, ηᵐ⁺¹) = @inbounds ηᵐ⁺¹[i, j, k] +@inline η★(i, j, k, grid, t::AdamsBashforth3Scheme, ηᵐ⁺¹) = @inbounds t.δ * ηᵐ⁺¹[i, j, k] + t.μ * t.ηᵐ[i, j, k] + t.γ * t.ηᵐ⁻¹[i, j, k] + t.ϵ * t.ηᵐ⁻²[i, j, k] + +@inline advance_previous_velocities!(::ForwardBackwardScheme, i, j, k, U) = nothing + +@inline function advance_previous_velocities!(t::AdamsBashforth3Scheme, i, j, k, U) + @inbounds t.Uᵐ⁻²[i, j, k] = t.Uᵐ⁻¹[i, j, k] + @inbounds t.Uᵐ⁻¹[i, j, k] = U[i, j, k] + + return nothing +end + +@inline advance_previous_free_surface!(::ForwardBackwardScheme, i, j, k, η) = nothing + +@inline function advance_previous_free_surface!(t::AdamsBashforth3Scheme, i, j, k, η) + @inbounds t.ηᵐ⁻²[i, j, k] = t.ηᵐ⁻¹[i, j, k] + @inbounds t.ηᵐ⁻¹[i, j, k] = t.ηᵐ[i, j, k] + @inbounds t.ηᵐ[i, j, k] = η[i, j, k] + + return nothing +end From dad5654346710f881a4578dc711e5a451189cbe7 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 1 Nov 2024 09:24:54 +0100 Subject: [PATCH 58/79] correction --- .../barotropic_correction.jl | 1 - .../barotropic_kernels.jl | 20 +++++++++---------- test/dependencies_for_runtests.jl | 2 +- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_correction.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_correction.jl index 2af652b573..d76ca1c624 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_correction.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_correction.jl @@ -31,7 +31,6 @@ end @kernel function _barotropic_split_explicit_corrector!(u, v, U, V, U̅, V̅, grid) i, j, k = @index(Global, NTuple) - k_top = grid.Nz+1 @inbounds begin Hᶠᶜ = static_column_depthᶠᶜᵃ(i, j, grid) diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_kernels.jl index d5e242d131..dd022e6f45 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_kernels.jl @@ -69,20 +69,20 @@ end k_top = grid.Nz+1 @inbounds begin - advance_previous_velocities!(timestepper, i, j, k_top-1, U) - advance_previous_velocities!(timestepper, i, j, k_top-1, V) + advance_previous_velocities!(timestepper, i, j, 1, U) + advance_previous_velocities!(timestepper, i, j, 1, V) Hᶠᶜ = static_column_depthᶠᶜᵃ(i, j, grid) Hᶜᶠ = static_column_depthᶜᶠᵃ(i, j, grid) # ∂τ(U) = - ∇η + G - U[i, j, k_top-1] += Δτ * (- g * Hᶠᶜ * ∂xTᶠᶜᶠ(i, j, k_top, grid, η★, timestepper, η) + Gᵁ[i, j, k_top-1]) - V[i, j, k_top-1] += Δτ * (- g * Hᶜᶠ * ∂yTᶜᶠᶠ(i, j, k_top, grid, η★, timestepper, η) + Gⱽ[i, j, k_top-1]) + U[i, j, 1] += Δτ * (- g * Hᶠᶜ * ∂xTᶠᶜᶠ(i, j, k_top, grid, η★, timestepper, η) + Gᵁ[i, j, 1]) + V[i, j, 1] += Δτ * (- g * Hᶜᶠ * ∂yTᶜᶠᶠ(i, j, k_top, grid, η★, timestepper, η) + Gⱽ[i, j, 1]) # time-averaging - η̅[i, j, k_top] += averaging_weight * η[i, j, k_top] - U̅[i, j, k_top-1] += averaging_weight * U[i, j, k_top-1] - V̅[i, j, k_top-1] += averaging_weight * V[i, j, k_top-1] + η̅[i, j, 1] += averaging_weight * η[i, j, k_top] + U̅[i, j, 1] += averaging_weight * U[i, j, 1] + V̅[i, j, 1] += averaging_weight * V[i, j, 1] end end @@ -187,9 +187,9 @@ function split_explicit_free_surface_step!(free_surface::SplitExplicitFreeSurfac end # Reset eta and velocities for the next timestep - set!(free_surface.η, filtered_state.η) - set!(velocities.U, filtered_state.U) - set!(velocities.U, filtered_state.V) + parent(free_surface.η) .= parent(filtered_state.η) + parent(velocities.U) .= parent(filtered_state.U) + parent(velocities.U) .= parent(filtered_state.V) fields_to_fill = (velocities.U, velocities.V, free_surface.η) # TODO: do this? fill_halo_regions!(fields_to_fill; async = true) diff --git a/test/dependencies_for_runtests.jl b/test/dependencies_for_runtests.jl index 3e15e0a3d8..54da7bbc52 100644 --- a/test/dependencies_for_runtests.jl +++ b/test/dependencies_for_runtests.jl @@ -5,7 +5,7 @@ using Random using Statistics using LinearAlgebra using Logging -using Enzyme +# using Enzyme using SparseArrays using JLD2 using FFTW From a253cc0050c499f92ae0c9fa1c390a4129d0f5a5 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 1 Nov 2024 10:31:24 +0100 Subject: [PATCH 59/79] use a module for now --- .../HydrostaticFreeSurfaceModels.jl | 9 +++----- .../SplitExplicitFreeSurfaces.jl | 23 +++++++++++++++++++ .../split_explicit_timesteppers.jl | 10 -------- 3 files changed, 26 insertions(+), 16 deletions(-) create mode 100644 src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl diff --git a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl index 9e83c7fbd4..4db580b2d2 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl @@ -46,12 +46,7 @@ include("matrix_implicit_free_surface_solver.jl") include("implicit_free_surface.jl") # Split-Explicit free-surface solver functionality -include("SplitExplicitFreeSurfaces/split_explicit_timesteppers.jl") -include("SplitExplicitFreeSurfaces/split_explicit_free_surface.jl") -include("SplitExplicitFreeSurfaces/setup_split_explicit.jl") -include("SplitExplicitFreeSurfaces/barotropic_kernels.jl") -include("SplitExplicitFreeSurfaces/barotropic_correction.jl") -include("SplitExplicitFreeSurfaces/distributed_split_explicit_free_surface.jl") +include("SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl") include("hydrostatic_free_surface_field_tuples.jl") include("hydrostatic_free_surface_model.jl") @@ -141,4 +136,6 @@ include("slice_ensemble_model_mode.jl") include("vertical_vorticity_field.jl") +using .SplitExplicitFreeSurfaces + end # module diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl new file mode 100644 index 0000000000..0a47968feb --- /dev/null +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl @@ -0,0 +1,23 @@ +module SplitExplicitFreeSurfaces + +export SplitExplicitFreeSurface, ForwardBackwardScheme, AdamsBashforth3Scheme + +using Oceananigans +using Oceananigans.Architectures +using Oceananigans.Fields +using Oceananigans.Grids +using Oceananigans.Operators +using Oceananigans.Grids: AbstractGrid + +using Adapt +using Base +using KernelAbstractions: @index, @kernel + +import Oceananigans.Models.HydrostaticFreeSurfaceModels: initialize_free_surface!, + setup_free_surface!, + materialize_free_surface, + ab2_step_free_surface! + + + +end \ No newline at end of file diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_timesteppers.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_timesteppers.jl index bc5db14714..1d9aa583a9 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_timesteppers.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_timesteppers.jl @@ -1,13 +1,3 @@ -using Oceananigans -using Oceananigans.Architectures -using Oceananigans.Fields -using Oceananigans.Grids -using Oceananigans.Grids: AbstractGrid - -using Adapt -using Base -using KernelAbstractions: @index, @kernel - function materialize_timestepper(name::Symbol, args...) fullname = Symbol(name, :Scheme) TS = getglobal(@__MODULE__, fullname) From e350d1c43586b0b0cc917af85d4caf723db598ad Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 1 Nov 2024 10:42:05 +0100 Subject: [PATCH 60/79] make module work --- .../HydrostaticFreeSurfaceModels.jl | 2 +- .../SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl index 4db580b2d2..1216269daa 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl @@ -136,6 +136,6 @@ include("slice_ensemble_model_mode.jl") include("vertical_vorticity_field.jl") -using .SplitExplicitFreeSurfaces +using .SplitExplicitFreeSurface end # module diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl index 0a47968feb..5719481257 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl @@ -18,6 +18,11 @@ import Oceananigans.Models.HydrostaticFreeSurfaceModels: initialize_free_surface materialize_free_surface, ab2_step_free_surface! - +include("split_explicit_timesteppers.jl") +include("split_explicit_free_surface.jl") +include("distributed_split_explicit_free_surface.jl") +include("setup_split_explicit.jl") +include("barotropic_kernels.jl") +include("barotropic_correction.jl") end \ No newline at end of file From 81a2c61a737fda10540d93042780b222eb6bcd68 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 1 Nov 2024 10:56:30 +0100 Subject: [PATCH 61/79] some more organizational stuff --- .../HydrostaticFreeSurfaceModels.jl | 4 +++ .../SplitExplicitFreeSurfaces.jl | 7 ++++ .../split_explicit_free_surface.jl | 13 ------- .../barotropic_pressure_correction.jl | 14 +++----- ...ute_hydrostatic_free_surface_tendencies.jl | 34 +++---------------- .../explicit_free_surface.jl | 29 ++++++++++++++++ .../hydrostatic_free_surface_ab2_step.jl | 2 -- .../hydrostatic_free_surface_model.jl | 1 - .../single_column_model_mode.jl | 5 +-- 9 files changed, 51 insertions(+), 58 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl index 1216269daa..2795c8a989 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl @@ -31,6 +31,10 @@ fill_horizontal_velocity_halos!(args...) = nothing free_surface_displacement_field(velocities, free_surface, grid) = ZFaceField(grid, indices = (:, :, size(grid, 3)+1)) free_surface_displacement_field(velocities, ::Nothing, grid) = nothing +# free surface initialization functions +initialize_free_surface!(free_surface, grid, velocities) = nothing +setup_free_surface!(model, free_surface, χ) = nothing + include("compute_w_from_continuity.jl") include("rigid_lid.jl") diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl index 5719481257..39b9008368 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl @@ -8,6 +8,7 @@ using Oceananigans.Fields using Oceananigans.Grids using Oceananigans.Operators using Oceananigans.Grids: AbstractGrid +using Oceananigans.Models.HydrostaticFreeSurfaceModels: AbstractFreeSurface using Adapt using Base @@ -25,4 +26,10 @@ include("setup_split_explicit.jl") include("barotropic_kernels.jl") include("barotropic_correction.jl") +# extend +@inline explicit_barotropic_pressure_x_gradient(i, j, k, grid, ::SplitExplicitFreeSurface) = zero(grid) +@inline explicit_barotropic_pressure_y_gradient(i, j, k, grid, ::SplitExplicitFreeSurface) = zero(grid) + + + end \ No newline at end of file diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_free_surface.jl index f8e10bbea4..26902d35a7 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_free_surface.jl @@ -1,10 +1,4 @@ -""" - struct SplitExplicitFreeSurface - -The split-explicit free surface solver. -$(FIELDS) -""" struct SplitExplicitFreeSurface{H, U, M, FT, K , S, T} <: AbstractFreeSurface{H, FT} η :: H barotropic_velocities :: U # A namedtuple with U, V @@ -238,13 +232,6 @@ end return Δτ, tuple(averaging_weights...) end -# Convenience Functions for grabbing free surface -free_surface(free_surface::SplitExplicitFreeSurface) = free_surface.η - -# extend -@inline explicit_barotropic_pressure_x_gradient(i, j, k, grid, ::SplitExplicitFreeSurface) = zero(grid) -@inline explicit_barotropic_pressure_y_gradient(i, j, k, grid, ::SplitExplicitFreeSurface) = zero(grid) - Base.summary(s::FixedTimeStepSize) = string("Barotropic time step equal to $(prettytime(s.Δt_barotropic))") Base.summary(s::FixedSubstepNumber) = string("Barotropic fractional step equal to $(s.fractional_step_size) times the baroclinic step") diff --git a/src/Models/HydrostaticFreeSurfaceModels/barotropic_pressure_correction.jl b/src/Models/HydrostaticFreeSurfaceModels/barotropic_pressure_correction.jl index a0536107aa..5e364da7cc 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/barotropic_pressure_correction.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/barotropic_pressure_correction.jl @@ -6,15 +6,14 @@ calculate_pressure_correction!(::HydrostaticFreeSurfaceModel, Δt) = nothing ##### Barotropic pressure correction for models with a free surface ##### -const HFSM = HydrostaticFreeSurfaceModel -const ExplicitFreeSurfaceHFSM = HFSM{<:Any, <:Any, <:Any, <:ExplicitFreeSurface} -const ImplicitFreeSurfaceHFSM = HFSM{<:Any, <:Any, <:Any, <:ImplicitFreeSurface} -const SplitExplicitFreeSurfaceHFSM = HFSM{<:Any, <:Any, <:Any, <:SplitExplicitFreeSurface} +pressure_correct_velocities!(model::HydrostaticFreeSurfaceModel, Δt; kwargs...) = + pressure_correct_velocities!(model, model.free_surface, Δt; kwargs...) -pressure_correct_velocities!(model::ExplicitFreeSurfaceHFSM, Δt; kwargs...) = nothing +# Fallback +pressure_correct_velocities!(model, free_surface, Δt; kwargs...) = nothing ##### -##### Barotropic pressure correction for models with a free surface +##### Barotropic pressure correction for models with an Implicit free surface ##### function pressure_correct_velocities!(model::ImplicitFreeSurfaceHFSM, Δt) @@ -30,9 +29,6 @@ function pressure_correct_velocities!(model::ImplicitFreeSurfaceHFSM, Δt) return nothing end -compute_free_surface_tendency!(grid, ::ImplicitFreeSurfaceHFSM , args...) = nothing -compute_free_surface_tendency!(grid, ::SplitExplicitFreeSurfaceHFSM, args...) = nothing - function pressure_correct_velocities!(model::SplitExplicitFreeSurfaceHFSM, Δt) u, v, _ = model.velocities grid = model.grid diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl index 449b263a16..2a2285f32e 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl @@ -115,24 +115,6 @@ function apply_flux_bcs!(Gcⁿ, c, arch, args) return nothing end -function compute_free_surface_tendency!(grid, model, kernel_parameters) - - arch = architecture(grid) - - args = tuple(model.velocities, - model.free_surface, - model.tracers, - model.auxiliary_fields, - model.forcing, - model.clock) - - launch!(arch, grid, kernel_parameters, - compute_hydrostatic_free_surface_Gη!, model.timestepper.Gⁿ.η, - grid, args) - - return nothing -end - """ Calculate momentum tendencies if momentum is not prescribed.""" function compute_hydrostatic_momentum_tendencies!(model, velocities, kernel_parameters; active_cells_map = nothing) @@ -169,11 +151,15 @@ function compute_hydrostatic_momentum_tendencies!(model, velocities, kernel_para active_cells_map, v_kernel_args; active_cells_map) - compute_free_surface_tendency!(grid, model, :xy) + compute_free_surface_tendency!(grid, model, model.free_surface, :xy) return nothing end +# Fallback +compute_free_surface_tendency!(grid, model, free_surface, args...) = nothing + + """ Apply boundary conditions by adding flux divergences to the right-hand-side. """ function compute_hydrostatic_boundary_tendency_contributions!(Gⁿ, arch, velocities, free_surface, tracers, args...) @@ -240,13 +226,3 @@ end i, j, k = active_linear_index_to_tuple(idx, active_cells_map) @inbounds Gc[i, j, k] = hydrostatic_free_surface_tracer_tendency(i, j, k, grid, args...) end - -##### -##### Tendency calculators for an explicit free surface -##### - -""" Calculate the right-hand-side of the free surface displacement (``η``) equation. """ -@kernel function compute_hydrostatic_free_surface_Gη!(Gη, grid, args) - i, j = @index(Global, NTuple) - @inbounds Gη[i, j, grid.Nz+1] = free_surface_tendency(i, j, grid, args...) -end diff --git a/src/Models/HydrostaticFreeSurfaceModels/explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/explicit_free_surface.jl index 56fd8d0049..f30e04d4db 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/explicit_free_surface.jl @@ -69,3 +69,32 @@ explicit_ab2_step_free_surface!(free_surface, model, Δt, χ) = η[i, j, Nz+1] += Δt * ((FT(1.5) + χ) * Gηⁿ[i, j, Nz+1] - (FT(0.5) + χ) * Gη⁻[i, j, Nz+1]) end end + +# Comopute free surface tendency +function compute_free_surface_tendency!(grid, model, ::ExplicitFreeSurface, kernel_parameters) + + arch = architecture(grid) + + args = tuple(model.velocities, + model.free_surface, + model.tracers, + model.auxiliary_fields, + model.forcing, + model.clock) + + launch!(arch, grid, kernel_parameters, + compute_hydrostatic_free_surface_Gη!, model.timestepper.Gⁿ.η, + grid, args) + + return nothing +end + +##### +##### Tendency calculators for an explicit free surface +##### + +""" Calculate the right-hand-side of the free surface displacement (``η``) equation. """ +@kernel function compute_hydrostatic_free_surface_Gη!(Gη, grid, args) + i, j = @index(Global, NTuple) + @inbounds Gη[i, j, grid.Nz+1] = free_surface_tendency(i, j, grid, args...) +end diff --git a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl index c76ac3b525..032063bd9a 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl @@ -9,8 +9,6 @@ import Oceananigans.TimeSteppers: ab2_step! ##### Step everything ##### -setup_free_surface!(model, free_surface, χ) = nothing - function ab2_step!(model::HydrostaticFreeSurfaceModel, Δt) χ = model.timestepper.χ diff --git a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl index 990dced852..1567cb3a17 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl @@ -230,7 +230,6 @@ validate_momentum_advection(momentum_advection::VectorInvariant, grid::Orthogona validate_momentum_advection(momentum_advection, grid::OrthogonalSphericalShellGrid) = error("$(typeof(momentum_advection)) is not supported with $(typeof(grid))") initialize!(model::HydrostaticFreeSurfaceModel) = initialize_free_surface!(model.free_surface, model.grid, model.velocities) -initialize_free_surface!(free_surface, grid, velocities) = nothing # return the total advective velocities @inline total_velocities(model::HydrostaticFreeSurfaceModel) = model.velocities diff --git a/src/Models/HydrostaticFreeSurfaceModels/single_column_model_mode.jl b/src/Models/HydrostaticFreeSurfaceModels/single_column_model_mode.jl index d4d3dec8d8..82cec71d8a 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/single_column_model_mode.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/single_column_model_mode.jl @@ -53,11 +53,8 @@ compute_w_from_continuity!(::PrescribedVelocityFields, arch, ::SingleColumnGrid; ##### Time-step optimizations ##### -compute_free_surface_tendency!(::SingleColumnGrid, args...) = nothing - # Disambiguation -compute_free_surface_tendency!(::SingleColumnGrid, ::ImplicitFreeSurfaceHFSM , args...) = nothing -compute_free_surface_tendency!(::SingleColumnGrid, ::SplitExplicitFreeSurfaceHFSM, args...) = nothing +compute_free_surface_tendency!(::SingleColumnGrid, model, ::ExplicitFreeSurface, args...) = nothing # Fast state update and halo filling From cf6d49d799d69379c4d17300570d23460d49f0dd Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 1 Nov 2024 10:57:47 +0100 Subject: [PATCH 62/79] some more changes --- .../SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl | 2 -- .../barotropic_pressure_correction.jl | 5 +++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl index 39b9008368..e295e31254 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl @@ -30,6 +30,4 @@ include("barotropic_correction.jl") @inline explicit_barotropic_pressure_x_gradient(i, j, k, grid, ::SplitExplicitFreeSurface) = zero(grid) @inline explicit_barotropic_pressure_y_gradient(i, j, k, grid, ::SplitExplicitFreeSurface) = zero(grid) - - end \ No newline at end of file diff --git a/src/Models/HydrostaticFreeSurfaceModels/barotropic_pressure_correction.jl b/src/Models/HydrostaticFreeSurfaceModels/barotropic_pressure_correction.jl index 5e364da7cc..cc01efeba5 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/barotropic_pressure_correction.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/barotropic_pressure_correction.jl @@ -1,3 +1,4 @@ +using .SplitExplicitFreeSurfaces: barotropic_split_explicit_corrector import Oceananigans.TimeSteppers: calculate_pressure_correction!, pressure_correct_velocities! calculate_pressure_correction!(::HydrostaticFreeSurfaceModel, Δt) = nothing @@ -16,7 +17,7 @@ pressure_correct_velocities!(model, free_surface, Δt; kwargs...) = nothing ##### Barotropic pressure correction for models with an Implicit free surface ##### -function pressure_correct_velocities!(model::ImplicitFreeSurfaceHFSM, Δt) +function pressure_correct_velocities!(model, ::ImplicitFreeSurface, Δt) launch!(model.architecture, model.grid, :xyz, _barotropic_pressure_correction, @@ -29,7 +30,7 @@ function pressure_correct_velocities!(model::ImplicitFreeSurfaceHFSM, Δt) return nothing end -function pressure_correct_velocities!(model::SplitExplicitFreeSurfaceHFSM, Δt) +function pressure_correct_velocities!(model, ::SplitExplicitFreeSurface, Δt) u, v, _ = model.velocities grid = model.grid barotropic_split_explicit_corrector!(u, v, model.free_surface, grid) From 4c844d85297147b69fdf0af3563836a8dfd0e799 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 1 Nov 2024 11:09:06 +0100 Subject: [PATCH 63/79] it compiles --- .../HydrostaticFreeSurfaceModels.jl | 4 ++-- .../SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl | 4 +++- .../distributed_split_explicit_free_surface.jl | 3 --- .../split_explicit_free_surface.jl | 1 - .../{barotropic_kernels.jl => substepping_kernels.jl} | 0 .../barotropic_pressure_correction.jl | 6 +++--- .../multi_region_split_explicit_free_surface.jl | 7 ++++--- 7 files changed, 12 insertions(+), 13 deletions(-) rename src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/{barotropic_kernels.jl => substepping_kernels.jl} (100%) diff --git a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl index 2795c8a989..b74d5a3dfd 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl @@ -52,6 +52,8 @@ include("implicit_free_surface.jl") # Split-Explicit free-surface solver functionality include("SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl") +using .SplitExplicitFreeSurfaces + include("hydrostatic_free_surface_field_tuples.jl") include("hydrostatic_free_surface_model.jl") include("show_hydrostatic_free_surface_model.jl") @@ -140,6 +142,4 @@ include("slice_ensemble_model_mode.jl") include("vertical_vorticity_field.jl") -using .SplitExplicitFreeSurface - end # module diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl index e295e31254..715fd00cab 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl @@ -1,10 +1,12 @@ module SplitExplicitFreeSurfaces export SplitExplicitFreeSurface, ForwardBackwardScheme, AdamsBashforth3Scheme +export FixedSubstepNumber, FixedTimeStepSize using Oceananigans using Oceananigans.Architectures using Oceananigans.Fields +using Oceananigans.Utils using Oceananigans.Grids using Oceananigans.Operators using Oceananigans.Grids: AbstractGrid @@ -23,7 +25,7 @@ include("split_explicit_timesteppers.jl") include("split_explicit_free_surface.jl") include("distributed_split_explicit_free_surface.jl") include("setup_split_explicit.jl") -include("barotropic_kernels.jl") +include("substepping_kernels.jl") include("barotropic_correction.jl") # extend diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/distributed_split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/distributed_split_explicit_free_surface.jl index eeab4b8c61..912a7fce6b 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/distributed_split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/distributed_split_explicit_free_surface.jl @@ -1,9 +1,6 @@ using Oceananigans.AbstractOperations: GridMetricOperation, Δz using Oceananigans.DistributedComputations: DistributedGrid, DistributedField using Oceananigans.DistributedComputations: SynchronizedDistributed, synchronize_communication! -using Oceananigans.Models.HydrostaticFreeSurfaceModels: SplitExplicitFreeSurface - -import Oceananigans.Models.HydrostaticFreeSurfaceModels: materialize_free_surface # Internal function for HydrostaticFreeSurfaceModel function materialize_free_surface(free_surface::SplitExplicitFreeSurface, velocities, grid::DistributedGrid) diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_free_surface.jl index 26902d35a7..0935a5ce26 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_free_surface.jl @@ -1,4 +1,3 @@ - struct SplitExplicitFreeSurface{H, U, M, FT, K , S, T} <: AbstractFreeSurface{H, FT} η :: H barotropic_velocities :: U # A namedtuple with U, V diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/substepping_kernels.jl similarity index 100% rename from src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_kernels.jl rename to src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/substepping_kernels.jl diff --git a/src/Models/HydrostaticFreeSurfaceModels/barotropic_pressure_correction.jl b/src/Models/HydrostaticFreeSurfaceModels/barotropic_pressure_correction.jl index cc01efeba5..004f1099d1 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/barotropic_pressure_correction.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/barotropic_pressure_correction.jl @@ -1,4 +1,4 @@ -using .SplitExplicitFreeSurfaces: barotropic_split_explicit_corrector +using .SplitExplicitFreeSurfaces: barotropic_split_explicit_corrector! import Oceananigans.TimeSteppers: calculate_pressure_correction!, pressure_correct_velocities! calculate_pressure_correction!(::HydrostaticFreeSurfaceModel, Δt) = nothing @@ -20,7 +20,7 @@ pressure_correct_velocities!(model, free_surface, Δt; kwargs...) = nothing function pressure_correct_velocities!(model, ::ImplicitFreeSurface, Δt) launch!(model.architecture, model.grid, :xyz, - _barotropic_pressure_correction, + _barotropic_pressure_correction!, model.velocities, model.grid, Δt, @@ -38,7 +38,7 @@ function pressure_correct_velocities!(model, ::SplitExplicitFreeSurface, Δt) return nothing end -@kernel function _barotropic_pressure_correction(U, grid, Δt, g, η) +@kernel function _barotropic_pressure_correction!(U, grid, Δt, g, η) i, j, k = @index(Global, NTuple) @inbounds begin diff --git a/src/MultiRegion/multi_region_split_explicit_free_surface.jl b/src/MultiRegion/multi_region_split_explicit_free_surface.jl index 5c2b05ee71..df691ae00b 100644 --- a/src/MultiRegion/multi_region_split_explicit_free_surface.jl +++ b/src/MultiRegion/multi_region_split_explicit_free_surface.jl @@ -1,8 +1,9 @@ using Oceananigans.Utils using Oceananigans.AbstractOperations: GridMetricOperation, Δz -using Oceananigans.Models.HydrostaticFreeSurfaceModels: SplitExplicitFreeSurface, - FixedSubstepNumber, FixedTimeStepSize, - calculate_substeps +using Oceananigans.Models.HydrostaticFreeSurfaceModels.SplitExplicitFreeSurfaces +using Oceananigans.Models.HydrostaticFreeSurfaceModels.SplitExplicitFreeSurfaces: calculate_substeps, + barotropic_bc, + materialize_timestepper import Oceananigans.Models.HydrostaticFreeSurfaceModels: materialize_free_surface From d6d0bc71619602da1b4fb87f73e95c8a27c09a58 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 1 Nov 2024 12:23:49 +0100 Subject: [PATCH 64/79] using free_surface_displacement_field --- .../SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl index 715fd00cab..bdc2657dc3 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl @@ -10,7 +10,8 @@ using Oceananigans.Utils using Oceananigans.Grids using Oceananigans.Operators using Oceananigans.Grids: AbstractGrid -using Oceananigans.Models.HydrostaticFreeSurfaceModels: AbstractFreeSurface +using Oceananigans.Models.HydrostaticFreeSurfaceModels: AbstractFreeSurface, + free_surface_displacement_field using Adapt using Base From bb5c992f06af8aa0fae37c0e80f0caef242d77fe Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 1 Nov 2024 14:09:58 +0100 Subject: [PATCH 65/79] include g_Earth --- .../SplitExplicitFreeSurfaces/split_explicit_free_surface.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_free_surface.jl index 0935a5ce26..6862da6258 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_free_surface.jl @@ -1,3 +1,5 @@ +using Oceananigans.BuoyancyModels: g_Earth + struct SplitExplicitFreeSurface{H, U, M, FT, K , S, T} <: AbstractFreeSurface{H, FT} η :: H barotropic_velocities :: U # A namedtuple with U, V From 611aaec3cfd86dea9a99bd978cbd13b5db757d85 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 1 Nov 2024 14:33:06 +0100 Subject: [PATCH 66/79] now it should run --- Project.toml | 1 + .../SplitExplicitFreeSurfaces.jl | 4 +++- .../split_explicit_free_surface.jl | 11 ++++++++--- .../split_explicit_timesteppers.jl | 12 ++++++++++++ .../SplitExplicitFreeSurfaces/substepping_kernels.jl | 6 +++--- .../show_hydrostatic_free_surface_model.jl | 2 +- 6 files changed, 28 insertions(+), 8 deletions(-) diff --git a/Project.toml b/Project.toml index 4d80cee5b7..d08be7879d 100644 --- a/Project.toml +++ b/Project.toml @@ -11,6 +11,7 @@ CubedSphere = "7445602f-e544-4518-8976-18f8e8ae6cdb" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Distances = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7" DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" +FFMPEG = "c87230d0-a227-11e9-1b43-d7ebe4e7570a" FFTW = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" Glob = "c27321d9-0574-5035-807b-f59d2c89b15c" IncompleteLU = "40713840-3770-5561-ab4c-a76e7d0d7895" diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl index bdc2657dc3..4bab4a4f94 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl @@ -20,7 +20,9 @@ using KernelAbstractions: @index, @kernel import Oceananigans.Models.HydrostaticFreeSurfaceModels: initialize_free_surface!, setup_free_surface!, materialize_free_surface, - ab2_step_free_surface! + ab2_step_free_surface!, + explicit_barotropic_pressure_x_gradient, + explicit_barotropic_pressure_y_gradient include("split_explicit_timesteppers.jl") include("split_explicit_free_surface.jl") diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_free_surface.jl index 6862da6258..dfa09cf828 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_free_surface.jl @@ -236,14 +236,19 @@ end Base.summary(s::FixedTimeStepSize) = string("Barotropic time step equal to $(prettytime(s.Δt_barotropic))") Base.summary(s::FixedSubstepNumber) = string("Barotropic fractional step equal to $(s.fractional_step_size) times the baroclinic step") -Base.summary(sefs::SplitExplicitFreeSurface) = string("SplitExplicitFreeSurface with $(summary(sefs.settings.substepping))") +Base.summary(sefs::SplitExplicitFreeSurface) = string("SplitExplicitFreeSurface with $(summary(sefs.substepping))") Base.show(io::IO, sefs::SplitExplicitFreeSurface) = print(io, "$(summary(sefs))\n") # Adapt Adapt.adapt_structure(to, free_surface::SplitExplicitFreeSurface) = - SplitExplicitFreeSurface(Adapt.adapt(to, free_surface.η), nothing, nothing, - free_surface.gravitational_acceleration, nothing) + SplitExplicitFreeSurface(Adapt.adapt(to, free_surface.η), + Adapt.adapt(to, free_surface.barotropic_velocities), + Adapt.adapt(to, free_surface.filtered_state), + free_surface.gravitational_acceleration, + nothing, + Adapt.adapt(to, free_surface.substepping), + Adapt.adapt(to, free_surface.timestepper)) for Type in (:SplitExplicitFreeSurface, :AdamsBashforth3Scheme, diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_timesteppers.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_timesteppers.jl index 1d9aa583a9..7d7de4d80c 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_timesteppers.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_timesteppers.jl @@ -28,6 +28,17 @@ end AdamsBashforth3Scheme(; β = 0.281105, α = 1.5 + β, θ = - 0.5 - 2β, γ = 0.088, δ = 0.614, ϵ = 0.013, μ = 1 - δ - γ - ϵ) = AdamsBashforth3Scheme(nothing, nothing, nothing, nothing, nothing, nothing, nothing, β, α, θ, γ, δ, ϵ, μ) +Adapt.adapt_structure(to, t::AdamsBashforth3Scheme) = + AdamsBashforth3Scheme( + Adapt.adapt(to, t.ηᵐ ), + Adapt.adapt(to, t.ηᵐ⁻¹), + Adapt.adapt(to, t.ηᵐ⁻²), + Adapt.adapt(to, t.Uᵐ⁻¹), + Adapt.adapt(to, t.Uᵐ⁻²), + Adapt.adapt(to, t.Vᵐ⁻¹), + Adapt.adapt(to, t.Vᵐ⁻²), + t.β, t.α, t.θ, t.γ, t.δ, t.ϵ, t.μ) + function materialize_timestepper(t::AdamsBashforth3Scheme, grid, free_surface, velocities, u_bc, v_bc) ηᵐ = free_surface_displacement_field(velocities, free_surface, grid) ηᵐ⁻¹ = free_surface_displacement_field(velocities, free_surface, grid) @@ -47,6 +58,7 @@ function materialize_timestepper(t::AdamsBashforth3Scheme, grid, free_surface, v δ = convert(FT, t.δ) ϵ = convert(FT, t.ϵ) μ = convert(FT, t.μ) + return AdamsBashforth3Scheme(ηᵐ, ηᵐ⁻¹, ηᵐ⁻², Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², β, α, θ, γ, δ, ϵ, μ) end diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/substepping_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/substepping_kernels.jl index dd022e6f45..5cfe1b9b3a 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/substepping_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/substepping_kernels.jl @@ -80,9 +80,9 @@ end V[i, j, 1] += Δτ * (- g * Hᶜᶠ * ∂yTᶜᶠᶠ(i, j, k_top, grid, η★, timestepper, η) + Gⱽ[i, j, 1]) # time-averaging - η̅[i, j, 1] += averaging_weight * η[i, j, k_top] - U̅[i, j, 1] += averaging_weight * U[i, j, 1] - V̅[i, j, 1] += averaging_weight * V[i, j, 1] + η̅[i, j, k_top] += averaging_weight * η[i, j, k_top] + U̅[i, j, 1] += averaging_weight * U[i, j, 1] + V̅[i, j, 1] += averaging_weight * V[i, j, 1] end end diff --git a/src/Models/HydrostaticFreeSurfaceModels/show_hydrostatic_free_surface_model.jl b/src/Models/HydrostaticFreeSurfaceModels/show_hydrostatic_free_surface_model.jl index 88cd4ab915..bf461a43f1 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/show_hydrostatic_free_surface_model.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/show_hydrostatic_free_surface_model.jl @@ -27,7 +27,7 @@ function Base.show(io::IO, model::HydrostaticFreeSurfaceModel) end if typeof(model.free_surface).name.wrapper == SplitExplicitFreeSurface - print(io, "│ └── substepping: $(summary(model.free_surface.settings.substepping))", "\n") + print(io, "│ └── substepping: $(summary(model.free_surface.substepping))", "\n") end end From aface2ef2013338ae41645be3d8a24263cb3aa7d Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 1 Nov 2024 15:05:44 +0100 Subject: [PATCH 67/79] a little bit of cleanup --- .../substepping_kernels.jl | 30 ++----------------- 1 file changed, 3 insertions(+), 27 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/substepping_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/substepping_kernels.jl index 5cfe1b9b3a..2d8a4fdbe8 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/substepping_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/substepping_kernels.jl @@ -24,26 +24,16 @@ using KernelAbstractions.Extras.LoopInfo: @unroll # the free surface field η and its average η̄ are located on `Face`s at the surface (grid.Nz +1). All other intermediate variables # (U, V, Ū, V̄) are barotropic fields (`ReducedField`) for which a k index is not defined -# Special ``partial'' divergence for free surface evolution -@inline div_Txᶜᶜᶠ(i, j, k, grid, U★::Function, args...) = 1 / Azᶜᶜᶠ(i, j, k, grid) * δxTᶜᵃᵃ(i, j, k, grid, Δy_qᶠᶜᶠ, U★, args...) -@inline div_Tyᶜᶜᶠ(i, j, k, grid, V★::Function, args...) = 1 / Azᶜᶜᶠ(i, j, k, grid) * δyTᵃᶜᵃ(i, j, k, grid, Δx_qᶜᶠᶠ, V★, args...) - @kernel function _split_explicit_free_surface!(grid, Δτ, η, U, V, timestepper) i, j = @index(Global, NTuple) - free_surface_evolution!(η, i, j, grid, Δτ, U, V, timestepper) -end - -@inline function free_surface_evolution!(η, i, j, grid, Δτ, U, V, timestepper) k_top = grid.Nz+1 - + @inbounds begin advance_previous_free_surface!(timestepper, i, j, k_top, η) - η[i, j, k_top] -= Δτ * (div_Txᶜᶜᶠ(i, j, k_top-1, grid, U★, timestepper, U) + - div_Tyᶜᶜᶠ(i, j, k_top-1, grid, U★, timestepper, V)) + η[i, j, k_top] -= Δτ * (δxTᶜᵃᵃ(i, j, k_top, grid, U★, timestepper, U) + + δyTᵃᶜᵃ(i, j, k_top, grid, U★, timestepper, V)) / Azᶜᶜᶠ(i, j, k_top, grid) end - - return nothing end @kernel function _split_explicit_barotropic_velocity!(averaging_weight, grid, Δτ, @@ -52,20 +42,6 @@ end Gᵁ, Gⱽ, g, timestepper) i, j = @index(Global, NTuple) - velocity_evolution!(η, U, V, - η̅, U̅, V̅, - i, j, grid, Δτ, - averaging_weight, - Gᵁ, Gⱽ, g, - timestepper) -end - -@inline function velocity_evolution!(η, U, V, - η̅, U̅, V̅, - i, j, grid, Δτ, - averaging_weight, - Gᵁ, Gⱽ, g, - timestepper) k_top = grid.Nz+1 @inbounds begin From 811a1a6f09d80880c5e3fe28d0ef4d8f5c67bba8 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 1 Nov 2024 16:33:55 +0100 Subject: [PATCH 68/79] tests should run now --- .../HydrostaticFreeSurfaceModels.jl | 4 +++- .../SplitExplicitFreeSurfaces.jl | 2 +- .../setup_split_explicit.jl | 21 ++++++++--------- ...jl => step_split_explicit_free_surface.jl} | 23 ++++++++----------- .../hydrostatic_free_surface_field_tuples.jl | 3 ++- .../{rigid_lid.jl => nothing_free_surface.jl} | 0 6 files changed, 26 insertions(+), 27 deletions(-) rename src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/{substepping_kernels.jl => step_split_explicit_free_surface.jl} (91%) rename src/Models/HydrostaticFreeSurfaceModels/{rigid_lid.jl => nothing_free_surface.jl} (100%) diff --git a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl index b74d5a3dfd..f88c3aa08b 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl @@ -36,7 +36,9 @@ initialize_free_surface!(free_surface, grid, velocities) = nothing setup_free_surface!(model, free_surface, χ) = nothing include("compute_w_from_continuity.jl") -include("rigid_lid.jl") + +# No free surface +include("nothing_free_surface.jl") # Explicit free-surface solver functionality include("explicit_free_surface.jl") diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl index 4bab4a4f94..e69666d507 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl @@ -28,7 +28,7 @@ include("split_explicit_timesteppers.jl") include("split_explicit_free_surface.jl") include("distributed_split_explicit_free_surface.jl") include("setup_split_explicit.jl") -include("substepping_kernels.jl") +include("step_split_explicit_free_surface.jl") include("barotropic_correction.jl") # extend diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/setup_split_explicit.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/setup_split_explicit.jl index 26a6d0b7d4..89f3aed570 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/setup_split_explicit.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/setup_split_explicit.jl @@ -3,7 +3,8 @@ function initialize_free_surface!(sefs::SplitExplicitFreeSurface, grid, velocities) barotropic_velocities = sefs.barotropic_velocities @apply_regionally compute_barotropic_mode!(barotropic_velocities.U, barotropic_velocities.V, grid, velocities.u, velocities.v) - fill_halo_regions!((barotropic_velocities.U, barotropic_velocities.V, sefs.η)) + fill_halo_regions!((barotropic_velocities.U, barotropic_velocities.V)) + return nothing end # `initialize_free_surface_state!` is called at the beginning of the substepping to @@ -38,14 +39,13 @@ end # Calculate RHS for the barotropic time step. @kernel function _compute_integrated_ab2_tendencies!(Gᵁ, Gⱽ, grid, ::Nothing, Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ) i, j = @index(Global, NTuple) - k_top = grid.Nz + 1 - @inbounds Gᵁ[i, j, k_top-1] = Δzᶠᶜᶜ(i, j, 1, grid) * ab2_step_Gu(i, j, 1, grid, Gu⁻, Guⁿ, χ) - @inbounds Gⱽ[i, j, k_top-1] = Δzᶜᶠᶜ(i, j, 1, grid) * ab2_step_Gv(i, j, 1, grid, Gv⁻, Gvⁿ, χ) + @inbounds Gᵁ[i, j, 1] = Δzᶠᶜᶜ(i, j, 1, grid) * ab2_step_Gu(i, j, 1, grid, Gu⁻, Guⁿ, χ) + @inbounds Gⱽ[i, j, 1] = Δzᶜᶠᶜ(i, j, 1, grid) * ab2_step_Gv(i, j, 1, grid, Gv⁻, Gvⁿ, χ) for k in 2:grid.Nz - @inbounds Gᵁ[i, j, k_top-1] += Δzᶠᶜᶜ(i, j, k, grid) * ab2_step_Gu(i, j, k, grid, Gu⁻, Guⁿ, χ) - @inbounds Gⱽ[i, j, k_top-1] += Δzᶜᶠᶜ(i, j, k, grid) * ab2_step_Gv(i, j, k, grid, Gv⁻, Gvⁿ, χ) + @inbounds Gᵁ[i, j, 1] += Δzᶠᶜᶜ(i, j, k, grid) * ab2_step_Gu(i, j, k, grid, Gu⁻, Guⁿ, χ) + @inbounds Gⱽ[i, j, 1] += Δzᶜᶠᶜ(i, j, k, grid) * ab2_step_Gv(i, j, k, grid, Gv⁻, Gvⁿ, χ) end end @@ -53,14 +53,13 @@ end @kernel function _compute_integrated_ab2_tendencies!(Gᵁ, Gⱽ, grid, active_cells_map, Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ) idx = @index(Global, Linear) i, j = active_linear_index_to_tuple(idx, active_cells_map) - k_top = grid.Nz+1 - @inbounds Gᵁ[i, j, k_top-1] = Δzᶠᶜᶜ(i, j, 1, grid) * ab2_step_Gu(i, j, 1, grid, Gu⁻, Guⁿ, χ) - @inbounds Gⱽ[i, j, k_top-1] = Δzᶜᶠᶜ(i, j, 1, grid) * ab2_step_Gv(i, j, 1, grid, Gv⁻, Gvⁿ, χ) + @inbounds Gᵁ[i, j, 1] = Δzᶠᶜᶜ(i, j, 1, grid) * ab2_step_Gu(i, j, 1, grid, Gu⁻, Guⁿ, χ) + @inbounds Gⱽ[i, j, 1] = Δzᶜᶠᶜ(i, j, 1, grid) * ab2_step_Gv(i, j, 1, grid, Gv⁻, Gvⁿ, χ) for k in 2:grid.Nz - @inbounds Gᵁ[i, j, k_top-1] += Δzᶠᶜᶜ(i, j, k, grid) * ab2_step_Gu(i, j, k, grid, Gu⁻, Guⁿ, χ) - @inbounds Gⱽ[i, j, k_top-1] += Δzᶜᶠᶜ(i, j, k, grid) * ab2_step_Gv(i, j, k, grid, Gv⁻, Gvⁿ, χ) + @inbounds Gᵁ[i, j, 1] += Δzᶠᶜᶜ(i, j, k, grid) * ab2_step_Gu(i, j, k, grid, Gu⁻, Guⁿ, χ) + @inbounds Gⱽ[i, j, 1] += Δzᶜᶠᶜ(i, j, k, grid) * ab2_step_Gv(i, j, k, grid, Gv⁻, Gvⁿ, χ) end end diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/substepping_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/step_split_explicit_free_surface.jl similarity index 91% rename from src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/substepping_kernels.jl rename to src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/step_split_explicit_free_surface.jl index 2d8a4fdbe8..b31b2367d9 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/substepping_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/step_split_explicit_free_surface.jl @@ -28,12 +28,9 @@ using KernelAbstractions.Extras.LoopInfo: @unroll i, j = @index(Global, NTuple) k_top = grid.Nz+1 - @inbounds begin - advance_previous_free_surface!(timestepper, i, j, k_top, η) - - η[i, j, k_top] -= Δτ * (δxTᶜᵃᵃ(i, j, k_top, grid, U★, timestepper, U) + - δyTᵃᶜᵃ(i, j, k_top, grid, U★, timestepper, V)) / Azᶜᶜᶠ(i, j, k_top, grid) - end + advance_previous_free_surface!(timestepper, i, j, k_top, η) + @inbounds η[i, j, k_top] -= Δτ * (δxTᶜᵃᵃ(i, j, k_top, grid, Δy_qᶠᶜᶠ, U★, timestepper, U) + + δyTᵃᶜᵃ(i, j, k_top, grid, Δx_qᶜᶠᶠ, U★, timestepper, V)) / Azᶜᶜᶠ(i, j, k_top, grid) end @kernel function _split_explicit_barotropic_velocity!(averaging_weight, grid, Δτ, @@ -44,13 +41,13 @@ end i, j = @index(Global, NTuple) k_top = grid.Nz+1 - @inbounds begin - advance_previous_velocities!(timestepper, i, j, 1, U) - advance_previous_velocities!(timestepper, i, j, 1, V) + advance_previous_velocities!(timestepper, i, j, 1, U) + advance_previous_velocities!(timestepper, i, j, 1, V) - Hᶠᶜ = static_column_depthᶠᶜᵃ(i, j, grid) - Hᶜᶠ = static_column_depthᶜᶠᵃ(i, j, grid) - + Hᶠᶜ = static_column_depthᶠᶜᵃ(i, j, grid) + Hᶜᶠ = static_column_depthᶜᶠᵃ(i, j, grid) + + @inbounds begin # ∂τ(U) = - ∇η + G U[i, j, 1] += Δτ * (- g * Hᶠᶜ * ∂xTᶠᶜᶠ(i, j, k_top, grid, η★, timestepper, η) + Gᵁ[i, j, 1]) V[i, j, 1] += Δτ * (- g * Hᶜᶠ * ∂yTᶜᶠᶠ(i, j, k_top, grid, η★, timestepper, η) + Gⱽ[i, j, 1]) @@ -165,7 +162,7 @@ function split_explicit_free_surface_step!(free_surface::SplitExplicitFreeSurfac # Reset eta and velocities for the next timestep parent(free_surface.η) .= parent(filtered_state.η) parent(velocities.U) .= parent(filtered_state.U) - parent(velocities.U) .= parent(filtered_state.V) + parent(velocities.V) .= parent(filtered_state.V) fields_to_fill = (velocities.U, velocities.V, free_surface.η) # TODO: do this? fill_halo_regions!(fields_to_fill; async = true) diff --git a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_field_tuples.jl b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_field_tuples.jl index 465cadd146..6824a8411f 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_field_tuples.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_field_tuples.jl @@ -28,6 +28,7 @@ function HydrostaticFreeSurfaceTendencyFields(velocities, free_surface::SplitExp v = YFaceField(grid) U = deepcopy(free_surface.barotropic_velocities.U) V = deepcopy(free_surface.barotropic_velocities.V) + η = free_surface_displacement_field(velocities, free_surface, grid) tracers = TracerFields(tracer_names, grid) - return merge((u=u, v=v, U=U, V=V), tracers) + return merge((u=u, v=v, U=U, V=V, η=η), tracers) end diff --git a/src/Models/HydrostaticFreeSurfaceModels/rigid_lid.jl b/src/Models/HydrostaticFreeSurfaceModels/nothing_free_surface.jl similarity index 100% rename from src/Models/HydrostaticFreeSurfaceModels/rigid_lid.jl rename to src/Models/HydrostaticFreeSurfaceModels/nothing_free_surface.jl From df17ecaac3a3c19ad1d494d9b718901511d98e0d Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 1 Nov 2024 16:35:52 +0100 Subject: [PATCH 69/79] Update Project.toml --- Project.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/Project.toml b/Project.toml index d08be7879d..4d80cee5b7 100644 --- a/Project.toml +++ b/Project.toml @@ -11,7 +11,6 @@ CubedSphere = "7445602f-e544-4518-8976-18f8e8ae6cdb" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Distances = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7" DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" -FFMPEG = "c87230d0-a227-11e9-1b43-d7ebe4e7570a" FFTW = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" Glob = "c27321d9-0574-5035-807b-f59d2c89b15c" IncompleteLU = "40713840-3770-5561-ab4c-a76e7d0d7895" From fee6ace83bc0d4c2f2401a47450ab80db9183da8 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 1 Nov 2024 16:38:54 +0100 Subject: [PATCH 70/79] conceptually this is better --- .../hydrostatic_free_surface_field_tuples.jl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_field_tuples.jl b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_field_tuples.jl index 6824a8411f..465cadd146 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_field_tuples.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_field_tuples.jl @@ -28,7 +28,6 @@ function HydrostaticFreeSurfaceTendencyFields(velocities, free_surface::SplitExp v = YFaceField(grid) U = deepcopy(free_surface.barotropic_velocities.U) V = deepcopy(free_surface.barotropic_velocities.V) - η = free_surface_displacement_field(velocities, free_surface, grid) tracers = TracerFields(tracer_names, grid) - return merge((u=u, v=v, U=U, V=V, η=η), tracers) + return merge((u=u, v=v, U=U, V=V), tracers) end From 3b45d3ba028bb66c3bdbd63760130ac6d6abdc06 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 1 Nov 2024 16:42:30 +0100 Subject: [PATCH 71/79] fix checkpointer test --- test/test_checkpointer.jl | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/test_checkpointer.jl b/test/test_checkpointer.jl index c32fad63b3..b775c0a2b7 100644 --- a/test/test_checkpointer.jl +++ b/test/test_checkpointer.jl @@ -13,9 +13,11 @@ function test_model_equality(test_model, true_model) field_names = keys(test_model_fields) for name in field_names - @test all(test_model_fields[name].data .≈ true_model_fields[name].data) - @test all(test_model.timestepper.Gⁿ[name].data .≈ true_model.timestepper.Gⁿ[name].data) - @test all(test_model.timestepper.G⁻[name].data .≈ true_model.timestepper.G⁻[name].data) + if name ∈ keys(test_model.timestepper.Gⁿ) + @test all(test_model_fields[name].data .≈ true_model_fields[name].data) + @test all(test_model.timestepper.Gⁿ[name].data .≈ true_model.timestepper.Gⁿ[name].data) + @test all(test_model.timestepper.G⁻[name].data .≈ true_model.timestepper.G⁻[name].data) + end end end From e06226b9d368ab3de979f6679f34a7d28a40047c Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 1 Nov 2024 16:48:30 +0100 Subject: [PATCH 72/79] fix split explicit settings --- ...test_split_explicit_free_surface_solver.jl | 55 +++++++++---------- 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/test/test_split_explicit_free_surface_solver.jl b/test/test_split_explicit_free_surface_solver.jl index 6f4af2ed4d..44e7023b2c 100644 --- a/test/test_split_explicit_free_surface_solver.jl +++ b/test/test_split_explicit_free_surface_solver.jl @@ -27,10 +27,13 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: calculate_substeps, sefs = materialize_free_surface(sefs, nothing, grid) sefs.η .= 0 + GU = Field{Face, Center, Nothing}(grid) + GV = Field{Center, Face, Nothing}(grid) @testset " One timestep test " begin - state = sefs.state - U, V, η̅, U̅, V̅ = state.U, state.V, state.η̅, state.U̅, state.V̅ + state = sefs.filtered_state + U, V = sefs.barotropic_velocities + η̅, U̅, V̅ = state.η, state.U, state.V η = sefs.η Δτ = 1.0 @@ -38,10 +41,10 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: calculate_substeps, η₀(x, y, z) = sin(x) set!(η, η₀) - Nsubsteps = calculate_substeps(sefs.settings.substepping, 1) - fractional_Δt, weights = calculate_adaptive_settings(sefs.settings.substepping, Nsubsteps) # barotropic time step in fraction of baroclinic step and averaging weights + Nsubsteps = calculate_substeps(sefs.substepping, 1) + fractional_Δt, weights = calculate_adaptive_settings(sefs.substepping, Nsubsteps) # barotropic time step in fraction of baroclinic step and averaging weights - iterate_split_explicit!(sefs, grid, Δτ, weights, Val(1)) + iterate_split_explicit!(sefs, grid, GU, GV, Δτ, weights, Val(1)) U_computed = Array(U.data.parent)[2:Nx+1, 2:Ny+1] U_exact = (reshape(-cos.(grid.xᶠᵃᵃ), (length(grid.xᶜᵃᵃ), 1)).+reshape(0 * grid.yᵃᶜᵃ, (1, length(grid.yᵃᶜᵃ))))[2:Nx+1, 2:Ny+1] @@ -50,10 +53,9 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: calculate_substeps, end @testset "Multi-timestep test " begin - state = sefs.state - auxiliary = sefs.auxiliary - U, V, η̅, U̅, V̅ = state.U, state.V, state.η̅, state.U̅, state.V̅ - Gᵁ, Gⱽ = auxiliary.Gᵁ, auxiliary.Gⱽ + state = sefs.filtered_state + U, V = sefs.barotropic_velocities + η̅, U̅, V̅ = state.η, state.U, state.V η = sefs.η T = 2π @@ -75,15 +77,15 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: calculate_substeps, η̅ .= 0 U̅ .= 0 V̅ .= 0 - Gᵁ .= 0 - Gⱽ .= 0 + GU .= 0 + GV .= 0 weights = sefs.settings.substepping.averaging_weights for _ in 1:Nt - iterate_split_explicit!(sefs, grid, Δτ, weights, Val(1)) + iterate_split_explicit!(sefs, grid, GU, GV, Δτ, weights, Val(1)) end - iterate_split_explicit!(sefs, grid, Δτ_end, weights, Val(1)) + iterate_split_explicit!(sefs, grid, GU, GV, Δτ_end, weights, Val(1)) U_computed = Array(deepcopy(interior(U))) η_computed = Array(deepcopy(interior(η))) @@ -102,13 +104,11 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: calculate_substeps, sefs.η .= 0 @testset "Averaging / Do Nothing test " begin - state = sefs.state - auxiliary = sefs.auxiliary - U, V, η̅, U̅, V̅ = state.U, state.V, state.η̅, state.U̅, state.V̅ - Gᵁ, Gⱽ = auxiliary.Gᵁ, auxiliary.Gⱽ - - g = sefs.gravitational_acceleration + state = sefs.filtered_state + U, V = sefs.barotropic_velocities + η̅, U̅, V̅ = state.η, state.U, state.V η = sefs.η + g = sefs.gravitational_acceleration Δτ = 2π / maximum([Nx, Ny]) * 1e-2 # the last factor is essentially the order of accuracy @@ -132,7 +132,7 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: calculate_substeps, fractional_Δt, weights = calculate_adaptive_settings(settings.substepping, Nsubsteps) # barotropic time step in fraction of baroclinic step and averaging weights for step in 1:Nsubsteps - iterate_split_explicit!(sefs, grid, Δτ, weights, Val(1)) + iterate_split_explicit!(sefs, grid, GU, GV, Δτ, weights, Val(1)) end U_computed = Array(deepcopy(interior(U))) @@ -169,10 +169,9 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: calculate_substeps, sefs = SplitExplicitFreeSurface(substeps=200) sefs = materialize_free_surface(sefs, nothing, grid) - state = sefs.state - auxiliary = sefs.auxiliary - U, V, η̅, U̅, V̅ = state.U, state.V, state.η̅, state.U̅, state.V̅ - Gᵁ, Gⱽ = auxiliary.Gᵁ, auxiliary.Gⱽ + tate = sefs.filtered_state + U, V = sefs.barotropic_velocities + η̅, U̅, V̅ = state.η, state.U, state.V η = sefs.η g = sefs.gravitational_acceleration @@ -189,17 +188,17 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: calculate_substeps, η̅ .= 0 U̅ .= 0 V̅ .= 0 - Gᵁ .= gu_c - Gⱽ .= gv_c + GU .= gu_c + GV .= gv_c settings = SplitExplicitSettings(grid; substeps = Nt + 1, averaging_kernel = constant_averaging_kernel) sefs = sefs(settings) weights = settings.substepping.averaging_weights for i in 1:Nt - iterate_split_explicit!(sefs, grid, Δτ, weights, Val(1)) + iterate_split_explicit!(sefs, grid, GU, GV, Δτ, weights, Val(1)) end - iterate_split_explicit!(sefs, grid, Δτ_end, weights, Val(1)) + iterate_split_explicit!(sefs, grid, GU, GV, Δτ_end, weights, Val(1)) η_mean_after = mean(Array(interior(η))) From b09ce27efdf4d54c19357ec3c43736484e82e01e Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 1 Nov 2024 17:10:23 +0100 Subject: [PATCH 73/79] fixing some more tests --- ...distributed_split_explicit_free_surface.jl | 4 +- .../split_explicit_timesteppers.jl | 52 ++++++++++++++++--- ...test_split_explicit_free_surface_solver.jl | 14 +++-- .../test_split_explicit_vertical_integrals.jl | 29 +++++------ 4 files changed, 64 insertions(+), 35 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/distributed_split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/distributed_split_explicit_free_surface.jl index 912a7fce6b..6afb0edb90 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/distributed_split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/distributed_split_explicit_free_surface.jl @@ -5,10 +5,8 @@ using Oceananigans.DistributedComputations: SynchronizedDistributed, synchronize # Internal function for HydrostaticFreeSurfaceModel function materialize_free_surface(free_surface::SplitExplicitFreeSurface, velocities, grid::DistributedGrid) - settings = free_surface.settings - old_halos = halo_size(grid) - Nsubsteps = length(settings.substepping.averaging_weights) + Nsubsteps = length(free_surface.substepping.averaging_weights) extended_halos = distributed_split_explicit_halos(old_halos, Nsubsteps+1, grid) extended_grid = with_halo(extended_halos, grid) diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_timesteppers.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_timesteppers.jl index 7d7de4d80c..eec1024baf 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_timesteppers.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_timesteppers.jl @@ -1,9 +1,15 @@ -function materialize_timestepper(name::Symbol, args...) - fullname = Symbol(name, :Scheme) - TS = getglobal(@__MODULE__, fullname) - return materialize_timestepper(TS, args...) -end +""" + struct ForwardBackwardScheme end +a timestepping scheme used for substepping in the split-explicit free surface solver, + +The equations are evolved as follows: +```math +ηᵐ⁺¹ = ηᵐ - Δτ g H (∂x Uᵐ + ∂y Vᵐ) +Uᵐ⁺¹ = Uᵐ - Δτ (∂x ηᵐ⁺¹ - Gᵁ) +Vᵐ⁺¹ = Vᵐ - Δτ (∂y ηᵐ⁺¹ - Gⱽ) +``` +""" struct ForwardBackwardScheme end materialize_timestepper(::ForwardBackwardScheme, grid, args...) = ForwardBackwardScheme() @@ -25,6 +31,27 @@ struct AdamsBashforth3Scheme{CC, FC, CF, FT} μ :: FT end +""" + AdamsBashforth3Scheme(; β=0.281105, α=1.5 + β, θ=-0.5 - 2β, γ=0.088, δ=0.614, ϵ=0.013, μ=1 - δ - γ - ϵ) + +Creates an instance of `AdamsBashforth3Scheme` with the specified parameters. +This scheme is used for substepping in the split-explicit free surface solver, where an AB3 extrapolation is used to +evaluate barotropic velocities and free surface at time-step `m+1/2`: + +The equations are evolved as follows: +```math +ηᵐ⁺¹ = ηᵐ - Δτ g H (∂x Ũ + Δτ ∂y Ṽ) +Uᵐ⁺¹ = Uᵐ - Δτ (∂x η̃ - Gᵁ) +Vᵐ⁺¹ = Vᵐ - Δτ (∂y η̃ - Gⱽ) +``` +where `η̃`, `Ũ` and `Ṽ` are the AB3 time-extrapolated values of free surface, +barotropic zonal and meridional velocities, respectively: +```math +Ũ = α Uᵐ + θ Uᵐ⁻¹ + β Uᵐ⁻² +Ṽ = α Vᵐ + θ Vᵐ⁻¹ + β Vᵐ⁻² +η̃ = δ ηᵐ⁺¹ + μ ηᵐ + γ ηᵐ⁻¹ + ϵ ηᵐ⁻² +``` +""" AdamsBashforth3Scheme(; β = 0.281105, α = 1.5 + β, θ = - 0.5 - 2β, γ = 0.088, δ = 0.614, ϵ = 0.013, μ = 1 - δ - γ - ϵ) = AdamsBashforth3Scheme(nothing, nothing, nothing, nothing, nothing, nothing, nothing, β, α, θ, γ, δ, ϵ, μ) @@ -62,6 +89,16 @@ function materialize_timestepper(t::AdamsBashforth3Scheme, grid, free_surface, v return AdamsBashforth3Scheme(ηᵐ, ηᵐ⁻¹, ηᵐ⁻², Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², β, α, θ, γ, δ, ϵ, μ) end +##### +##### Timestepper extrapolations and utils +##### + +function materialize_timestepper(name::Symbol, args...) + fullname = Symbol(name, :Scheme) + TS = getglobal(@__MODULE__, fullname) + return materialize_timestepper(TS, args...) +end + # The functions `η★` `U★` and `V★` represent the value of free surface, barotropic zonal and meridional velocity at time step m+1/2 @inline U★(i, j, k, grid, t::ForwardBackwardScheme, Uᵐ) = @inbounds Uᵐ[i, j, k] @inline U★(i, j, k, grid, t::AdamsBashforth3Scheme, Uᵐ) = @inbounds t.α * Uᵐ[i, j, k] + t.θ * t.Uᵐ⁻¹[i, j, k] + t.β * t.Uᵐ⁻²[i, j, k] @@ -69,7 +106,8 @@ end @inline η★(i, j, k, grid, t::ForwardBackwardScheme, ηᵐ⁺¹) = @inbounds ηᵐ⁺¹[i, j, k] @inline η★(i, j, k, grid, t::AdamsBashforth3Scheme, ηᵐ⁺¹) = @inbounds t.δ * ηᵐ⁺¹[i, j, k] + t.μ * t.ηᵐ[i, j, k] + t.γ * t.ηᵐ⁻¹[i, j, k] + t.ϵ * t.ηᵐ⁻²[i, j, k] -@inline advance_previous_velocities!(::ForwardBackwardScheme, i, j, k, U) = nothing +@inline advance_previous_velocities!(::ForwardBackwardScheme, i, j, k, U) = nothing +@inline advance_previous_free_surface!(::ForwardBackwardScheme, i, j, k, η) = nothing @inline function advance_previous_velocities!(t::AdamsBashforth3Scheme, i, j, k, U) @inbounds t.Uᵐ⁻²[i, j, k] = t.Uᵐ⁻¹[i, j, k] @@ -78,8 +116,6 @@ end return nothing end -@inline advance_previous_free_surface!(::ForwardBackwardScheme, i, j, k, η) = nothing - @inline function advance_previous_free_surface!(t::AdamsBashforth3Scheme, i, j, k, η) @inbounds t.ηᵐ⁻²[i, j, k] = t.ηᵐ⁻¹[i, j, k] @inbounds t.ηᵐ⁻¹[i, j, k] = t.ηᵐ[i, j, k] diff --git a/test/test_split_explicit_free_surface_solver.jl b/test/test_split_explicit_free_surface_solver.jl index 44e7023b2c..dcc7c10f46 100644 --- a/test/test_split_explicit_free_surface_solver.jl +++ b/test/test_split_explicit_free_surface_solver.jl @@ -80,7 +80,7 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: calculate_substeps, GU .= 0 GV .= 0 - weights = sefs.settings.substepping.averaging_weights + weights = sefs.substepping.averaging_weights for _ in 1:Nt iterate_split_explicit!(sefs, grid, GU, GV, Δτ, weights, Val(1)) @@ -126,10 +126,9 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: calculate_substeps, fill!(Gᵁ, 0) fill!(Gⱽ, 0) - settings = sefs.settings - Nsubsteps = calculate_substeps(settings.substepping, 1) - fractional_Δt, weights = calculate_adaptive_settings(settings.substepping, Nsubsteps) # barotropic time step in fraction of baroclinic step and averaging weights + Nsubsteps = calculate_substeps(sefs.substepping, 1) + fractional_Δt, weights = calculate_adaptive_settings(sefs.substepping, Nsubsteps) # barotropic time step in fraction of baroclinic step and averaging weights for step in 1:Nsubsteps iterate_split_explicit!(sefs, grid, GU, GV, Δτ, weights, Val(1)) @@ -191,10 +190,9 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: calculate_substeps, GU .= gu_c GV .= gv_c - settings = SplitExplicitSettings(grid; substeps = Nt + 1, averaging_kernel = constant_averaging_kernel) - sefs = sefs(settings) - - weights = settings.substepping.averaging_weights + sefs = SplitExplicitFreeSurface(grid; substeps = Nt + 1, averaging_kernel = constant_averaging_kernel) + + weights = sefs.substepping.averaging_weights for i in 1:Nt iterate_split_explicit!(sefs, grid, GU, GV, Δτ, weights, Val(1)) end diff --git a/test/test_split_explicit_vertical_integrals.jl b/test/test_split_explicit_vertical_integrals.jl index d7912e2619..0489ac5deb 100644 --- a/test/test_split_explicit_vertical_integrals.jl +++ b/test/test_split_explicit_vertical_integrals.jl @@ -16,27 +16,25 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: compute_barotropic_mode! grid = RectilinearGrid(arch, topology = topology, size = (Nx, Ny, Nz), x = (0, Lx), y = (0, Ly), z = (-Lz, 0)) - tmp = SplitExplicitFreeSurface(substeps = 200) - - sefs = SplitExplicitState(grid, tmp.settings.timestepper) - sefs = SplitExplicitAuxiliaryFields(grid) sefs = SplitExplicitFreeSurface(substeps = 200) sefs = materialize_free_surface(sefs, nothing, grid) - state = sefs.state - auxiliary = sefs.auxiliary - U, V, η̅, U̅, V̅ = state.U, state.V, state.η̅, state.U̅, state.V̅ - Gᵁ, Gⱽ = auxiliary.Gᵁ, auxiliary.Gⱽ + state = sefs.filtered_state + barotropic_velocities = sefs.barotropic_velocities + η̅, U̅, V̅ = state.η, state.U, state.V u = Field{Face, Center, Center}(grid) v = Field{Center, Face, Center}(grid) + GU = Field{Face, Center, Nothing}(grid) + GV = Field{Center, Face, Nothing}(grid) + @testset "Average to zero" begin # set equal to something else η̅ .= U̅ .= V̅ .= 1.0 # now set equal to zero - initialize_free_surface_state!(sefs.state, sefs.η, sefs.settings.timestepper) + initialize_free_surface_state!(state, sefs.η, barotropic_velocities, sefs.timestepper) # don't forget the halo points fill_halo_regions!(η̅) @@ -115,11 +113,10 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: compute_barotropic_mode! sefs = SplitExplicitFreeSurface(grid, cfl=0.7) sefs = materialize_free_surface(sefs, nothing, grid) - state = sefs.state - auxiliary = sefs.auxiliary - U, V, η̅, U̅, V̅ = state.U, state.V, state.η̅, state.U̅, state.V̅ - Gᵁ, Gⱽ = auxiliary.Gᵁ, auxiliary.Gⱽ - + state = sefs.filtered_state + U, V = sefs.barotropic_velocities + η̅, U̅, V̅ = state.η, state.U, state.V + u = Field{Face, Center, Center}(grid) v = Field{Center, Face, Center}(grid) u_corrected = similar(u) @@ -129,14 +126,14 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: compute_barotropic_mode! set_U̅(x, y, z) = cos(x) * Lz set_u_corrected(x, y, z) = z + Lz / 2 + cos(x) set!(u, set_u) - set!(U̅, set_U̅) + set!(U, set_U̅) set!(u_corrected, set_u_corrected) set_v(x, y, z) = (z + Lz / 2) * sin(y) + sin(x) set_V̅(x, y, z) = (cos(x) + x) * Lz set_v_corrected(x, y, z) = (z + Lz / 2) * sin(y) + cos(x) + x set!(v, set_v) - set!(V̅, set_V̅) + set!(V, set_V̅) set!(v_corrected, set_v_corrected) Δz = zeros(Nz) From dd9bd45a6511c32b799476a69e6027380cdf5442 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 1 Nov 2024 17:43:03 +0100 Subject: [PATCH 74/79] import with_halo --- .../distributed_split_explicit_free_surface.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/distributed_split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/distributed_split_explicit_free_surface.jl index 6afb0edb90..81c6b67fad 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/distributed_split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/distributed_split_explicit_free_surface.jl @@ -1,3 +1,4 @@ +using Oceananigans.Grids: with_halo using Oceananigans.AbstractOperations: GridMetricOperation, Δz using Oceananigans.DistributedComputations: DistributedGrid, DistributedField using Oceananigans.DistributedComputations: SynchronizedDistributed, synchronize_communication! From 1b15f915d9b2ebe95af49983ec21e9b2e911aaf6 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 1 Nov 2024 18:04:47 +0100 Subject: [PATCH 75/79] back on Enzyme --- geostrophic_adjustment.jl | 95 +++++++++++++++++++++++++++++++ test/dependencies_for_runtests.jl | 2 +- 2 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 geostrophic_adjustment.jl diff --git a/geostrophic_adjustment.jl b/geostrophic_adjustment.jl new file mode 100644 index 0000000000..8465f6fa2f --- /dev/null +++ b/geostrophic_adjustment.jl @@ -0,0 +1,95 @@ +# Run this script with +# +# $ mpiexec -n 4 julia --project distributed_nonhydrostatic_two_dimensional_turbulence.jl +# +# for example. +# +# You also probably should set +# +# $ export JULIA_NUM_THREADS=1 + +using MPI +using Oceananigans +using Oceananigans.DistributedComputations +using Oceananigans.DistributedComputations: Sizes +using Oceananigans.Grids: topology, architecture +using Oceananigans.Units: kilometers, meters +using Printf +using JLD2 + +arch = CPU() + +# Distribute problem irregularly +Nx = 80 +Lh = 100kilometers +Lz = 400meters + +topo = (Bounded, Periodic, Bounded) +grid = RectilinearGrid(arch, + size = (Nx, 3, 2), + x = (0, Lh), + y = (0, Lh), + z = (-Lz, 0), + topology = topo) + +# bottom(x, y) = x > 80kilometers && x < 90kilometers ? 100 : -500meters +# grid = ImmersedBoundaryGrid(grid, GridFittedBottom(bottom); active_cells_map = true) + +coriolis = FPlane(f=1e-4) + +model = HydrostaticFreeSurfaceModel(; grid, + coriolis, + free_surface = SplitExplicitFreeSurface(grid; substeps=10)) + +gaussian(x, L) = exp(-x^2 / 2L^2) + +U = 0.1 # geostrophic velocity +L = Lh / 40 # gaussian width +x₀ = Lh / 4 # gaussian center +vᵍ(x, y, z) = -U * (x - x₀) / L * gaussian(x - x₀, L) + +g = model.free_surface.gravitational_acceleration +η = model.free_surface.η +η₀ = coriolis.f * U * L / g # geostrophic free surface amplitude + +ηᵍ(x) = η₀ * gaussian(x - x₀, L) +ηⁱ(x, y, z) = 2 * ηᵍ(x) + +set!(model, v = vᵍ) +set!(model, η = ηⁱ) + +gravity_wave_speed = sqrt(g * grid.Lz) # hydrostatic (shallow water) gravity wave speed +Δt = 2 * model.grid.Δxᶜᵃᵃ / gravity_wave_speed +simulation = Simulation(model; Δt, stop_iteration = 1000) + +ut = [] +vt = [] +ηt = [] + +save_u(sim) = push!(ut, deepcopy(sim.model.velocities.u)) +save_v(sim) = push!(vt, deepcopy(sim.model.velocities.v)) +save_η(sim) = push!(ηt, deepcopy(sim.model.free_surface.η)) + +function progress_message(sim) + @info @sprintf("[%.2f%%], iteration: %d, time: %.3f, max|w|: %.2e", + 100 * sim.model.clock.time / sim.stop_time, sim.model.clock.iteration, + sim.model.clock.time, maximum(abs, sim.model.velocities.u)) +end + +simulation.callbacks[:save_η] = Callback(save_η, IterationInterval(1)) +simulation.callbacks[:save_v] = Callback(save_v, IterationInterval(1)) +simulation.callbacks[:save_u] = Callback(save_u, IterationInterval(1)) +simulation.callbacks[:progress] = Callback(progress_message, IterationInterval(10)) + +run!(simulation) + +iter = Observable(1) +η_img = @lift(interior(ηt[$iter], :, 1, 1)) +fig = Figure() +ax = Axis(fig[1, 1]) +lines!(ax, η_img, color=:black) + +GLMakie.record(fig, "test.mp4", 1:length(ηt), framerate=2) do i + @info "Animating iteration $i/$(length(ηt))..." + iter[] = i +end diff --git a/test/dependencies_for_runtests.jl b/test/dependencies_for_runtests.jl index 54da7bbc52..3e15e0a3d8 100644 --- a/test/dependencies_for_runtests.jl +++ b/test/dependencies_for_runtests.jl @@ -5,7 +5,7 @@ using Random using Statistics using LinearAlgebra using Logging -# using Enzyme +using Enzyme using SparseArrays using JLD2 using FFTW From 2e21d038022a0c22d2da187f76258e042e9ed6d2 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 4 Nov 2024 09:23:46 +0100 Subject: [PATCH 76/79] fix tets --- .../multi_region_split_explicit_free_surface.jl | 4 ++-- test/test_split_explicit_free_surface_solver.jl | 12 ++++++------ test/test_split_explicit_vertical_integrals.jl | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/MultiRegion/multi_region_split_explicit_free_surface.jl b/src/MultiRegion/multi_region_split_explicit_free_surface.jl index df691ae00b..6e0ba28164 100644 --- a/src/MultiRegion/multi_region_split_explicit_free_surface.jl +++ b/src/MultiRegion/multi_region_split_explicit_free_surface.jl @@ -16,10 +16,10 @@ function materialize_free_surface(free_surface::SplitExplicitFreeSurface, veloci switch_device!(grid.devices[1]) old_halos = halo_size(getregion(grid, 1)) - Nsubsteps = calculate_substeps(settings.substepping) + Nsubsteps = calculate_substeps(free_surface.substepping) extended_halos = multiregion_split_explicit_halos(old_halos, Nsubsteps+1, grid.partition) - extended_grid = with_halo(new_halos, grid) + extended_grid = with_halo(extended_halos, grid) η = free_surface_displacement_field(velocities, free_surface, extended_grid) η̅ = free_surface_displacement_field(velocities, free_surface, extended_grid) diff --git a/test/test_split_explicit_free_surface_solver.jl b/test/test_split_explicit_free_surface_solver.jl index dcc7c10f46..21090b004c 100644 --- a/test/test_split_explicit_free_surface_solver.jl +++ b/test/test_split_explicit_free_surface_solver.jl @@ -1,12 +1,12 @@ include("dependencies_for_runtests.jl") using Oceananigans.Models.HydrostaticFreeSurfaceModels -using Oceananigans.Models.HydrostaticFreeSurfaceModels: calculate_substeps, - calculate_adaptive_settings, - constant_averaging_kernel, - materialize_free_surface, - SplitExplicitFreeSurface, - iterate_split_explicit! +using Oceananigans.Models.HydrostaticFreeSurfaceModels.SplitExplicitFreeSurfaces: calculate_substeps, + calculate_adaptive_settings, + constant_averaging_kernel, + materialize_free_surface, + SplitExplicitFreeSurface, + iterate_split_explicit! @testset "Split-Explicit Dynamics" begin diff --git a/test/test_split_explicit_vertical_integrals.jl b/test/test_split_explicit_vertical_integrals.jl index 0489ac5deb..2df00eb194 100644 --- a/test/test_split_explicit_vertical_integrals.jl +++ b/test/test_split_explicit_vertical_integrals.jl @@ -2,9 +2,9 @@ include("dependencies_for_runtests.jl") using Oceananigans.Models.HydrostaticFreeSurfaceModels using Oceananigans.Models.HydrostaticFreeSurfaceModels: SplitExplicitFreeSurface -using Oceananigans.Models.HydrostaticFreeSurfaceModels: compute_barotropic_mode!, - barotropic_split_explicit_corrector!, - initialize_free_surface_state! +using Oceananigans.Models.HydrostaticFreeSurfaceModels.SplitExplicitFreeSurfaces: compute_barotropic_mode!, + barotropic_split_explicit_corrector!, + initialize_free_surface_state! @testset "Barotropic Kernels" begin From 07cf9abf0155167992b9ea3997645b440f1ace87 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 4 Nov 2024 10:51:09 +0100 Subject: [PATCH 77/79] some fixes --- ...distributed_split_explicit_free_surface.jl | 2 ++ .../split_explicit_free_surface.jl | 10 ++++---- .../step_split_explicit_free_surface.jl | 5 ++-- ...te_hydrostatic_free_surface_model_state.jl | 25 ++++++++++++++++++- 4 files changed, 34 insertions(+), 8 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/distributed_split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/distributed_split_explicit_free_surface.jl index 81c6b67fad..3bd3f97d72 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/distributed_split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/distributed_split_explicit_free_surface.jl @@ -91,6 +91,8 @@ wait_free_surface_communication!(::DistributedSplitExplicit, model, ::Synchroniz function wait_free_surface_communication!(free_surface::DistributedSplitExplicit, model, arch) + barotropic_velocities = free_surface.barotropic_velocities + for field in (barotropic_velocities.U, barotropic_velocities.V) synchronize_communication!(field) end diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_free_surface.jl index dfa09cf828..147671b4e9 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_free_surface.jl @@ -135,11 +135,11 @@ end @inline north_barotropic_bc(baroclinic_velocity) = baroclinic_velocity.boundary_conditions.north @inline barotropic_bc(baroclinic_velocity) = FieldBoundaryConditions( - west = west_barotropic_bc(baroclinic_velocity), - east = east_barotropic_bc(baroclinic_velocity), - south = south_barotropic_bc(baroclinic_velocity), - north = north_barotropic_bc(baroclinic_velocity), - top = nothing, + west = west_barotropic_bc(baroclinic_velocity), + east = east_barotropic_bc(baroclinic_velocity), + south = south_barotropic_bc(baroclinic_velocity), + north = north_barotropic_bc(baroclinic_velocity), + top = nothing, bottom = nothing ) diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/step_split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/step_split_explicit_free_surface.jl index b31b2367d9..550c05e439 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/step_split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/step_split_explicit_free_surface.jl @@ -11,6 +11,7 @@ using Oceananigans.ImmersedBoundaries: mask_immersed_field!, retrieve_surface_ac using Oceananigans.ImmersedBoundaries: active_linear_index_to_tuple, ActiveCellsIBG, ActiveZColumnsIBG using Oceananigans.DistributedComputations: child_architecture using Oceananigans.DistributedComputations: Distributed +using Oceananigans: fields using Printf using KernelAbstractions: @index, @kernel @@ -164,8 +165,8 @@ function split_explicit_free_surface_step!(free_surface::SplitExplicitFreeSurfac parent(velocities.U) .= parent(filtered_state.U) parent(velocities.V) .= parent(filtered_state.V) - fields_to_fill = (velocities.U, velocities.V, free_surface.η) # TODO: do this? - fill_halo_regions!(fields_to_fill; async = true) + fields_to_fill = (velocities.U, velocities.V, free_surface.η) + fill_halo_regions!(fields_to_fill, model.clock, fields(model); async = true) # Preparing velocities for the barotropic correction @apply_regionally begin diff --git a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl index cf6b20eade..64d2f36c33 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl @@ -38,7 +38,9 @@ function update_state!(model::HydrostaticFreeSurfaceModel, grid, callbacks; comp # Update the boundary conditions @apply_regionally update_boundary_condition!(fields(model), model) - fill_halo_regions!(prognostic_fields(model), model.clock, fields(model); async = true) + # Depends on which free surface model is used + update_model_halos!(model, free_surface) + @apply_regionally replace_horizontal_vector_halos!(model.velocities, model.grid) @apply_regionally compute_auxiliaries!(model) @@ -54,6 +56,27 @@ function update_state!(model::HydrostaticFreeSurfaceModel, grid, callbacks; comp return nothing end +function update_model_halos!(model, free_surface) + u, v, _ = model.velocities + η = free_surface.η + + fields_to_fill = merge((; u, v, η), model.tracers) + + fill_halo_regions!(fields_to_fill, model.clock, fields(model); async = true) + + return nothing +end + +function update_model_halos!(model, ::SplitExplicitFreeSurface) + u, v, _ = model.velocities + + fields_to_fill = merge((; u, v), model.tracers) + + fill_halo_regions!(fields_to_fill, model.clock, fields(model); async = true) + + return nothing +end + # Mask immersed fields function mask_immersed_model_fields!(model, grid) η = displacement(model.free_surface) From bc3833e1e4e53df8832d9acfbbf989b49157bf3d Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 4 Nov 2024 12:56:48 +0100 Subject: [PATCH 78/79] these are prognostic fields --- .../step_split_explicit_free_surface.jl | 4 +--- ...te_hydrostatic_free_surface_model_state.jl | 24 +------------------ 2 files changed, 2 insertions(+), 26 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/step_split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/step_split_explicit_free_surface.jl index 550c05e439..0dc5df93b9 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/step_split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/step_split_explicit_free_surface.jl @@ -161,13 +161,11 @@ function split_explicit_free_surface_step!(free_surface::SplitExplicitFreeSurfac end # Reset eta and velocities for the next timestep + # The halos are updated in the `update_state!` function parent(free_surface.η) .= parent(filtered_state.η) parent(velocities.U) .= parent(filtered_state.U) parent(velocities.V) .= parent(filtered_state.V) - fields_to_fill = (velocities.U, velocities.V, free_surface.η) - fill_halo_regions!(fields_to_fill, model.clock, fields(model); async = true) - # Preparing velocities for the barotropic correction @apply_regionally begin mask_immersed_field!(model.velocities.u) diff --git a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl index 64d2f36c33..89aa0d6ab8 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl @@ -38,8 +38,7 @@ function update_state!(model::HydrostaticFreeSurfaceModel, grid, callbacks; comp # Update the boundary conditions @apply_regionally update_boundary_condition!(fields(model), model) - # Depends on which free surface model is used - update_model_halos!(model, free_surface) + fill_halo_regions!(prognostic_fields(model), model.clock, fields(model); async = true) @apply_regionally replace_horizontal_vector_halos!(model.velocities, model.grid) @apply_regionally compute_auxiliaries!(model) @@ -56,27 +55,6 @@ function update_state!(model::HydrostaticFreeSurfaceModel, grid, callbacks; comp return nothing end -function update_model_halos!(model, free_surface) - u, v, _ = model.velocities - η = free_surface.η - - fields_to_fill = merge((; u, v, η), model.tracers) - - fill_halo_regions!(fields_to_fill, model.clock, fields(model); async = true) - - return nothing -end - -function update_model_halos!(model, ::SplitExplicitFreeSurface) - u, v, _ = model.velocities - - fields_to_fill = merge((; u, v), model.tracers) - - fill_halo_regions!(fields_to_fill, model.clock, fields(model); async = true) - - return nothing -end - # Mask immersed fields function mask_immersed_model_fields!(model, grid) η = displacement(model.free_surface) From 581848bd2831c89c83a94cfba8be9c728072ebc3 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 4 Nov 2024 12:57:44 +0100 Subject: [PATCH 79/79] comment --- .../step_split_explicit_free_surface.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/step_split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/step_split_explicit_free_surface.jl index 0dc5df93b9..14501e12a7 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/step_split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/step_split_explicit_free_surface.jl @@ -136,7 +136,7 @@ function split_explicit_free_surface_step!(free_surface::SplitExplicitFreeSurfac timestepper = free_surface.timestepper velocities = free_surface.barotropic_velocities - # Wait for previous set up + # Wait for setup step to finish wait_free_surface_communication!(free_surface, model, architecture(free_surface_grid)) # Calculate the substepping parameterers