From 2cd21fe14a97997a91cce7010631155f41cdb7db Mon Sep 17 00:00:00 2001 From: Jago Stong-Wright Date: Wed, 13 Mar 2024 12:00:22 -0400 Subject: [PATCH 001/113] Generalised `Open` boundary condition --- src/BoundaryConditions/boundary_condition.jl | 4 ++-- .../boundary_condition_classifications.jl | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/BoundaryConditions/boundary_condition.jl b/src/BoundaryConditions/boundary_condition.jl index 86a81cbf25..45167daebc 100644 --- a/src/BoundaryConditions/boundary_condition.jl +++ b/src/BoundaryConditions/boundary_condition.jl @@ -93,14 +93,14 @@ const DCBC = BoundaryCondition{<:DistributedCommunication} # More readable BC constructors for the public API. PeriodicBoundaryCondition() = BoundaryCondition(Periodic, nothing) NoFluxBoundaryCondition() = BoundaryCondition(Flux, nothing) - ImpenetrableBoundaryCondition() = BoundaryCondition(Open, nothing) + ImpenetrableBoundaryCondition() = BoundaryCondition(Open{nothing}, nothing) MultiRegionCommunicationBoundaryCondition() = BoundaryCondition(MultiRegionCommunication, nothing) DistributedCommunicationBoundaryCondition() = BoundaryCondition(DistributedCommunication, nothing) FluxBoundaryCondition(val; kwargs...) = BoundaryCondition(Flux, val; kwargs...) ValueBoundaryCondition(val; kwargs...) = BoundaryCondition(Value, val; kwargs...) GradientBoundaryCondition(val; kwargs...) = BoundaryCondition(Gradient, val; kwargs...) - OpenBoundaryCondition(val; kwargs...) = BoundaryCondition(Open, val; kwargs...) + OpenBoundaryCondition(val; kwargs...) = BoundaryCondition(Open{nothing}, val; kwargs...) MultiRegionCommunicationBoundaryCondition(val; kwargs...) = BoundaryCondition(MultiRegionCommunication, val; kwargs...) DistributedCommunicationBoundaryCondition(val; kwargs...) = BoundaryCondition(DistributedCommunication, val; kwargs...) diff --git a/src/BoundaryConditions/boundary_condition_classifications.jl b/src/BoundaryConditions/boundary_condition_classifications.jl index d3d97bb685..957cb148b5 100644 --- a/src/BoundaryConditions/boundary_condition_classifications.jl +++ b/src/BoundaryConditions/boundary_condition_classifications.jl @@ -59,7 +59,11 @@ For fields located at `Faces`, `Open` also specifies field value _on_ the bounda Open boundary conditions are used to specify the component of a velocity field normal to a boundary and can also be used to describe nested or linked simulation domains. """ -struct Open <: AbstractBoundaryConditionClassification end +struct Open{MS} <: AbstractBoundaryConditionClassification + matching_scheme :: MS +end + +Open{MS}() where MS = Open(MS) """ struct MultiRegionCommunication <: AbstractBoundaryConditionClassification From 2d360909908256a161eac36e283c65e922d092f2 Mon Sep 17 00:00:00 2001 From: Jago Stong-Wright Date: Mon, 18 Mar 2024 16:26:07 -0400 Subject: [PATCH 002/113] `adapt` seems to be defined twice for boundary conditions --- src/BoundaryConditions/boundary_condition.jl | 15 ++++---- .../boundary_condition_classifications.jl | 6 ++-- .../show_boundary_conditions.jl | 36 +++++++++---------- 3 files changed, 27 insertions(+), 30 deletions(-) diff --git a/src/BoundaryConditions/boundary_condition.jl b/src/BoundaryConditions/boundary_condition.jl index 45167daebc..ac3f686e4a 100644 --- a/src/BoundaryConditions/boundary_condition.jl +++ b/src/BoundaryConditions/boundary_condition.jl @@ -67,13 +67,13 @@ function BoundaryCondition(Classification::DataType, condition::Function; end # Adapt boundary condition struct to be GPU friendly and passable to GPU kernels. -Adapt.adapt_structure(to, b::BoundaryCondition{Classification}) where Classification = - BoundaryCondition(Classification(), Adapt.adapt(to, b.condition)) +Adapt.adapt_structure(to, b::BoundaryCondition) = + BoundaryCondition(Adapt.adapt(b.classifction), Adapt.adapt(to, b.condition)) # Adapt boundary condition struct to be GPU friendly and passable to GPU kernels. -on_architecture(to, b::BoundaryCondition{Classification}) where Classification = - BoundaryCondition(Classification(), on_architecture(to, b.condition)) +on_architecture(to, b::BoundaryCondition) = + BoundaryCondition(on_architecture(to, b.classification), on_architecture(to, b.condition)) ##### ##### Some abbreviations to make life easier. @@ -93,14 +93,14 @@ const DCBC = BoundaryCondition{<:DistributedCommunication} # More readable BC constructors for the public API. PeriodicBoundaryCondition() = BoundaryCondition(Periodic, nothing) NoFluxBoundaryCondition() = BoundaryCondition(Flux, nothing) - ImpenetrableBoundaryCondition() = BoundaryCondition(Open{nothing}, nothing) + ImpenetrableBoundaryCondition() = BoundaryCondition(Open{Nothing}, nothing) MultiRegionCommunicationBoundaryCondition() = BoundaryCondition(MultiRegionCommunication, nothing) DistributedCommunicationBoundaryCondition() = BoundaryCondition(DistributedCommunication, nothing) FluxBoundaryCondition(val; kwargs...) = BoundaryCondition(Flux, val; kwargs...) ValueBoundaryCondition(val; kwargs...) = BoundaryCondition(Value, val; kwargs...) GradientBoundaryCondition(val; kwargs...) = BoundaryCondition(Gradient, val; kwargs...) - OpenBoundaryCondition(val; kwargs...) = BoundaryCondition(Open{nothing}, val; kwargs...) + OpenBoundaryCondition(val; kwargs...) = BoundaryCondition(Open{Nothing}, val; kwargs...) MultiRegionCommunicationBoundaryCondition(val; kwargs...) = BoundaryCondition(MultiRegionCommunication, val; kwargs...) DistributedCommunicationBoundaryCondition(val; kwargs...) = BoundaryCondition(DistributedCommunication, val; kwargs...) @@ -125,9 +125,6 @@ DistributedCommunicationBoundaryCondition(val; kwargs...) = BoundaryCondition(Di const NumberRef = Base.RefValue{<:Number} @inline getbc(bc::BC{<:Any, <:NumberRef}, args...) = bc.condition[] -Adapt.adapt_structure(to, bc::BoundaryCondition) = BoundaryCondition(Adapt.adapt(to, bc.classification), - Adapt.adapt(to, bc.condition)) - ##### ##### Validation with topology ##### diff --git a/src/BoundaryConditions/boundary_condition_classifications.jl b/src/BoundaryConditions/boundary_condition_classifications.jl index 957cb148b5..ea83557498 100644 --- a/src/BoundaryConditions/boundary_condition_classifications.jl +++ b/src/BoundaryConditions/boundary_condition_classifications.jl @@ -59,11 +59,11 @@ For fields located at `Faces`, `Open` also specifies field value _on_ the bounda Open boundary conditions are used to specify the component of a velocity field normal to a boundary and can also be used to describe nested or linked simulation domains. """ -struct Open{MS} <: AbstractBoundaryConditionClassification - matching_scheme :: MS +struct Open{MS} <: AbstractBoundaryConditionClassification + matching_scheme::MS end -Open{MS}() where MS = Open(MS) +Open{ms}() where ms = Open(ms()) """ struct MultiRegionCommunication <: AbstractBoundaryConditionClassification diff --git a/src/BoundaryConditions/show_boundary_conditions.jl b/src/BoundaryConditions/show_boundary_conditions.jl index e6ab12a6bd..8752b4669d 100644 --- a/src/BoundaryConditions/show_boundary_conditions.jl +++ b/src/BoundaryConditions/show_boundary_conditions.jl @@ -4,29 +4,29 @@ using Oceananigans.Utils: prettysummary const DFBC = DefaultBoundaryCondition const IBC = BoundaryCondition{Open, Nothing} # ImpenetrableBoundaryCondition -bc_str(::FBC) = "Flux" -bc_str(::PBC) = "Periodic" -bc_str(::OBC) = "Open" -bc_str(::VBC) = "Value" -bc_str(::GBC) = "Gradient" -bc_str(::ZFBC) = "ZeroFlux" -bc_str(::IBC) = "Impenetrable" -bc_str(::DFBC) = "Default" -bc_str(::MCBC) = "MultiRegionCommunication" -bc_str(::DCBC) = "DistributedCommunication" -bc_str(::Nothing) = "Nothing" +bc_str(::FBC) = "Flux" +bc_str(::PBC) = "Periodic" +bc_str(::OBC{Open{MS}}) where MS = "Open{$MS}" +bc_str(::VBC) = "Value" +bc_str(::GBC) = "Gradient" +bc_str(::ZFBC) = "ZeroFlux" +bc_str(::IBC) = "Impenetrable" +bc_str(::DFBC) = "Default" +bc_str(::MCBC) = "MultiRegionCommunication" +bc_str(::DCBC) = "DistributedCommunication" +bc_str(::Nothing) = "Nothing" ##### ##### BoundaryCondition ##### -Base.summary(bc::DFBC) = string("DefaultBoundaryCondition (", summary(bc.boundary_condition), ")") -Base.summary(bc::OBC) = string("OpenBoundaryCondition: ", prettysummary(bc.condition)) -Base.summary(bc::FBC) = string("FluxBoundaryCondition: ", prettysummary(bc.condition)) -Base.summary(bc::VBC) = string("ValueBoundaryCondition: ", prettysummary(bc.condition)) -Base.summary(bc::GBC) = string("GradientBoundaryCondition: ", prettysummary(bc.condition)) -Base.summary(::PBC) = string("PeriodicBoundaryCondition") -Base.summary(bc::DCBC) = string("DistributedBoundaryCondition: ", prettysummary(bc.condition)) +Base.summary(bc::DFBC) = string("DefaultBoundaryCondition (", summary(bc.boundary_condition), ")") +Base.summary(bc::OBC{Open{MS}}) where MS = string("OpenBoundaryCondition{$MS}: ", prettysummary(bc.condition)) +Base.summary(bc::FBC) = string("FluxBoundaryCondition: ", prettysummary(bc.condition)) +Base.summary(bc::VBC) = string("ValueBoundaryCondition: ", prettysummary(bc.condition)) +Base.summary(bc::GBC) = string("GradientBoundaryCondition: ", prettysummary(bc.condition)) +Base.summary(::PBC) = string("PeriodicBoundaryCondition") +Base.summary(bc::DCBC) = string("DistributedBoundaryCondition: ", prettysummary(bc.condition)) show(io::IO, bc::BoundaryCondition) = print(io, summary(bc)) From 581159f46e7b05477505a04f1d69a60b42b0f5d1 Mon Sep 17 00:00:00 2001 From: Jago Stong-Wright Date: Mon, 18 Mar 2024 16:32:12 -0400 Subject: [PATCH 003/113] added internal wave example --- .../open_boundaries/mode_1_internal_wave.jl | 108 ++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 validation/open_boundaries/mode_1_internal_wave.jl diff --git a/validation/open_boundaries/mode_1_internal_wave.jl b/validation/open_boundaries/mode_1_internal_wave.jl new file mode 100644 index 0000000000..74a4b3eee0 --- /dev/null +++ b/validation/open_boundaries/mode_1_internal_wave.jl @@ -0,0 +1,108 @@ +using Oceananigans +using Oceananigans.Fields: FunctionField + +# TODO: clean up all the global variables in this script + +h = 10 +k = π/h + +λ = 2π / k + +grid = RectilinearGrid(topology = (Bounded, Flat, Bounded), size = (256, 128), extent = (3*λ, h)) + +xc, yc, zf = nodes(grid, Center(), Center(), Face()) +xf, yc, zc = nodes(grid, Face(), Center(), Center()) + +N² = 0.2 +w₀ = .03 + +ω = @show sqrt(k^2 * N² / (k^2 + (π/h)^2)) + +u(x, z, t) = w₀ * π / (k * h) * cos(π * z / h) * cos(k * x - ω * t) + +w(x, z, t) = w₀ * sin(π * z / h) * sin(k * x - ω * t) + +b(x, z, t) = - N² * w₀ / ω * sin(π * z / h) * cos(k * x - ω * t) + N² * z + +@inline u_east(z, t) = u(xf[grid.Nx], z, t) +@inline u_west(z, t) = u(0, z, t) + +@inline w_east(z, t) = w(xc[grid.Nx], z, t) +@inline w_west(z, t) = w(0, z, t) + +@inline b_east(z, t) = b(xc[grid.Nx], z, t) +@inline b_west(z, t) = b(0, z, t) + +u_boundaries = FieldBoundaryConditions(east = OpenBoundaryCondition(u_east), + west = OpenBoundaryCondition(u_west)) + +w_boundaries = FieldBoundaryConditions(east = OpenBoundaryCondition(w_east), + west = OpenBoundaryCondition(w_west)) + +b_boundaries = FieldBoundaryConditions(east = OpenBoundaryCondition(b_east), + west = OpenBoundaryCondition(b_west)) + +sponge_thickness = λ/2 + +mask(x, z) = ((sponge_thickness - x) / sponge_thickness) * (x < sponge_thickness) + ((x - grid.Lx + sponge_thickness) / sponge_thickness) * (x > grid.Lx - sponge_thickness) + +cₚ = ω/k + +cᵣ = 4 * (w₀ * π / (k * h) + cₚ) + +u_forcing = Relaxation(; rate = cᵣ / sponge_thickness, mask, target = u) +w_forcing = Relaxation(; rate = cᵣ / sponge_thickness, mask, target = w) +b_forcing = Relaxation(; rate = cᵣ / sponge_thickness, mask, target = b) + +model = NonhydrostaticModel(; grid, + forcing = (; u = u_forcing, w = w_forcing, b = b_forcing), + boundary_conditions = (; u = u_boundaries, w = w_boundaries, b = b_boundaries), + advection = WENO(; order = 3), + buoyancy = BuoyancyTracer(), + tracers = :b) + +set!(model, u = (x, z) -> u(x, z, 0), w = (x, z) -> w(x, z, 0), b = (x, z) -> b(x, z, 0)) + +simulation = Simulation(model, Δt = 0.1, stop_time = 400) + +simulation.output_writers[:velocities] = JLD2OutputWriter(model, + merge(model.velocities, model.tracers), + filename = "internal_wave.jld2", + schedule = TimeInterval(0.5), + overwrite_existing=true) + +prog(sim) = @info prettytime(time(sim))*" with Δt = "*prettytime(sim.Δt) + +simulation.callbacks[:progress] = Callback(prog, TimeInterval(5)) + +conjure_time_step_wizard!(simulation, cfl=0.5) + +run!(simulation) + +using CairoMakie + +fig = Figure(); + +n = Observable(1) + +u_ts = FieldTimeSeries("internal_wave.jld2", "u"); +w_ts = FieldTimeSeries("internal_wave.jld2", "w"); +b_ts = FieldTimeSeries("internal_wave.jld2", "b"); + +u_plt = @lift interior(u_ts[$n], :, 1, :) +w_plt = @lift interior(w_ts[$n], :, 1, :) +b_plt = @lift interior(b_ts[$n], :, 1, :) .- [N² * z for x in xc, z in zc] + +title = @lift "t = $(u_ts.times[$n])" + +ax = Axis(fig[1, 1], aspect = DataAspect(); title) +ax2 = Axis(fig[2, 1], aspect = DataAspect()) +ax3 = Axis(fig[3, 1], aspect = DataAspect()) + +hm = heatmap!(ax, xf./h, zc./h, u_plt, colorrange = w₀ * π / (k * h).*(-1, 1), colormap = Reverse(:roma)) +hm2 = heatmap!(ax2, xc./h, zf./h, w_plt, colorrange = w₀.*(-1, 1), colormap = Reverse(:roma)) +hm3 = heatmap!(ax3, xc./h, zc./h, b_plt, colorrange = N² * w₀ / ω.*(-1, 1), colormap = Reverse(:roma)) + +record(fig, "internal_wave.mp4", 1:length(u_ts.times); framerate = 20) do i; + n[] = i +end \ No newline at end of file From dea4e2b4f1a1ae63cf13fe079d99e187bda2ee84 Mon Sep 17 00:00:00 2001 From: Jago Stong-Wright Date: Tue, 19 Mar 2024 12:38:15 -0400 Subject: [PATCH 004/113] corrected spelling in adapt --- src/BoundaryConditions/boundary_condition.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BoundaryConditions/boundary_condition.jl b/src/BoundaryConditions/boundary_condition.jl index ac3f686e4a..166b2d34ae 100644 --- a/src/BoundaryConditions/boundary_condition.jl +++ b/src/BoundaryConditions/boundary_condition.jl @@ -68,7 +68,7 @@ end # Adapt boundary condition struct to be GPU friendly and passable to GPU kernels. Adapt.adapt_structure(to, b::BoundaryCondition) = - BoundaryCondition(Adapt.adapt(b.classifction), Adapt.adapt(to, b.condition)) + BoundaryCondition(Adapt.adapt(b.classification), Adapt.adapt(to, b.condition)) # Adapt boundary condition struct to be GPU friendly and passable to GPU kernels. From d7ec52243bc60cc7b7897e58c52976fe13fa5b52 Mon Sep 17 00:00:00 2001 From: Jago Stong-Wright Date: Tue, 19 Mar 2024 16:20:15 -0400 Subject: [PATCH 005/113] mistake --- src/BoundaryConditions/boundary_condition.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BoundaryConditions/boundary_condition.jl b/src/BoundaryConditions/boundary_condition.jl index 166b2d34ae..e625d69e43 100644 --- a/src/BoundaryConditions/boundary_condition.jl +++ b/src/BoundaryConditions/boundary_condition.jl @@ -68,7 +68,7 @@ end # Adapt boundary condition struct to be GPU friendly and passable to GPU kernels. Adapt.adapt_structure(to, b::BoundaryCondition) = - BoundaryCondition(Adapt.adapt(b.classification), Adapt.adapt(to, b.condition)) + BoundaryCondition(Adapt.adapt(to, b.classification), Adapt.adapt(to, b.condition)) # Adapt boundary condition struct to be GPU friendly and passable to GPU kernels. From 98cc3b4ae99235c86672408470cac427b48b076b Mon Sep 17 00:00:00 2001 From: Jago Stong-Wright Date: Thu, 21 Mar 2024 15:18:46 -0400 Subject: [PATCH 006/113] corrected syntax --- test/test_boundary_conditions_integration.jl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/test_boundary_conditions_integration.jl b/test/test_boundary_conditions_integration.jl index d817dc9180..7ac312fd09 100644 --- a/test/test_boundary_conditions_integration.jl +++ b/test/test_boundary_conditions_integration.jl @@ -127,19 +127,19 @@ test_boundary_conditions(C, FT, ArrayType) = (integer_bc(C, FT, ArrayType), top = simple_function_bc(Value), north = simple_function_bc(Value), south = simple_function_bc(Value), - east = simple_function_bc(Open), - west = simple_function_bc(Open)) + east = simple_function_bc(Open{Nothing}), + west = simple_function_bc(Open{Nothing})) v_boundary_conditions = FieldBoundaryConditions(bottom = simple_function_bc(Value), top = simple_function_bc(Value), - north = simple_function_bc(Open), - south = simple_function_bc(Open), + north = simple_function_bc(Open{Nothing}), + south = simple_function_bc(Open{Nothing}), east = simple_function_bc(Value), west = simple_function_bc(Value)) - w_boundary_conditions = FieldBoundaryConditions(bottom = simple_function_bc(Open), - top = simple_function_bc(Open), + w_boundary_conditions = FieldBoundaryConditions(bottom = simple_function_bc(Open{Nothing}), + top = simple_function_bc(Open{Nothing}), north = simple_function_bc(Value), south = simple_function_bc(Value), east = simple_function_bc(Value), @@ -203,7 +203,7 @@ test_boundary_conditions(C, FT, ArrayType) = (integer_bc(C, FT, ArrayType), @test test_boundary_condition(arch, FT, topo, :top, :T, boundary_condition) end - for boundary_condition in test_boundary_conditions(Open, FT, array_type(arch)) + for boundary_condition in test_boundary_conditions(Open{Nothing}, FT, array_type(arch)) @test test_boundary_condition(arch, FT, topo, :east, :u, boundary_condition) @test test_boundary_condition(arch, FT, topo, :south, :v, boundary_condition) @test test_boundary_condition(arch, FT, topo, :top, :w, boundary_condition) From 14f795859cf5e5ca8f9a07572ce0ca57a91f450f Mon Sep 17 00:00:00 2001 From: Jago Stong-Wright Date: Tue, 26 Mar 2024 18:24:34 -0400 Subject: [PATCH 007/113] changed validation name --- .../{mode_1_internal_wave.jl => mode_one_internal_wave.jl} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename validation/open_boundaries/{mode_1_internal_wave.jl => mode_one_internal_wave.jl} (100%) diff --git a/validation/open_boundaries/mode_1_internal_wave.jl b/validation/open_boundaries/mode_one_internal_wave.jl similarity index 100% rename from validation/open_boundaries/mode_1_internal_wave.jl rename to validation/open_boundaries/mode_one_internal_wave.jl From ec5b515ee50aa5554df31dfbb1498e2335b05603 Mon Sep 17 00:00:00 2001 From: Jago Stong-Wright Date: Tue, 26 Mar 2024 18:24:42 -0400 Subject: [PATCH 008/113] clarified sponge --- validation/open_boundaries/mode_one_internal_wave.jl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/validation/open_boundaries/mode_one_internal_wave.jl b/validation/open_boundaries/mode_one_internal_wave.jl index 74a4b3eee0..cb663c7ae1 100644 --- a/validation/open_boundaries/mode_one_internal_wave.jl +++ b/validation/open_boundaries/mode_one_internal_wave.jl @@ -42,9 +42,12 @@ w_boundaries = FieldBoundaryConditions(east = OpenBoundaryCondition(w_east), b_boundaries = FieldBoundaryConditions(east = OpenBoundaryCondition(b_east), west = OpenBoundaryCondition(b_west)) -sponge_thickness = λ/2 +const sponge_thickness = λ/2 -mask(x, z) = ((sponge_thickness - x) / sponge_thickness) * (x < sponge_thickness) + ((x - grid.Lx + sponge_thickness) / sponge_thickness) * (x > grid.Lx - sponge_thickness) +left_sponge(x) = ((sponge_thickness - x) / sponge_thickness) * (x < sponge_thickness) +right_sponge(x) = ((x - grid.Lx + sponge_thickness) / sponge_thickness) * (x > grid.Lx - sponge_thickness) + +mask(x, z) = left_sponge(x) + right_sponge(x) cₚ = ω/k From 3aa6b6ba1598288042b03b2fe7ed4baab50d405e Mon Sep 17 00:00:00 2001 From: Jago Stong-Wright Date: Thu, 28 Mar 2024 13:03:14 -0400 Subject: [PATCH 009/113] fixed docs --- docs/src/model_setup/boundary_conditions.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/model_setup/boundary_conditions.md b/docs/src/model_setup/boundary_conditions.md index 9ead613e45..5bcfb49f17 100644 --- a/docs/src/model_setup/boundary_conditions.md +++ b/docs/src/model_setup/boundary_conditions.md @@ -91,8 +91,8 @@ julia> model.velocities.v.boundary_conditions Oceananigans.FieldBoundaryConditions, with boundary conditions ├── west: PeriodicBoundaryCondition ├── east: PeriodicBoundaryCondition -├── south: OpenBoundaryCondition: Nothing -├── north: OpenBoundaryCondition: Nothing +├── south: OpenBoundaryCondition{Nothing}: Nothing +├── north: OpenBoundaryCondition{Nothing}: Nothing ├── bottom: ValueBoundaryCondition: 0.0 ├── top: FluxBoundaryCondition: Nothing └── immersed: FluxBoundaryCondition: Nothing From 8892a7fec83c84c1d02dbac7960a6335eb085238 Mon Sep 17 00:00:00 2001 From: Jago Stong-Wright Date: Thu, 28 Mar 2024 14:00:13 -0400 Subject: [PATCH 010/113] a slightly less ugly way of passing around boundary condition classifcations and keeping their properties --- src/BoundaryConditions/boundary_condition.jl | 10 ++- .../boundary_condition_classifications.jl | 2 +- .../continuous_boundary_function.jl | 2 +- .../open_boundary_matching_schemes.jl | 64 +++++++++++++++++++ 4 files changed, 73 insertions(+), 5 deletions(-) create mode 100644 src/BoundaryConditions/open_boundary_matching_schemes.jl diff --git a/src/BoundaryConditions/boundary_condition.jl b/src/BoundaryConditions/boundary_condition.jl index e625d69e43..da3838cc03 100644 --- a/src/BoundaryConditions/boundary_condition.jl +++ b/src/BoundaryConditions/boundary_condition.jl @@ -50,7 +50,7 @@ where `i`, and `j` are indices that vary along the boundary. If `discrete_form = condition(i, j, grid, clock, model_fields, parameters) ``` """ -function BoundaryCondition(Classification::DataType, condition::Function; +function BoundaryCondition(Classification::Union{DataType, Open}, condition::Function; parameters = nothing, discrete_form = false, field_dependencies=()) @@ -93,17 +93,21 @@ const DCBC = BoundaryCondition{<:DistributedCommunication} # More readable BC constructors for the public API. PeriodicBoundaryCondition() = BoundaryCondition(Periodic, nothing) NoFluxBoundaryCondition() = BoundaryCondition(Flux, nothing) - ImpenetrableBoundaryCondition() = BoundaryCondition(Open{Nothing}, nothing) + ImpenetrableBoundaryCondition() = BoundaryCondition(Open(Nothing), nothing) MultiRegionCommunicationBoundaryCondition() = BoundaryCondition(MultiRegionCommunication, nothing) DistributedCommunicationBoundaryCondition() = BoundaryCondition(DistributedCommunication, nothing) FluxBoundaryCondition(val; kwargs...) = BoundaryCondition(Flux, val; kwargs...) ValueBoundaryCondition(val; kwargs...) = BoundaryCondition(Value, val; kwargs...) GradientBoundaryCondition(val; kwargs...) = BoundaryCondition(Gradient, val; kwargs...) - OpenBoundaryCondition(val; kwargs...) = BoundaryCondition(Open{Nothing}, val; kwargs...) + OpenBoundaryCondition(val; kwargs...) = BoundaryCondition(Open(Nothing), val; kwargs...) MultiRegionCommunicationBoundaryCondition(val; kwargs...) = BoundaryCondition(MultiRegionCommunication, val; kwargs...) DistributedCommunicationBoundaryCondition(val; kwargs...) = BoundaryCondition(DistributedCommunication, val; kwargs...) +# Open boundary constructors +NudgingOpenBoundaryCondition(val, inflow_nudging_timescale, outflow_nudging_timescale; kwargs...) = + BoundaryCondition(Open(Nudging(inflow_nudging_timescale, outflow_nudging_timescale)), val; kwargs...) + # Support for various types of boundary conditions. # # Notes: diff --git a/src/BoundaryConditions/boundary_condition_classifications.jl b/src/BoundaryConditions/boundary_condition_classifications.jl index ea83557498..f59ec06cb8 100644 --- a/src/BoundaryConditions/boundary_condition_classifications.jl +++ b/src/BoundaryConditions/boundary_condition_classifications.jl @@ -63,7 +63,7 @@ struct Open{MS} <: AbstractBoundaryConditionClassification matching_scheme::MS end -Open{ms}() where ms = Open(ms()) +(open::Open)() = open """ struct MultiRegionCommunication <: AbstractBoundaryConditionClassification diff --git a/src/BoundaryConditions/continuous_boundary_function.jl b/src/BoundaryConditions/continuous_boundary_function.jl index 3023dbd2ac..922739641b 100644 --- a/src/BoundaryConditions/continuous_boundary_function.jl +++ b/src/BoundaryConditions/continuous_boundary_function.jl @@ -88,7 +88,7 @@ function regularize_boundary_condition(bc::BoundaryCondition{C, <:ContinuousBoun boundary_func.field_dependencies, indices, interps) - return BoundaryCondition(C, regularized_boundary_func) + return BoundaryCondition(bc.classification, regularized_boundary_func) end @inline domain_boundary_indices(::LeftBoundary, N) = 1, 1 diff --git a/src/BoundaryConditions/open_boundary_matching_schemes.jl b/src/BoundaryConditions/open_boundary_matching_schemes.jl new file mode 100644 index 0000000000..2a67e2f796 --- /dev/null +++ b/src/BoundaryConditions/open_boundary_matching_schemes.jl @@ -0,0 +1,64 @@ +abstract type MatchingScheme end + +# generic infrastructure +const MOBC = BoundaryCondition{Open{<:MatchingScheme}} + +# nudging + +struct Nudging{IT, OT} <: MatchingScheme + inflow_nudging_timescale :: IT + outflow_nudging_timescale :: OT +end + +(nudging::Nudging)() = nudging + +const NOBC = BoundaryCondition{<:Open{<:Nudging}} + +@inline function _fill_west_halo!(j, k, grid, c, bc::NOBC, loc, clock, model_fields) + Δt = clock.last_Δt + + matching_scheme = bc.classification.matching_scheme + + i, i′ = domain_boundary_indices(LeftBoundary(), grid.Nx) + + external_state = getbc(bc, j, k, grid, clock, model_fields) + + wall_normal_velocity = @inbounds model_fields.u[i, j, k] + + relaxation_timescale = wall_normal_velocity > 0 * matching_scheme.outflow_nudging_timescale + + wall_normal_velocity < 0 * matching_scheme.inflow_nudging_timescale + + relaxation_rate = min(1, Δt / relaxation_timescale) + + internal_state = @inbounds c[i, j, k] + + @inbounds c[i′, j, k] = 0#internal_state * (1 - relaxation_rate) + external_state * relaxation_rate +end + +@inline function _fill_east_halo!(j, k, grid, c, bc::NOBC, loc, clock, model_fields) + Δt = clock.last_Δt + + matching_scheme = bc.classification.matching_scheme + + i, i′ = domain_boundary_indices(RightBoundary(), grid.Nx) + + external_state = getbc(bc, j, k, grid, clock, model_fields) + + wall_normal_velocity = - @inbounds model_fields.u[i + 1, j, k] + + relaxation_timescale = wall_normal_velocity > 0 * matching_scheme.outflow_nudging_timescale + + wall_normal_velocity < 0 * matching_scheme.inflow_nudging_timescale + + relaxation_rate = min(1, Δt / relaxation_timescale) + + internal_state = @inbounds c[i + 1, j, k] + + @inbounds c[i′ + 1, j, k] = 0#internal_state * (1 - relaxation_rate) + external_state * relaxation_rate +end + +#= +@inline _fill_south_halo!(i, k, grid, c, bc::MOBC, loc, args...) = @inbounds c[i, 1, k] = getbc(bc, i, k, grid, args...) +@inline _fill_north_halo!(i, k, grid, c, bc::MOBC, loc, args...) = @inbounds c[i, grid.Ny + 1, k] = getbc(bc, i, k, grid, args...) +@inline _fill_bottom_halo!(i, j, grid, c, bc::MOBC, loc, args...) = @inbounds c[i, j, 1] = getbc(bc, i, j, grid, args...) +@inline _fill_top_halo!(i, j, grid, c, bc::MOBC, loc, args...) = @inbounds c[i, j, grid.Nz + 1] = getbc(bc, i, j, grid, args...) +=# \ No newline at end of file From 2555886f74bbcff3c79042e7e9d4940cdb64621c Mon Sep 17 00:00:00 2001 From: Jago Stong-Wright Date: Thu, 28 Mar 2024 14:00:42 -0400 Subject: [PATCH 011/113] added `update_boundary_condition!` and `update_boundary_conditions!` --- src/BoundaryConditions/BoundaryConditions.jl | 2 ++ .../update_boundary_conditions.jl | 15 +++++++++++++++ .../NonhydrostaticModels/NonhydrostaticModels.jl | 1 + .../update_nonhydrostatic_model_state.jl | 5 ++++- 4 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 src/BoundaryConditions/update_boundary_conditions.jl diff --git a/src/BoundaryConditions/BoundaryConditions.jl b/src/BoundaryConditions/BoundaryConditions.jl index 13dd9a6e19..1f035f4323 100644 --- a/src/BoundaryConditions/BoundaryConditions.jl +++ b/src/BoundaryConditions/BoundaryConditions.jl @@ -34,4 +34,6 @@ include("fill_halo_regions_nothing.jl") include("apply_flux_bcs.jl") +include("update_boundary_conditions.jl") + end # module diff --git a/src/BoundaryConditions/update_boundary_conditions.jl b/src/BoundaryConditions/update_boundary_conditions.jl new file mode 100644 index 0000000000..73bd4f4f0d --- /dev/null +++ b/src/BoundaryConditions/update_boundary_conditions.jl @@ -0,0 +1,15 @@ +update_boundary_condition!(::Union{BoundaryCondition, Nothing}, args...) = nothing + +function update_boundary_conditions!(bcs::FieldBoundaryConditions, field, model) + update_boundary_condition!(bcs.west, field, model) + update_boundary_condition!(bcs.east, field, model) + update_boundary_condition!(bcs.south, field, model) + update_boundary_condition!(bcs.north, field, model) + update_boundary_condition!(bcs.bottom, field, model) + update_boundary_condition!(bcs.top, field, model) + update_boundary_condition!(bcs.immersed, field, model) +end + +update_boundary_conditions!(fields::Union{NamedTuple, Tuple}, model) = + Tuple(update_boundary_conditions!(field.boundary_conditions, field, model) for field in fields) + diff --git a/src/Models/NonhydrostaticModels/NonhydrostaticModels.jl b/src/Models/NonhydrostaticModels/NonhydrostaticModels.jl index 5fccd29f82..96f127b6f6 100644 --- a/src/Models/NonhydrostaticModels/NonhydrostaticModels.jl +++ b/src/Models/NonhydrostaticModels/NonhydrostaticModels.jl @@ -10,6 +10,7 @@ using Oceananigans.Utils using Oceananigans.Grids using Oceananigans.Solvers +using Oceananigans.BoundaryConditions: update_boundary_conditions! using Oceananigans.DistributedComputations: Distributed, DistributedFFTBasedPoissonSolver, reconstruct_global_grid using Oceananigans.Grids: XYRegularRG, XZRegularRG, YZRegularRG, XYZRegularRG using Oceananigans.ImmersedBoundaries: ImmersedBoundaryGrid diff --git a/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl b/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl index b59a74eafe..364a050462 100644 --- a/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl +++ b/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl @@ -26,6 +26,9 @@ function update_state!(model::NonhydrostaticModel, callbacks=[]; compute_tendenc # Update all FieldTimeSeries used in the model update_model_field_time_series!(model, model.clock) + # Update the boundary conditions + update_boundary_conditions!(fields(model), model) + # Fill halos for velocities and tracers fill_halo_regions!(merge(model.velocities, model.tracers), model.clock, fields(model); async = true) @@ -61,4 +64,4 @@ function compute_auxiliaries!(model::NonhydrostaticModel; p_parameters = tuple(p update_hydrostatic_pressure!(model; parameters = ppar) end return nothing -end +end \ No newline at end of file From 42da0171e96b42591807111e1c9c8b2166294c1b Mon Sep 17 00:00:00 2001 From: Jago Stong-Wright Date: Fri, 29 Mar 2024 14:30:40 -0400 Subject: [PATCH 012/113] more update boundary condition changes --- src/BoundaryConditions/BoundaryConditions.jl | 2 ++ src/BoundaryConditions/boundary_condition.jl | 8 ++------ .../update_boundary_conditions.jl | 16 ++++++++-------- ...pdate_hydrostatic_free_surface_model_state.jl | 4 ++++ .../NonhydrostaticModels/NonhydrostaticModels.jl | 1 - .../update_nonhydrostatic_model_state.jl | 3 ++- 6 files changed, 18 insertions(+), 16 deletions(-) diff --git a/src/BoundaryConditions/BoundaryConditions.jl b/src/BoundaryConditions/BoundaryConditions.jl index 1f035f4323..3a54b41464 100644 --- a/src/BoundaryConditions/BoundaryConditions.jl +++ b/src/BoundaryConditions/BoundaryConditions.jl @@ -34,6 +34,8 @@ include("fill_halo_regions_nothing.jl") include("apply_flux_bcs.jl") +include("open_boundary_matching_schemes.jl") + include("update_boundary_conditions.jl") end # module diff --git a/src/BoundaryConditions/boundary_condition.jl b/src/BoundaryConditions/boundary_condition.jl index da3838cc03..6e96e12221 100644 --- a/src/BoundaryConditions/boundary_condition.jl +++ b/src/BoundaryConditions/boundary_condition.jl @@ -93,21 +93,17 @@ const DCBC = BoundaryCondition{<:DistributedCommunication} # More readable BC constructors for the public API. PeriodicBoundaryCondition() = BoundaryCondition(Periodic, nothing) NoFluxBoundaryCondition() = BoundaryCondition(Flux, nothing) - ImpenetrableBoundaryCondition() = BoundaryCondition(Open(Nothing), nothing) + ImpenetrableBoundaryCondition() = BoundaryCondition(Open(nothing), nothing) MultiRegionCommunicationBoundaryCondition() = BoundaryCondition(MultiRegionCommunication, nothing) DistributedCommunicationBoundaryCondition() = BoundaryCondition(DistributedCommunication, nothing) FluxBoundaryCondition(val; kwargs...) = BoundaryCondition(Flux, val; kwargs...) ValueBoundaryCondition(val; kwargs...) = BoundaryCondition(Value, val; kwargs...) GradientBoundaryCondition(val; kwargs...) = BoundaryCondition(Gradient, val; kwargs...) - OpenBoundaryCondition(val; kwargs...) = BoundaryCondition(Open(Nothing), val; kwargs...) + OpenBoundaryCondition(val; kwargs...) = BoundaryCondition(Open(nothing), val; kwargs...) MultiRegionCommunicationBoundaryCondition(val; kwargs...) = BoundaryCondition(MultiRegionCommunication, val; kwargs...) DistributedCommunicationBoundaryCondition(val; kwargs...) = BoundaryCondition(DistributedCommunication, val; kwargs...) -# Open boundary constructors -NudgingOpenBoundaryCondition(val, inflow_nudging_timescale, outflow_nudging_timescale; kwargs...) = - BoundaryCondition(Open(Nudging(inflow_nudging_timescale, outflow_nudging_timescale)), val; kwargs...) - # Support for various types of boundary conditions. # # Notes: diff --git a/src/BoundaryConditions/update_boundary_conditions.jl b/src/BoundaryConditions/update_boundary_conditions.jl index 73bd4f4f0d..65ab8ffdb9 100644 --- a/src/BoundaryConditions/update_boundary_conditions.jl +++ b/src/BoundaryConditions/update_boundary_conditions.jl @@ -1,13 +1,13 @@ -update_boundary_condition!(::Union{BoundaryCondition, Nothing}, args...) = nothing +update_boundary_condition!(bc::Union{BoundaryCondition, Nothing}, args...) = nothing function update_boundary_conditions!(bcs::FieldBoundaryConditions, field, model) - update_boundary_condition!(bcs.west, field, model) - update_boundary_condition!(bcs.east, field, model) - update_boundary_condition!(bcs.south, field, model) - update_boundary_condition!(bcs.north, field, model) - update_boundary_condition!(bcs.bottom, field, model) - update_boundary_condition!(bcs.top, field, model) - update_boundary_condition!(bcs.immersed, field, model) + update_boundary_condition!(bcs.west, field, model, Val(:west)) + update_boundary_condition!(bcs.east, field, model, Val(:east)) + update_boundary_condition!(bcs.south, field, model, Val(:south)) + update_boundary_condition!(bcs.north, field, model, Val(:north)) + update_boundary_condition!(bcs.bottom, field, model, Val(:bottom)) + update_boundary_condition!(bcs.top, field, model, Val(:top)) + update_boundary_condition!(bcs.immersed, field, model, Val(:immersed)) end update_boundary_conditions!(fields::Union{NamedTuple, Tuple}, model) = diff --git a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl index c1e477138e..bd76ce33b5 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl @@ -3,6 +3,7 @@ using Oceananigans.BoundaryConditions using Oceananigans: UpdateStateCallsite using Oceananigans.Biogeochemistry: update_biogeochemical_state! +using Oceananigans.BoundaryConditions: update_boundary_conditions! using Oceananigans.TurbulenceClosures: compute_diffusivities! using Oceananigans.ImmersedBoundaries: mask_immersed_field!, mask_immersed_field_xy!, inactive_node using Oceananigans.Models.NonhydrostaticModels: update_hydrostatic_pressure!, p_kernel_parameters @@ -35,6 +36,9 @@ function update_state!(model::HydrostaticFreeSurfaceModel, grid, callbacks; comp # Update possible FieldTimeSeries used in the model @apply_regionally update_model_field_time_series!(model, model.clock) + # Update the boundary conditions + update_boundary_conditions!(fields(model), model) + fill_halo_regions!(prognostic_fields(model), model.clock, fields(model); async = true) @apply_regionally replace_horizontal_vector_halos!(model.velocities, model.grid) @apply_regionally compute_auxiliaries!(model) diff --git a/src/Models/NonhydrostaticModels/NonhydrostaticModels.jl b/src/Models/NonhydrostaticModels/NonhydrostaticModels.jl index 96f127b6f6..5fccd29f82 100644 --- a/src/Models/NonhydrostaticModels/NonhydrostaticModels.jl +++ b/src/Models/NonhydrostaticModels/NonhydrostaticModels.jl @@ -10,7 +10,6 @@ using Oceananigans.Utils using Oceananigans.Grids using Oceananigans.Solvers -using Oceananigans.BoundaryConditions: update_boundary_conditions! using Oceananigans.DistributedComputations: Distributed, DistributedFFTBasedPoissonSolver, reconstruct_global_grid using Oceananigans.Grids: XYRegularRG, XZRegularRG, YZRegularRG, XYZRegularRG using Oceananigans.ImmersedBoundaries: ImmersedBoundaryGrid diff --git a/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl b/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl index 364a050462..c9fe26e5a3 100644 --- a/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl +++ b/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl @@ -2,6 +2,7 @@ using Oceananigans: UpdateStateCallsite using Oceananigans.Architectures using Oceananigans.BoundaryConditions using Oceananigans.Biogeochemistry: update_biogeochemical_state! +using Oceananigans.BoundaryConditions: update_boundary_conditions! using Oceananigans.TurbulenceClosures: compute_diffusivities! using Oceananigans.Fields: compute! using Oceananigans.ImmersedBoundaries: mask_immersed_field! @@ -24,7 +25,7 @@ function update_state!(model::NonhydrostaticModel, callbacks=[]; compute_tendenc end # Update all FieldTimeSeries used in the model - update_model_field_time_series!(model, model.clock) + #update_model_field_time_series!(model, model.clock) # Update the boundary conditions update_boundary_conditions!(fields(model), model) From 4bf02522aa63679a03c3b32d7e2ee9517fc37b3c Mon Sep 17 00:00:00 2001 From: Jago Stong-Wright Date: Tue, 23 Apr 2024 19:23:46 +0100 Subject: [PATCH 013/113] add mean outflow open boundary condition --- src/BoundaryConditions/BoundaryConditions.jl | 1 + .../open_boundary_matching_schemes.jl | 107 +++++++++++------- 2 files changed, 69 insertions(+), 39 deletions(-) diff --git a/src/BoundaryConditions/BoundaryConditions.jl b/src/BoundaryConditions/BoundaryConditions.jl index 3a54b41464..6c0b5e9d13 100644 --- a/src/BoundaryConditions/BoundaryConditions.jl +++ b/src/BoundaryConditions/BoundaryConditions.jl @@ -4,6 +4,7 @@ export BCType, Flux, Gradient, Value, Open, BoundaryCondition, getbc, setbc!, PeriodicBoundaryCondition, OpenBoundaryCondition, NoFluxBoundaryCondition, MultiRegionCommunicationBoundaryCondition, + MeanOutflowOpenBoundaryCondition, FluxBoundaryCondition, ValueBoundaryCondition, GradientBoundaryCondition, DistributedCommunicationBoundaryCondition, validate_boundary_condition_topology, validate_boundary_condition_architecture, FieldBoundaryConditions, diff --git a/src/BoundaryConditions/open_boundary_matching_schemes.jl b/src/BoundaryConditions/open_boundary_matching_schemes.jl index 2a67e2f796..2530cd109a 100644 --- a/src/BoundaryConditions/open_boundary_matching_schemes.jl +++ b/src/BoundaryConditions/open_boundary_matching_schemes.jl @@ -1,64 +1,93 @@ -abstract type MatchingScheme end +using Oceananigans.Architectures: architecture, on_architecture +using Oceananigans.Operators: Axᶜᶜᶠ, Ayᶜᶠᶜ, Azᶜᶜᶠ + +""" + MeanOutflow + +Advect out of the domain with the boundary mean flow, and relax inflows to external state. +""" +struct MeanOutflow{BF, IT} + boundary_flux :: BF + mean_outflow_velocity :: AbstractArray + inflow_relaxation_timescale :: IT +end + +(ms::MeanOutflow)() = ms -# generic infrastructure -const MOBC = BoundaryCondition{Open{<:MatchingScheme}} +Adapt.adapt_structure(to, mo::MeanOutflow) = + MeanOutflow(nothing, adapt(to, mo.mean_outflow_velocity), adapt(to, mo.inflow_relaxation_timescale)) -# nudging +function MeanOutflowOpenBoundaryCondition(grid, side, val; inflow_relaxation_timescale = 1, kwargs...) + boundary_flux = boundary_flux_field(grid, Val(side)) -struct Nudging{IT, OT} <: MatchingScheme - inflow_nudging_timescale :: IT - outflow_nudging_timescale :: OT + classifcation = Open(MeanOutflow(boundary_flux, [0.], inflow_relaxation_timescale)) + + return BoundaryCondition(classifcation, val; kwargs...) end -(nudging::Nudging)() = nudging +# fields is not yet defiled so have to use an array +boundary_flux_field(grid, ::Union{Val{:west}, Val{:east}}) = on_architecture(architecture(grid), zeros(size(grid, 2), size(grid, 3))) +boundary_flux_field(grid, ::Union{Val{:south}, Val{:north}}) = on_architecture(architecture(grid), zeros(size(grid, 1), size(grid, 3))) +boundary_flux_field(grid, ::Union{Val{:bottom}, Val{:top}}) = on_architecture(architecture(grid), zeros(size(grid, 2), size(grid, 3))) -const NOBC = BoundaryCondition{<:Open{<:Nudging}} +const MOOBC = BoundaryCondition{<:Open{<:MeanOutflow}} -@inline function _fill_west_halo!(j, k, grid, c, bc::NOBC, loc, clock, model_fields) - Δt = clock.last_Δt +boundary_normal_velocity(velocities, ::Union{Val{:west}, Val{:east}}) = velocities.u +boundary_normal_velocity(velocities, ::Union{Val{:south}, Val{:north}}) = velocities.v +boundary_normal_velocity(velocities, ::Union{Val{:bottom}, Val{:top}}) = velocities.w - matching_scheme = bc.classification.matching_scheme +function update_boundary_condition!(bc::MOOBC, field, model, side) + ms = bc.classification.matching_scheme - i, i′ = domain_boundary_indices(LeftBoundary(), grid.Nx) + u = boundary_normal_velocity(model.velocities, side) + F = ms.boundary_flux - external_state = getbc(bc, j, k, grid, clock, model_fields) - - wall_normal_velocity = @inbounds model_fields.u[i, j, k] + arch = architecture(model) + grid = model.grid - relaxation_timescale = wall_normal_velocity > 0 * matching_scheme.outflow_nudging_timescale + - wall_normal_velocity < 0 * matching_scheme.inflow_nudging_timescale + launch!(arch, grid, :yz, _update_boundary_flux!, F, grid, u, side) - relaxation_rate = min(1, Δt / relaxation_timescale) + ms.mean_outflow_velocity[1] = sum(F) / (grid.Ly * grid.Lz) +end - internal_state = @inbounds c[i, j, k] +@kernel function _update_boundary_flux!(F, grid, u, ::Val{:west}) + j, k = @index(Global, NTuple) - @inbounds c[i′, j, k] = 0#internal_state * (1 - relaxation_rate) + external_state * relaxation_rate + @inbounds F[j, k] = -u[1, j, k] * Axᶜᶜᶠ(1, j, k, grid) end -@inline function _fill_east_halo!(j, k, grid, c, bc::NOBC, loc, clock, model_fields) - Δt = clock.last_Δt +@kernel function _update_boundary_flux!(F, grid, u, ::Val{:east}) + j, k = @index(Global, NTuple) - matching_scheme = bc.classification.matching_scheme + i = grid.Nx - i, i′ = domain_boundary_indices(RightBoundary(), grid.Nx) + @inbounds F[j, k] = u[i, j, k] * Axᶜᶜᶠ(i, j, k, grid) +end - external_state = getbc(bc, j, k, grid, clock, model_fields) - - wall_normal_velocity = - @inbounds model_fields.u[i + 1, j, k] +@kernel function _update_boundary_flux!(F, grid, u, ::Val{:south}) + i, k = @index(Global, NTuple) - relaxation_timescale = wall_normal_velocity > 0 * matching_scheme.outflow_nudging_timescale + - wall_normal_velocity < 0 * matching_scheme.inflow_nudging_timescale + @inbounds F[i, k] = -u[i, 1, k] * Ayᶜᶠᶜ(i, 1, k, grid) +end + +@kernel function _update_boundary_flux!(F, grid, u, ::Val{:north}) + i, k = @index(Global, NTuple) - relaxation_rate = min(1, Δt / relaxation_timescale) + j = grid.Ny + + @inbounds F[i, k] = u[i, j, k] * Ayᶜᶠᶜ(i, j, k, grid) +end - internal_state = @inbounds c[i + 1, j, k] +@kernel function _update_boundary_flux!(F, grid, u, ::Val{:bottom}) + i, j = @index(Global, NTuple) - @inbounds c[i′ + 1, j, k] = 0#internal_state * (1 - relaxation_rate) + external_state * relaxation_rate + @inbounds F[i, j] = -u[i, j, 1] * Azᶜᶜᶠ(i, j, 1, grid) end -#= -@inline _fill_south_halo!(i, k, grid, c, bc::MOBC, loc, args...) = @inbounds c[i, 1, k] = getbc(bc, i, k, grid, args...) -@inline _fill_north_halo!(i, k, grid, c, bc::MOBC, loc, args...) = @inbounds c[i, grid.Ny + 1, k] = getbc(bc, i, k, grid, args...) -@inline _fill_bottom_halo!(i, j, grid, c, bc::MOBC, loc, args...) = @inbounds c[i, j, 1] = getbc(bc, i, j, grid, args...) -@inline _fill_top_halo!(i, j, grid, c, bc::MOBC, loc, args...) = @inbounds c[i, j, grid.Nz + 1] = getbc(bc, i, j, grid, args...) -=# \ No newline at end of file +@kernel function _update_boundary_flux!(F, grid, u, ::Val{:top}) + i, j = @index(Global, NTuple) + + k = grid.Nz + + @inbounds F[i, j] = u[i, j, k] * Azᶜᶜᶠ(i, j, k, grid) +end From c06aded210d0175c4cba3c393197d2b29886cf86 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Wed, 22 May 2024 17:38:29 +0100 Subject: [PATCH 014/113] change boundary conditions setup to allow classifcations to have properties --- src/BoundaryConditions/boundary_condition.jl | 30 +++--- .../boundary_condition_classifications.jl | 5 + .../open_boundary_matching_schemes.jl | 93 ------------------- test/test_boundary_conditions.jl | 4 +- test/test_boundary_conditions_integration.jl | 62 ++++++------- 5 files changed, 52 insertions(+), 142 deletions(-) delete mode 100644 src/BoundaryConditions/open_boundary_matching_schemes.jl diff --git a/src/BoundaryConditions/boundary_condition.jl b/src/BoundaryConditions/boundary_condition.jl index 6e96e12221..0b146699fb 100644 --- a/src/BoundaryConditions/boundary_condition.jl +++ b/src/BoundaryConditions/boundary_condition.jl @@ -1,4 +1,3 @@ -import Adapt import Oceananigans.Architectures: on_architecture """ @@ -18,7 +17,7 @@ Construct a boundary condition of type `BC` with a number or array as a `conditi Boundary condition types include `Periodic`, `Flux`, `Value`, `Gradient`, and `Open`. """ -BoundaryCondition(Classification::DataType, condition) = BoundaryCondition(Classification(), condition) +#BoundaryCondition(Classification::DataType, condition) = BoundaryCondition(Classification(), condition) """ BoundaryCondition(Classification::DataType, condition::Function; @@ -50,7 +49,7 @@ where `i`, and `j` are indices that vary along the boundary. If `discrete_form = condition(i, j, grid, clock, model_fields, parameters) ``` """ -function BoundaryCondition(Classification::Union{DataType, Open}, condition::Function; +function BoundaryCondition(Classification::AbstractBoundaryConditionClassification, condition::Function; parameters = nothing, discrete_form = false, field_dependencies=()) @@ -63,14 +62,13 @@ function BoundaryCondition(Classification::Union{DataType, Open}, condition::Fun condition = ContinuousBoundaryFunction(condition, parameters, field_dependencies) end - return BoundaryCondition(Classification(), condition) + return BoundaryCondition(Classification, condition) end # Adapt boundary condition struct to be GPU friendly and passable to GPU kernels. Adapt.adapt_structure(to, b::BoundaryCondition) = BoundaryCondition(Adapt.adapt(to, b.classification), Adapt.adapt(to, b.condition)) - # Adapt boundary condition struct to be GPU friendly and passable to GPU kernels. on_architecture(to, b::BoundaryCondition) = BoundaryCondition(on_architecture(to, b.classification), on_architecture(to, b.condition)) @@ -91,18 +89,18 @@ const MCBC = BoundaryCondition{<:MultiRegionCommunication} const DCBC = BoundaryCondition{<:DistributedCommunication} # More readable BC constructors for the public API. - PeriodicBoundaryCondition() = BoundaryCondition(Periodic, nothing) - NoFluxBoundaryCondition() = BoundaryCondition(Flux, nothing) - ImpenetrableBoundaryCondition() = BoundaryCondition(Open(nothing), nothing) -MultiRegionCommunicationBoundaryCondition() = BoundaryCondition(MultiRegionCommunication, nothing) -DistributedCommunicationBoundaryCondition() = BoundaryCondition(DistributedCommunication, nothing) - - FluxBoundaryCondition(val; kwargs...) = BoundaryCondition(Flux, val; kwargs...) - ValueBoundaryCondition(val; kwargs...) = BoundaryCondition(Value, val; kwargs...) - GradientBoundaryCondition(val; kwargs...) = BoundaryCondition(Gradient, val; kwargs...) + PeriodicBoundaryCondition() = BoundaryCondition(Periodic(), nothing) + NoFluxBoundaryCondition() = BoundaryCondition(Flux(), nothing) + ImpenetrableBoundaryCondition() = OpenBoundaryCondition(0) +MultiRegionCommunicationBoundaryCondition() = BoundaryCondition(MultiRegionCommunication(), nothing) +DistributedCommunicationBoundaryCondition() = BoundaryCondition(DistributedCommunication(), nothing) + + FluxBoundaryCondition(val; kwargs...) = BoundaryCondition(Flux(), val; kwargs...) + ValueBoundaryCondition(val; kwargs...) = BoundaryCondition(Value(), val; kwargs...) + GradientBoundaryCondition(val; kwargs...) = BoundaryCondition(Gradient(), val; kwargs...) OpenBoundaryCondition(val; kwargs...) = BoundaryCondition(Open(nothing), val; kwargs...) -MultiRegionCommunicationBoundaryCondition(val; kwargs...) = BoundaryCondition(MultiRegionCommunication, val; kwargs...) -DistributedCommunicationBoundaryCondition(val; kwargs...) = BoundaryCondition(DistributedCommunication, val; kwargs...) +MultiRegionCommunicationBoundaryCondition(val; kwargs...) = BoundaryCondition(MultiRegionCommunication(), val; kwargs...) +DistributedCommunicationBoundaryCondition(val; kwargs...) = BoundaryCondition(DistributedCommunication(), val; kwargs...) # Support for various types of boundary conditions. # diff --git a/src/BoundaryConditions/boundary_condition_classifications.jl b/src/BoundaryConditions/boundary_condition_classifications.jl index f59ec06cb8..549a4daee3 100644 --- a/src/BoundaryConditions/boundary_condition_classifications.jl +++ b/src/BoundaryConditions/boundary_condition_classifications.jl @@ -63,8 +63,13 @@ struct Open{MS} <: AbstractBoundaryConditionClassification matching_scheme::MS end +Open() = Open(nothing) + (open::Open)() = open +Adapt.adapt_structure(to, bc::Open) = + Open(adapt(to, bc.matching_scheme)) + """ struct MultiRegionCommunication <: AbstractBoundaryConditionClassification diff --git a/src/BoundaryConditions/open_boundary_matching_schemes.jl b/src/BoundaryConditions/open_boundary_matching_schemes.jl deleted file mode 100644 index 2530cd109a..0000000000 --- a/src/BoundaryConditions/open_boundary_matching_schemes.jl +++ /dev/null @@ -1,93 +0,0 @@ -using Oceananigans.Architectures: architecture, on_architecture -using Oceananigans.Operators: Axᶜᶜᶠ, Ayᶜᶠᶜ, Azᶜᶜᶠ - -""" - MeanOutflow - -Advect out of the domain with the boundary mean flow, and relax inflows to external state. -""" -struct MeanOutflow{BF, IT} - boundary_flux :: BF - mean_outflow_velocity :: AbstractArray - inflow_relaxation_timescale :: IT -end - -(ms::MeanOutflow)() = ms - -Adapt.adapt_structure(to, mo::MeanOutflow) = - MeanOutflow(nothing, adapt(to, mo.mean_outflow_velocity), adapt(to, mo.inflow_relaxation_timescale)) - -function MeanOutflowOpenBoundaryCondition(grid, side, val; inflow_relaxation_timescale = 1, kwargs...) - boundary_flux = boundary_flux_field(grid, Val(side)) - - classifcation = Open(MeanOutflow(boundary_flux, [0.], inflow_relaxation_timescale)) - - return BoundaryCondition(classifcation, val; kwargs...) -end - -# fields is not yet defiled so have to use an array -boundary_flux_field(grid, ::Union{Val{:west}, Val{:east}}) = on_architecture(architecture(grid), zeros(size(grid, 2), size(grid, 3))) -boundary_flux_field(grid, ::Union{Val{:south}, Val{:north}}) = on_architecture(architecture(grid), zeros(size(grid, 1), size(grid, 3))) -boundary_flux_field(grid, ::Union{Val{:bottom}, Val{:top}}) = on_architecture(architecture(grid), zeros(size(grid, 2), size(grid, 3))) - -const MOOBC = BoundaryCondition{<:Open{<:MeanOutflow}} - -boundary_normal_velocity(velocities, ::Union{Val{:west}, Val{:east}}) = velocities.u -boundary_normal_velocity(velocities, ::Union{Val{:south}, Val{:north}}) = velocities.v -boundary_normal_velocity(velocities, ::Union{Val{:bottom}, Val{:top}}) = velocities.w - -function update_boundary_condition!(bc::MOOBC, field, model, side) - ms = bc.classification.matching_scheme - - u = boundary_normal_velocity(model.velocities, side) - F = ms.boundary_flux - - arch = architecture(model) - grid = model.grid - - launch!(arch, grid, :yz, _update_boundary_flux!, F, grid, u, side) - - ms.mean_outflow_velocity[1] = sum(F) / (grid.Ly * grid.Lz) -end - -@kernel function _update_boundary_flux!(F, grid, u, ::Val{:west}) - j, k = @index(Global, NTuple) - - @inbounds F[j, k] = -u[1, j, k] * Axᶜᶜᶠ(1, j, k, grid) -end - -@kernel function _update_boundary_flux!(F, grid, u, ::Val{:east}) - j, k = @index(Global, NTuple) - - i = grid.Nx - - @inbounds F[j, k] = u[i, j, k] * Axᶜᶜᶠ(i, j, k, grid) -end - -@kernel function _update_boundary_flux!(F, grid, u, ::Val{:south}) - i, k = @index(Global, NTuple) - - @inbounds F[i, k] = -u[i, 1, k] * Ayᶜᶠᶜ(i, 1, k, grid) -end - -@kernel function _update_boundary_flux!(F, grid, u, ::Val{:north}) - i, k = @index(Global, NTuple) - - j = grid.Ny - - @inbounds F[i, k] = u[i, j, k] * Ayᶜᶠᶜ(i, j, k, grid) -end - -@kernel function _update_boundary_flux!(F, grid, u, ::Val{:bottom}) - i, j = @index(Global, NTuple) - - @inbounds F[i, j] = -u[i, j, 1] * Azᶜᶜᶠ(i, j, 1, grid) -end - -@kernel function _update_boundary_flux!(F, grid, u, ::Val{:top}) - i, j = @index(Global, NTuple) - - k = grid.Nz - - @inbounds F[i, j] = u[i, j, k] * Azᶜᶜᶠ(i, j, k, grid) -end diff --git a/test/test_boundary_conditions.jl b/test/test_boundary_conditions.jl index 06e3859f58..7a7eaaa791 100644 --- a/test/test_boundary_conditions.jl +++ b/test/test_boundary_conditions.jl @@ -21,7 +21,7 @@ end @testset "Boundary condition instantiation" begin @info " Testing boundary condition instantiation..." - for C in (Value, Gradient, Flux) + for C in (Value(), Gradient(), Flux()) @test can_instantiate_boundary_condition(integer_bc, C) @test can_instantiate_boundary_condition(irrational_bc, C) @test can_instantiate_boundary_condition(simple_function_bc, C) @@ -235,7 +235,7 @@ end @test T_bcs.top.condition.func === simple_bc @test T_bcs.bottom.condition.func === simple_bc - one_bc = BoundaryCondition(Value, 1.0) + one_bc = BoundaryCondition(Value(), 1.0) T_bcs = FieldBoundaryConditions( east = one_bc, west = one_bc, diff --git a/test/test_boundary_conditions_integration.jl b/test/test_boundary_conditions_integration.jl index cb39f1077a..16143f817f 100644 --- a/test/test_boundary_conditions_integration.jl +++ b/test/test_boundary_conditions_integration.jl @@ -58,7 +58,7 @@ function fluxes_with_diffusivity_boundary_conditions_are_correct(arch, FT) grid = RectilinearGrid(arch, FT, size=(16, 16, 16), extent=(1, 1, Lz)) buoyancy_bcs = FieldBoundaryConditions(bottom=GradientBoundaryCondition(bz)) - κₑ_bcs = FieldBoundaryConditions(grid, (Center, Center, Center), bottom=ValueBoundaryCondition(κ₀)) + κₑ_bcs = FieldBoundaryConditions(grid, (Center, Center, Center), bottom=Value()BoundaryCondition(κ₀)) model_bcs = (b=buoyancy_bcs, κₑ=(b=κₑ_bcs,)) model = NonhydrostaticModel( @@ -123,34 +123,34 @@ test_boundary_conditions(C, FT, ArrayType) = (integer_bc(C, FT, ArrayType), grid = RectilinearGrid(arch, FT, size=(1, 1, 1), extent=(1, π, 42), topology=(Bounded, Bounded, Bounded)) - u_boundary_conditions = FieldBoundaryConditions(bottom = simple_function_bc(Value), - top = simple_function_bc(Value), - north = simple_function_bc(Value), - south = simple_function_bc(Value), - east = simple_function_bc(Open{Nothing}), - west = simple_function_bc(Open{Nothing})) - - v_boundary_conditions = FieldBoundaryConditions(bottom = simple_function_bc(Value), - top = simple_function_bc(Value), - north = simple_function_bc(Open{Nothing}), - south = simple_function_bc(Open{Nothing}), - east = simple_function_bc(Value), - west = simple_function_bc(Value)) - - - w_boundary_conditions = FieldBoundaryConditions(bottom = simple_function_bc(Open{Nothing}), - top = simple_function_bc(Open{Nothing}), - north = simple_function_bc(Value), - south = simple_function_bc(Value), - east = simple_function_bc(Value), - west = simple_function_bc(Value)) - - T_boundary_conditions = FieldBoundaryConditions(bottom = simple_function_bc(Value), - top = simple_function_bc(Value), - north = simple_function_bc(Value), - south = simple_function_bc(Value), - east = simple_function_bc(Value), - west = simple_function_bc(Value)) + u_boundary_conditions = FieldBoundaryConditions(bottom = simple_function_bc(Value()), + top = simple_function_bc(Value()), + north = simple_function_bc(Value()), + south = simple_function_bc(Value()), + east = simple_function_bc(Open()), + west = simple_function_bc(Open())) + + v_boundary_conditions = FieldBoundaryConditions(bottom = simple_function_bc(Value()), + top = simple_function_bc(Value()), + north = simple_function_bc(Open()), + south = simple_function_bc(Open()), + east = simple_function_bc(Value()), + west = simple_function_bc(Value())) + + + w_boundary_conditions = FieldBoundaryConditions(bottom = simple_function_bc(Open()), + top = simple_function_bc(Open()), + north = simple_function_bc(Value()), + south = simple_function_bc(Value()), + east = simple_function_bc(Value()), + west = simple_function_bc(Value())) + + T_boundary_conditions = FieldBoundaryConditions(bottom = simple_function_bc(Value()), + top = simple_function_bc(Value()), + north = simple_function_bc(Value()), + south = simple_function_bc(Value()), + east = simple_function_bc(Value()), + west = simple_function_bc(Value())) boundary_conditions = (u=u_boundary_conditions, v=v_boundary_conditions, @@ -197,13 +197,13 @@ test_boundary_conditions(C, FT, ArrayType) = (integer_bc(C, FT, ArrayType), topo = (Bounded, Bounded, Bounded) - for C in (Gradient, Flux, Value), boundary_condition in test_boundary_conditions(C, FT, array_type(arch)) + for C in (Gradient(), Flux(), Value()), boundary_condition in test_boundary_conditions(C, FT, array_type(arch)) @test test_boundary_condition(arch, FT, topo, :east, :T, boundary_condition) @test test_boundary_condition(arch, FT, topo, :south, :T, boundary_condition) @test test_boundary_condition(arch, FT, topo, :top, :T, boundary_condition) end - for boundary_condition in test_boundary_conditions(Open{Nothing}, FT, array_type(arch)) + for boundary_condition in test_boundary_conditions(Open(), FT, array_type(arch)) @test test_boundary_condition(arch, FT, topo, :east, :u, boundary_condition) @test test_boundary_condition(arch, FT, topo, :south, :v, boundary_condition) @test test_boundary_condition(arch, FT, topo, :top, :w, boundary_condition) From e6e33f4761aa3d12470b24164a80d84e85890dbc Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Wed, 22 May 2024 17:45:50 +0100 Subject: [PATCH 015/113] forgot to add Adapt import --- src/BoundaryConditions/BoundaryConditions.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/BoundaryConditions/BoundaryConditions.jl b/src/BoundaryConditions/BoundaryConditions.jl index 6c0b5e9d13..4435b7a3d2 100644 --- a/src/BoundaryConditions/BoundaryConditions.jl +++ b/src/BoundaryConditions/BoundaryConditions.jl @@ -11,7 +11,7 @@ export apply_x_bcs!, apply_y_bcs!, apply_z_bcs!, fill_halo_regions! -using CUDA +using CUDA, Adapt using KernelAbstractions: @index, @kernel using Oceananigans.Architectures: CPU, GPU, device @@ -19,6 +19,8 @@ using Oceananigans.Utils: work_layout, launch! using Oceananigans.Operators: Ax, Ay, Az, volume using Oceananigans.Grids +import Adapt: adapt_structure + include("boundary_condition_classifications.jl") include("boundary_condition.jl") include("discrete_boundary_function.jl") From c9aa31869a924df27b4359d6fb5df53c93ff1afd Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Wed, 22 May 2024 17:55:01 +0100 Subject: [PATCH 016/113] forgot another local change --- src/BoundaryConditions/BoundaryConditions.jl | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/BoundaryConditions/BoundaryConditions.jl b/src/BoundaryConditions/BoundaryConditions.jl index 4435b7a3d2..efa4f07571 100644 --- a/src/BoundaryConditions/BoundaryConditions.jl +++ b/src/BoundaryConditions/BoundaryConditions.jl @@ -37,8 +37,6 @@ include("fill_halo_regions_nothing.jl") include("apply_flux_bcs.jl") -include("open_boundary_matching_schemes.jl") - include("update_boundary_conditions.jl") end # module From f00eea3641198985dbe055ed93f8cb66e7ed3bf7 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Wed, 22 May 2024 19:01:51 +0100 Subject: [PATCH 017/113] fixed immersed boundary conditions --- src/ImmersedBoundaries/immersed_boundary_condition.jl | 5 ++++- test/test_boundary_conditions_integration.jl | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/ImmersedBoundaries/immersed_boundary_condition.jl b/src/ImmersedBoundaries/immersed_boundary_condition.jl index 34a7981c3c..423b2cd381 100644 --- a/src/ImmersedBoundaries/immersed_boundary_condition.jl +++ b/src/ImmersedBoundaries/immersed_boundary_condition.jl @@ -4,7 +4,7 @@ using Oceananigans.BoundaryConditions: DefaultBoundaryCondition using Oceananigans.TurbulenceClosures: AbstractScalarDiffusivity, h_diffusivity, z_diffusivity using Oceananigans.Operators: index_left, index_right, Δx, Δy, Δz, div -import Oceananigans.BoundaryConditions: regularize_immersed_boundary_condition, bc_str +import Oceananigans.BoundaryConditions: regularize_immersed_boundary_condition, bc_str, update_boundary_condition! import Oceananigans.TurbulenceClosures: immersed_∂ⱼ_τ₁ⱼ, immersed_∂ⱼ_τ₂ⱼ, @@ -272,6 +272,9 @@ Adapt.adapt_structure(to, bc::ImmersedBoundaryCondition) = ImmersedBoundaryCondi Adapt.adapt(to, bc.north), Adapt.adapt(to, bc.bottom), Adapt.adapt(to, bc.top)) + +update_boundary_condition!(bc::ImmersedBoundaryCondition, args...) = nothing + ##### ##### Alternative implementation for immersed flux divergence ##### diff --git a/test/test_boundary_conditions_integration.jl b/test/test_boundary_conditions_integration.jl index 16143f817f..22d1bc8dad 100644 --- a/test/test_boundary_conditions_integration.jl +++ b/test/test_boundary_conditions_integration.jl @@ -27,7 +27,7 @@ function test_nonhydrostatic_flux_budget(grid, name, side, L) FT = eltype(grid) flux = FT(π) direction = side ∈ (:west, :south, :bottom, :immersed) ? 1 : -1 - bc_kwarg = Dict(side => BoundaryCondition(Flux, flux * direction)) + bc_kwarg = Dict(side => BoundaryCondition(Flux(), flux * direction)) field_bcs = FieldBoundaryConditions(; bc_kwarg...) boundary_conditions = (; name => field_bcs) @@ -58,7 +58,7 @@ function fluxes_with_diffusivity_boundary_conditions_are_correct(arch, FT) grid = RectilinearGrid(arch, FT, size=(16, 16, 16), extent=(1, 1, Lz)) buoyancy_bcs = FieldBoundaryConditions(bottom=GradientBoundaryCondition(bz)) - κₑ_bcs = FieldBoundaryConditions(grid, (Center, Center, Center), bottom=Value()BoundaryCondition(κ₀)) + κₑ_bcs = FieldBoundaryConditions(grid, (Center, Center, Center), bottom=ValueBoundaryCondition(κ₀)) model_bcs = (b=buoyancy_bcs, κₑ=(b=κₑ_bcs,)) model = NonhydrostaticModel( From 9d106cdccaca379f2a74f6f37a6298a3da05d0ac Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Wed, 22 May 2024 19:07:40 +0100 Subject: [PATCH 018/113] fixed tests --- test/test_boundary_conditions_integration.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_boundary_conditions_integration.jl b/test/test_boundary_conditions_integration.jl index 22d1bc8dad..285c75d665 100644 --- a/test/test_boundary_conditions_integration.jl +++ b/test/test_boundary_conditions_integration.jl @@ -70,7 +70,7 @@ function fluxes_with_diffusivity_boundary_conditions_are_correct(arch, FT) set!(model, b=b₀) b = model.tracers.b - mean_b₀ = mean(interior(b)) + mean_b₀ = mean(b) τκ = Lz^2 / κ₀ # Diffusion time-scale Δt = 1e-6 * τκ # Time step much less than diffusion time-scale @@ -98,7 +98,7 @@ function fluxes_with_diffusivity_boundary_conditions_are_correct(arch, FT) # mean(interior(b)) - mean_b₀ = -3.141592656086267e-5 # (flux * model.clock.time) / Lz = -3.141592653589793e-5 - return isapprox(mean(interior(b)) - mean_b₀, flux * model.clock.time / Lz, atol=1e-6) + return isapprox(mean(b) - mean_b₀, flux * model.clock.time / Lz, atol=1e-6) end test_boundary_conditions(C, FT, ArrayType) = (integer_bc(C, FT, ArrayType), From 7294f478263cd5cf4b33f2b373ccbb1022d83315 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Fri, 24 May 2024 15:51:41 +0100 Subject: [PATCH 019/113] fixed for etc. --- src/BoundaryConditions/update_boundary_conditions.jl | 3 ++- src/Fields/field.jl | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/BoundaryConditions/update_boundary_conditions.jl b/src/BoundaryConditions/update_boundary_conditions.jl index 65ab8ffdb9..2846bbbf0d 100644 --- a/src/BoundaryConditions/update_boundary_conditions.jl +++ b/src/BoundaryConditions/update_boundary_conditions.jl @@ -11,5 +11,6 @@ function update_boundary_conditions!(bcs::FieldBoundaryConditions, field, model) end update_boundary_conditions!(fields::Union{NamedTuple, Tuple}, model) = - Tuple(update_boundary_conditions!(field.boundary_conditions, field, model) for field in fields) + Tuple(update_boundary_conditions!(get_boundary_conditions(field), field, model) for field in fields) +get_boundary_conditions(field) = nothing \ No newline at end of file diff --git a/src/Fields/field.jl b/src/Fields/field.jl index 81021f3743..043c903760 100644 --- a/src/Fields/field.jl +++ b/src/Fields/field.jl @@ -7,7 +7,7 @@ using KernelAbstractions: @kernel, @index using Base: @propagate_inbounds import Oceananigans.Architectures: on_architecture -import Oceananigans.BoundaryConditions: fill_halo_regions!, getbc +import Oceananigans.BoundaryConditions: fill_halo_regions!, getbc, get_boundary_conditions import Statistics: norm, mean, mean! import Base: == @@ -775,3 +775,5 @@ function fill_halo_regions!(field::Field, args...; kwargs...) return nothing end +# return a fields boundary conditions +get_boundary_conditions(field::Field) = field.boundary_conditions \ No newline at end of file From 1369d64463eea01245fd8ac1abadf314c80eaac3 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Tue, 28 May 2024 14:56:59 +0100 Subject: [PATCH 020/113] added kwarg to to know if the pressure is corrected --- .../update_boundary_conditions.jl | 24 +++++++++---------- src/Fields/field.jl | 2 +- .../pressure_correction.jl | 2 ++ .../solve_for_pressure.jl | 2 ++ .../update_nonhydrostatic_model_state.jl | 2 +- 5 files changed, 18 insertions(+), 14 deletions(-) diff --git a/src/BoundaryConditions/update_boundary_conditions.jl b/src/BoundaryConditions/update_boundary_conditions.jl index 2846bbbf0d..8576f2f7c1 100644 --- a/src/BoundaryConditions/update_boundary_conditions.jl +++ b/src/BoundaryConditions/update_boundary_conditions.jl @@ -1,16 +1,16 @@ -update_boundary_condition!(bc::Union{BoundaryCondition, Nothing}, args...) = nothing +update_boundary_condition!(bc::Union{BoundaryCondition, Nothing}, args...; kwargs...) = nothing -function update_boundary_conditions!(bcs::FieldBoundaryConditions, field, model) - update_boundary_condition!(bcs.west, field, model, Val(:west)) - update_boundary_condition!(bcs.east, field, model, Val(:east)) - update_boundary_condition!(bcs.south, field, model, Val(:south)) - update_boundary_condition!(bcs.north, field, model, Val(:north)) - update_boundary_condition!(bcs.bottom, field, model, Val(:bottom)) - update_boundary_condition!(bcs.top, field, model, Val(:top)) - update_boundary_condition!(bcs.immersed, field, model, Val(:immersed)) +function update_boundary_conditions!(bcs::FieldBoundaryConditions, field, model; kwargs...) + update_boundary_condition!(bcs.west, field, model, Val(:west); kwargs...) + update_boundary_condition!(bcs.east, field, model, Val(:east); kwargs...) + update_boundary_condition!(bcs.south, field, model, Val(:south); kwargs...) + update_boundary_condition!(bcs.north, field, model, Val(:north); kwargs...) + update_boundary_condition!(bcs.bottom, field, model, Val(:bottom); kwargs...) + update_boundary_condition!(bcs.top, field, model, Val(:top); kwargs...) + update_boundary_condition!(bcs.immersed, field, model, Val(:immersed); kwargs...) end -update_boundary_conditions!(fields::Union{NamedTuple, Tuple}, model) = - Tuple(update_boundary_conditions!(get_boundary_conditions(field), field, model) for field in fields) +update_boundary_conditions!(fields::Union{NamedTuple, Tuple}, model; kwargs...) = + Tuple(update_boundary_conditions!(get_boundary_conditions(field), field, model; kwargs...) for field in fields) -get_boundary_conditions(field) = nothing \ No newline at end of file +@inline get_boundary_conditions(field) = nothing \ No newline at end of file diff --git a/src/Fields/field.jl b/src/Fields/field.jl index 043c903760..b1c45b8de3 100644 --- a/src/Fields/field.jl +++ b/src/Fields/field.jl @@ -776,4 +776,4 @@ function fill_halo_regions!(field::Field, args...; kwargs...) end # return a fields boundary conditions -get_boundary_conditions(field::Field) = field.boundary_conditions \ No newline at end of file +@inline get_boundary_conditions(field::Field) = field.boundary_conditions \ No newline at end of file diff --git a/src/Models/NonhydrostaticModels/pressure_correction.jl b/src/Models/NonhydrostaticModels/pressure_correction.jl index c657995799..4dd5a33a26 100644 --- a/src/Models/NonhydrostaticModels/pressure_correction.jl +++ b/src/Models/NonhydrostaticModels/pressure_correction.jl @@ -10,6 +10,8 @@ function calculate_pressure_correction!(model::NonhydrostaticModel, Δt) # Mask immersed velocities foreach(mask_immersed_field!, model.velocities) + update_boundary_conditions!(fields(model), model; pressure_corrected = false) + fill_halo_regions!(model.velocities, model.clock, fields(model)) solve_for_pressure!(model.pressures.pNHS, model.pressure_solver, Δt, model.velocities) diff --git a/src/Models/NonhydrostaticModels/solve_for_pressure.jl b/src/Models/NonhydrostaticModels/solve_for_pressure.jl index 23bd6b2599..1e020d2d4f 100644 --- a/src/Models/NonhydrostaticModels/solve_for_pressure.jl +++ b/src/Models/NonhydrostaticModels/solve_for_pressure.jl @@ -3,6 +3,8 @@ using Oceananigans.Solvers: FFTBasedPoissonSolver, FourierTridiagonalPoissonSolv using Oceananigans.DistributedComputations: DistributedFFTBasedPoissonSolver using Oceananigans.Grids: XDirection, YDirection, ZDirection +using Statistics: mean + using PencilArrays: Permutation ##### diff --git a/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl b/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl index c9fe26e5a3..06561b2e41 100644 --- a/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl +++ b/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl @@ -28,7 +28,7 @@ function update_state!(model::NonhydrostaticModel, callbacks=[]; compute_tendenc #update_model_field_time_series!(model, model.clock) # Update the boundary conditions - update_boundary_conditions!(fields(model), model) + update_boundary_conditions!(fields(model), model; pressure_corrected = true) # Fill halos for velocities and tracers fill_halo_regions!(merge(model.velocities, model.tracers), model.clock, fields(model); async = true) From 26e7f4aa2e1a0d617855adc1fabe77f5698bd085 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Tue, 28 May 2024 17:08:14 +0100 Subject: [PATCH 021/113] addded no wall normal velocity gradcient --- src/BoundaryConditions/BoundaryConditions.jl | 1 + .../no_gradient_open_boudnary.jl | 54 +++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 src/BoundaryConditions/no_gradient_open_boudnary.jl diff --git a/src/BoundaryConditions/BoundaryConditions.jl b/src/BoundaryConditions/BoundaryConditions.jl index efa4f07571..5bb0789a69 100644 --- a/src/BoundaryConditions/BoundaryConditions.jl +++ b/src/BoundaryConditions/BoundaryConditions.jl @@ -39,4 +39,5 @@ include("apply_flux_bcs.jl") include("update_boundary_conditions.jl") +include("no_gradient_open_boudnary.jl") end # module diff --git a/src/BoundaryConditions/no_gradient_open_boudnary.jl b/src/BoundaryConditions/no_gradient_open_boudnary.jl new file mode 100644 index 0000000000..f031b7aa4f --- /dev/null +++ b/src/BoundaryConditions/no_gradient_open_boudnary.jl @@ -0,0 +1,54 @@ +""" + ZeroGradient + +Zero gradient perepndicular velocity boundary condition. This is compatible +with the pressure solve (???) and implies that there is a pressure gradient +on the boundary equal to the pressure gradient at the boundary adjacent face +node. + +```math +u_b^\\star = u_{b-1}^\\star,\\\\ +u_b^{n+1} = u_{b-1}^{n+1} = u_{b-1}^\\star - \\Delta t\\partial_x P^{fcc}_{b-1} += u_{b}^\\star - \\Delta t\\partial_x P^{fcc}_{b},\\\\ +\\to \\partial_x P^{fcc}_{b} = \\partial_x P^{fcc}_{b-1} \\therefore \\partial_x^2 P^{ccc}_{b-1} = 0. +``` +""" +struct ZeroGradient end + +const ZGOBC = BoundaryCondition{<:Open{<:ZeroGradient}} + +function ZeroGradientOpenBoundaryCondition() + classifcation = Open(ZeroGradient()) + + return BoundaryCondition(classifcation, nothing) +end + +@inline function _fill_west_halo!(j, k, grid, c, bc::ZGOBC, loc, clock, model_fields) + @inbounds c[0, j, k] = c[1, j, k] +end + +@inline function _fill_east_halo!(j, k, grid, c, bc::ZGOBC, loc, clock, model_fields) + i = grid.Nx + 1 + + @inbounds c[i, j, k] = c[i - 1, j, k] #2 * c[i - 1, j, k] - c[i - 2, j, k] +end + +@inline function _fill_south_halo!(i, k, grid, c, bc::ZGOBC, loc, clock, model_fields) + @inbounds c[i, 0, k] = c[i, 1, k] +end + +@inline function _fill_north_halo!(i, k, grid, c, bc::ZGOBC, loc, clock, model_fields) + j = grid.Ny + 1 + + @inbounds c[i, j, k] = c[i, j - 1, k] +end + +@inline function _fill_bottom_halo!(i, j, grid, c, bc::ZGOBC, loc, clock, model_fields) + @inbounds c[i, j, 0] = c[i, j, 1] +end + +@inline function _fill_top_halo!(i, j, grid, c, bc::ZGOBC, loc, clock, model_fields) + k = grid.Nz + 1 + + @inbounds c[i, j, k] = c[i, j, k - 1] +end \ No newline at end of file From 15a527a1278553bc4ff1e78e0a2e5ee5f7be8554 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Wed, 29 May 2024 16:21:15 +0100 Subject: [PATCH 022/113] moved clock tick to before pressure correction --- src/TimeSteppers/quasi_adams_bashforth_2.jl | 4 +++- src/TimeSteppers/runge_kutta_3.jl | 11 +++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/TimeSteppers/quasi_adams_bashforth_2.jl b/src/TimeSteppers/quasi_adams_bashforth_2.jl index 780df779b6..b89619f7da 100644 --- a/src/TimeSteppers/quasi_adams_bashforth_2.jl +++ b/src/TimeSteppers/quasi_adams_bashforth_2.jl @@ -92,11 +92,13 @@ function time_step!(model::AbstractModel{<:QuasiAdamsBashforth2TimeStepper}, Δt model.clock.iteration == 0 && update_state!(model, callbacks) ab2_step!(model, Δt, χ) # full step for tracers, fractional step for velocities. + + tick!(model.clock, Δt) + calculate_pressure_correction!(model, Δt) @apply_regionally correct_velocities_and_store_tendecies!(model, Δt) - tick!(model.clock, Δt) update_state!(model, callbacks; compute_tendencies) step_lagrangian_particles!(model, Δt) diff --git a/src/TimeSteppers/runge_kutta_3.jl b/src/TimeSteppers/runge_kutta_3.jl index 9c706c88ff..ab17a65bdd 100644 --- a/src/TimeSteppers/runge_kutta_3.jl +++ b/src/TimeSteppers/runge_kutta_3.jl @@ -101,10 +101,11 @@ function time_step!(model::AbstractModel{<:RungeKutta3TimeStepper}, Δt; callbac rk3_substep!(model, Δt, γ¹, nothing) + tick!(model.clock, first_stage_Δt; stage=true) + calculate_pressure_correction!(model, first_stage_Δt) pressure_correct_velocities!(model, first_stage_Δt) - tick!(model.clock, first_stage_Δt; stage=true) store_tendencies!(model) update_state!(model, callbacks) step_lagrangian_particles!(model, first_stage_Δt) @@ -115,10 +116,11 @@ function time_step!(model::AbstractModel{<:RungeKutta3TimeStepper}, Δt; callbac rk3_substep!(model, Δt, γ², ζ²) + tick!(model.clock, second_stage_Δt; stage=true) + calculate_pressure_correction!(model, second_stage_Δt) pressure_correct_velocities!(model, second_stage_Δt) - tick!(model.clock, second_stage_Δt; stage=true) store_tendencies!(model) update_state!(model, callbacks) step_lagrangian_particles!(model, second_stage_Δt) @@ -129,10 +131,11 @@ function time_step!(model::AbstractModel{<:RungeKutta3TimeStepper}, Δt; callbac rk3_substep!(model, Δt, γ³, ζ³) + tick!(model.clock, third_stage_Δt) + calculate_pressure_correction!(model, third_stage_Δt) pressure_correct_velocities!(model, third_stage_Δt) - - tick!(model.clock, third_stage_Δt) + update_state!(model, callbacks; compute_tendencies) step_lagrangian_particles!(model, third_stage_Δt) From b38400b85f49ec0d909424472dc892af6f666225 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Thu, 30 May 2024 15:23:02 +0100 Subject: [PATCH 023/113] Revert "added kwarg to to know if the pressure is corrected" This reverts commit 1369d64463eea01245fd8ac1abadf314c80eaac3. --- .../update_boundary_conditions.jl | 24 +++++++++---------- src/Fields/field.jl | 2 +- .../pressure_correction.jl | 2 -- .../solve_for_pressure.jl | 2 -- .../update_nonhydrostatic_model_state.jl | 2 +- 5 files changed, 14 insertions(+), 18 deletions(-) diff --git a/src/BoundaryConditions/update_boundary_conditions.jl b/src/BoundaryConditions/update_boundary_conditions.jl index 8576f2f7c1..2846bbbf0d 100644 --- a/src/BoundaryConditions/update_boundary_conditions.jl +++ b/src/BoundaryConditions/update_boundary_conditions.jl @@ -1,16 +1,16 @@ -update_boundary_condition!(bc::Union{BoundaryCondition, Nothing}, args...; kwargs...) = nothing +update_boundary_condition!(bc::Union{BoundaryCondition, Nothing}, args...) = nothing -function update_boundary_conditions!(bcs::FieldBoundaryConditions, field, model; kwargs...) - update_boundary_condition!(bcs.west, field, model, Val(:west); kwargs...) - update_boundary_condition!(bcs.east, field, model, Val(:east); kwargs...) - update_boundary_condition!(bcs.south, field, model, Val(:south); kwargs...) - update_boundary_condition!(bcs.north, field, model, Val(:north); kwargs...) - update_boundary_condition!(bcs.bottom, field, model, Val(:bottom); kwargs...) - update_boundary_condition!(bcs.top, field, model, Val(:top); kwargs...) - update_boundary_condition!(bcs.immersed, field, model, Val(:immersed); kwargs...) +function update_boundary_conditions!(bcs::FieldBoundaryConditions, field, model) + update_boundary_condition!(bcs.west, field, model, Val(:west)) + update_boundary_condition!(bcs.east, field, model, Val(:east)) + update_boundary_condition!(bcs.south, field, model, Val(:south)) + update_boundary_condition!(bcs.north, field, model, Val(:north)) + update_boundary_condition!(bcs.bottom, field, model, Val(:bottom)) + update_boundary_condition!(bcs.top, field, model, Val(:top)) + update_boundary_condition!(bcs.immersed, field, model, Val(:immersed)) end -update_boundary_conditions!(fields::Union{NamedTuple, Tuple}, model; kwargs...) = - Tuple(update_boundary_conditions!(get_boundary_conditions(field), field, model; kwargs...) for field in fields) +update_boundary_conditions!(fields::Union{NamedTuple, Tuple}, model) = + Tuple(update_boundary_conditions!(get_boundary_conditions(field), field, model) for field in fields) -@inline get_boundary_conditions(field) = nothing \ No newline at end of file +get_boundary_conditions(field) = nothing \ No newline at end of file diff --git a/src/Fields/field.jl b/src/Fields/field.jl index b1c45b8de3..043c903760 100644 --- a/src/Fields/field.jl +++ b/src/Fields/field.jl @@ -776,4 +776,4 @@ function fill_halo_regions!(field::Field, args...; kwargs...) end # return a fields boundary conditions -@inline get_boundary_conditions(field::Field) = field.boundary_conditions \ No newline at end of file +get_boundary_conditions(field::Field) = field.boundary_conditions \ No newline at end of file diff --git a/src/Models/NonhydrostaticModels/pressure_correction.jl b/src/Models/NonhydrostaticModels/pressure_correction.jl index 4dd5a33a26..c657995799 100644 --- a/src/Models/NonhydrostaticModels/pressure_correction.jl +++ b/src/Models/NonhydrostaticModels/pressure_correction.jl @@ -10,8 +10,6 @@ function calculate_pressure_correction!(model::NonhydrostaticModel, Δt) # Mask immersed velocities foreach(mask_immersed_field!, model.velocities) - update_boundary_conditions!(fields(model), model; pressure_corrected = false) - fill_halo_regions!(model.velocities, model.clock, fields(model)) solve_for_pressure!(model.pressures.pNHS, model.pressure_solver, Δt, model.velocities) diff --git a/src/Models/NonhydrostaticModels/solve_for_pressure.jl b/src/Models/NonhydrostaticModels/solve_for_pressure.jl index 1e020d2d4f..23bd6b2599 100644 --- a/src/Models/NonhydrostaticModels/solve_for_pressure.jl +++ b/src/Models/NonhydrostaticModels/solve_for_pressure.jl @@ -3,8 +3,6 @@ using Oceananigans.Solvers: FFTBasedPoissonSolver, FourierTridiagonalPoissonSolv using Oceananigans.DistributedComputations: DistributedFFTBasedPoissonSolver using Oceananigans.Grids: XDirection, YDirection, ZDirection -using Statistics: mean - using PencilArrays: Permutation ##### diff --git a/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl b/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl index 06561b2e41..c9fe26e5a3 100644 --- a/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl +++ b/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl @@ -28,7 +28,7 @@ function update_state!(model::NonhydrostaticModel, callbacks=[]; compute_tendenc #update_model_field_time_series!(model, model.clock) # Update the boundary conditions - update_boundary_conditions!(fields(model), model; pressure_corrected = true) + update_boundary_conditions!(fields(model), model) # Fill halos for velocities and tracers fill_halo_regions!(merge(model.velocities, model.tracers), model.clock, fields(model); async = true) From c8f6cd6bb600e8ec2f03c7f18518d9f838f4940f Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Thu, 30 May 2024 16:00:21 +0100 Subject: [PATCH 024/113] refuse to fill bounded-wall normal velocity points after the pressure correction --- src/BoundaryConditions/fill_halo_regions.jl | 50 +++++++++---------- .../fill_halo_regions_open.jl | 26 +++++----- .../fill_halo_regions_value_gradient.jl | 12 ++--- .../pressure_correction.jl | 4 ++ .../update_nonhydrostatic_model_state.jl | 3 +- 5 files changed, 51 insertions(+), 44 deletions(-) diff --git a/src/BoundaryConditions/fill_halo_regions.jl b/src/BoundaryConditions/fill_halo_regions.jl index cc92479f40..b652041eb3 100644 --- a/src/BoundaryConditions/fill_halo_regions.jl +++ b/src/BoundaryConditions/fill_halo_regions.jl @@ -46,7 +46,6 @@ const MaybeTupledData = Union{OffsetArray, NTuple{<:Any, OffsetArray}} "Fill halo regions in ``x``, ``y``, and ``z`` for a given field's data." function fill_halo_regions!(c::MaybeTupledData, boundary_conditions, indices, loc, grid, args...; kwargs...) - arch = architecture(grid) fill_halos!, bcs = permute_boundary_conditions(boundary_conditions) @@ -66,6 +65,7 @@ function fill_halo_event!(c, fill_halos!, bcs, indices, loc, arch, grid, args... # We assume that the kernel size is the same for west and east boundaries, # south and north boundaries, and bottom and top boundaries size = fill_halo_size(c, fill_halos!, indices, bcs[1], loc, grid) + offset = fill_halo_offset(size, fill_halos!, indices) fill_halos!(c, bcs..., size, offset, loc, arch, grid, args...; kwargs...) @@ -191,22 +191,22 @@ fill_first(bc1, bc2) = true ##### Double-sided fill_halo! kernels ##### -@kernel function _fill_west_and_east_halo!(c, west_bc, east_bc, loc, grid, args) +@kernel function _fill_west_and_east_halo!(c, west_bc, east_bc, loc, grid, pressure_corrected, args) j, k = @index(Global, NTuple) - _fill_west_halo!(j, k, grid, c, west_bc, loc, args...) - _fill_east_halo!(j, k, grid, c, east_bc, loc, args...) + _fill_west_halo!(j, k, grid, c, west_bc, loc, pressure_corrected, args...) + _fill_east_halo!(j, k, grid, c, east_bc, loc, pressure_corrected, args...) end -@kernel function _fill_south_and_north_halo!(c, south_bc, north_bc, loc, grid, args) +@kernel function _fill_south_and_north_halo!(c, south_bc, north_bc, loc, grid, pressure_corrected, args) i, k = @index(Global, NTuple) - _fill_south_halo!(i, k, grid, c, south_bc, loc, args...) - _fill_north_halo!(i, k, grid, c, north_bc, loc, args...) + _fill_south_halo!(i, k, grid, c, south_bc, loc, pressure_corrected, args...) + _fill_north_halo!(i, k, grid, c, north_bc, loc, pressure_corrected, args...) end -@kernel function _fill_bottom_and_top_halo!(c, bottom_bc, top_bc, loc, grid, args) +@kernel function _fill_bottom_and_top_halo!(c, bottom_bc, top_bc, loc, grid, pressure_corrected, args) i, j = @index(Global, NTuple) - _fill_bottom_halo!(i, j, grid, c, bottom_bc, loc, args...) - _fill_top_halo!(i, j, grid, c, top_bc, loc, args...) + _fill_bottom_halo!(i, j, grid, c, bottom_bc, loc, pressure_corrected, args...) + _fill_top_halo!(i, j, grid, c, top_bc, loc, pressure_corrected, args...) end ##### @@ -251,38 +251,38 @@ end # support tupled halo filling import Oceananigans.Utils: @constprop -@kernel function _fill_west_and_east_halo!(c::NTuple, west_bc, east_bc, loc, grid, args) +@kernel function _fill_west_and_east_halo!(c::NTuple, west_bc, east_bc, loc, grid, pressure_corrected, args) j, k = @index(Global, NTuple) ntuple(Val(length(west_bc))) do n Base.@_inline_meta @constprop(:aggressive) # TODO constprop failure on `loc[n]` @inbounds begin - _fill_west_halo!(j, k, grid, c[n], west_bc[n], loc[n], args...) - _fill_east_halo!(j, k, grid, c[n], east_bc[n], loc[n], args...) + _fill_west_halo!(j, k, grid, c[n], west_bc[n], loc[n], pressure_corrected, args...) + _fill_east_halo!(j, k, grid, c[n], east_bc[n], loc[n], pressure_corrected, args...) end end end -@kernel function _fill_south_and_north_halo!(c::NTuple, south_bc, north_bc, loc, grid, args) +@kernel function _fill_south_and_north_halo!(c::NTuple, south_bc, north_bc, loc, grid, pressure_corrected, args) i, k = @index(Global, NTuple) ntuple(Val(length(south_bc))) do n Base.@_inline_meta @constprop(:aggressive) # TODO constprop failure on `loc[n]` @inbounds begin - _fill_south_halo!(i, k, grid, c[n], south_bc[n], loc[n], args...) - _fill_north_halo!(i, k, grid, c[n], north_bc[n], loc[n], args...) + _fill_south_halo!(i, k, grid, c[n], south_bc[n], loc[n], pressure_corrected, args...) + _fill_north_halo!(i, k, grid, c[n], north_bc[n], loc[n], pressure_corrected, args...) end end end -@kernel function _fill_bottom_and_top_halo!(c::NTuple, bottom_bc, top_bc, loc, grid, args) +@kernel function _fill_bottom_and_top_halo!(c::NTuple, bottom_bc, top_bc, loc, grid, pressure_corrected, args) i, j = @index(Global, NTuple) ntuple(Val(length(bottom_bc))) do n Base.@_inline_meta @constprop(:aggressive) # TODO constprop failure on `loc[n]` @inbounds begin - _fill_bottom_halo!(i, j, grid, c[n], bottom_bc[n], loc[n], args...) - _fill_top_halo!(i, j, grid, c[n], top_bc[n], loc[n], args...) + _fill_bottom_halo!(i, j, grid, c[n], bottom_bc[n], loc[n], pressure_corrected, args...) + _fill_top_halo!(i, j, grid, c[n], top_bc[n], loc[n], pressure_corrected, args...) end end end @@ -319,19 +319,19 @@ fill_top_halo!(c, bc, size, offset, loc, arch, grid, args...; kwargs...) = ##### Kernel launchers for double-sided fill_halos ##### -function fill_west_and_east_halo!(c, west_bc, east_bc, size, offset, loc, arch, grid, args...; kwargs...) +function fill_west_and_east_halo!(c, west_bc, east_bc, size, offset, loc, arch, grid, args...; pressure_corrected = false, kwargs...) return launch!(arch, grid, KernelParameters(size, offset), - _fill_west_and_east_halo!, c, west_bc, east_bc, loc, grid, Tuple(args); kwargs...) + _fill_west_and_east_halo!, c, west_bc, east_bc, loc, grid, Val(pressure_corrected), Tuple(args); kwargs...) end -function fill_south_and_north_halo!(c, south_bc, north_bc, size, offset, loc, arch, grid, args...; kwargs...) +function fill_south_and_north_halo!(c, south_bc, north_bc, size, offset, loc, arch, grid, args...; pressure_corrected = false, kwargs...) return launch!(arch, grid, KernelParameters(size, offset), - _fill_south_and_north_halo!, c, south_bc, north_bc, loc, grid, Tuple(args); kwargs...) + _fill_south_and_north_halo!, c, south_bc, north_bc, loc, grid, Val(pressure_corrected), Tuple(args); kwargs...) end -function fill_bottom_and_top_halo!(c, bottom_bc, top_bc, size, offset, loc, arch, grid, args...; kwargs...) +function fill_bottom_and_top_halo!(c, bottom_bc, top_bc, size, offset, loc, arch, grid, args...; pressure_corrected = false, kwargs...) return launch!(arch, grid, KernelParameters(size, offset), - _fill_bottom_and_top_halo!, c, bottom_bc, top_bc, loc, grid, Tuple(args); kwargs...) + _fill_bottom_and_top_halo!, c, bottom_bc, top_bc, loc, grid, Val(pressure_corrected), Tuple(args); kwargs...) end ##### diff --git a/src/BoundaryConditions/fill_halo_regions_open.jl b/src/BoundaryConditions/fill_halo_regions_open.jl index 75ff5eac05..9e0373d488 100644 --- a/src/BoundaryConditions/fill_halo_regions_open.jl +++ b/src/BoundaryConditions/fill_halo_regions_open.jl @@ -24,16 +24,18 @@ end @inbounds w[i, j, k_boundary] = getbc(bc, i, j, grid, args...) end -@inline fill_west_halo!(u, bc::OBC, kernel_size, offset, loc, arch, grid, args...; kwargs...) = launch!(arch, grid, KernelParameters(kernel_size, offset), set_west_or_east_u!, u, 1, bc, grid, Tuple(args); kwargs...) -@inline fill_east_halo!(u, bc::OBC, kernel_size, offset, loc, arch, grid, args...; kwargs...) = launch!(arch, grid, KernelParameters(kernel_size, offset), set_west_or_east_u!, u, grid.Nx + 1, bc, grid, Tuple(args); kwargs...) -@inline fill_south_halo!(v, bc::OBC, kernel_size, offset, loc, arch, grid, args...; kwargs...) = launch!(arch, grid, KernelParameters(kernel_size, offset), set_south_or_north_v!, v, 1, bc, grid, Tuple(args); kwargs...) -@inline fill_north_halo!(v, bc::OBC, kernel_size, offset, loc, arch, grid, args...; kwargs...) = launch!(arch, grid, KernelParameters(kernel_size, offset), set_south_or_north_v!, v, grid.Ny + 1, bc, grid, Tuple(args); kwargs...) -@inline fill_bottom_halo!(w, bc::OBC, kernel_size, offset, loc, arch, grid, args...; kwargs...) = launch!(arch, grid, KernelParameters(kernel_size, offset), set_bottom_or_top_w!, w, 1, bc, grid, Tuple(args); kwargs...) -@inline fill_top_halo!(w, bc::OBC, kernel_size, offset, loc, arch, grid, args...; kwargs...) = launch!(arch, grid, KernelParameters(kernel_size, offset), set_bottom_or_top_w!, w, grid.Nz + 1, bc, grid, Tuple(args); kwargs...) +@inline _fill_west_halo!(j, k, grid, c, bc::OBC, loc, pressure_corrected, args...) = @inbounds c[1, j, k] = getbc(bc, j, k, grid, args...) +@inline _fill_east_halo!(j, k, grid, c, bc::OBC, loc, pressure_corrected, args...) = @inbounds c[grid.Nx + 1, j, k] = getbc(bc, j, k, grid, args...) +@inline _fill_south_halo!(i, k, grid, c, bc::OBC, loc, pressure_corrected, args...) = @inbounds c[i, 1, k] = getbc(bc, i, k, grid, args...) +@inline _fill_north_halo!(i, k, grid, c, bc::OBC, loc, pressure_corrected, args...) = @inbounds c[i, grid.Ny + 1, k] = getbc(bc, i, k, grid, args...) +@inline _fill_bottom_halo!(i, j, grid, c, bc::OBC, loc, pressure_corrected, args...) = @inbounds c[i, j, 1] = getbc(bc, i, j, grid, args...) +@inline _fill_top_halo!(i, j, grid, c, bc::OBC, loc, pressure_corrected, args...) = @inbounds c[i, j, grid.Nz + 1] = getbc(bc, i, j, grid, args...) -@inline _fill_west_halo!(j, k, grid, c, bc::OBC, loc, args...) = @inbounds c[1, j, k] = getbc(bc, j, k, grid, args...) -@inline _fill_east_halo!(j, k, grid, c, bc::OBC, loc, args...) = @inbounds c[grid.Nx + 1, j, k] = getbc(bc, j, k, grid, args...) -@inline _fill_south_halo!(i, k, grid, c, bc::OBC, loc, args...) = @inbounds c[i, 1, k] = getbc(bc, i, k, grid, args...) -@inline _fill_north_halo!(i, k, grid, c, bc::OBC, loc, args...) = @inbounds c[i, grid.Ny + 1, k] = getbc(bc, i, k, grid, args...) -@inline _fill_bottom_halo!(i, j, grid, c, bc::OBC, loc, args...) = @inbounds c[i, j, 1] = getbc(bc, i, j, grid, args...) -@inline _fill_top_halo!(i, j, grid, c, bc::OBC, loc, args...) = @inbounds c[i, j, grid.Nz + 1] = getbc(bc, i, j, grid, args...) +# refuse if wall normal + +@inline _fill_west_halo!(j, k, grid::AbstractGrid{<:Any, Bounded}, c, bc::OBC, loc::Tuple{Face, Center, Center}, pressure_corrected::Val{true}, args...) = nothing +@inline _fill_east_halo!(j, k, grid::AbstractGrid{<:Any, Bounded}, c, bc::OBC, loc::Tuple{Face, Center, Center}, pressure_corrected::Val{true}, args...) = nothing +@inline _fill_south_halo!(j, k, grid::AbstractGrid{<:Any, <:Any, Bounded}, c, bc::OBC, loc::Tuple{Center, Face, Center}, pressure_corrected::Val{true}, args...) = nothing +@inline _fill_north_halo!(j, k, grid::AbstractGrid{<:Any, <:Any, Bounded}, c, bc::OBC, loc::Tuple{Center, Face, Center}, pressure_corrected::Val{true}, args...) = nothing +@inline _fill_bottom_halo!(j, k, grid::AbstractGrid{<:Any, <:Any, <:Any, Bounded}, c, bc::OBC, loc::Tuple{Center, Center, Face}, pressure_corrected::Val{true}, args...) = nothing +@inline _fill_top_halo!(j, k, grid::AbstractGrid{<:Any, <:Any, <:Any, Bounded}, c, bc::OBC, loc::Tuple{Center, Center, Face}, pressure_corrected::Val{true}, args...) = nothing \ No newline at end of file diff --git a/src/BoundaryConditions/fill_halo_regions_value_gradient.jl b/src/BoundaryConditions/fill_halo_regions_value_gradient.jl index 7d08b13012..a6f85f8c32 100644 --- a/src/BoundaryConditions/fill_halo_regions_value_gradient.jl +++ b/src/BoundaryConditions/fill_halo_regions_value_gradient.jl @@ -12,7 +12,7 @@ using Oceananigans.Operators: Δx, Δy, Δz @inline left_gradient(bc::VBC, c¹, Δ, i, j, args...) = ( c¹ - getbc(bc, i, j, args...) ) / (Δ/2) @inline right_gradient(bc::VBC, cᴺ, Δ, i, j, args...) = ( getbc(bc, i, j, args...) - cᴺ ) / (Δ/2) -@inline function _fill_west_halo!(j, k, grid, c, bc::Union{VBC, GBC}, loc, args...) +@inline function _fill_west_halo!(j, k, grid, c, bc::Union{VBC, GBC}, loc, pressure_corrected, args...) # ↑ x ↑ interior # ----- interior face @@ -26,7 +26,7 @@ using Oceananigans.Operators: Δx, Δy, Δz @inbounds c[iᴴ, j, k] = linearly_extrapolate(c[iᴵ, j, k], ∇c, -Δ) # extrapolate westward in -x direction. end -@inline function _fill_east_halo!(j, k, grid, c, bc::Union{VBC, GBC}, loc, args...) +@inline function _fill_east_halo!(j, k, grid, c, bc::Union{VBC, GBC}, loc, pressure_corrected, args...) # ↑ x ↑ iᴴ = grid.Nx + 1 # * halo cell @@ -41,7 +41,7 @@ end @inbounds c[iᴴ, j, k] = linearly_extrapolate(c[iᴵ, j, k], ∇c, Δ) # extrapolate eastward in +x direction. end -@inline function _fill_south_halo!(i, k, grid, c, bc::Union{VBC, GBC}, loc, args...) +@inline function _fill_south_halo!(i, k, grid, c, bc::Union{VBC, GBC}, loc, pressure_corrected, args...) # ↑ y ↑ interior # ----- interior face @@ -55,7 +55,7 @@ end @inbounds c[i, jᴴ, k] = linearly_extrapolate(c[i, jᴵ, k], ∇c, -Δ) # extrapolate southward in -y direction. end -@inline function _fill_north_halo!(i, k, grid, c, bc::Union{VBC, GBC}, loc, args...) +@inline function _fill_north_halo!(i, k, grid, c, bc::Union{VBC, GBC}, loc, pressure_corrected, args...) # ↑ y ↑ jᴴ = grid.Ny + 1 # * halo cell @@ -70,7 +70,7 @@ end @inbounds c[i, jᴴ, k] = linearly_extrapolate(c[i, jᴵ, k], ∇c, Δ) # extrapolate northward in +y direction. end -@inline function _fill_bottom_halo!(i, j, grid, c, bc::Union{VBC, GBC}, loc, args...) +@inline function _fill_bottom_halo!(i, j, grid, c, bc::Union{VBC, GBC}, loc, pressure_corrected, args...) # ↑ z ↑ interior # ----- interior face @@ -84,7 +84,7 @@ end @inbounds c[i, j, kᴴ] = linearly_extrapolate(c[i, j, kᴵ], ∇c, -Δ) # extrapolate downward in -z direction. end -@inline function _fill_top_halo!(i, j, grid, c, bc::Union{VBC, GBC}, loc, args...) +@inline function _fill_top_halo!(i, j, grid, c, bc::Union{VBC, GBC}, loc, pressure_corrected, args...) # ↑ z ↑ kᴴ = grid.Nz + 1 # * halo cell diff --git a/src/Models/NonhydrostaticModels/pressure_correction.jl b/src/Models/NonhydrostaticModels/pressure_correction.jl index c657995799..03deb77020 100644 --- a/src/Models/NonhydrostaticModels/pressure_correction.jl +++ b/src/Models/NonhydrostaticModels/pressure_correction.jl @@ -10,6 +10,10 @@ function calculate_pressure_correction!(model::NonhydrostaticModel, Δt) # Mask immersed velocities foreach(mask_immersed_field!, model.velocities) + # Update the boundary conditions + update_boundary_conditions!(fields(model), model) + + # this is where we fill u⋆ fill_halo_regions!(model.velocities, model.clock, fields(model)) solve_for_pressure!(model.pressures.pNHS, model.pressure_solver, Δt, model.velocities) diff --git a/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl b/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl index c9fe26e5a3..76c13447c0 100644 --- a/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl +++ b/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl @@ -31,7 +31,8 @@ function update_state!(model::NonhydrostaticModel, callbacks=[]; compute_tendenc update_boundary_conditions!(fields(model), model) # Fill halos for velocities and tracers - fill_halo_regions!(merge(model.velocities, model.tracers), model.clock, fields(model); async = true) + # this is where we need to NOT update boundary normal velocities + fill_halo_regions!(merge(model.velocities, model.tracers), model.clock, fields(model); async = true, pressure_corrected = true) # Compute auxiliary fields for aux_field in model.auxiliary_fields From a8e5c2461811daee63cfc1bae1d2bf52d73dd300 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Fri, 31 May 2024 13:23:22 +0100 Subject: [PATCH 025/113] Revert "refuse to fill bounded-wall normal velocity points after the pressure correction" This reverts commit c8f6cd6bb600e8ec2f03c7f18518d9f838f4940f. --- src/BoundaryConditions/fill_halo_regions.jl | 50 +++++++++---------- .../fill_halo_regions_open.jl | 26 +++++----- .../fill_halo_regions_value_gradient.jl | 12 ++--- .../pressure_correction.jl | 4 -- .../update_nonhydrostatic_model_state.jl | 3 +- 5 files changed, 44 insertions(+), 51 deletions(-) diff --git a/src/BoundaryConditions/fill_halo_regions.jl b/src/BoundaryConditions/fill_halo_regions.jl index b652041eb3..cc92479f40 100644 --- a/src/BoundaryConditions/fill_halo_regions.jl +++ b/src/BoundaryConditions/fill_halo_regions.jl @@ -46,6 +46,7 @@ const MaybeTupledData = Union{OffsetArray, NTuple{<:Any, OffsetArray}} "Fill halo regions in ``x``, ``y``, and ``z`` for a given field's data." function fill_halo_regions!(c::MaybeTupledData, boundary_conditions, indices, loc, grid, args...; kwargs...) + arch = architecture(grid) fill_halos!, bcs = permute_boundary_conditions(boundary_conditions) @@ -65,7 +66,6 @@ function fill_halo_event!(c, fill_halos!, bcs, indices, loc, arch, grid, args... # We assume that the kernel size is the same for west and east boundaries, # south and north boundaries, and bottom and top boundaries size = fill_halo_size(c, fill_halos!, indices, bcs[1], loc, grid) - offset = fill_halo_offset(size, fill_halos!, indices) fill_halos!(c, bcs..., size, offset, loc, arch, grid, args...; kwargs...) @@ -191,22 +191,22 @@ fill_first(bc1, bc2) = true ##### Double-sided fill_halo! kernels ##### -@kernel function _fill_west_and_east_halo!(c, west_bc, east_bc, loc, grid, pressure_corrected, args) +@kernel function _fill_west_and_east_halo!(c, west_bc, east_bc, loc, grid, args) j, k = @index(Global, NTuple) - _fill_west_halo!(j, k, grid, c, west_bc, loc, pressure_corrected, args...) - _fill_east_halo!(j, k, grid, c, east_bc, loc, pressure_corrected, args...) + _fill_west_halo!(j, k, grid, c, west_bc, loc, args...) + _fill_east_halo!(j, k, grid, c, east_bc, loc, args...) end -@kernel function _fill_south_and_north_halo!(c, south_bc, north_bc, loc, grid, pressure_corrected, args) +@kernel function _fill_south_and_north_halo!(c, south_bc, north_bc, loc, grid, args) i, k = @index(Global, NTuple) - _fill_south_halo!(i, k, grid, c, south_bc, loc, pressure_corrected, args...) - _fill_north_halo!(i, k, grid, c, north_bc, loc, pressure_corrected, args...) + _fill_south_halo!(i, k, grid, c, south_bc, loc, args...) + _fill_north_halo!(i, k, grid, c, north_bc, loc, args...) end -@kernel function _fill_bottom_and_top_halo!(c, bottom_bc, top_bc, loc, grid, pressure_corrected, args) +@kernel function _fill_bottom_and_top_halo!(c, bottom_bc, top_bc, loc, grid, args) i, j = @index(Global, NTuple) - _fill_bottom_halo!(i, j, grid, c, bottom_bc, loc, pressure_corrected, args...) - _fill_top_halo!(i, j, grid, c, top_bc, loc, pressure_corrected, args...) + _fill_bottom_halo!(i, j, grid, c, bottom_bc, loc, args...) + _fill_top_halo!(i, j, grid, c, top_bc, loc, args...) end ##### @@ -251,38 +251,38 @@ end # support tupled halo filling import Oceananigans.Utils: @constprop -@kernel function _fill_west_and_east_halo!(c::NTuple, west_bc, east_bc, loc, grid, pressure_corrected, args) +@kernel function _fill_west_and_east_halo!(c::NTuple, west_bc, east_bc, loc, grid, args) j, k = @index(Global, NTuple) ntuple(Val(length(west_bc))) do n Base.@_inline_meta @constprop(:aggressive) # TODO constprop failure on `loc[n]` @inbounds begin - _fill_west_halo!(j, k, grid, c[n], west_bc[n], loc[n], pressure_corrected, args...) - _fill_east_halo!(j, k, grid, c[n], east_bc[n], loc[n], pressure_corrected, args...) + _fill_west_halo!(j, k, grid, c[n], west_bc[n], loc[n], args...) + _fill_east_halo!(j, k, grid, c[n], east_bc[n], loc[n], args...) end end end -@kernel function _fill_south_and_north_halo!(c::NTuple, south_bc, north_bc, loc, grid, pressure_corrected, args) +@kernel function _fill_south_and_north_halo!(c::NTuple, south_bc, north_bc, loc, grid, args) i, k = @index(Global, NTuple) ntuple(Val(length(south_bc))) do n Base.@_inline_meta @constprop(:aggressive) # TODO constprop failure on `loc[n]` @inbounds begin - _fill_south_halo!(i, k, grid, c[n], south_bc[n], loc[n], pressure_corrected, args...) - _fill_north_halo!(i, k, grid, c[n], north_bc[n], loc[n], pressure_corrected, args...) + _fill_south_halo!(i, k, grid, c[n], south_bc[n], loc[n], args...) + _fill_north_halo!(i, k, grid, c[n], north_bc[n], loc[n], args...) end end end -@kernel function _fill_bottom_and_top_halo!(c::NTuple, bottom_bc, top_bc, loc, grid, pressure_corrected, args) +@kernel function _fill_bottom_and_top_halo!(c::NTuple, bottom_bc, top_bc, loc, grid, args) i, j = @index(Global, NTuple) ntuple(Val(length(bottom_bc))) do n Base.@_inline_meta @constprop(:aggressive) # TODO constprop failure on `loc[n]` @inbounds begin - _fill_bottom_halo!(i, j, grid, c[n], bottom_bc[n], loc[n], pressure_corrected, args...) - _fill_top_halo!(i, j, grid, c[n], top_bc[n], loc[n], pressure_corrected, args...) + _fill_bottom_halo!(i, j, grid, c[n], bottom_bc[n], loc[n], args...) + _fill_top_halo!(i, j, grid, c[n], top_bc[n], loc[n], args...) end end end @@ -319,19 +319,19 @@ fill_top_halo!(c, bc, size, offset, loc, arch, grid, args...; kwargs...) = ##### Kernel launchers for double-sided fill_halos ##### -function fill_west_and_east_halo!(c, west_bc, east_bc, size, offset, loc, arch, grid, args...; pressure_corrected = false, kwargs...) +function fill_west_and_east_halo!(c, west_bc, east_bc, size, offset, loc, arch, grid, args...; kwargs...) return launch!(arch, grid, KernelParameters(size, offset), - _fill_west_and_east_halo!, c, west_bc, east_bc, loc, grid, Val(pressure_corrected), Tuple(args); kwargs...) + _fill_west_and_east_halo!, c, west_bc, east_bc, loc, grid, Tuple(args); kwargs...) end -function fill_south_and_north_halo!(c, south_bc, north_bc, size, offset, loc, arch, grid, args...; pressure_corrected = false, kwargs...) +function fill_south_and_north_halo!(c, south_bc, north_bc, size, offset, loc, arch, grid, args...; kwargs...) return launch!(arch, grid, KernelParameters(size, offset), - _fill_south_and_north_halo!, c, south_bc, north_bc, loc, grid, Val(pressure_corrected), Tuple(args); kwargs...) + _fill_south_and_north_halo!, c, south_bc, north_bc, loc, grid, Tuple(args); kwargs...) end -function fill_bottom_and_top_halo!(c, bottom_bc, top_bc, size, offset, loc, arch, grid, args...; pressure_corrected = false, kwargs...) +function fill_bottom_and_top_halo!(c, bottom_bc, top_bc, size, offset, loc, arch, grid, args...; kwargs...) return launch!(arch, grid, KernelParameters(size, offset), - _fill_bottom_and_top_halo!, c, bottom_bc, top_bc, loc, grid, Val(pressure_corrected), Tuple(args); kwargs...) + _fill_bottom_and_top_halo!, c, bottom_bc, top_bc, loc, grid, Tuple(args); kwargs...) end ##### diff --git a/src/BoundaryConditions/fill_halo_regions_open.jl b/src/BoundaryConditions/fill_halo_regions_open.jl index 9e0373d488..75ff5eac05 100644 --- a/src/BoundaryConditions/fill_halo_regions_open.jl +++ b/src/BoundaryConditions/fill_halo_regions_open.jl @@ -24,18 +24,16 @@ end @inbounds w[i, j, k_boundary] = getbc(bc, i, j, grid, args...) end -@inline _fill_west_halo!(j, k, grid, c, bc::OBC, loc, pressure_corrected, args...) = @inbounds c[1, j, k] = getbc(bc, j, k, grid, args...) -@inline _fill_east_halo!(j, k, grid, c, bc::OBC, loc, pressure_corrected, args...) = @inbounds c[grid.Nx + 1, j, k] = getbc(bc, j, k, grid, args...) -@inline _fill_south_halo!(i, k, grid, c, bc::OBC, loc, pressure_corrected, args...) = @inbounds c[i, 1, k] = getbc(bc, i, k, grid, args...) -@inline _fill_north_halo!(i, k, grid, c, bc::OBC, loc, pressure_corrected, args...) = @inbounds c[i, grid.Ny + 1, k] = getbc(bc, i, k, grid, args...) -@inline _fill_bottom_halo!(i, j, grid, c, bc::OBC, loc, pressure_corrected, args...) = @inbounds c[i, j, 1] = getbc(bc, i, j, grid, args...) -@inline _fill_top_halo!(i, j, grid, c, bc::OBC, loc, pressure_corrected, args...) = @inbounds c[i, j, grid.Nz + 1] = getbc(bc, i, j, grid, args...) +@inline fill_west_halo!(u, bc::OBC, kernel_size, offset, loc, arch, grid, args...; kwargs...) = launch!(arch, grid, KernelParameters(kernel_size, offset), set_west_or_east_u!, u, 1, bc, grid, Tuple(args); kwargs...) +@inline fill_east_halo!(u, bc::OBC, kernel_size, offset, loc, arch, grid, args...; kwargs...) = launch!(arch, grid, KernelParameters(kernel_size, offset), set_west_or_east_u!, u, grid.Nx + 1, bc, grid, Tuple(args); kwargs...) +@inline fill_south_halo!(v, bc::OBC, kernel_size, offset, loc, arch, grid, args...; kwargs...) = launch!(arch, grid, KernelParameters(kernel_size, offset), set_south_or_north_v!, v, 1, bc, grid, Tuple(args); kwargs...) +@inline fill_north_halo!(v, bc::OBC, kernel_size, offset, loc, arch, grid, args...; kwargs...) = launch!(arch, grid, KernelParameters(kernel_size, offset), set_south_or_north_v!, v, grid.Ny + 1, bc, grid, Tuple(args); kwargs...) +@inline fill_bottom_halo!(w, bc::OBC, kernel_size, offset, loc, arch, grid, args...; kwargs...) = launch!(arch, grid, KernelParameters(kernel_size, offset), set_bottom_or_top_w!, w, 1, bc, grid, Tuple(args); kwargs...) +@inline fill_top_halo!(w, bc::OBC, kernel_size, offset, loc, arch, grid, args...; kwargs...) = launch!(arch, grid, KernelParameters(kernel_size, offset), set_bottom_or_top_w!, w, grid.Nz + 1, bc, grid, Tuple(args); kwargs...) -# refuse if wall normal - -@inline _fill_west_halo!(j, k, grid::AbstractGrid{<:Any, Bounded}, c, bc::OBC, loc::Tuple{Face, Center, Center}, pressure_corrected::Val{true}, args...) = nothing -@inline _fill_east_halo!(j, k, grid::AbstractGrid{<:Any, Bounded}, c, bc::OBC, loc::Tuple{Face, Center, Center}, pressure_corrected::Val{true}, args...) = nothing -@inline _fill_south_halo!(j, k, grid::AbstractGrid{<:Any, <:Any, Bounded}, c, bc::OBC, loc::Tuple{Center, Face, Center}, pressure_corrected::Val{true}, args...) = nothing -@inline _fill_north_halo!(j, k, grid::AbstractGrid{<:Any, <:Any, Bounded}, c, bc::OBC, loc::Tuple{Center, Face, Center}, pressure_corrected::Val{true}, args...) = nothing -@inline _fill_bottom_halo!(j, k, grid::AbstractGrid{<:Any, <:Any, <:Any, Bounded}, c, bc::OBC, loc::Tuple{Center, Center, Face}, pressure_corrected::Val{true}, args...) = nothing -@inline _fill_top_halo!(j, k, grid::AbstractGrid{<:Any, <:Any, <:Any, Bounded}, c, bc::OBC, loc::Tuple{Center, Center, Face}, pressure_corrected::Val{true}, args...) = nothing \ No newline at end of file +@inline _fill_west_halo!(j, k, grid, c, bc::OBC, loc, args...) = @inbounds c[1, j, k] = getbc(bc, j, k, grid, args...) +@inline _fill_east_halo!(j, k, grid, c, bc::OBC, loc, args...) = @inbounds c[grid.Nx + 1, j, k] = getbc(bc, j, k, grid, args...) +@inline _fill_south_halo!(i, k, grid, c, bc::OBC, loc, args...) = @inbounds c[i, 1, k] = getbc(bc, i, k, grid, args...) +@inline _fill_north_halo!(i, k, grid, c, bc::OBC, loc, args...) = @inbounds c[i, grid.Ny + 1, k] = getbc(bc, i, k, grid, args...) +@inline _fill_bottom_halo!(i, j, grid, c, bc::OBC, loc, args...) = @inbounds c[i, j, 1] = getbc(bc, i, j, grid, args...) +@inline _fill_top_halo!(i, j, grid, c, bc::OBC, loc, args...) = @inbounds c[i, j, grid.Nz + 1] = getbc(bc, i, j, grid, args...) diff --git a/src/BoundaryConditions/fill_halo_regions_value_gradient.jl b/src/BoundaryConditions/fill_halo_regions_value_gradient.jl index a6f85f8c32..7d08b13012 100644 --- a/src/BoundaryConditions/fill_halo_regions_value_gradient.jl +++ b/src/BoundaryConditions/fill_halo_regions_value_gradient.jl @@ -12,7 +12,7 @@ using Oceananigans.Operators: Δx, Δy, Δz @inline left_gradient(bc::VBC, c¹, Δ, i, j, args...) = ( c¹ - getbc(bc, i, j, args...) ) / (Δ/2) @inline right_gradient(bc::VBC, cᴺ, Δ, i, j, args...) = ( getbc(bc, i, j, args...) - cᴺ ) / (Δ/2) -@inline function _fill_west_halo!(j, k, grid, c, bc::Union{VBC, GBC}, loc, pressure_corrected, args...) +@inline function _fill_west_halo!(j, k, grid, c, bc::Union{VBC, GBC}, loc, args...) # ↑ x ↑ interior # ----- interior face @@ -26,7 +26,7 @@ using Oceananigans.Operators: Δx, Δy, Δz @inbounds c[iᴴ, j, k] = linearly_extrapolate(c[iᴵ, j, k], ∇c, -Δ) # extrapolate westward in -x direction. end -@inline function _fill_east_halo!(j, k, grid, c, bc::Union{VBC, GBC}, loc, pressure_corrected, args...) +@inline function _fill_east_halo!(j, k, grid, c, bc::Union{VBC, GBC}, loc, args...) # ↑ x ↑ iᴴ = grid.Nx + 1 # * halo cell @@ -41,7 +41,7 @@ end @inbounds c[iᴴ, j, k] = linearly_extrapolate(c[iᴵ, j, k], ∇c, Δ) # extrapolate eastward in +x direction. end -@inline function _fill_south_halo!(i, k, grid, c, bc::Union{VBC, GBC}, loc, pressure_corrected, args...) +@inline function _fill_south_halo!(i, k, grid, c, bc::Union{VBC, GBC}, loc, args...) # ↑ y ↑ interior # ----- interior face @@ -55,7 +55,7 @@ end @inbounds c[i, jᴴ, k] = linearly_extrapolate(c[i, jᴵ, k], ∇c, -Δ) # extrapolate southward in -y direction. end -@inline function _fill_north_halo!(i, k, grid, c, bc::Union{VBC, GBC}, loc, pressure_corrected, args...) +@inline function _fill_north_halo!(i, k, grid, c, bc::Union{VBC, GBC}, loc, args...) # ↑ y ↑ jᴴ = grid.Ny + 1 # * halo cell @@ -70,7 +70,7 @@ end @inbounds c[i, jᴴ, k] = linearly_extrapolate(c[i, jᴵ, k], ∇c, Δ) # extrapolate northward in +y direction. end -@inline function _fill_bottom_halo!(i, j, grid, c, bc::Union{VBC, GBC}, loc, pressure_corrected, args...) +@inline function _fill_bottom_halo!(i, j, grid, c, bc::Union{VBC, GBC}, loc, args...) # ↑ z ↑ interior # ----- interior face @@ -84,7 +84,7 @@ end @inbounds c[i, j, kᴴ] = linearly_extrapolate(c[i, j, kᴵ], ∇c, -Δ) # extrapolate downward in -z direction. end -@inline function _fill_top_halo!(i, j, grid, c, bc::Union{VBC, GBC}, loc, pressure_corrected, args...) +@inline function _fill_top_halo!(i, j, grid, c, bc::Union{VBC, GBC}, loc, args...) # ↑ z ↑ kᴴ = grid.Nz + 1 # * halo cell diff --git a/src/Models/NonhydrostaticModels/pressure_correction.jl b/src/Models/NonhydrostaticModels/pressure_correction.jl index 03deb77020..c657995799 100644 --- a/src/Models/NonhydrostaticModels/pressure_correction.jl +++ b/src/Models/NonhydrostaticModels/pressure_correction.jl @@ -10,10 +10,6 @@ function calculate_pressure_correction!(model::NonhydrostaticModel, Δt) # Mask immersed velocities foreach(mask_immersed_field!, model.velocities) - # Update the boundary conditions - update_boundary_conditions!(fields(model), model) - - # this is where we fill u⋆ fill_halo_regions!(model.velocities, model.clock, fields(model)) solve_for_pressure!(model.pressures.pNHS, model.pressure_solver, Δt, model.velocities) diff --git a/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl b/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl index 76c13447c0..c9fe26e5a3 100644 --- a/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl +++ b/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl @@ -31,8 +31,7 @@ function update_state!(model::NonhydrostaticModel, callbacks=[]; compute_tendenc update_boundary_conditions!(fields(model), model) # Fill halos for velocities and tracers - # this is where we need to NOT update boundary normal velocities - fill_halo_regions!(merge(model.velocities, model.tracers), model.clock, fields(model); async = true, pressure_corrected = true) + fill_halo_regions!(merge(model.velocities, model.tracers), model.clock, fields(model); async = true) # Compute auxiliary fields for aux_field in model.auxiliary_fields From dc6fa323b053b0ba8ad8ddae05fcc29c49ed261c Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Fri, 31 May 2024 14:06:35 +0100 Subject: [PATCH 026/113] different implementation to only fill bounde wall normal velocity halos before pressure correction --- .../fill_halo_regions_open.jl | 112 +++++++++++++----- .../no_gradient_open_boudnary.jl | 38 +++--- .../pressure_correction.jl | 3 + 3 files changed, 100 insertions(+), 53 deletions(-) diff --git a/src/BoundaryConditions/fill_halo_regions_open.jl b/src/BoundaryConditions/fill_halo_regions_open.jl index 75ff5eac05..a0478c5756 100644 --- a/src/BoundaryConditions/fill_halo_regions_open.jl +++ b/src/BoundaryConditions/fill_halo_regions_open.jl @@ -1,39 +1,91 @@ -##### -##### Outer functions for filling halo regions for open boundary conditions. -##### - -# TODO: support true open boundary conditions. -# For this we need to have separate functions for each of the six boundaries, -# and need to unroll a loop over the boundary normal direction. -# The syntax for `getbc` is also different for OpenBoundaryCondition than for others, -# because the boundary-normal index can vary (and array boundary conditions need to be -# 3D in general). - -@kernel function set_west_or_east_u!(u, i_boundary, bc, grid, args) +function fill_bounded_wall_normal_halo_regions!(velocities, clock, fields) + grid = velocities.u.grid + + TX, TY, TZ = topology(grid) + + if TX() isa Bounded + fill_u_open_halo_regions!(velocities.u, grid, clock, fields) + end + + if TY() isa Bounded + fill_v_open_halo_regions!(velocities.v, grid, clock, fields) + end + + if TZ() isa Bounded + fill_w_open_halo_regions!(velocities.w, grid, clock, fields) + end + + return nothing +end + +function fill_u_open_halo_regions!(u, grid, args...) + arch = architecture(grid) + + west_bc = u.boundary_conditions.west + east_bc = u.boundary_conditions.east + + launch!(arch, grid, :yz, _fill_west_and_east_open_halo!, u, west_bc, east_bc, location(u), grid, args) +end + +function fill_v_open_halo_regions!(v, grid, args...) + arch = architecture(grid) + + south_bc = v.boundary_conditions.south + north_bc = v.boundary_conditions.north + + launch!(arch, grid, :yz, _fill_south_and_north_open_halo!, v, south_bc, north_bc, location(v), grid, args) +end + +function fill_w_open_halo_regions!(w, grid, args...) + arch = architecture(grid) + + bottom_bc = w.boundary_conditions.bottom + top_bc = w.boundary_conditions.top + + launch!(arch, grid, :yz, _fill_bottom_and_top_open_halo!, w, bottom_bc, top_bc, location(w), grid, args) +end + +@kernel function _fill_west_and_east_open_halo!(c, west_bc, east_bc, loc, grid, args) j, k = @index(Global, NTuple) - @inbounds u[i_boundary, j, k] = getbc(bc, j, k, grid, args...) + _fill_west_open_halo!(j, k, grid, c, west_bc, loc, args...) + _fill_east_open_halo!(j, k, grid, c, east_bc, loc, args...) end -@kernel function set_south_or_north_v!(v, j_boundary, bc, grid, args) +@kernel function _fill_south_and_top_open_halo!(c, south_bc, top_bc, loc, grid, args) i, k = @index(Global, NTuple) - @inbounds v[i, j_boundary, k] = getbc(bc, i, k, grid, args...) + _fill_south_open_halo!(i, k, grid, c, south_bc, loc, args...) + _fill_north_open_halo!(i, k, grid, c, north_bc, loc, args...) end -@kernel function set_bottom_or_top_w!(w, k_boundary, bc, grid, args) +@kernel function _fill_bottom_and_top_open_halo!(c, bottom_bc, top_bc, loc, grid, args) i, j = @index(Global, NTuple) - @inbounds w[i, j, k_boundary] = getbc(bc, i, j, grid, args...) + _fill_bottom_open_halo!(i, j, grid, c, bottom_bc, loc, args...) + _fill_top_open_halo!(i, j, grid, c, top_bc, loc, args...) end -@inline fill_west_halo!(u, bc::OBC, kernel_size, offset, loc, arch, grid, args...; kwargs...) = launch!(arch, grid, KernelParameters(kernel_size, offset), set_west_or_east_u!, u, 1, bc, grid, Tuple(args); kwargs...) -@inline fill_east_halo!(u, bc::OBC, kernel_size, offset, loc, arch, grid, args...; kwargs...) = launch!(arch, grid, KernelParameters(kernel_size, offset), set_west_or_east_u!, u, grid.Nx + 1, bc, grid, Tuple(args); kwargs...) -@inline fill_south_halo!(v, bc::OBC, kernel_size, offset, loc, arch, grid, args...; kwargs...) = launch!(arch, grid, KernelParameters(kernel_size, offset), set_south_or_north_v!, v, 1, bc, grid, Tuple(args); kwargs...) -@inline fill_north_halo!(v, bc::OBC, kernel_size, offset, loc, arch, grid, args...; kwargs...) = launch!(arch, grid, KernelParameters(kernel_size, offset), set_south_or_north_v!, v, grid.Ny + 1, bc, grid, Tuple(args); kwargs...) -@inline fill_bottom_halo!(w, bc::OBC, kernel_size, offset, loc, arch, grid, args...; kwargs...) = launch!(arch, grid, KernelParameters(kernel_size, offset), set_bottom_or_top_w!, w, 1, bc, grid, Tuple(args); kwargs...) -@inline fill_top_halo!(w, bc::OBC, kernel_size, offset, loc, arch, grid, args...; kwargs...) = launch!(arch, grid, KernelParameters(kernel_size, offset), set_bottom_or_top_w!, w, grid.Nz + 1, bc, grid, Tuple(args); kwargs...) - -@inline _fill_west_halo!(j, k, grid, c, bc::OBC, loc, args...) = @inbounds c[1, j, k] = getbc(bc, j, k, grid, args...) -@inline _fill_east_halo!(j, k, grid, c, bc::OBC, loc, args...) = @inbounds c[grid.Nx + 1, j, k] = getbc(bc, j, k, grid, args...) -@inline _fill_south_halo!(i, k, grid, c, bc::OBC, loc, args...) = @inbounds c[i, 1, k] = getbc(bc, i, k, grid, args...) -@inline _fill_north_halo!(i, k, grid, c, bc::OBC, loc, args...) = @inbounds c[i, grid.Ny + 1, k] = getbc(bc, i, k, grid, args...) -@inline _fill_bottom_halo!(i, j, grid, c, bc::OBC, loc, args...) = @inbounds c[i, j, 1] = getbc(bc, i, j, grid, args...) -@inline _fill_top_halo!(i, j, grid, c, bc::OBC, loc, args...) = @inbounds c[i, j, grid.Nz + 1] = getbc(bc, i, j, grid, args...) +# fallback for normal boundary conditions + +@inline _fill_west_open_halo!(j, k, grid, c, bc, loc, args...) = nothing +@inline _fill_east_open_halo!(j, k, grid, c, bc, loc, args...) = nothing +@inline _fill_south_open_halo!(i, k, grid, c, bc, loc, args...) = nothing +@inline _fill_north_open_halo!(i, k, grid, c, bc, loc, args...) = nothing +@inline _fill_bottom_open_halo!(i, j, grid, c, bc, loc, args...) = nothing +@inline _fill_top_open_halo!(i, j, grid, c, bc, loc, args...) = nothing + +# and don't do anything on the normal fill call + +@inline _fill_west_halo!(j, k, grid, c, bc, loc, args...) = nothing +@inline _fill_east_halo!(j, k, grid, c, bc, loc, args...) = nothing +@inline _fill_south_halo!(i, k, grid, c, bc, loc, args...) = nothing +@inline _fill_north_halo!(i, k, grid, c, bc, loc, args...) = nothing +@inline _fill_bottom_halo!(i, j, grid, c, bc, loc, args...) = nothing +@inline _fill_top_halo!(i, j, grid, c, bc, loc, args...) = nothing + +# generic for open boundary conditions + +@inline _fill_west_open_halo!(j, k, grid, c, bc::OBC, loc, args...) = @inbounds c[1, j, k] = getbc(bc, j, k, grid, args...) +@inline _fill_east_open_halo!(j, k, grid, c, bc::OBC, loc, args...) = @inbounds c[grid.Nx + 1, j, k] = getbc(bc, j, k, grid, args...) +@inline _fill_south_open_halo!(i, k, grid, c, bc::OBC, loc, args...) = @inbounds c[i, 1, k] = getbc(bc, i, k, grid, args...) +@inline _fill_north_open_halo!(i, k, grid, c, bc::OBC, loc, args...) = @inbounds c[i, grid.Ny + 1, k] = getbc(bc, i, k, grid, args...) +@inline _fill_bottom_open_halo!(i, j, grid, c, bc::OBC, loc, args...) = @inbounds c[i, j, 1] = getbc(bc, i, j, grid, args...) +@inline _fill_top_open_halo!(i, j, grid, c, bc::OBC, loc, args...) = @inbounds c[i, j, grid.Nz + 1] = getbc(bc, i, j, grid, args...) diff --git a/src/BoundaryConditions/no_gradient_open_boudnary.jl b/src/BoundaryConditions/no_gradient_open_boudnary.jl index f031b7aa4f..ba3237ea4f 100644 --- a/src/BoundaryConditions/no_gradient_open_boudnary.jl +++ b/src/BoundaryConditions/no_gradient_open_boudnary.jl @@ -1,17 +1,9 @@ +using Oceananigans.Operators: ∂xᶜᶜᶜ + """ ZeroGradient -Zero gradient perepndicular velocity boundary condition. This is compatible -with the pressure solve (???) and implies that there is a pressure gradient -on the boundary equal to the pressure gradient at the boundary adjacent face -node. - -```math -u_b^\\star = u_{b-1}^\\star,\\\\ -u_b^{n+1} = u_{b-1}^{n+1} = u_{b-1}^\\star - \\Delta t\\partial_x P^{fcc}_{b-1} -= u_{b}^\\star - \\Delta t\\partial_x P^{fcc}_{b},\\\\ -\\to \\partial_x P^{fcc}_{b} = \\partial_x P^{fcc}_{b-1} \\therefore \\partial_x^2 P^{ccc}_{b-1} = 0. -``` +Zero gradient perepndicular velocity boundary condition. """ struct ZeroGradient end @@ -23,32 +15,32 @@ function ZeroGradientOpenBoundaryCondition() return BoundaryCondition(classifcation, nothing) end -@inline function _fill_west_halo!(j, k, grid, c, bc::ZGOBC, loc, clock, model_fields) - @inbounds c[0, j, k] = c[1, j, k] +@inline function _fill_west_open_halo!(j, k, grid, c, bc::ZGOBC, loc, clock, model_fields) + @inbounds c[0, j, k] = c[2, j, k] end -@inline function _fill_east_halo!(j, k, grid, c, bc::ZGOBC, loc, clock, model_fields) +@inline function _fill_east_open_halo!(j, k, grid, c, bc::ZGOBC, loc, clock, model_fields) i = grid.Nx + 1 - @inbounds c[i, j, k] = c[i - 1, j, k] #2 * c[i - 1, j, k] - c[i - 2, j, k] + @inbounds c[i, j, k] = c[i - 2, j, k] end -@inline function _fill_south_halo!(i, k, grid, c, bc::ZGOBC, loc, clock, model_fields) - @inbounds c[i, 0, k] = c[i, 1, k] +@inline function _fill_south_open_halo!(i, k, grid, c, bc::ZGOBC, loc, clock, model_fields) + @inbounds c[i, 0, k] = c[i, 2, k] end -@inline function _fill_north_halo!(i, k, grid, c, bc::ZGOBC, loc, clock, model_fields) +@inline function _fill_north_open_halo!(i, k, grid, c, bc::ZGOBC, loc, clock, model_fields) j = grid.Ny + 1 - @inbounds c[i, j, k] = c[i, j - 1, k] + @inbounds c[i, j, k] = c[i, j - 2, k] end -@inline function _fill_bottom_halo!(i, j, grid, c, bc::ZGOBC, loc, clock, model_fields) - @inbounds c[i, j, 0] = c[i, j, 1] +@inline function _fill_bottom_open_halo!(i, j, grid, c, bc::ZGOBC, loc, clock, model_fields) + @inbounds c[i, j, 0] = c[i, j, 2] end -@inline function _fill_top_halo!(i, j, grid, c, bc::ZGOBC, loc, clock, model_fields) +@inline function _fill_top_open_halo!(i, j, grid, c, bc::ZGOBC, loc, clock, model_fields) k = grid.Nz + 1 - @inbounds c[i, j, k] = c[i, j, k - 1] + @inbounds c[i, j, k] = c[i, j, k - 2] end \ No newline at end of file diff --git a/src/Models/NonhydrostaticModels/pressure_correction.jl b/src/Models/NonhydrostaticModels/pressure_correction.jl index c657995799..44e392e91f 100644 --- a/src/Models/NonhydrostaticModels/pressure_correction.jl +++ b/src/Models/NonhydrostaticModels/pressure_correction.jl @@ -1,3 +1,5 @@ +using Oceananigans.BoundaryConditions: fill_bounded_wall_normal_halo_regions! + import Oceananigans.TimeSteppers: calculate_pressure_correction!, pressure_correct_velocities! """ @@ -11,6 +13,7 @@ function calculate_pressure_correction!(model::NonhydrostaticModel, Δt) foreach(mask_immersed_field!, model.velocities) fill_halo_regions!(model.velocities, model.clock, fields(model)) + fill_bounded_wall_normal_halo_regions!(model.velocities, model.clock, fields(model)) solve_for_pressure!(model.pressures.pNHS, model.pressure_solver, Δt, model.velocities) From d951d2f583fb213656b3d683e9d4356c83eed8f6 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Fri, 31 May 2024 15:00:00 +0100 Subject: [PATCH 027/113] corrections --- src/BoundaryConditions/fill_halo_regions_open.jl | 8 ++++---- src/BoundaryConditions/no_gradient_open_boudnary.jl | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/BoundaryConditions/fill_halo_regions_open.jl b/src/BoundaryConditions/fill_halo_regions_open.jl index a0478c5756..5ce9cf28f4 100644 --- a/src/BoundaryConditions/fill_halo_regions_open.jl +++ b/src/BoundaryConditions/fill_halo_regions_open.jl @@ -20,11 +20,11 @@ end function fill_u_open_halo_regions!(u, grid, args...) arch = architecture(grid) - + west_bc = u.boundary_conditions.west east_bc = u.boundary_conditions.east - launch!(arch, grid, :yz, _fill_west_and_east_open_halo!, u, west_bc, east_bc, location(u), grid, args) + launch!(arch, grid, :yz, _fill_west_and_east_open_halo!, u, west_bc, east_bc, instantiated_location(u), grid, args) end function fill_v_open_halo_regions!(v, grid, args...) @@ -33,7 +33,7 @@ function fill_v_open_halo_regions!(v, grid, args...) south_bc = v.boundary_conditions.south north_bc = v.boundary_conditions.north - launch!(arch, grid, :yz, _fill_south_and_north_open_halo!, v, south_bc, north_bc, location(v), grid, args) + launch!(arch, grid, :yz, _fill_south_and_north_open_halo!, v, south_bc, north_bc, instantiated_location(v), grid, args) end function fill_w_open_halo_regions!(w, grid, args...) @@ -42,7 +42,7 @@ function fill_w_open_halo_regions!(w, grid, args...) bottom_bc = w.boundary_conditions.bottom top_bc = w.boundary_conditions.top - launch!(arch, grid, :yz, _fill_bottom_and_top_open_halo!, w, bottom_bc, top_bc, location(w), grid, args) + launch!(arch, grid, :yz, _fill_bottom_and_top_open_halo!, w, bottom_bc, top_bc, instantiated_location(w), grid, args) end @kernel function _fill_west_and_east_open_halo!(c, west_bc, east_bc, loc, grid, args) diff --git a/src/BoundaryConditions/no_gradient_open_boudnary.jl b/src/BoundaryConditions/no_gradient_open_boudnary.jl index ba3237ea4f..ebe9aa2d48 100644 --- a/src/BoundaryConditions/no_gradient_open_boudnary.jl +++ b/src/BoundaryConditions/no_gradient_open_boudnary.jl @@ -16,31 +16,31 @@ function ZeroGradientOpenBoundaryCondition() end @inline function _fill_west_open_halo!(j, k, grid, c, bc::ZGOBC, loc, clock, model_fields) - @inbounds c[0, j, k] = c[2, j, k] + @inbounds c[0, j, k] = c[1, j, k] end @inline function _fill_east_open_halo!(j, k, grid, c, bc::ZGOBC, loc, clock, model_fields) i = grid.Nx + 1 - @inbounds c[i, j, k] = c[i - 2, j, k] + @inbounds c[i, j, k] = c[i - 1, j, k] end @inline function _fill_south_open_halo!(i, k, grid, c, bc::ZGOBC, loc, clock, model_fields) - @inbounds c[i, 0, k] = c[i, 2, k] + @inbounds c[i, 0, k] = c[i, 1, k] end @inline function _fill_north_open_halo!(i, k, grid, c, bc::ZGOBC, loc, clock, model_fields) j = grid.Ny + 1 - @inbounds c[i, j, k] = c[i, j - 2, k] + @inbounds c[i, j, k] = c[i, j - 1, k] end @inline function _fill_bottom_open_halo!(i, j, grid, c, bc::ZGOBC, loc, clock, model_fields) - @inbounds c[i, j, 0] = c[i, j, 2] + @inbounds c[i, j, 0] = c[i, j, 1] end @inline function _fill_top_open_halo!(i, j, grid, c, bc::ZGOBC, loc, clock, model_fields) k = grid.Nz + 1 - @inbounds c[i, j, k] = c[i, j, k - 2] + @inbounds c[i, j, k] = c[i, j, k - 1] end \ No newline at end of file From 9f72248b1d6127c843b2f9505da528697f89b37f Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Fri, 31 May 2024 17:26:28 +0100 Subject: [PATCH 028/113] correction --- .../no_gradient_open_boudnary.jl | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/src/BoundaryConditions/no_gradient_open_boudnary.jl b/src/BoundaryConditions/no_gradient_open_boudnary.jl index ebe9aa2d48..701df87ea5 100644 --- a/src/BoundaryConditions/no_gradient_open_boudnary.jl +++ b/src/BoundaryConditions/no_gradient_open_boudnary.jl @@ -4,6 +4,19 @@ using Oceananigans.Operators: ∂xᶜᶜᶜ ZeroGradient Zero gradient perepndicular velocity boundary condition. + +*Given constant spacing* +```math +f′(xᵢ) ≈ f′(xᵢ₋₁) + f′′(xᵢ₋₁)(xᵢ₋₁ - xᵢ) + O(Δx²) = f′(xᵢ₋₁) + f′′(xᵢ₋₁)Δx + O(Δx²), + +f′(xᵢ₋₁) ≈ (f(xᵢ) - f(xᵢ₋₂)) / 2Δx + +f′(xᵢ) ≈ (f(xᵢ) - f(xᵢ₋₂)) / 2Δx + O(Δx) = 0 ∴ f(xᵢ) ≈ f(xᵢ₋₂) + O(Δx) + +f′′(xᵢ₋₁) ≈ (f′(xᵢ) - f′(xᵢ₋₂)) / 2Δx = - f′(xᵢ₋₂) / 2Δx ≈ - (f(xᵢ₋₁) - f(xᵢ₋₃)) / (2Δx)² + +∴ f(xᵢ) ≈ f(xᵢ₋₂) + (f(xᵢ₋₁) - f(xᵢ₋₃))/2 + O(Δx²) +``` """ struct ZeroGradient end @@ -16,31 +29,31 @@ function ZeroGradientOpenBoundaryCondition() end @inline function _fill_west_open_halo!(j, k, grid, c, bc::ZGOBC, loc, clock, model_fields) - @inbounds c[0, j, k] = c[1, j, k] + @inbounds c[0, j, k] = c[2, j, k] end @inline function _fill_east_open_halo!(j, k, grid, c, bc::ZGOBC, loc, clock, model_fields) i = grid.Nx + 1 - @inbounds c[i, j, k] = c[i - 1, j, k] + @inbounds c[i, j, k] = c[i - 2, j, k] end @inline function _fill_south_open_halo!(i, k, grid, c, bc::ZGOBC, loc, clock, model_fields) - @inbounds c[i, 0, k] = c[i, 1, k] + @inbounds c[i, 0, k] = c[i, 2, k] end @inline function _fill_north_open_halo!(i, k, grid, c, bc::ZGOBC, loc, clock, model_fields) j = grid.Ny + 1 - @inbounds c[i, j, k] = c[i, j - 1, k] + @inbounds c[i, j, k] = c[i, j - 2, k] end @inline function _fill_bottom_open_halo!(i, j, grid, c, bc::ZGOBC, loc, clock, model_fields) - @inbounds c[i, j, 0] = c[i, j, 1] + @inbounds c[i, j, 0] = c[i, j, 2] end @inline function _fill_top_open_halo!(i, j, grid, c, bc::ZGOBC, loc, clock, model_fields) k = grid.Nz + 1 - @inbounds c[i, j, k] = c[i, j, k - 1] + @inbounds c[i, j, k] = c[i, j, k - 2] end \ No newline at end of file From ee83c9129cc6ed8350abc22416691fd047c711df Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Sat, 1 Jun 2024 18:05:43 +0100 Subject: [PATCH 029/113] refactor open halo filling to multiple dispatch rather than if else --- .../fill_halo_regions_open.jl | 54 +++++++++---------- 1 file changed, 24 insertions(+), 30 deletions(-) diff --git a/src/BoundaryConditions/fill_halo_regions_open.jl b/src/BoundaryConditions/fill_halo_regions_open.jl index 5ce9cf28f4..62025040cd 100644 --- a/src/BoundaryConditions/fill_halo_regions_open.jl +++ b/src/BoundaryConditions/fill_halo_regions_open.jl @@ -1,49 +1,43 @@ function fill_bounded_wall_normal_halo_regions!(velocities, clock, fields) grid = velocities.u.grid - TX, TY, TZ = topology(grid) - - if TX() isa Bounded - fill_u_open_halo_regions!(velocities.u, grid, clock, fields) - end - - if TY() isa Bounded - fill_v_open_halo_regions!(velocities.v, grid, clock, fields) - end - - if TZ() isa Bounded - fill_w_open_halo_regions!(velocities.w, grid, clock, fields) - end + fill_open_halo_regions!(velocities.u, grid, clock, fields) + fill_open_halo_regions!(velocities.v, grid, clock, fields) + fill_open_halo_regions!(velocities.w, grid, clock, fields) return nothing end -function fill_u_open_halo_regions!(u, grid, args...) - arch = architecture(grid) - - west_bc = u.boundary_conditions.west - east_bc = u.boundary_conditions.east +const bounded_grid = Union{AbstractGrid{Any, Bounded}, AbstractGrid{Any, Any, Bounded}, AbstractGrid{Any, Any, Any, Bounded}} - launch!(arch, grid, :yz, _fill_west_and_east_open_halo!, u, west_bc, east_bc, instantiated_location(u), grid, args) -end +@inline fill_open_halo_regions!(field, grid, args...) = nothing -function fill_v_open_halo_regions!(v, grid, args...) +function fill_open_halo_regions!(field, grid::bounded_grid, loc, args...) arch = architecture(grid) - south_bc = v.boundary_conditions.south - north_bc = v.boundary_conditions.north + left_bc = left_boundary_condition(field) + right_bc = right_boundary_condition(field) - launch!(arch, grid, :yz, _fill_south_and_north_open_halo!, v, south_bc, north_bc, instantiated_location(v), grid, args) + launch!(arch, grid, side_plan(field), fill_open_halo!(field), field, left_bc, right_bc, loc, grid, args) + + return nothing end -function fill_w_open_halo_regions!(w, grid, args...) - arch = architecture(grid) +@inline left_boundary_condition(field, loc::Tuple{Face, Center, Center}) = field.boundary_conditions.west +@inline left_boundary_condition(field, loc::Tuple{Center, Face, Center}) = field.boundary_conditions.south +@inline left_boundary_condition(field, loc::Tuple{Center, Center, Face}) = field.boundary_conditions.bottom - bottom_bc = w.boundary_conditions.bottom - top_bc = w.boundary_conditions.top +@inline right_boundary_condition(field, loc::Tuple{Face, Center, Center}) = field.boundary_conditions.east +@inline right_boundary_condition(field, loc::Tuple{Center, Face, Center}) = field.boundary_conditions.north +@inline right_boundary_condition(field, loc::Tuple{Center, Center, Face}) = field.boundary_conditions.top - launch!(arch, grid, :yz, _fill_bottom_and_top_open_halo!, w, bottom_bc, top_bc, instantiated_location(w), grid, args) -end +@inline side_plan(loc::Tuple{Face, Center, Center}) = :yz +@inline side_plan(loc::Tuple{Center, Face, Center}) = :xz +@inline side_plan(loc::Tuple{Center, Center, Face}) = :xy + +@inline fill_open_halo!(loc::Tuple{Face, Center, Center}) = _fill_west_and_east_open_halo! +@inline fill_open_halo!(loc::Tuple{Center, Face, Center}) = _fill_south_and_north_open_halo! +@inline fill_open_halo!(loc::Tuple{Center, Center, Face}) = _fill_bottom_and_top_open_halo! @kernel function _fill_west_and_east_open_halo!(c, west_bc, east_bc, loc, grid, args) j, k = @index(Global, NTuple) From d20012396eaedd4924cfb415b69c17b900683176 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Sat, 1 Jun 2024 18:09:42 +0100 Subject: [PATCH 030/113] formatting --- src/BoundaryConditions/boundary_condition_classifications.jl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/BoundaryConditions/boundary_condition_classifications.jl b/src/BoundaryConditions/boundary_condition_classifications.jl index 549a4daee3..2966eff2f3 100644 --- a/src/BoundaryConditions/boundary_condition_classifications.jl +++ b/src/BoundaryConditions/boundary_condition_classifications.jl @@ -67,8 +67,7 @@ Open() = Open(nothing) (open::Open)() = open -Adapt.adapt_structure(to, bc::Open) = - Open(adapt(to, bc.matching_scheme)) +Adapt.adapt_structure(to, bc::Open) = Open(adapt(to, bc.matching_scheme)) """ struct MultiRegionCommunication <: AbstractBoundaryConditionClassification From 087288e7381ea92c2daf2448741639004f30b410 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Mon, 3 Jun 2024 12:41:37 +0100 Subject: [PATCH 031/113] formatting --- src/BoundaryConditions/fill_halo_regions_open.jl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/BoundaryConditions/fill_halo_regions_open.jl b/src/BoundaryConditions/fill_halo_regions_open.jl index 62025040cd..940f88c61b 100644 --- a/src/BoundaryConditions/fill_halo_regions_open.jl +++ b/src/BoundaryConditions/fill_halo_regions_open.jl @@ -8,11 +8,13 @@ function fill_bounded_wall_normal_halo_regions!(velocities, clock, fields) return nothing end -const bounded_grid = Union{AbstractGrid{Any, Bounded}, AbstractGrid{Any, Any, Bounded}, AbstractGrid{Any, Any, Any, Bounded}} +const BoundedGrid = Union{AbstractGrid{<:Any, <:Bounded}, + AbstractGrid{<:Any, <:Any, <:Bounded}, + AbstractGrid{<:Any, <:Any, <:Any, <:Bounded}} @inline fill_open_halo_regions!(field, grid, args...) = nothing -function fill_open_halo_regions!(field, grid::bounded_grid, loc, args...) +function fill_open_halo_regions!(field, grid::BoundedGrid, loc, args...) arch = architecture(grid) left_bc = left_boundary_condition(field) From 319615e4cdfe778a1cc98f190d1e886f802eb761 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Mon, 3 Jun 2024 13:12:27 +0100 Subject: [PATCH 032/113] fixed new way to fill open halos --- src/BoundaryConditions/fill_halo_regions_open.jl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/BoundaryConditions/fill_halo_regions_open.jl b/src/BoundaryConditions/fill_halo_regions_open.jl index 940f88c61b..d2e48c68c6 100644 --- a/src/BoundaryConditions/fill_halo_regions_open.jl +++ b/src/BoundaryConditions/fill_halo_regions_open.jl @@ -1,9 +1,9 @@ function fill_bounded_wall_normal_halo_regions!(velocities, clock, fields) grid = velocities.u.grid - fill_open_halo_regions!(velocities.u, grid, clock, fields) - fill_open_halo_regions!(velocities.v, grid, clock, fields) - fill_open_halo_regions!(velocities.w, grid, clock, fields) + fill_open_halo_regions!(velocities.u, grid, instantiated_location(velocities.u), clock, fields) + fill_open_halo_regions!(velocities.v, grid, instantiated_location(velocities.v), clock, fields) + fill_open_halo_regions!(velocities.w, grid, instantiated_location(velocities.w), clock, fields) return nothing end @@ -17,10 +17,10 @@ const BoundedGrid = Union{AbstractGrid{<:Any, <:Bounded}, function fill_open_halo_regions!(field, grid::BoundedGrid, loc, args...) arch = architecture(grid) - left_bc = left_boundary_condition(field) - right_bc = right_boundary_condition(field) + left_bc = left_boundary_condition(field, loc) + right_bc = right_boundary_condition(field, loc) - launch!(arch, grid, side_plan(field), fill_open_halo!(field), field, left_bc, right_bc, loc, grid, args) + launch!(arch, grid, side_plan(loc), fill_open_halo!(loc), field, left_bc, right_bc, loc, grid, args) return nothing end @@ -47,7 +47,7 @@ end _fill_east_open_halo!(j, k, grid, c, east_bc, loc, args...) end -@kernel function _fill_south_and_top_open_halo!(c, south_bc, top_bc, loc, grid, args) +@kernel function _fill_south_and_north_open_halo!(c, south_bc, north_bc, loc, grid, args) i, k = @index(Global, NTuple) _fill_south_open_halo!(i, k, grid, c, south_bc, loc, args...) _fill_north_open_halo!(i, k, grid, c, north_bc, loc, args...) From 6a7c91e17398e96950266e17fb17d46375f14c8d Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Mon, 3 Jun 2024 15:26:41 +0100 Subject: [PATCH 033/113] backwars compatible, do we want this or to update the tests? --- src/BoundaryConditions/boundary_condition.jl | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/src/BoundaryConditions/boundary_condition.jl b/src/BoundaryConditions/boundary_condition.jl index 0b146699fb..6ee7d6e833 100644 --- a/src/BoundaryConditions/boundary_condition.jl +++ b/src/BoundaryConditions/boundary_condition.jl @@ -11,21 +11,12 @@ struct BoundaryCondition{C<:AbstractBoundaryConditionClassification, T} end """ - BoundaryCondition(Classification::DataType, condition) - -Construct a boundary condition of type `BC` with a number or array as a `condition`. - -Boundary condition types include `Periodic`, `Flux`, `Value`, `Gradient`, and `Open`. -""" -#BoundaryCondition(Classification::DataType, condition) = BoundaryCondition(Classification(), condition) - -""" - BoundaryCondition(Classification::DataType, condition::Function; + BoundaryCondition(classification::AbstractBoundaryConditionClassification, condition::Function; parameters = nothing, discrete_form = false, field_dependencies=()) -Construct a boundary condition of type `Classification` with a function boundary `condition`. +Construct a boundary condition of type `classification` with a function boundary `condition`. By default, the function boudnary `condition` is assumed to have the 'continuous form' `condition(ξ, η, t)`, where `t` is time and `ξ` and `η` vary along the boundary. @@ -49,7 +40,7 @@ where `i`, and `j` are indices that vary along the boundary. If `discrete_form = condition(i, j, grid, clock, model_fields, parameters) ``` """ -function BoundaryCondition(Classification::AbstractBoundaryConditionClassification, condition::Function; +function BoundaryCondition(classification::AbstractBoundaryConditionClassification, condition::Function; parameters = nothing, discrete_form = false, field_dependencies=()) @@ -62,9 +53,12 @@ function BoundaryCondition(Classification::AbstractBoundaryConditionClassificati condition = ContinuousBoundaryFunction(condition, parameters, field_dependencies) end - return BoundaryCondition(Classification, condition) + return BoundaryCondition(classification, condition) end +# for backwards compatability, maybe we should remove... +BoundaryCondition(Classification::DataType, condition; kwargs...) = BoundaryCondition(Classification(), condition; kwargs...) + # Adapt boundary condition struct to be GPU friendly and passable to GPU kernels. Adapt.adapt_structure(to, b::BoundaryCondition) = BoundaryCondition(Adapt.adapt(to, b.classification), Adapt.adapt(to, b.condition)) From cc177aba903e83646c8520a459e6b1f70a0942b6 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Tue, 4 Jun 2024 12:15:26 +0100 Subject: [PATCH 034/113] renamed functions --- src/BoundaryConditions/fill_halo_regions_open.jl | 12 ++++++------ .../NonhydrostaticModels/pressure_correction.jl | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/BoundaryConditions/fill_halo_regions_open.jl b/src/BoundaryConditions/fill_halo_regions_open.jl index d2e48c68c6..02ed7f5a3f 100644 --- a/src/BoundaryConditions/fill_halo_regions_open.jl +++ b/src/BoundaryConditions/fill_halo_regions_open.jl @@ -1,9 +1,9 @@ -function fill_bounded_wall_normal_halo_regions!(velocities, clock, fields) +function fill_boundary_normal_velocities!(velocities, clock, fields) grid = velocities.u.grid - fill_open_halo_regions!(velocities.u, grid, instantiated_location(velocities.u), clock, fields) - fill_open_halo_regions!(velocities.v, grid, instantiated_location(velocities.v), clock, fields) - fill_open_halo_regions!(velocities.w, grid, instantiated_location(velocities.w), clock, fields) + fill_open_boundary_regions!(velocities.u, grid, instantiated_location(velocities.u), clock, fields) + fill_open_boundary_regions!(velocities.v, grid, instantiated_location(velocities.v), clock, fields) + fill_open_boundary_regions!(velocities.w, grid, instantiated_location(velocities.w), clock, fields) return nothing end @@ -12,9 +12,9 @@ const BoundedGrid = Union{AbstractGrid{<:Any, <:Bounded}, AbstractGrid{<:Any, <:Any, <:Bounded}, AbstractGrid{<:Any, <:Any, <:Any, <:Bounded}} -@inline fill_open_halo_regions!(field, grid, args...) = nothing +@inline fill_open_boundary_regions!(field, grid, args...) = nothing -function fill_open_halo_regions!(field, grid::BoundedGrid, loc, args...) +function fill_open_boundary_regions!(field, grid::BoundedGrid, loc, args...) arch = architecture(grid) left_bc = left_boundary_condition(field, loc) diff --git a/src/Models/NonhydrostaticModels/pressure_correction.jl b/src/Models/NonhydrostaticModels/pressure_correction.jl index 44e392e91f..8a7fe819ef 100644 --- a/src/Models/NonhydrostaticModels/pressure_correction.jl +++ b/src/Models/NonhydrostaticModels/pressure_correction.jl @@ -1,4 +1,4 @@ -using Oceananigans.BoundaryConditions: fill_bounded_wall_normal_halo_regions! +using Oceananigans.BoundaryConditions: fill_boundary_normal_velocities! import Oceananigans.TimeSteppers: calculate_pressure_correction!, pressure_correct_velocities! @@ -13,7 +13,7 @@ function calculate_pressure_correction!(model::NonhydrostaticModel, Δt) foreach(mask_immersed_field!, model.velocities) fill_halo_regions!(model.velocities, model.clock, fields(model)) - fill_bounded_wall_normal_halo_regions!(model.velocities, model.clock, fields(model)) + fill_boundary_normal_velocities!(model.velocities, model.clock, fields(model)) solve_for_pressure!(model.pressures.pNHS, model.pressure_solver, Δt, model.velocities) From 980e253814fac925f972587c293403c299abff1f Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Tue, 4 Jun 2024 12:15:59 +0100 Subject: [PATCH 035/113] change no graidnet bc to return nothing --- .../no_gradient_open_boudnary.jl | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/BoundaryConditions/no_gradient_open_boudnary.jl b/src/BoundaryConditions/no_gradient_open_boudnary.jl index 701df87ea5..82f6b5b931 100644 --- a/src/BoundaryConditions/no_gradient_open_boudnary.jl +++ b/src/BoundaryConditions/no_gradient_open_boudnary.jl @@ -28,32 +28,32 @@ function ZeroGradientOpenBoundaryCondition() return BoundaryCondition(classifcation, nothing) end -@inline function _fill_west_open_halo!(j, k, grid, c, bc::ZGOBC, loc, clock, model_fields) - @inbounds c[0, j, k] = c[2, j, k] -end +@inline _fill_west_open_halo!(j, k, grid, c, bc::ZGOBC, loc, clock, model_fields) = @inbounds c[0, j, k] = c[2, j, k] @inline function _fill_east_open_halo!(j, k, grid, c, bc::ZGOBC, loc, clock, model_fields) i = grid.Nx + 1 @inbounds c[i, j, k] = c[i - 2, j, k] + + return nothing end -@inline function _fill_south_open_halo!(i, k, grid, c, bc::ZGOBC, loc, clock, model_fields) - @inbounds c[i, 0, k] = c[i, 2, k] -end +@inline _fill_south_open_halo!(i, k, grid, c, bc::ZGOBC, loc, clock, model_fields) = @inbounds c[i, 0, k] = c[i, 2, k] @inline function _fill_north_open_halo!(i, k, grid, c, bc::ZGOBC, loc, clock, model_fields) j = grid.Ny + 1 @inbounds c[i, j, k] = c[i, j - 2, k] -end -@inline function _fill_bottom_open_halo!(i, j, grid, c, bc::ZGOBC, loc, clock, model_fields) - @inbounds c[i, j, 0] = c[i, j, 2] + return nothing end +@inline _fill_bottom_open_halo!(i, j, grid, c, bc::ZGOBC, loc, clock, model_fields) = @inbounds c[i, j, 0] = c[i, j, 2] + @inline function _fill_top_open_halo!(i, j, grid, c, bc::ZGOBC, loc, clock, model_fields) k = grid.Nz + 1 @inbounds c[i, j, k] = c[i, j, k - 2] + + return nothing end \ No newline at end of file From 56e68373fc6ef57db75ac7279db4a0cc9286f2e0 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Tue, 4 Jun 2024 12:59:50 +0100 Subject: [PATCH 036/113] Fixed set model for new boundary filling --- src/Models/NonhydrostaticModels/nonhydrostatic_model.jl | 2 +- src/Models/NonhydrostaticModels/pressure_correction.jl | 2 -- src/Models/NonhydrostaticModels/set_nonhydrostatic_model.jl | 2 ++ 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Models/NonhydrostaticModels/nonhydrostatic_model.jl b/src/Models/NonhydrostaticModels/nonhydrostatic_model.jl index b5a22163c3..dff22500d3 100644 --- a/src/Models/NonhydrostaticModels/nonhydrostatic_model.jl +++ b/src/Models/NonhydrostaticModels/nonhydrostatic_model.jl @@ -6,7 +6,7 @@ using Oceananigans.DistributedComputations: Distributed using Oceananigans.Advection: CenteredSecondOrder using Oceananigans.BuoyancyModels: validate_buoyancy, regularize_buoyancy, SeawaterBuoyancy using Oceananigans.Biogeochemistry: validate_biogeochemistry, AbstractBiogeochemistry, biogeochemical_auxiliary_fields -using Oceananigans.BoundaryConditions: regularize_field_boundary_conditions +using Oceananigans.BoundaryConditions: regularize_field_boundary_conditions, fill_boundary_normal_velocities! using Oceananigans.Fields: BackgroundFields, Field, tracernames, VelocityFields, TracerFields, CenterField using Oceananigans.Forcings: model_forcing using Oceananigans.Grids: inflate_halo_size, with_halo, architecture diff --git a/src/Models/NonhydrostaticModels/pressure_correction.jl b/src/Models/NonhydrostaticModels/pressure_correction.jl index 8a7fe819ef..e2fef72559 100644 --- a/src/Models/NonhydrostaticModels/pressure_correction.jl +++ b/src/Models/NonhydrostaticModels/pressure_correction.jl @@ -1,5 +1,3 @@ -using Oceananigans.BoundaryConditions: fill_boundary_normal_velocities! - import Oceananigans.TimeSteppers: calculate_pressure_correction!, pressure_correct_velocities! """ diff --git a/src/Models/NonhydrostaticModels/set_nonhydrostatic_model.jl b/src/Models/NonhydrostaticModels/set_nonhydrostatic_model.jl index 19003a9438..31dfde5736 100644 --- a/src/Models/NonhydrostaticModels/set_nonhydrostatic_model.jl +++ b/src/Models/NonhydrostaticModels/set_nonhydrostatic_model.jl @@ -46,6 +46,8 @@ function set!(model::NonhydrostaticModel; enforce_incompressibility=true, kwargs foreach(mask_immersed_field!, model.velocities) update_state!(model) + fill_boundary_normal_velocities!(model.velocities, model.clock, fields(model)) + if enforce_incompressibility FT = eltype(model.grid) calculate_pressure_correction!(model, one(FT)) From 924c48c24902f1b409625ebd6dc3fe76bfbbfea3 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Tue, 4 Jun 2024 15:59:48 +0100 Subject: [PATCH 037/113] fixed poisson solver test --- test/dependencies_for_poisson_solvers.jl | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/test/dependencies_for_poisson_solvers.jl b/test/dependencies_for_poisson_solvers.jl index 24ff92678a..1a985c4c81 100644 --- a/test/dependencies_for_poisson_solvers.jl +++ b/test/dependencies_for_poisson_solvers.jl @@ -1,4 +1,5 @@ using CUDA +using Oceananigans.BoundaryConditions: fill_boundary_normal_velocities! using Oceananigans.Solvers: solve!, set_source_term! using Oceananigans.Solvers: poisson_eigenvalues using Oceananigans.Models.NonhydrostaticModels: solve_for_pressure! @@ -17,20 +18,22 @@ function random_divergent_source_term(grid) v_bcs = regularize_field_boundary_conditions(default_bcs, grid, :v) w_bcs = regularize_field_boundary_conditions(default_bcs, grid, :w) - Ru = CenterField(grid, boundary_conditions=u_bcs) - Rv = CenterField(grid, boundary_conditions=v_bcs) - Rw = CenterField(grid, boundary_conditions=w_bcs) + Ru, Rv, Rw = VelocityFields(grid, (; u = u_bcs, v = v_bcs, w = w_bcs)) + U = (u=Ru, v=Rv, w=Rw) Nx, Ny, Nz = size(grid) - set!(Ru, rand(Nx, Ny, Nz)) - set!(Rv, rand(Nx, Ny, Nz)) - set!(Rw, rand(Nx, Ny, Nz)) + + set!(Ru, rand(size(Ru)...)) + set!(Rv, rand(size(Rv)...)) + set!(Rw, rand(size(Rw)...)) fill_halo_regions!(Ru) fill_halo_regions!(Rv) fill_halo_regions!(Rw) + fill_boundary_normal_velocities!(U, Clock(; time = 0.0), nothing) + # Compute the right hand side R = ∇⋅U ArrayType = array_type(arch) R = zeros(Nx, Ny, Nz) |> ArrayType From 0f982a48b67e1afd2c3db3065623f36de0bfba01 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Tue, 4 Jun 2024 17:00:43 +0100 Subject: [PATCH 038/113] fixed stretched poisson solver test --- test/dependencies_for_poisson_solvers.jl | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/test/dependencies_for_poisson_solvers.jl b/test/dependencies_for_poisson_solvers.jl index 1a985c4c81..df5ed57bb1 100644 --- a/test/dependencies_for_poisson_solvers.jl +++ b/test/dependencies_for_poisson_solvers.jl @@ -49,22 +49,26 @@ function random_divergence_free_source_term(grid) w_bcs = regularize_field_boundary_conditions(default_bcs, grid, :w) # Random right hand side - Ru = CenterField(grid, boundary_conditions=u_bcs) - Rv = CenterField(grid, boundary_conditions=v_bcs) - Rw = CenterField(grid, boundary_conditions=w_bcs) + Ru, Rv, Rw = VelocityFields(grid, (; u = u_bcs, v = v_bcs, w = w_bcs)) + U = (u=Ru, v=Rv, w=Rw) Nx, Ny, Nz = size(grid) - set!(Ru, rand(Nx, Ny, Nz)) - set!(Rv, rand(Nx, Ny, Nz)) - set!(Rw, zeros(Nx, Ny, Nz)) + set!(Ru, rand(size(Ru)...)) + set!(Rv, rand(size(Rv)...)) + set!(Rw, rand(size(Rw)...)) + + fill_halo_regions!(Ru) + fill_halo_regions!(Rv) + arch = architecture(grid) - fill_halo_regions!((Ru, Rv, Rw)) compute_w_from_continuity!(U, arch, grid) fill_halo_regions!(Rw) + fill_boundary_normal_velocities!(U, Clock(; time = 0.0), nothing) + # Compute the right hand side R = ∇⋅U ArrayType = array_type(arch) R = zeros(Nx, Ny, Nz) |> ArrayType From 6af937c8f921420cae7fa73266830607caffdc30 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Wed, 5 Jun 2024 11:06:10 +0100 Subject: [PATCH 039/113] simplified test updates --- src/BoundaryConditions/boundary_condition.jl | 3 --- .../ocean_large_eddy_simulation_regression_test.jl | 6 +++--- test/regression_tests/rayleigh_benard_regression_test.jl | 4 ++-- test/test_boundary_conditions.jl | 4 ++-- 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/BoundaryConditions/boundary_condition.jl b/src/BoundaryConditions/boundary_condition.jl index 6ee7d6e833..139c676033 100644 --- a/src/BoundaryConditions/boundary_condition.jl +++ b/src/BoundaryConditions/boundary_condition.jl @@ -56,9 +56,6 @@ function BoundaryCondition(classification::AbstractBoundaryConditionClassificati return BoundaryCondition(classification, condition) end -# for backwards compatability, maybe we should remove... -BoundaryCondition(Classification::DataType, condition; kwargs...) = BoundaryCondition(Classification(), condition; kwargs...) - # Adapt boundary condition struct to be GPU friendly and passable to GPU kernels. Adapt.adapt_structure(to, b::BoundaryCondition) = BoundaryCondition(Adapt.adapt(to, b.classification), Adapt.adapt(to, b.condition)) diff --git a/test/regression_tests/ocean_large_eddy_simulation_regression_test.jl b/test/regression_tests/ocean_large_eddy_simulation_regression_test.jl index 395d8ecb60..7de3484cad 100644 --- a/test/regression_tests/ocean_large_eddy_simulation_regression_test.jl +++ b/test/regression_tests/ocean_large_eddy_simulation_regression_test.jl @@ -23,9 +23,9 @@ function run_ocean_large_eddy_simulation_regression_test(arch, grid_type, closur end # Boundary conditions - u_bcs = FieldBoundaryConditions(top = BoundaryCondition(Flux, Qᵘ)) - T_bcs = FieldBoundaryConditions(top = BoundaryCondition(Flux, Qᵀ), bottom = BoundaryCondition(Gradient, ∂T∂z)) - S_bcs = FieldBoundaryConditions(top = BoundaryCondition(Flux, 5e-8)) + u_bcs = FieldBoundaryConditions(top = BoundaryCondition(Flux(), Qᵘ)) + T_bcs = FieldBoundaryConditions(top = BoundaryCondition(Flux(), Qᵀ), bottom = BoundaryCondition(Gradient(), ∂T∂z)) + S_bcs = FieldBoundaryConditions(top = BoundaryCondition(Flux(), 5e-8)) equation_of_state = LinearEquationOfState(thermal_expansion=2e-4, haline_contraction=8e-4) diff --git a/test/regression_tests/rayleigh_benard_regression_test.jl b/test/regression_tests/rayleigh_benard_regression_test.jl index 2f64eb30aa..08ab23c6ab 100644 --- a/test/regression_tests/rayleigh_benard_regression_test.jl +++ b/test/regression_tests/rayleigh_benard_regression_test.jl @@ -43,8 +43,8 @@ function run_rayleigh_benard_regression_test(arch, grid_type) cforcing = Forcing(Fc, discrete_form=true) - bbcs = FieldBoundaryConditions(top = BoundaryCondition(Value, 0.0), - bottom = BoundaryCondition(Value, Δb)) + bbcs = FieldBoundaryConditions(top = BoundaryCondition(Value(), 0.0), + bottom = BoundaryCondition(Value(), Δb)) model = NonhydrostaticModel(; grid, closure = ScalarDiffusivity(ν=ν, κ=κ), diff --git a/test/test_boundary_conditions.jl b/test/test_boundary_conditions.jl index 7a7eaaa791..aea30f1961 100644 --- a/test/test_boundary_conditions.jl +++ b/test/test_boundary_conditions.jl @@ -7,7 +7,7 @@ simple_bc(ξ, η, t) = exp(ξ) * cos(η) * sin(t) function can_instantiate_boundary_condition(bc, C, FT=Float64, ArrayType=Array) success = try - bc(C, FT, ArrayType) + bc(C(), FT, ArrayType) true catch false @@ -21,7 +21,7 @@ end @testset "Boundary condition instantiation" begin @info " Testing boundary condition instantiation..." - for C in (Value(), Gradient(), Flux()) + for C in (Value, Gradient, Flux) @test can_instantiate_boundary_condition(integer_bc, C) @test can_instantiate_boundary_condition(irrational_bc, C) @test can_instantiate_boundary_condition(simple_function_bc, C) From ee478d43e2e1c2f002e6d170645cc16c5cfd774c Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Wed, 5 Jun 2024 11:20:46 +0100 Subject: [PATCH 040/113] fixed naming for boundary condition updates --- .../update_boundary_conditions.jl | 16 +++++++------- .../immersed_boundary_condition.jl | 2 +- test/utils_for_runtests.jl | 22 +++++++++---------- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/BoundaryConditions/update_boundary_conditions.jl b/src/BoundaryConditions/update_boundary_conditions.jl index 2846bbbf0d..53a9497808 100644 --- a/src/BoundaryConditions/update_boundary_conditions.jl +++ b/src/BoundaryConditions/update_boundary_conditions.jl @@ -1,13 +1,13 @@ -update_boundary_condition!(bc::Union{BoundaryCondition, Nothing}, args...) = nothing +update_boundary_conditions!(bc::Union{BoundaryCondition, Nothing}, args...) = nothing function update_boundary_conditions!(bcs::FieldBoundaryConditions, field, model) - update_boundary_condition!(bcs.west, field, model, Val(:west)) - update_boundary_condition!(bcs.east, field, model, Val(:east)) - update_boundary_condition!(bcs.south, field, model, Val(:south)) - update_boundary_condition!(bcs.north, field, model, Val(:north)) - update_boundary_condition!(bcs.bottom, field, model, Val(:bottom)) - update_boundary_condition!(bcs.top, field, model, Val(:top)) - update_boundary_condition!(bcs.immersed, field, model, Val(:immersed)) + update_boundary_conditions!(bcs.west, field, model, Val(:west)) + update_boundary_conditions!(bcs.east, field, model, Val(:east)) + update_boundary_conditions!(bcs.south, field, model, Val(:south)) + update_boundary_conditions!(bcs.north, field, model, Val(:north)) + update_boundary_conditions!(bcs.bottom, field, model, Val(:bottom)) + update_boundary_conditions!(bcs.top, field, model, Val(:top)) + update_boundary_conditions!(bcs.immersed, field, model, Val(:immersed)) end update_boundary_conditions!(fields::Union{NamedTuple, Tuple}, model) = diff --git a/src/ImmersedBoundaries/immersed_boundary_condition.jl b/src/ImmersedBoundaries/immersed_boundary_condition.jl index 423b2cd381..f8fbbbb505 100644 --- a/src/ImmersedBoundaries/immersed_boundary_condition.jl +++ b/src/ImmersedBoundaries/immersed_boundary_condition.jl @@ -273,7 +273,7 @@ Adapt.adapt_structure(to, bc::ImmersedBoundaryCondition) = ImmersedBoundaryCondi Adapt.adapt(to, bc.bottom), Adapt.adapt(to, bc.top)) -update_boundary_condition!(bc::ImmersedBoundaryCondition, args...) = nothing +update_boundary_conditions!(bc::ImmersedBoundaryCondition, args...) = nothing ##### ##### Alternative implementation for immersed flux divergence diff --git a/test/utils_for_runtests.jl b/test/utils_for_runtests.jl index d4a92035e2..cfe9ab9689 100644 --- a/test/utils_for_runtests.jl +++ b/test/utils_for_runtests.jl @@ -167,14 +167,14 @@ field_dependent_fun(ξ, η, t, u, v, w) = - w * sqrt(u^2 + v^2) exploding_fun(ξ, η, t, T, S, p) = - p.μ * cosh(S - p.S0) * exp((T - p.T0) / p.λ) # Many bc. Very many - integer_bc(C, FT=Float64, ArrayType=Array) = BoundaryCondition(C, 1) - float_bc(C, FT=Float64, ArrayType=Array) = BoundaryCondition(C, FT(π)) - irrational_bc(C, FT=Float64, ArrayType=Array) = BoundaryCondition(C, π) - array_bc(C, FT=Float64, ArrayType=Array) = BoundaryCondition(C, ArrayType(rand(FT, 1, 1))) - simple_function_bc(C, FT=Float64, ArrayType=Array) = BoundaryCondition(C, (ξ, η, t) -> exp(ξ) * cos(η) * sin(t)) - parameterized_function_bc(C, FT=Float64, ArrayType=Array) = BoundaryCondition(C, parameterized_fun, parameters=(μ=0.1, ω=2π)) -field_dependent_function_bc(C, FT=Float64, ArrayType=Array) = BoundaryCondition(C, field_dependent_fun, field_dependencies=(:u, :v, :w)) - discrete_function_bc(C, FT=Float64, ArrayType=Array) = BoundaryCondition(C, discrete_func, discrete_form=true) - - parameterized_discrete_function_bc(C, FT=Float64, ArrayType=Array) = BoundaryCondition(C, parameterized_discrete_func, discrete_form=true, parameters=(μ=0.1,)) -parameterized_field_dependent_function_bc(C, FT=Float64, ArrayType=Array) = BoundaryCondition(C, exploding_fun, field_dependencies=(:T, :S), parameters=(S0=35, T0=100, μ=2π, λ=FT(2))) + integer_bc(C, FT=Float64, ArrayType=Array) = BoundaryCondition(C(), 1) + float_bc(C, FT=Float64, ArrayType=Array) = BoundaryCondition(C(), FT(π)) + irrational_bc(C, FT=Float64, ArrayType=Array) = BoundaryCondition(C(), π) + array_bc(C, FT=Float64, ArrayType=Array) = BoundaryCondition(C(), ArrayType(rand(FT, 1, 1))) + simple_function_bc(C, FT=Float64, ArrayType=Array) = BoundaryCondition(C(), (ξ, η, t) -> exp(ξ) * cos(η) * sin(t)) + parameterized_function_bc(C, FT=Float64, ArrayType=Array) = BoundaryCondition(C(), parameterized_fun, parameters=(μ=0.1, ω=2π)) +field_dependent_function_bc(C, FT=Float64, ArrayType=Array) = BoundaryCondition(C(), field_dependent_fun, field_dependencies=(:u, :v, :w)) + discrete_function_bc(C, FT=Float64, ArrayType=Array) = BoundaryCondition(C(), discrete_func, discrete_form=true) + + parameterized_discrete_function_bc(C, FT=Float64, ArrayType=Array) = BoundaryCondition(C(), parameterized_discrete_func, discrete_form=true, parameters=(μ=0.1,)) +parameterized_field_dependent_function_bc(C, FT=Float64, ArrayType=Array) = BoundaryCondition(C(), exploding_fun, field_dependencies=(:T, :S), parameters=(S0=35, T0=100, μ=2π, λ=FT(2))) From dbf175a73ef9c1c3860f8e086b55af49e2ed47c6 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Wed, 5 Jun 2024 11:43:23 +0100 Subject: [PATCH 041/113] fixed bc tests again --- test/test_boundary_conditions.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_boundary_conditions.jl b/test/test_boundary_conditions.jl index aea30f1961..4c608c7fa0 100644 --- a/test/test_boundary_conditions.jl +++ b/test/test_boundary_conditions.jl @@ -7,7 +7,7 @@ simple_bc(ξ, η, t) = exp(ξ) * cos(η) * sin(t) function can_instantiate_boundary_condition(bc, C, FT=Float64, ArrayType=Array) success = try - bc(C(), FT, ArrayType) + bc(C, FT, ArrayType) true catch false From 14c32ec9393ce6c27685da4ce099e7e10ac67bf2 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Wed, 5 Jun 2024 11:52:23 +0100 Subject: [PATCH 042/113] fixed bc integration test --- test/test_boundary_conditions_integration.jl | 60 ++++++++++---------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/test/test_boundary_conditions_integration.jl b/test/test_boundary_conditions_integration.jl index 285c75d665..8336df0cdf 100644 --- a/test/test_boundary_conditions_integration.jl +++ b/test/test_boundary_conditions_integration.jl @@ -123,34 +123,34 @@ test_boundary_conditions(C, FT, ArrayType) = (integer_bc(C, FT, ArrayType), grid = RectilinearGrid(arch, FT, size=(1, 1, 1), extent=(1, π, 42), topology=(Bounded, Bounded, Bounded)) - u_boundary_conditions = FieldBoundaryConditions(bottom = simple_function_bc(Value()), - top = simple_function_bc(Value()), - north = simple_function_bc(Value()), - south = simple_function_bc(Value()), - east = simple_function_bc(Open()), - west = simple_function_bc(Open())) - - v_boundary_conditions = FieldBoundaryConditions(bottom = simple_function_bc(Value()), - top = simple_function_bc(Value()), - north = simple_function_bc(Open()), - south = simple_function_bc(Open()), - east = simple_function_bc(Value()), - west = simple_function_bc(Value())) - - - w_boundary_conditions = FieldBoundaryConditions(bottom = simple_function_bc(Open()), - top = simple_function_bc(Open()), - north = simple_function_bc(Value()), - south = simple_function_bc(Value()), - east = simple_function_bc(Value()), - west = simple_function_bc(Value())) - - T_boundary_conditions = FieldBoundaryConditions(bottom = simple_function_bc(Value()), - top = simple_function_bc(Value()), - north = simple_function_bc(Value()), - south = simple_function_bc(Value()), - east = simple_function_bc(Value()), - west = simple_function_bc(Value())) + u_boundary_conditions = FieldBoundaryConditions(bottom = simple_function_bc(Value), + top = simple_function_bc(Value), + north = simple_function_bc(Value), + south = simple_function_bc(Value), + east = simple_function_bc(Open), + west = simple_function_bc(Open)) + + v_boundary_conditions = FieldBoundaryConditions(bottom = simple_function_bc(Value), + top = simple_function_bc(Value), + north = simple_function_bc(Open), + south = simple_function_bc(Open), + east = simple_function_bc(Value), + west = simple_function_bc(Value)) + + + w_boundary_conditions = FieldBoundaryConditions(bottom = simple_function_bc(Open), + top = simple_function_bc(Open), + north = simple_function_bc(Value), + south = simple_function_bc(Value), + east = simple_function_bc(Value), + west = simple_function_bc(Value)) + + T_boundary_conditions = FieldBoundaryConditions(bottom = simple_function_bc(Value), + top = simple_function_bc(Value), + north = simple_function_bc(Value), + south = simple_function_bc(Value), + east = simple_function_bc(Value), + west = simple_function_bc(Value)) boundary_conditions = (u=u_boundary_conditions, v=v_boundary_conditions, @@ -197,13 +197,13 @@ test_boundary_conditions(C, FT, ArrayType) = (integer_bc(C, FT, ArrayType), topo = (Bounded, Bounded, Bounded) - for C in (Gradient(), Flux(), Value()), boundary_condition in test_boundary_conditions(C, FT, array_type(arch)) + for C in (Gradient, Flux, Value), boundary_condition in test_boundary_conditions(C, FT, array_type(arch)) @test test_boundary_condition(arch, FT, topo, :east, :T, boundary_condition) @test test_boundary_condition(arch, FT, topo, :south, :T, boundary_condition) @test test_boundary_condition(arch, FT, topo, :top, :T, boundary_condition) end - for boundary_condition in test_boundary_conditions(Open(), FT, array_type(arch)) + for boundary_condition in test_boundary_conditions(Open, FT, array_type(arch)) @test test_boundary_condition(arch, FT, topo, :east, :u, boundary_condition) @test test_boundary_condition(arch, FT, topo, :south, :v, boundary_condition) @test test_boundary_condition(arch, FT, topo, :top, :w, boundary_condition) From 4c3623c31af2325d4374b524cf76b47361d6c2a8 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Wed, 5 Jun 2024 12:30:31 +0100 Subject: [PATCH 043/113] fixed import --- src/ImmersedBoundaries/immersed_boundary_condition.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ImmersedBoundaries/immersed_boundary_condition.jl b/src/ImmersedBoundaries/immersed_boundary_condition.jl index f8fbbbb505..fb2aa5f22e 100644 --- a/src/ImmersedBoundaries/immersed_boundary_condition.jl +++ b/src/ImmersedBoundaries/immersed_boundary_condition.jl @@ -4,7 +4,7 @@ using Oceananigans.BoundaryConditions: DefaultBoundaryCondition using Oceananigans.TurbulenceClosures: AbstractScalarDiffusivity, h_diffusivity, z_diffusivity using Oceananigans.Operators: index_left, index_right, Δx, Δy, Δz, div -import Oceananigans.BoundaryConditions: regularize_immersed_boundary_condition, bc_str, update_boundary_condition! +import Oceananigans.BoundaryConditions: regularize_immersed_boundary_condition, bc_str, update_boundary_conditions! import Oceananigans.TurbulenceClosures: immersed_∂ⱼ_τ₁ⱼ, immersed_∂ⱼ_τ₂ⱼ, From aed409d2eafc9e511bc5cac3a56bd87b498b4616 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Wed, 5 Jun 2024 13:40:35 +0100 Subject: [PATCH 044/113] added \`fill_open_boundary_region\` method for manual call on \`Field\` --- src/BoundaryConditions/fill_halo_regions_open.jl | 2 ++ test/test_broadcasting.jl | 3 +++ 2 files changed, 5 insertions(+) diff --git a/src/BoundaryConditions/fill_halo_regions_open.jl b/src/BoundaryConditions/fill_halo_regions_open.jl index 02ed7f5a3f..6fd67b8f64 100644 --- a/src/BoundaryConditions/fill_halo_regions_open.jl +++ b/src/BoundaryConditions/fill_halo_regions_open.jl @@ -14,6 +14,8 @@ const BoundedGrid = Union{AbstractGrid{<:Any, <:Bounded}, @inline fill_open_boundary_regions!(field, grid, args...) = nothing +@inline fill_open_boundary_regions!(field) = fill_open_boundary_regions!(field, field.grid, instantiated_location(field)) + function fill_open_boundary_regions!(field, grid::BoundedGrid, loc, args...) arch = architecture(grid) diff --git a/test/test_broadcasting.jl b/test/test_broadcasting.jl index 6f2e8a6788..e1aaaa520a 100644 --- a/test/test_broadcasting.jl +++ b/test/test_broadcasting.jl @@ -1,5 +1,7 @@ include("dependencies_for_runtests.jl") +using Oceananigans.BoundaryConditions: fill_open_boundary_regions! + @testset "Field broadcasting" begin @info " Testing broadcasting with fields..." @@ -46,6 +48,7 @@ include("dependencies_for_runtests.jl") b2 .= 1 fill_halo_regions!(b2) # sets b2[1, 1, 1] = b[1, 1, 4] = 0 + fill_open_boundary_regions!(b2) CUDA.@allowscalar begin @test b2[1, 1, 1] == 0 From 3dd71adb08d8c64ebdd9dc22aad8a3540b95e1b0 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Wed, 5 Jun 2024 14:56:45 +0100 Subject: [PATCH 045/113] moved `fill_open_boundary_region\!` --- src/BoundaryConditions/fill_halo_regions.jl | 8 +++-- .../fill_halo_regions_open.jl | 33 ++++++++++++------- .../no_gradient_open_boudnary.jl | 4 +-- .../nonhydrostatic_model.jl | 2 +- .../pressure_correction.jl | 1 - .../set_nonhydrostatic_model.jl | 5 +-- .../update_nonhydrostatic_model_state.jl | 2 +- test/dependencies_for_poisson_solvers.jl | 5 --- 8 files changed, 34 insertions(+), 26 deletions(-) diff --git a/src/BoundaryConditions/fill_halo_regions.jl b/src/BoundaryConditions/fill_halo_regions.jl index cc92479f40..70cad7dd17 100644 --- a/src/BoundaryConditions/fill_halo_regions.jl +++ b/src/BoundaryConditions/fill_halo_regions.jl @@ -45,8 +45,7 @@ end const MaybeTupledData = Union{OffsetArray, NTuple{<:Any, OffsetArray}} "Fill halo regions in ``x``, ``y``, and ``z`` for a given field's data." -function fill_halo_regions!(c::MaybeTupledData, boundary_conditions, indices, loc, grid, args...; kwargs...) - +function fill_halo_regions!(c::MaybeTupledData, boundary_conditions, indices, loc, grid, args...; fill_open_boundaries = true, kwargs...) arch = architecture(grid) fill_halos!, bcs = permute_boundary_conditions(boundary_conditions) @@ -57,6 +56,10 @@ function fill_halo_regions!(c::MaybeTupledData, boundary_conditions, indices, lo fill_halo_event!(c, fill_halos![task], bcs[task], indices, loc, arch, grid, args...; kwargs...) end + if fill_open_boundaries + fill_open_boundary_regions!(c, boundary_conditions, loc, grid, args...; kwargs...) + end + return nothing end @@ -208,7 +211,6 @@ end _fill_bottom_halo!(i, j, grid, c, bottom_bc, loc, args...) _fill_top_halo!(i, j, grid, c, top_bc, loc, args...) end - ##### ##### Single-sided fill_halo! kernels ##### diff --git a/src/BoundaryConditions/fill_halo_regions_open.jl b/src/BoundaryConditions/fill_halo_regions_open.jl index 6fd67b8f64..2fe9e5fc1f 100644 --- a/src/BoundaryConditions/fill_halo_regions_open.jl +++ b/src/BoundaryConditions/fill_halo_regions_open.jl @@ -1,4 +1,4 @@ -function fill_boundary_normal_velocities!(velocities, clock, fields) +#=function fill_boundary_normal_velocities!(velocities, clock, fields) grid = velocities.u.grid fill_open_boundary_regions!(velocities.u, grid, instantiated_location(velocities.u), clock, fields) @@ -6,7 +6,7 @@ function fill_boundary_normal_velocities!(velocities, clock, fields) fill_open_boundary_regions!(velocities.w, grid, instantiated_location(velocities.w), clock, fields) return nothing -end +end=# const BoundedGrid = Union{AbstractGrid{<:Any, <:Bounded}, AbstractGrid{<:Any, <:Any, <:Bounded}, @@ -16,33 +16,44 @@ const BoundedGrid = Union{AbstractGrid{<:Any, <:Bounded}, @inline fill_open_boundary_regions!(field) = fill_open_boundary_regions!(field, field.grid, instantiated_location(field)) -function fill_open_boundary_regions!(field, grid::BoundedGrid, loc, args...) +fill_open_boundary_regions!(field, grid::BoundedGrid, loc, args...) = fill_open_boundary_regions!(field, field.boundary_conditions, loc, grid, args...) + +function fill_open_boundary_regions!(field, boundary_conditions, loc, grid, args...; kwargs...) arch = architecture(grid) - left_bc = left_boundary_condition(field, loc) - right_bc = right_boundary_condition(field, loc) + left_bc = left_boundary_condition(boundary_conditions, loc) + right_bc = right_boundary_condition(boundary_conditions, loc) launch!(arch, grid, side_plan(loc), fill_open_halo!(loc), field, left_bc, right_bc, loc, grid, args) return nothing end -@inline left_boundary_condition(field, loc::Tuple{Face, Center, Center}) = field.boundary_conditions.west -@inline left_boundary_condition(field, loc::Tuple{Center, Face, Center}) = field.boundary_conditions.south -@inline left_boundary_condition(field, loc::Tuple{Center, Center, Face}) = field.boundary_conditions.bottom +fill_open_boundary_regions!(fields::NTuple, boundary_conditions, loc, grid, args...; kwargs...) = + [fill_open_boundary_regions!(field, boundary_conditions[n], loc[n], grid, args...; kwargs...) for (n, field) in enumerate(fields)] -@inline right_boundary_condition(field, loc::Tuple{Face, Center, Center}) = field.boundary_conditions.east -@inline right_boundary_condition(field, loc::Tuple{Center, Face, Center}) = field.boundary_conditions.north -@inline right_boundary_condition(field, loc::Tuple{Center, Center, Face}) = field.boundary_conditions.top +@inline left_boundary_condition(boundary_conditions, loc) = nothing +@inline left_boundary_condition(boundary_conditions, loc::Tuple{Face, Center, Center}) = boundary_conditions.west +@inline left_boundary_condition(boundary_conditions, loc::Tuple{Center, Face, Center}) = boundary_conditions.south +@inline left_boundary_condition(boundary_conditions, loc::Tuple{Center, Center, Face}) = boundary_conditions.bottom +@inline right_boundary_condition(boundary_conditions, loc) = nothing +@inline right_boundary_condition(boundary_conditions, loc::Tuple{Face, Center, Center}) = boundary_conditions.east +@inline right_boundary_condition(boundary_conditions, loc::Tuple{Center, Face, Center}) = boundary_conditions.north +@inline right_boundary_condition(boundary_conditions, loc::Tuple{Center, Center, Face}) = boundary_conditions.top + +@inline side_plan(loc) = :xy @inline side_plan(loc::Tuple{Face, Center, Center}) = :yz @inline side_plan(loc::Tuple{Center, Face, Center}) = :xz @inline side_plan(loc::Tuple{Center, Center, Face}) = :xy +@inline fill_open_halo!(loc) = _no_fill! @inline fill_open_halo!(loc::Tuple{Face, Center, Center}) = _fill_west_and_east_open_halo! @inline fill_open_halo!(loc::Tuple{Center, Face, Center}) = _fill_south_and_north_open_halo! @inline fill_open_halo!(loc::Tuple{Center, Center, Face}) = _fill_bottom_and_top_open_halo! +@kernel _no_fill!(args...) = nothing + @kernel function _fill_west_and_east_open_halo!(c, west_bc, east_bc, loc, grid, args) j, k = @index(Global, NTuple) _fill_west_open_halo!(j, k, grid, c, west_bc, loc, args...) diff --git a/src/BoundaryConditions/no_gradient_open_boudnary.jl b/src/BoundaryConditions/no_gradient_open_boudnary.jl index 82f6b5b931..16e83a4d38 100644 --- a/src/BoundaryConditions/no_gradient_open_boudnary.jl +++ b/src/BoundaryConditions/no_gradient_open_boudnary.jl @@ -33,8 +33,8 @@ end @inline function _fill_east_open_halo!(j, k, grid, c, bc::ZGOBC, loc, clock, model_fields) i = grid.Nx + 1 - @inbounds c[i, j, k] = c[i - 2, j, k] - + @inbounds c[i, j, k] = c[i - 2, j, k] + (c[i - 1, j, k] - c[i - 3, j, k]) / 2 + return nothing end diff --git a/src/Models/NonhydrostaticModels/nonhydrostatic_model.jl b/src/Models/NonhydrostaticModels/nonhydrostatic_model.jl index dff22500d3..b5a22163c3 100644 --- a/src/Models/NonhydrostaticModels/nonhydrostatic_model.jl +++ b/src/Models/NonhydrostaticModels/nonhydrostatic_model.jl @@ -6,7 +6,7 @@ using Oceananigans.DistributedComputations: Distributed using Oceananigans.Advection: CenteredSecondOrder using Oceananigans.BuoyancyModels: validate_buoyancy, regularize_buoyancy, SeawaterBuoyancy using Oceananigans.Biogeochemistry: validate_biogeochemistry, AbstractBiogeochemistry, biogeochemical_auxiliary_fields -using Oceananigans.BoundaryConditions: regularize_field_boundary_conditions, fill_boundary_normal_velocities! +using Oceananigans.BoundaryConditions: regularize_field_boundary_conditions using Oceananigans.Fields: BackgroundFields, Field, tracernames, VelocityFields, TracerFields, CenterField using Oceananigans.Forcings: model_forcing using Oceananigans.Grids: inflate_halo_size, with_halo, architecture diff --git a/src/Models/NonhydrostaticModels/pressure_correction.jl b/src/Models/NonhydrostaticModels/pressure_correction.jl index e2fef72559..c657995799 100644 --- a/src/Models/NonhydrostaticModels/pressure_correction.jl +++ b/src/Models/NonhydrostaticModels/pressure_correction.jl @@ -11,7 +11,6 @@ function calculate_pressure_correction!(model::NonhydrostaticModel, Δt) foreach(mask_immersed_field!, model.velocities) fill_halo_regions!(model.velocities, model.clock, fields(model)) - fill_boundary_normal_velocities!(model.velocities, model.clock, fields(model)) solve_for_pressure!(model.pressures.pNHS, model.pressure_solver, Δt, model.velocities) diff --git a/src/Models/NonhydrostaticModels/set_nonhydrostatic_model.jl b/src/Models/NonhydrostaticModels/set_nonhydrostatic_model.jl index 31dfde5736..65e6386f22 100644 --- a/src/Models/NonhydrostaticModels/set_nonhydrostatic_model.jl +++ b/src/Models/NonhydrostaticModels/set_nonhydrostatic_model.jl @@ -1,3 +1,4 @@ +using Oceananigans.BoundaryConditions: fill_halo_regions! using Oceananigans.TimeSteppers: update_state!, calculate_pressure_correction!, pressure_correct_velocities! import Oceananigans.Fields: set! @@ -39,6 +40,8 @@ function set!(model::NonhydrostaticModel; enforce_incompressibility=true, kwargs throw(ArgumentError("name $fldname not found in model.velocities or model.tracers.")) end set!(ϕ, value) + + fill_halo_regions!(ϕ) end # Apply a mask @@ -46,8 +49,6 @@ function set!(model::NonhydrostaticModel; enforce_incompressibility=true, kwargs foreach(mask_immersed_field!, model.velocities) update_state!(model) - fill_boundary_normal_velocities!(model.velocities, model.clock, fields(model)) - if enforce_incompressibility FT = eltype(model.grid) calculate_pressure_correction!(model, one(FT)) diff --git a/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl b/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl index c9fe26e5a3..0a187cdb29 100644 --- a/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl +++ b/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl @@ -31,7 +31,7 @@ function update_state!(model::NonhydrostaticModel, callbacks=[]; compute_tendenc update_boundary_conditions!(fields(model), model) # Fill halos for velocities and tracers - fill_halo_regions!(merge(model.velocities, model.tracers), model.clock, fields(model); async = true) + fill_halo_regions!(merge(model.velocities, model.tracers), model.clock, fields(model); fill_open_boundaries = false, async = true) # Compute auxiliary fields for aux_field in model.auxiliary_fields diff --git a/test/dependencies_for_poisson_solvers.jl b/test/dependencies_for_poisson_solvers.jl index df5ed57bb1..a4a404e034 100644 --- a/test/dependencies_for_poisson_solvers.jl +++ b/test/dependencies_for_poisson_solvers.jl @@ -1,5 +1,4 @@ using CUDA -using Oceananigans.BoundaryConditions: fill_boundary_normal_velocities! using Oceananigans.Solvers: solve!, set_source_term! using Oceananigans.Solvers: poisson_eigenvalues using Oceananigans.Models.NonhydrostaticModels: solve_for_pressure! @@ -32,8 +31,6 @@ function random_divergent_source_term(grid) fill_halo_regions!(Rv) fill_halo_regions!(Rw) - fill_boundary_normal_velocities!(U, Clock(; time = 0.0), nothing) - # Compute the right hand side R = ∇⋅U ArrayType = array_type(arch) R = zeros(Nx, Ny, Nz) |> ArrayType @@ -67,8 +64,6 @@ function random_divergence_free_source_term(grid) compute_w_from_continuity!(U, arch, grid) fill_halo_regions!(Rw) - fill_boundary_normal_velocities!(U, Clock(; time = 0.0), nothing) - # Compute the right hand side R = ∇⋅U ArrayType = array_type(arch) R = zeros(Nx, Ny, Nz) |> ArrayType From 7b4ee13e0cf6c2c71d98f655ab30aacb293816a9 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Thu, 6 Jun 2024 16:12:19 +0100 Subject: [PATCH 046/113] corrected order of boundary filling --- src/BoundaryConditions/fill_halo_regions.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/BoundaryConditions/fill_halo_regions.jl b/src/BoundaryConditions/fill_halo_regions.jl index 70cad7dd17..970148fb08 100644 --- a/src/BoundaryConditions/fill_halo_regions.jl +++ b/src/BoundaryConditions/fill_halo_regions.jl @@ -48,6 +48,10 @@ const MaybeTupledData = Union{OffsetArray, NTuple{<:Any, OffsetArray}} function fill_halo_regions!(c::MaybeTupledData, boundary_conditions, indices, loc, grid, args...; fill_open_boundaries = true, kwargs...) arch = architecture(grid) + if fill_open_boundaries + fill_open_boundary_regions!(c, boundary_conditions, loc, grid, args...; kwargs...) + end + fill_halos!, bcs = permute_boundary_conditions(boundary_conditions) number_of_tasks = length(fill_halos!) @@ -56,10 +60,6 @@ function fill_halo_regions!(c::MaybeTupledData, boundary_conditions, indices, lo fill_halo_event!(c, fill_halos![task], bcs[task], indices, loc, arch, grid, args...; kwargs...) end - if fill_open_boundaries - fill_open_boundary_regions!(c, boundary_conditions, loc, grid, args...; kwargs...) - end - return nothing end From 730a50ceba8ef00da2ad65d78813d6b232cb4a49 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Thu, 6 Jun 2024 16:33:27 +0100 Subject: [PATCH 047/113] solved multi region bug for update boundary conditions --- .../update_hydrostatic_free_surface_model_state.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl index 5c302a0d1f..61483b8207 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl @@ -35,7 +35,7 @@ function update_state!(model::HydrostaticFreeSurfaceModel, grid, callbacks; comp @apply_regionally update_model_field_time_series!(model, model.clock) # Update the boundary conditions - update_boundary_conditions!(fields(model), model) + @apply_regionally update_boundary_conditions!(fields(model), model) fill_halo_regions!(prognostic_fields(model), model.clock, fields(model); async = true) @apply_regionally replace_horizontal_vector_halos!(model.velocities, model.grid) From 44687b05a0aad361b37ae557315e51e19d5b18cc Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Thu, 6 Jun 2024 16:54:43 +0100 Subject: [PATCH 048/113] renamed `ZeroGradient` matching scheme --- .../no_gradient_open_boudnary.jl | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/BoundaryConditions/no_gradient_open_boudnary.jl b/src/BoundaryConditions/no_gradient_open_boudnary.jl index 16e83a4d38..3d2687ecc1 100644 --- a/src/BoundaryConditions/no_gradient_open_boudnary.jl +++ b/src/BoundaryConditions/no_gradient_open_boudnary.jl @@ -1,7 +1,7 @@ using Oceananigans.Operators: ∂xᶜᶜᶜ """ - ZeroGradient + FlatExtrapolation Zero gradient perepndicular velocity boundary condition. @@ -18,19 +18,19 @@ f′′(xᵢ₋₁) ≈ (f′(xᵢ) - f′(xᵢ₋₂)) / 2Δx = - f′(xᵢ₋ ∴ f(xᵢ) ≈ f(xᵢ₋₂) + (f(xᵢ₋₁) - f(xᵢ₋₃))/2 + O(Δx²) ``` """ -struct ZeroGradient end +struct FlatExtrapolation end -const ZGOBC = BoundaryCondition{<:Open{<:ZeroGradient}} +const FEOBC = BoundaryCondition{<:Open{<:FlatExtrapolation}} -function ZeroGradientOpenBoundaryCondition() - classifcation = Open(ZeroGradient()) +function FlatExtrapolationOpenBoundaryCondition() + classifcation = Open(FlatExtrapolation()) return BoundaryCondition(classifcation, nothing) end -@inline _fill_west_open_halo!(j, k, grid, c, bc::ZGOBC, loc, clock, model_fields) = @inbounds c[0, j, k] = c[2, j, k] +@inline _fill_west_open_halo!(j, k, grid, c, bc::FEOBC, loc, clock, model_fields) = @inbounds c[0, j, k] = c[2, j, k] + (c[1, j, k] - c[3, j, k]) / 2 -@inline function _fill_east_open_halo!(j, k, grid, c, bc::ZGOBC, loc, clock, model_fields) +@inline function _fill_east_open_halo!(j, k, grid, c, bc::FEOBC, loc, clock, model_fields) i = grid.Nx + 1 @inbounds c[i, j, k] = c[i - 2, j, k] + (c[i - 1, j, k] - c[i - 3, j, k]) / 2 @@ -38,22 +38,22 @@ end return nothing end -@inline _fill_south_open_halo!(i, k, grid, c, bc::ZGOBC, loc, clock, model_fields) = @inbounds c[i, 0, k] = c[i, 2, k] +@inline _fill_south_open_halo!(i, k, grid, c, bc::FEOBC, loc, clock, model_fields) = @inbounds c[i, 0, k] = c[i, 2, k] + (c[i, 1, k] - c[i, 3, k]) / 2 -@inline function _fill_north_open_halo!(i, k, grid, c, bc::ZGOBC, loc, clock, model_fields) +@inline function _fill_north_open_halo!(i, k, grid, c, bc::FEOBC, loc, clock, model_fields) j = grid.Ny + 1 - @inbounds c[i, j, k] = c[i, j - 2, k] + @inbounds c[i, j, k] = c[i, j - 2, k] + (c[i, j - 1, k] - c[i, j - 3, k]) / 2 return nothing end -@inline _fill_bottom_open_halo!(i, j, grid, c, bc::ZGOBC, loc, clock, model_fields) = @inbounds c[i, j, 0] = c[i, j, 2] +@inline _fill_bottom_open_halo!(i, j, grid, c, bc::FEOBC, loc, clock, model_fields) = @inbounds c[i, j, 0] = c[i, j, 2] + (c[i, k, 1] - c[i, j, 3]) / 2 -@inline function _fill_top_open_halo!(i, j, grid, c, bc::ZGOBC, loc, clock, model_fields) +@inline function _fill_top_open_halo!(i, j, grid, c, bc::FEOBC, loc, clock, model_fields) k = grid.Nz + 1 - @inbounds c[i, j, k] = c[i, j, k - 2] + @inbounds c[i, j, k] = c[i, j, k - 2] + (c[i, j, k - 1] - c[i, j, k - 3]) / 2 return nothing end \ No newline at end of file From 361d6d807c7c61abadf3b23566f649c7338b885b Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Thu, 6 Jun 2024 16:55:59 +0100 Subject: [PATCH 049/113] renamed flat extrapolation file --- src/BoundaryConditions/BoundaryConditions.jl | 2 +- ...y.jl => flat_extrapolation_open_boundary_matching_scheme.jl} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename src/BoundaryConditions/{no_gradient_open_boudnary.jl => flat_extrapolation_open_boundary_matching_scheme.jl} (100%) diff --git a/src/BoundaryConditions/BoundaryConditions.jl b/src/BoundaryConditions/BoundaryConditions.jl index 5bb0789a69..a61cf86ac8 100644 --- a/src/BoundaryConditions/BoundaryConditions.jl +++ b/src/BoundaryConditions/BoundaryConditions.jl @@ -39,5 +39,5 @@ include("apply_flux_bcs.jl") include("update_boundary_conditions.jl") -include("no_gradient_open_boudnary.jl") +include("flat_extrapolation_open_boundary_matching_scheme.jl") end # module diff --git a/src/BoundaryConditions/no_gradient_open_boudnary.jl b/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl similarity index 100% rename from src/BoundaryConditions/no_gradient_open_boudnary.jl rename to src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl From 952477507b0ee4764abd268ade57c336b58ad084 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Thu, 6 Jun 2024 21:13:38 +0100 Subject: [PATCH 050/113] fixed open halo filling for tracers --- src/BoundaryConditions/fill_halo_regions.jl | 6 +++--- src/BoundaryConditions/fill_halo_regions_open.jl | 13 +++++++++++-- .../update_nonhydrostatic_model_state.jl | 2 +- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/BoundaryConditions/fill_halo_regions.jl b/src/BoundaryConditions/fill_halo_regions.jl index 970148fb08..d7c003546d 100644 --- a/src/BoundaryConditions/fill_halo_regions.jl +++ b/src/BoundaryConditions/fill_halo_regions.jl @@ -45,13 +45,13 @@ end const MaybeTupledData = Union{OffsetArray, NTuple{<:Any, OffsetArray}} "Fill halo regions in ``x``, ``y``, and ``z`` for a given field's data." -function fill_halo_regions!(c::MaybeTupledData, boundary_conditions, indices, loc, grid, args...; fill_open_boundaries = true, kwargs...) +function fill_halo_regions!(c::MaybeTupledData, boundary_conditions, indices, loc, grid, args...; fill_boundary_normal_velocities = true, kwargs...) arch = architecture(grid) - if fill_open_boundaries + if fill_boundary_normal_velocities fill_open_boundary_regions!(c, boundary_conditions, loc, grid, args...; kwargs...) end - + fill_halos!, bcs = permute_boundary_conditions(boundary_conditions) number_of_tasks = length(fill_halos!) diff --git a/src/BoundaryConditions/fill_halo_regions_open.jl b/src/BoundaryConditions/fill_halo_regions_open.jl index 2fe9e5fc1f..c0d81bba32 100644 --- a/src/BoundaryConditions/fill_halo_regions_open.jl +++ b/src/BoundaryConditions/fill_halo_regions_open.jl @@ -72,7 +72,7 @@ end _fill_top_open_halo!(i, j, grid, c, top_bc, loc, args...) end -# fallback for normal boundary conditions +# fallback for non-open boundary conditions @inline _fill_west_open_halo!(j, k, grid, c, bc, loc, args...) = nothing @inline _fill_east_open_halo!(j, k, grid, c, bc, loc, args...) = nothing @@ -81,7 +81,7 @@ end @inline _fill_bottom_open_halo!(i, j, grid, c, bc, loc, args...) = nothing @inline _fill_top_open_halo!(i, j, grid, c, bc, loc, args...) = nothing -# and don't do anything on the normal fill call +# and don't do anything on the non-open boundary fill call @inline _fill_west_halo!(j, k, grid, c, bc, loc, args...) = nothing @inline _fill_east_halo!(j, k, grid, c, bc, loc, args...) = nothing @@ -90,6 +90,15 @@ end @inline _fill_bottom_halo!(i, j, grid, c, bc, loc, args...) = nothing @inline _fill_top_halo!(i, j, grid, c, bc, loc, args...) = nothing +# except when its a center field (so presumably are tracers) + +@inline _fill_west_halo!(j, k, grid, c, bc, loc::Tuple{Center, Center, Center}, args...) = @inbounds c[1, j, k] = getbc(bc, j, k, grid, args...) +@inline _fill_east_halo!(j, k, grid, c, bc, loc::Tuple{Center, Center, Center}, args...) = @inbounds c[grid.Nx + 1, j, k] = getbc(bc, j, k, grid, args...) +@inline _fill_south_halo!(i, k, grid, c, bc, loc::Tuple{Center, Center, Center}, args...) = @inbounds c[i, 1, k] = getbc(bc, i, k, grid, args...) +@inline _fill_north_halo!(i, k, grid, c, bc, loc::Tuple{Center, Center, Center}, args...) = @inbounds c[i, grid.Ny + 1, k] = getbc(bc, i, k, grid, args...) +@inline _fill_bottom_halo!(i, j, grid, c, bc, loc::Tuple{Center, Center, Center}, args...) = @inbounds c[i, j, 1] = getbc(bc, i, j, grid, args...) +@inline _fill_top_halo!(i, j, grid, c, bc, loc::Tuple{Center, Center, Center}, args...) = @inbounds c[i, j, grid.Nz + 1] = getbc(bc, i, j, grid, args...) + # generic for open boundary conditions @inline _fill_west_open_halo!(j, k, grid, c, bc::OBC, loc, args...) = @inbounds c[1, j, k] = getbc(bc, j, k, grid, args...) diff --git a/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl b/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl index 0a187cdb29..2562f4f9c4 100644 --- a/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl +++ b/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl @@ -31,7 +31,7 @@ function update_state!(model::NonhydrostaticModel, callbacks=[]; compute_tendenc update_boundary_conditions!(fields(model), model) # Fill halos for velocities and tracers - fill_halo_regions!(merge(model.velocities, model.tracers), model.clock, fields(model); fill_open_boundaries = false, async = true) + fill_halo_regions!(merge(model.velocities, model.tracers), model.clock, fields(model); fill_boundary_normal_velocities = false, async = true) # Compute auxiliary fields for aux_field in model.auxiliary_fields From f88b5c67c2e91aa8e55e4155f6c7d5ee57fde204 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Thu, 6 Jun 2024 21:42:07 +0100 Subject: [PATCH 051/113] correction --- src/BoundaryConditions/fill_halo_regions_open.jl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/BoundaryConditions/fill_halo_regions_open.jl b/src/BoundaryConditions/fill_halo_regions_open.jl index c0d81bba32..64812120a4 100644 --- a/src/BoundaryConditions/fill_halo_regions_open.jl +++ b/src/BoundaryConditions/fill_halo_regions_open.jl @@ -92,12 +92,12 @@ end # except when its a center field (so presumably are tracers) -@inline _fill_west_halo!(j, k, grid, c, bc, loc::Tuple{Center, Center, Center}, args...) = @inbounds c[1, j, k] = getbc(bc, j, k, grid, args...) -@inline _fill_east_halo!(j, k, grid, c, bc, loc::Tuple{Center, Center, Center}, args...) = @inbounds c[grid.Nx + 1, j, k] = getbc(bc, j, k, grid, args...) -@inline _fill_south_halo!(i, k, grid, c, bc, loc::Tuple{Center, Center, Center}, args...) = @inbounds c[i, 1, k] = getbc(bc, i, k, grid, args...) -@inline _fill_north_halo!(i, k, grid, c, bc, loc::Tuple{Center, Center, Center}, args...) = @inbounds c[i, grid.Ny + 1, k] = getbc(bc, i, k, grid, args...) -@inline _fill_bottom_halo!(i, j, grid, c, bc, loc::Tuple{Center, Center, Center}, args...) = @inbounds c[i, j, 1] = getbc(bc, i, j, grid, args...) -@inline _fill_top_halo!(i, j, grid, c, bc, loc::Tuple{Center, Center, Center}, args...) = @inbounds c[i, j, grid.Nz + 1] = getbc(bc, i, j, grid, args...) +@inline _fill_west_halo!(j, k, grid, c, bc::OBC, loc::Tuple{Center, Center, Center}, args...) = @inbounds c[1, j, k] = getbc(bc, j, k, grid, args...) +@inline _fill_east_halo!(j, k, grid, c, bc::OBC, loc::Tuple{Center, Center, Center}, args...) = @inbounds c[grid.Nx + 1, j, k] = getbc(bc, j, k, grid, args...) +@inline _fill_south_halo!(i, k, grid, c, bc::OBC, loc::Tuple{Center, Center, Center}, args...) = @inbounds c[i, 1, k] = getbc(bc, i, k, grid, args...) +@inline _fill_north_halo!(i, k, grid, c, bc::OBC, loc::Tuple{Center, Center, Center}, args...) = @inbounds c[i, grid.Ny + 1, k] = getbc(bc, i, k, grid, args...) +@inline _fill_bottom_halo!(i, j, grid, c, bc::OBC, loc::Tuple{Center, Center, Center}, args...) = @inbounds c[i, j, 1] = getbc(bc, i, j, grid, args...) +@inline _fill_top_halo!(i, j, grid, c, bc::OBC, loc::Tuple{Center, Center, Center}, args...) = @inbounds c[i, j, grid.Nz + 1] = getbc(bc, i, j, grid, args...) # generic for open boundary conditions From dbbe26bc94b1e78976087289f77062a039d7b401 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Thu, 6 Jun 2024 21:51:49 +0100 Subject: [PATCH 052/113] fixed set nonhydrostatic model --- src/Models/NonhydrostaticModels/set_nonhydrostatic_model.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Models/NonhydrostaticModels/set_nonhydrostatic_model.jl b/src/Models/NonhydrostaticModels/set_nonhydrostatic_model.jl index 65e6386f22..e8011c9058 100644 --- a/src/Models/NonhydrostaticModels/set_nonhydrostatic_model.jl +++ b/src/Models/NonhydrostaticModels/set_nonhydrostatic_model.jl @@ -41,7 +41,7 @@ function set!(model::NonhydrostaticModel; enforce_incompressibility=true, kwargs end set!(ϕ, value) - fill_halo_regions!(ϕ) + fill_halo_regions!(ϕ, model.clock, fields(model)) end # Apply a mask From da50c49e6ed74f5b125793ab01c3752dc5409165 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Fri, 7 Jun 2024 12:11:45 +0100 Subject: [PATCH 053/113] corrected doc string --- docs/src/model_setup/boundary_conditions.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/model_setup/boundary_conditions.md b/docs/src/model_setup/boundary_conditions.md index 48310aa9be..3c5447e3c2 100644 --- a/docs/src/model_setup/boundary_conditions.md +++ b/docs/src/model_setup/boundary_conditions.md @@ -92,8 +92,8 @@ julia> model.velocities.v.boundary_conditions Oceananigans.FieldBoundaryConditions, with boundary conditions ├── west: PeriodicBoundaryCondition ├── east: PeriodicBoundaryCondition -├── south: OpenBoundaryCondition{Nothing}: Nothing -├── north: OpenBoundaryCondition{Nothing}: Nothing +├── south: OpenBoundaryCondition{Nothing}: 0.0 +├── north: OpenBoundaryCondition{Nothing}: 0.0 ├── bottom: ValueBoundaryCondition: 0.0 ├── top: FluxBoundaryCondition: Nothing └── immersed: FluxBoundaryCondition: Nothing From 92c16f33178d5fd6b352ff639d124be62920b7d0 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Fri, 7 Jun 2024 15:20:42 +0100 Subject: [PATCH 054/113] fix (?) multi region grid fills --- src/BoundaryConditions/fill_halo_regions_open.jl | 6 ++---- src/MultiRegion/multi_region_boundary_conditions.jl | 10 ++++++++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/BoundaryConditions/fill_halo_regions_open.jl b/src/BoundaryConditions/fill_halo_regions_open.jl index 64812120a4..24778f9375 100644 --- a/src/BoundaryConditions/fill_halo_regions_open.jl +++ b/src/BoundaryConditions/fill_halo_regions_open.jl @@ -12,11 +12,9 @@ const BoundedGrid = Union{AbstractGrid{<:Any, <:Bounded}, AbstractGrid{<:Any, <:Any, <:Bounded}, AbstractGrid{<:Any, <:Any, <:Any, <:Bounded}} -@inline fill_open_boundary_regions!(field, grid, args...) = nothing +@inline fill_open_boundary_regions!(field, args...) = fill_open_boundary_regions!(field, field.grid, instantiated_location(field)) -@inline fill_open_boundary_regions!(field) = fill_open_boundary_regions!(field, field.grid, instantiated_location(field)) - -fill_open_boundary_regions!(field, grid::BoundedGrid, loc, args...) = fill_open_boundary_regions!(field, field.boundary_conditions, loc, grid, args...) +@inline fill_open_boundary_regions!(field, grid::BoundedGrid, loc, args...) = fill_open_boundary_regions!(field, field.boundary_conditions, loc, grid, args...) function fill_open_boundary_regions!(field, boundary_conditions, loc, grid, args...; kwargs...) arch = architecture(grid) diff --git a/src/MultiRegion/multi_region_boundary_conditions.jl b/src/MultiRegion/multi_region_boundary_conditions.jl index 1910c8847e..97e58c5d69 100644 --- a/src/MultiRegion/multi_region_boundary_conditions.jl +++ b/src/MultiRegion/multi_region_boundary_conditions.jl @@ -12,7 +12,8 @@ using Oceananigans.BoundaryConditions: extract_north_bc, extract_top_bc, extract_bottom_bc, fill_halo_event!, MCBCT, - MCBC + MCBC, + fill_open_boundary_regions! import Oceananigans.Fields: tupled_fill_halo_regions!, boundary_conditions, data, fill_send_buffers! @@ -38,9 +39,14 @@ import Oceananigans.BoundaryConditions: end end -function fill_halo_regions!(field::MultiRegionField, args...; kwargs...) +function fill_halo_regions!(field::MultiRegionField, args...; fill_boundary_normal_velocities = true, kwargs...) reduced_dims = reduced_dimensions(field) + if fill_boundary_normal_velocities + clock, fields = args + apply_regionally!(fill_open_boundary_regions!, field, clock, fields) + end + return fill_halo_regions!(field.data, field.boundary_conditions, field.indices, From bf312b859f4976d61ab770c1e80e103a2ec6ad2e Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Thu, 20 Jun 2024 16:02:23 +0100 Subject: [PATCH 055/113] maybe made windowed fields work --- src/BoundaryConditions/fill_halo_regions.jl | 2 +- .../fill_halo_regions_open.jl | 42 ++++++++----------- 2 files changed, 18 insertions(+), 26 deletions(-) diff --git a/src/BoundaryConditions/fill_halo_regions.jl b/src/BoundaryConditions/fill_halo_regions.jl index d7c003546d..3f3b9a399d 100644 --- a/src/BoundaryConditions/fill_halo_regions.jl +++ b/src/BoundaryConditions/fill_halo_regions.jl @@ -49,7 +49,7 @@ function fill_halo_regions!(c::MaybeTupledData, boundary_conditions, indices, lo arch = architecture(grid) if fill_boundary_normal_velocities - fill_open_boundary_regions!(c, boundary_conditions, loc, grid, args...; kwargs...) + fill_open_boundary_regions!(c, boundary_conditions, indices, loc, grid, args...; kwargs...) end fill_halos!, bcs = permute_boundary_conditions(boundary_conditions) diff --git a/src/BoundaryConditions/fill_halo_regions_open.jl b/src/BoundaryConditions/fill_halo_regions_open.jl index 24778f9375..efc886e700 100644 --- a/src/BoundaryConditions/fill_halo_regions_open.jl +++ b/src/BoundaryConditions/fill_halo_regions_open.jl @@ -1,34 +1,29 @@ -#=function fill_boundary_normal_velocities!(velocities, clock, fields) - grid = velocities.u.grid - - fill_open_boundary_regions!(velocities.u, grid, instantiated_location(velocities.u), clock, fields) - fill_open_boundary_regions!(velocities.v, grid, instantiated_location(velocities.v), clock, fields) - fill_open_boundary_regions!(velocities.w, grid, instantiated_location(velocities.w), clock, fields) - - return nothing -end=# - const BoundedGrid = Union{AbstractGrid{<:Any, <:Bounded}, AbstractGrid{<:Any, <:Any, <:Bounded}, AbstractGrid{<:Any, <:Any, <:Any, <:Bounded}} -@inline fill_open_boundary_regions!(field, args...) = fill_open_boundary_regions!(field, field.grid, instantiated_location(field)) +@inline fill_open_boundary_regions!(field, args...) = + fill_open_boundary_regions!(field, field.boundary_conditions, field.indices, instantiated_location(field), field.grid) -@inline fill_open_boundary_regions!(field, grid::BoundedGrid, loc, args...) = fill_open_boundary_regions!(field, field.boundary_conditions, loc, grid, args...) +# what is this for +#@inline fill_open_boundary_regions!(field, grid::BoundedGrid, loc, args...) = fill_open_boundary_regions!(field, field.boundary_conditions, field.indices, loc, grid, args...) -function fill_open_boundary_regions!(field, boundary_conditions, loc, grid, args...; kwargs...) +function fill_open_boundary_regions!(field, boundary_conditions, indices, loc, grid, args...; kwargs...) arch = architecture(grid) left_bc = left_boundary_condition(boundary_conditions, loc) right_bc = right_boundary_condition(boundary_conditions, loc) - launch!(arch, grid, side_plan(loc), fill_open_halo!(loc), field, left_bc, right_bc, loc, grid, args) + open_fill, normal_fill = fill_open_halo(loc) + fill_size = fill_halo_size(field, normal_fill, indices, boundary_conditions) + + launch!(arch, grid, fill_size, open_fill, field, left_bc, right_bc, loc, grid, args) return nothing end -fill_open_boundary_regions!(fields::NTuple, boundary_conditions, loc, grid, args...; kwargs...) = - [fill_open_boundary_regions!(field, boundary_conditions[n], loc[n], grid, args...; kwargs...) for (n, field) in enumerate(fields)] +fill_open_boundary_regions!(fields::NTuple, boundary_conditions, indices, loc, grid, args...; kwargs...) = + [fill_open_boundary_regions!(field, boundary_conditions[n], indices, loc[n], grid, args...; kwargs...) for (n, field) in enumerate(fields)] @inline left_boundary_condition(boundary_conditions, loc) = nothing @inline left_boundary_condition(boundary_conditions, loc::Tuple{Face, Center, Center}) = boundary_conditions.west @@ -40,18 +35,15 @@ fill_open_boundary_regions!(fields::NTuple, boundary_conditions, loc, grid, args @inline right_boundary_condition(boundary_conditions, loc::Tuple{Center, Face, Center}) = boundary_conditions.north @inline right_boundary_condition(boundary_conditions, loc::Tuple{Center, Center, Face}) = boundary_conditions.top -@inline side_plan(loc) = :xy -@inline side_plan(loc::Tuple{Face, Center, Center}) = :yz -@inline side_plan(loc::Tuple{Center, Face, Center}) = :xz -@inline side_plan(loc::Tuple{Center, Center, Face}) = :xy - -@inline fill_open_halo!(loc) = _no_fill! -@inline fill_open_halo!(loc::Tuple{Face, Center, Center}) = _fill_west_and_east_open_halo! -@inline fill_open_halo!(loc::Tuple{Center, Face, Center}) = _fill_south_and_north_open_halo! -@inline fill_open_halo!(loc::Tuple{Center, Center, Face}) = _fill_bottom_and_top_open_halo! +@inline fill_open_halo(loc) = _no_fill!, _no_fill! +@inline fill_open_halo(loc::Tuple{Face, Center, Center}) = _fill_west_and_east_open_halo!, fill_west_and_east_halo! +@inline fill_open_halo(loc::Tuple{Center, Face, Center}) = _fill_south_and_north_open_halo!, fill_south_and_north_halo! +@inline fill_open_halo(loc::Tuple{Center, Center, Face}) = _fill_bottom_and_top_open_halo!, fill_bottom_and_top_halo! @kernel _no_fill!(args...) = nothing +@inline fill_halo_size(field, ::typeof(_no_fill!), args...) = (0, 0) + @kernel function _fill_west_and_east_open_halo!(c, west_bc, east_bc, loc, grid, args) j, k = @index(Global, NTuple) _fill_west_open_halo!(j, k, grid, c, west_bc, loc, args...) From 19aad6cbd86390f82d90517a94d08ba31bb1e2eb Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Thu, 20 Jun 2024 16:47:04 +0100 Subject: [PATCH 056/113] fixed multiregion open fills (more tests now pass locally ...) --- src/MultiRegion/multi_region_boundary_conditions.jl | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/MultiRegion/multi_region_boundary_conditions.jl b/src/MultiRegion/multi_region_boundary_conditions.jl index 97e58c5d69..26d9d2932e 100644 --- a/src/MultiRegion/multi_region_boundary_conditions.jl +++ b/src/MultiRegion/multi_region_boundary_conditions.jl @@ -39,14 +39,9 @@ import Oceananigans.BoundaryConditions: end end -function fill_halo_regions!(field::MultiRegionField, args...; fill_boundary_normal_velocities = true, kwargs...) +function fill_halo_regions!(field::MultiRegionField, args...; kwargs...) reduced_dims = reduced_dimensions(field) - if fill_boundary_normal_velocities - clock, fields = args - apply_regionally!(fill_open_boundary_regions!, field, clock, fields) - end - return fill_halo_regions!(field.data, field.boundary_conditions, field.indices, @@ -104,9 +99,13 @@ function multi_region_permute_boundary_conditions(bcs) return (fill_halos!, boundary_conditions) end -function fill_halo_regions!(c::MultiRegionObject, bcs, indices, loc, mrg::MultiRegionGrid, buffers, args...; kwargs...) +function fill_halo_regions!(c::MultiRegionObject, bcs, indices, loc, mrg::MultiRegionGrid, buffers, args...; fill_boundary_normal_velocities = true, kwargs...) arch = architecture(mrg) @apply_regionally fill_halos!, bcs = multi_region_permute_boundary_conditions(bcs) + + if fill_boundary_normal_velocities + apply_regionally!(fill_open_boundary_regions!, field, args...) + end # The number of tasks is fixed to 3 (see `multi_region_permute_boundary_conditions`). # When we want to allow asynchronous communication, we will might need to split the halos sides From f439402d8f5d5b4080eedfcd9fc90383ce392fdd Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Thu, 20 Jun 2024 17:07:21 +0100 Subject: [PATCH 057/113] Revert previous test change --- test/test_broadcasting.jl | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/test_broadcasting.jl b/test/test_broadcasting.jl index e1aaaa520a..6f2e8a6788 100644 --- a/test/test_broadcasting.jl +++ b/test/test_broadcasting.jl @@ -1,7 +1,5 @@ include("dependencies_for_runtests.jl") -using Oceananigans.BoundaryConditions: fill_open_boundary_regions! - @testset "Field broadcasting" begin @info " Testing broadcasting with fields..." @@ -48,7 +46,6 @@ using Oceananigans.BoundaryConditions: fill_open_boundary_regions! b2 .= 1 fill_halo_regions!(b2) # sets b2[1, 1, 1] = b[1, 1, 4] = 0 - fill_open_boundary_regions!(b2) CUDA.@allowscalar begin @test b2[1, 1, 1] == 0 From 2c5e6b516592106d65f4a43875660794162aa460 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Thu, 20 Jun 2024 17:16:46 +0100 Subject: [PATCH 058/113] fixed more tests --- src/BoundaryConditions/fill_halo_regions_open.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/BoundaryConditions/fill_halo_regions_open.jl b/src/BoundaryConditions/fill_halo_regions_open.jl index efc886e700..d3f0040a3c 100644 --- a/src/BoundaryConditions/fill_halo_regions_open.jl +++ b/src/BoundaryConditions/fill_halo_regions_open.jl @@ -3,7 +3,7 @@ const BoundedGrid = Union{AbstractGrid{<:Any, <:Bounded}, AbstractGrid{<:Any, <:Any, <:Any, <:Bounded}} @inline fill_open_boundary_regions!(field, args...) = - fill_open_boundary_regions!(field, field.boundary_conditions, field.indices, instantiated_location(field), field.grid) + fill_open_boundary_regions!(field.data, field.boundary_conditions, field.indices, instantiated_location(field), field.grid) # what is this for #@inline fill_open_boundary_regions!(field, grid::BoundedGrid, loc, args...) = fill_open_boundary_regions!(field, field.boundary_conditions, field.indices, loc, grid, args...) @@ -15,7 +15,7 @@ function fill_open_boundary_regions!(field, boundary_conditions, indices, loc, g right_bc = right_boundary_condition(boundary_conditions, loc) open_fill, normal_fill = fill_open_halo(loc) - fill_size = fill_halo_size(field, normal_fill, indices, boundary_conditions) + fill_size = fill_halo_size(field, normal_fill, indices, boundary_conditions, loc, grid) launch!(arch, grid, fill_size, open_fill, field, left_bc, right_bc, loc, grid, args) From f3a7ff8125f0b4028bdeffab3a5922956b690812 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Thu, 20 Jun 2024 17:46:57 +0100 Subject: [PATCH 059/113] reverted an accidental (?) change --- src/BoundaryConditions/fill_halo_regions_open.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BoundaryConditions/fill_halo_regions_open.jl b/src/BoundaryConditions/fill_halo_regions_open.jl index d3f0040a3c..524d8ecd68 100644 --- a/src/BoundaryConditions/fill_halo_regions_open.jl +++ b/src/BoundaryConditions/fill_halo_regions_open.jl @@ -3,7 +3,7 @@ const BoundedGrid = Union{AbstractGrid{<:Any, <:Bounded}, AbstractGrid{<:Any, <:Any, <:Any, <:Bounded}} @inline fill_open_boundary_regions!(field, args...) = - fill_open_boundary_regions!(field.data, field.boundary_conditions, field.indices, instantiated_location(field), field.grid) + fill_open_boundary_regions!(field, field.boundary_conditions, field.indices, instantiated_location(field), field.grid) # what is this for #@inline fill_open_boundary_regions!(field, grid::BoundedGrid, loc, args...) = fill_open_boundary_regions!(field, field.boundary_conditions, field.indices, loc, grid, args...) From bfb3f08e59d41912838571673a2071fca9fe994e Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Thu, 20 Jun 2024 18:04:39 +0100 Subject: [PATCH 060/113] MRG tests pass locally --- src/BoundaryConditions/fill_halo_regions_open.jl | 8 ++++++++ src/MultiRegion/multi_region_boundary_conditions.jl | 10 ++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/BoundaryConditions/fill_halo_regions_open.jl b/src/BoundaryConditions/fill_halo_regions_open.jl index 524d8ecd68..e15d5a2a95 100644 --- a/src/BoundaryConditions/fill_halo_regions_open.jl +++ b/src/BoundaryConditions/fill_halo_regions_open.jl @@ -25,15 +25,23 @@ end fill_open_boundary_regions!(fields::NTuple, boundary_conditions, indices, loc, grid, args...; kwargs...) = [fill_open_boundary_regions!(field, boundary_conditions[n], indices, loc[n], grid, args...; kwargs...) for (n, field) in enumerate(fields)] +const FACE_FIELD_LOCS = Union{<:Tuple{Face, Center, Center}, <:Tuple{Center, Face, Center}, <:Tuple{Center, Center, Face}} + @inline left_boundary_condition(boundary_conditions, loc) = nothing @inline left_boundary_condition(boundary_conditions, loc::Tuple{Face, Center, Center}) = boundary_conditions.west @inline left_boundary_condition(boundary_conditions, loc::Tuple{Center, Face, Center}) = boundary_conditions.south @inline left_boundary_condition(boundary_conditions, loc::Tuple{Center, Center, Face}) = boundary_conditions.bottom +@inline left_boundary_condition(boundary_conditions::Tuple, loc::Tuple{Face, Center, Center}) = @inbounds boundary_conditions[1] +@inline left_boundary_condition(boundary_conditions::Tuple, loc::Tuple{Center, Face, Center}) = @inbounds boundary_conditions[1] +@inline left_boundary_condition(boundary_conditions::Tuple, loc::Tuple{Center, Center, Face}) = @inbounds boundary_conditions[1] @inline right_boundary_condition(boundary_conditions, loc) = nothing @inline right_boundary_condition(boundary_conditions, loc::Tuple{Face, Center, Center}) = boundary_conditions.east @inline right_boundary_condition(boundary_conditions, loc::Tuple{Center, Face, Center}) = boundary_conditions.north @inline right_boundary_condition(boundary_conditions, loc::Tuple{Center, Center, Face}) = boundary_conditions.top +@inline right_boundary_condition(boundary_conditions::Tuple, loc::Tuple{Face, Center, Center}) = @inbounds boundary_conditions[2] +@inline right_boundary_condition(boundary_conditions::Tuple, loc::Tuple{Center, Face, Center}) = @inbounds boundary_conditions[2] +@inline right_boundary_condition(boundary_conditions::Tuple, loc::Tuple{Center, Center, Face}) = @inbounds boundary_conditions[2] @inline fill_open_halo(loc) = _no_fill!, _no_fill! @inline fill_open_halo(loc::Tuple{Face, Center, Center}) = _fill_west_and_east_open_halo!, fill_west_and_east_halo! diff --git a/src/MultiRegion/multi_region_boundary_conditions.jl b/src/MultiRegion/multi_region_boundary_conditions.jl index 26d9d2932e..138aec4c42 100644 --- a/src/MultiRegion/multi_region_boundary_conditions.jl +++ b/src/MultiRegion/multi_region_boundary_conditions.jl @@ -102,10 +102,6 @@ end function fill_halo_regions!(c::MultiRegionObject, bcs, indices, loc, mrg::MultiRegionGrid, buffers, args...; fill_boundary_normal_velocities = true, kwargs...) arch = architecture(mrg) @apply_regionally fill_halos!, bcs = multi_region_permute_boundary_conditions(bcs) - - if fill_boundary_normal_velocities - apply_regionally!(fill_open_boundary_regions!, field, args...) - end # The number of tasks is fixed to 3 (see `multi_region_permute_boundary_conditions`). # When we want to allow asynchronous communication, we will might need to split the halos sides @@ -117,9 +113,15 @@ function fill_halo_regions!(c::MultiRegionObject, bcs, indices, loc, mrg::MultiR fill_multiregion_send_buffers!(c, buffers, mrg, bcs_side) end buff = Reference(buffers.regional_objects) + + if fill_boundary_normal_velocities + apply_regionally!(fill_open_boundary_regions!, c, bcs_side, indices, loc, mrg, args...) + end + apply_regionally!(fill_halo_event!, c, fill_halo_side!, bcs_side, indices, loc, arch, mrg, buff, args...; kwargs...) + end return nothing From 8d8b049091ac4862263001d54aa1ec52496c8f2f Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Thu, 20 Jun 2024 18:06:45 +0100 Subject: [PATCH 061/113] Removed redundant code --- src/BoundaryConditions/fill_halo_regions_open.jl | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/BoundaryConditions/fill_halo_regions_open.jl b/src/BoundaryConditions/fill_halo_regions_open.jl index e15d5a2a95..0c3884cb5c 100644 --- a/src/BoundaryConditions/fill_halo_regions_open.jl +++ b/src/BoundaryConditions/fill_halo_regions_open.jl @@ -1,13 +1,6 @@ -const BoundedGrid = Union{AbstractGrid{<:Any, <:Bounded}, - AbstractGrid{<:Any, <:Any, <:Bounded}, - AbstractGrid{<:Any, <:Any, <:Any, <:Bounded}} - @inline fill_open_boundary_regions!(field, args...) = fill_open_boundary_regions!(field, field.boundary_conditions, field.indices, instantiated_location(field), field.grid) -# what is this for -#@inline fill_open_boundary_regions!(field, grid::BoundedGrid, loc, args...) = fill_open_boundary_regions!(field, field.boundary_conditions, field.indices, loc, grid, args...) - function fill_open_boundary_regions!(field, boundary_conditions, indices, loc, grid, args...; kwargs...) arch = architecture(grid) From d468538cb5de4847031efe86cd123a1eadcbdcd4 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Thu, 20 Jun 2024 19:03:37 +0100 Subject: [PATCH 062/113] added back Impenetrable BC summary --- src/BoundaryConditions/show_boundary_conditions.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/BoundaryConditions/show_boundary_conditions.jl b/src/BoundaryConditions/show_boundary_conditions.jl index 8752b4669d..040bf9b6b1 100644 --- a/src/BoundaryConditions/show_boundary_conditions.jl +++ b/src/BoundaryConditions/show_boundary_conditions.jl @@ -22,6 +22,7 @@ bc_str(::Nothing) = "Nothing" Base.summary(bc::DFBC) = string("DefaultBoundaryCondition (", summary(bc.boundary_condition), ")") Base.summary(bc::OBC{Open{MS}}) where MS = string("OpenBoundaryCondition{$MS}: ", prettysummary(bc.condition)) +Base.summary(bc::IBC) = string("ImpenetrableCondition") Base.summary(bc::FBC) = string("FluxBoundaryCondition: ", prettysummary(bc.condition)) Base.summary(bc::VBC) = string("ValueBoundaryCondition: ", prettysummary(bc.condition)) Base.summary(bc::GBC) = string("GradientBoundaryCondition: ", prettysummary(bc.condition)) From 727a3afc718e0f2dfe8c06b77ac1e5e6898c6427 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Thu, 20 Jun 2024 22:06:09 +0100 Subject: [PATCH 063/113] change some @test_skip to @test --- test/test_computed_field.jl | 26 +++++++++++++------------- test/test_time_stepping.jl | 4 +--- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/test/test_computed_field.jl b/test/test_computed_field.jl index 2f76ceaf23..7bba5040bf 100644 --- a/test/test_computed_field.jl +++ b/test/test_computed_field.jl @@ -561,11 +561,11 @@ for arch in archs tke_window = Field(tke_ccc, indices=(2:3, 2:3, 2:3)) if (grid isa ImmersedBoundaryGrid) & (arch==GPU()) - @test_broken try compute!(computed_tke); true; catch; false end - @test_broken try compute!(Field(tke)); true; catch; false; end - @test_broken try compute!(tke_window); true; catch; false; end - @test_broken all(interior(computed_tke, 2:3, 2:3, 2:3) .== 9/2) - @test_broken all(interior(tke_window) .== 9/2) + @test try compute!(computed_tke); true; catch; false end + @test try compute!(Field(tke)); true; catch; false; end + @test try compute!(tke_window); true; catch; false; end + @test all(interior(computed_tke, 2:3, 2:3, 2:3) .== 9/2) + @test all(interior(tke_window) .== 9/2) else @test try compute!(computed_tke); true; catch; false end @test try compute!(Field(tke)); true; catch; false; end @@ -581,17 +581,17 @@ for arch in archs tke_x = Field(tke_ccc, indices=(2:3, 2, 2)) if (grid isa ImmersedBoundaryGrid) & (arch==GPU()) - @test_broken try compute!(tke_xy); true; catch; false; end - @test_broken all(interior(tke_xy, 2:3, 2:3, 1) .== 9/2) + @test try compute!(tke_xy); true; catch; false; end + @test all(interior(tke_xy, 2:3, 2:3, 1) .== 9/2) - @test_broken try compute!(tke_xz); true; catch; false; end - @test_broken all(interior(tke_xz) .== 9/2) + @test try compute!(tke_xz); true; catch; false; end + @test all(interior(tke_xz) .== 9/2) - @test_broken try compute!(tke_yz); true; catch; false; end - @test_broken all(interior(tke_yz) .== 9/2) + @test try compute!(tke_yz); true; catch; false; end + @test all(interior(tke_yz) .== 9/2) - @test_broken try compute!(tke_x); true; catch; false; end - @test_broken all(interior(tke_x) .== 9/2) + @test try compute!(tke_x); true; catch; false; end + @test all(interior(tke_x) .== 9/2) else @test try compute!(tke_xy); true; catch; false; end @test all(interior(tke_xy, 2:3, 2:3, 1) .== 9/2) diff --git a/test/test_time_stepping.jl b/test/test_time_stepping.jl index 9b07247248..6ad370cb30 100644 --- a/test/test_time_stepping.jl +++ b/test/test_time_stepping.jl @@ -374,9 +374,7 @@ timesteppers = (:QuasiAdamsBashforth2, :RungeKutta3) for Closure in Closures @info " Testing that time stepping works [$(typeof(arch)), $FT, $Closure]..." if Closure === TwoDimensionalLeith - # TwoDimensionalLeith is slow on the CPU and doesn't compile right now on the GPU. - # See: https://github.com/CliMA/Oceananigans.jl/pull/1074 - @test_skip time_stepping_works_with_closure(arch, FT, Closure) + @test time_stepping_works_with_closure(arch, FT, Closure) elseif Closure === CATKEVerticalDiffusivity # CATKE isn't supported with NonhydrostaticModel yet @test_skip time_stepping_works_with_closure(arch, FT, Closure) From 43d6ac1e2b3cb5b912843d32c040721d1559a41a Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Thu, 20 Jun 2024 22:40:48 +0100 Subject: [PATCH 064/113] Update src/BoundaryConditions/show_boundary_conditions.jl Co-authored-by: Gregory L. Wagner --- src/BoundaryConditions/show_boundary_conditions.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BoundaryConditions/show_boundary_conditions.jl b/src/BoundaryConditions/show_boundary_conditions.jl index 040bf9b6b1..24a568f057 100644 --- a/src/BoundaryConditions/show_boundary_conditions.jl +++ b/src/BoundaryConditions/show_boundary_conditions.jl @@ -22,7 +22,7 @@ bc_str(::Nothing) = "Nothing" Base.summary(bc::DFBC) = string("DefaultBoundaryCondition (", summary(bc.boundary_condition), ")") Base.summary(bc::OBC{Open{MS}}) where MS = string("OpenBoundaryCondition{$MS}: ", prettysummary(bc.condition)) -Base.summary(bc::IBC) = string("ImpenetrableCondition") +Base.summary(bc::IBC) = string("ImpenetrableBoundaryCondition") Base.summary(bc::FBC) = string("FluxBoundaryCondition: ", prettysummary(bc.condition)) Base.summary(bc::VBC) = string("ValueBoundaryCondition: ", prettysummary(bc.condition)) Base.summary(bc::GBC) = string("GradientBoundaryCondition: ", prettysummary(bc.condition)) From 42f9216c4e49e9489d3fe13bd961ef1eb0b45af6 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Fri, 21 Jun 2024 13:09:11 +0100 Subject: [PATCH 065/113] renamed `left/right_boundary_condition` --- .../fill_halo_regions_open.jl | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/BoundaryConditions/fill_halo_regions_open.jl b/src/BoundaryConditions/fill_halo_regions_open.jl index 0c3884cb5c..5009dba797 100644 --- a/src/BoundaryConditions/fill_halo_regions_open.jl +++ b/src/BoundaryConditions/fill_halo_regions_open.jl @@ -4,8 +4,8 @@ function fill_open_boundary_regions!(field, boundary_conditions, indices, loc, grid, args...; kwargs...) arch = architecture(grid) - left_bc = left_boundary_condition(boundary_conditions, loc) - right_bc = right_boundary_condition(boundary_conditions, loc) + left_bc = left_velocity_open_boundary_condition(boundary_conditions, loc) + right_bc = right_velocity_open_boundary_condition(boundary_conditions, loc) open_fill, normal_fill = fill_open_halo(loc) fill_size = fill_halo_size(field, normal_fill, indices, boundary_conditions, loc, grid) @@ -20,21 +20,21 @@ fill_open_boundary_regions!(fields::NTuple, boundary_conditions, indices, loc, g const FACE_FIELD_LOCS = Union{<:Tuple{Face, Center, Center}, <:Tuple{Center, Face, Center}, <:Tuple{Center, Center, Face}} -@inline left_boundary_condition(boundary_conditions, loc) = nothing -@inline left_boundary_condition(boundary_conditions, loc::Tuple{Face, Center, Center}) = boundary_conditions.west -@inline left_boundary_condition(boundary_conditions, loc::Tuple{Center, Face, Center}) = boundary_conditions.south -@inline left_boundary_condition(boundary_conditions, loc::Tuple{Center, Center, Face}) = boundary_conditions.bottom -@inline left_boundary_condition(boundary_conditions::Tuple, loc::Tuple{Face, Center, Center}) = @inbounds boundary_conditions[1] -@inline left_boundary_condition(boundary_conditions::Tuple, loc::Tuple{Center, Face, Center}) = @inbounds boundary_conditions[1] -@inline left_boundary_condition(boundary_conditions::Tuple, loc::Tuple{Center, Center, Face}) = @inbounds boundary_conditions[1] - -@inline right_boundary_condition(boundary_conditions, loc) = nothing -@inline right_boundary_condition(boundary_conditions, loc::Tuple{Face, Center, Center}) = boundary_conditions.east -@inline right_boundary_condition(boundary_conditions, loc::Tuple{Center, Face, Center}) = boundary_conditions.north -@inline right_boundary_condition(boundary_conditions, loc::Tuple{Center, Center, Face}) = boundary_conditions.top -@inline right_boundary_condition(boundary_conditions::Tuple, loc::Tuple{Face, Center, Center}) = @inbounds boundary_conditions[2] -@inline right_boundary_condition(boundary_conditions::Tuple, loc::Tuple{Center, Face, Center}) = @inbounds boundary_conditions[2] -@inline right_boundary_condition(boundary_conditions::Tuple, loc::Tuple{Center, Center, Face}) = @inbounds boundary_conditions[2] +@inline left_velocity_open_boundary_condition(boundary_conditions, loc) = nothing +@inline left_velocity_open_boundary_condition(boundary_conditions, loc::Tuple{Face, Center, Center}) = boundary_conditions.west +@inline left_velocity_open_boundary_condition(boundary_conditions, loc::Tuple{Center, Face, Center}) = boundary_conditions.south +@inline left_velocity_open_boundary_condition(boundary_conditions, loc::Tuple{Center, Center, Face}) = boundary_conditions.bottom +@inline left_velocity_open_boundary_condition(boundary_conditions::Tuple, loc::Tuple{Face, Center, Center}) = @inbounds boundary_conditions[1] +@inline left_velocity_open_boundary_condition(boundary_conditions::Tuple, loc::Tuple{Center, Face, Center}) = @inbounds boundary_conditions[1] +@inline left_velocity_open_boundary_condition(boundary_conditions::Tuple, loc::Tuple{Center, Center, Face}) = @inbounds boundary_conditions[1] + +@inline right_velocity_open_boundary_condition(boundary_conditions, loc) = nothing +@inline right_velocity_open_boundary_condition(boundary_conditions, loc::Tuple{Face, Center, Center}) = boundary_conditions.east +@inline right_velocity_open_boundary_condition(boundary_conditions, loc::Tuple{Center, Face, Center}) = boundary_conditions.north +@inline right_velocity_open_boundary_condition(boundary_conditions, loc::Tuple{Center, Center, Face}) = boundary_conditions.top +@inline right_velocity_open_boundary_condition(boundary_conditions::Tuple, loc::Tuple{Face, Center, Center}) = @inbounds boundary_conditions[2] +@inline right_velocity_open_boundary_condition(boundary_conditions::Tuple, loc::Tuple{Center, Face, Center}) = @inbounds boundary_conditions[2] +@inline right_velocity_open_boundary_condition(boundary_conditions::Tuple, loc::Tuple{Center, Center, Face}) = @inbounds boundary_conditions[2] @inline fill_open_halo(loc) = _no_fill!, _no_fill! @inline fill_open_halo(loc::Tuple{Face, Center, Center}) = _fill_west_and_east_open_halo!, fill_west_and_east_halo! From 399799b77bc569d839ce912886e5a1afdb464cd2 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Tue, 25 Jun 2024 18:02:31 +0100 Subject: [PATCH 066/113] maybe made distributed work --- src/BoundaryConditions/fill_halo_regions_open.jl | 2 -- src/DistributedComputations/halo_communication.jl | 6 +++++- src/MultiRegion/multi_region_boundary_conditions.jl | 12 ++++++------ 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/BoundaryConditions/fill_halo_regions_open.jl b/src/BoundaryConditions/fill_halo_regions_open.jl index 5009dba797..a5ada919ea 100644 --- a/src/BoundaryConditions/fill_halo_regions_open.jl +++ b/src/BoundaryConditions/fill_halo_regions_open.jl @@ -18,8 +18,6 @@ end fill_open_boundary_regions!(fields::NTuple, boundary_conditions, indices, loc, grid, args...; kwargs...) = [fill_open_boundary_regions!(field, boundary_conditions[n], indices, loc[n], grid, args...; kwargs...) for (n, field) in enumerate(fields)] -const FACE_FIELD_LOCS = Union{<:Tuple{Face, Center, Center}, <:Tuple{Center, Face, Center}, <:Tuple{Center, Center, Face}} - @inline left_velocity_open_boundary_condition(boundary_conditions, loc) = nothing @inline left_velocity_open_boundary_condition(boundary_conditions, loc::Tuple{Face, Center, Center}) = boundary_conditions.west @inline left_velocity_open_boundary_condition(boundary_conditions, loc::Tuple{Center, Face, Center}) = boundary_conditions.south diff --git a/src/DistributedComputations/halo_communication.jl b/src/DistributedComputations/halo_communication.jl index 06264dd39c..f2534da37d 100644 --- a/src/DistributedComputations/halo_communication.jl +++ b/src/DistributedComputations/halo_communication.jl @@ -97,12 +97,16 @@ function fill_halo_regions!(field::DistributedField, args...; kwargs...) kwargs...) end -function fill_halo_regions!(c::OffsetArray, bcs, indices, loc, grid::DistributedGrid, buffers, args...; kwargs...) +function fill_halo_regions!(c::OffsetArray, bcs, indices, loc, grid::DistributedGrid, buffers, args...; fill_boundary_normal_velocities = true, kwargs...) arch = architecture(grid) fill_halos!, bcs = permute_boundary_conditions(bcs) number_of_tasks = length(fill_halos!) + if fill_boundary_normal_velocities + fill_open_boundary_regions!(c, bcs, indices, loc, grid, args...; kwargs...) + end + for task = 1:number_of_tasks fill_halo_event!(c, fill_halos![task], bcs[task], indices, loc, arch, grid, buffers, args...; kwargs...) end diff --git a/src/MultiRegion/multi_region_boundary_conditions.jl b/src/MultiRegion/multi_region_boundary_conditions.jl index 138aec4c42..598c73647d 100644 --- a/src/MultiRegion/multi_region_boundary_conditions.jl +++ b/src/MultiRegion/multi_region_boundary_conditions.jl @@ -102,7 +102,11 @@ end function fill_halo_regions!(c::MultiRegionObject, bcs, indices, loc, mrg::MultiRegionGrid, buffers, args...; fill_boundary_normal_velocities = true, kwargs...) arch = architecture(mrg) @apply_regionally fill_halos!, bcs = multi_region_permute_boundary_conditions(bcs) - + + if fill_boundary_normal_velocities + apply_regionally!(fill_open_boundary_regions!, c, bcs, indices, loc, mrg, args...) + end + # The number of tasks is fixed to 3 (see `multi_region_permute_boundary_conditions`). # When we want to allow asynchronous communication, we will might need to split the halos sides # and the number of tasks might increase. @@ -112,12 +116,8 @@ function fill_halo_regions!(c::MultiRegionObject, bcs, indices, loc, mrg::MultiR fill_halo_side! = getindex(fill_halos!, task) fill_multiregion_send_buffers!(c, buffers, mrg, bcs_side) end - buff = Reference(buffers.regional_objects) - - if fill_boundary_normal_velocities - apply_regionally!(fill_open_boundary_regions!, c, bcs_side, indices, loc, mrg, args...) - end + buff = Reference(buffers.regional_objects) apply_regionally!(fill_halo_event!, c, fill_halo_side!, bcs_side, indices, loc, arch, mrg, buff, args...; kwargs...) From 38fe629589ae4e7b2a3ad4be0baf694aebf0a139 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Tue, 25 Jun 2024 20:32:07 +0100 Subject: [PATCH 067/113] oops --- src/DistributedComputations/halo_communication.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/DistributedComputations/halo_communication.jl b/src/DistributedComputations/halo_communication.jl index f2534da37d..d749e24839 100644 --- a/src/DistributedComputations/halo_communication.jl +++ b/src/DistributedComputations/halo_communication.jl @@ -12,6 +12,7 @@ using Oceananigans.BoundaryConditions: fill_halo_size, fill_halo_offset, permute_boundary_conditions, + fill_open_boundary_regions!, PBCT, DCBCT, DCBC import Oceananigans.BoundaryConditions: From 2332b009222455170f40cac7fd51eebbcf3cb66c Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Tue, 25 Jun 2024 20:38:13 +0100 Subject: [PATCH 068/113] corrected multi region change --- src/MultiRegion/multi_region_boundary_conditions.jl | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/MultiRegion/multi_region_boundary_conditions.jl b/src/MultiRegion/multi_region_boundary_conditions.jl index 598c73647d..4c21a6d8f7 100644 --- a/src/MultiRegion/multi_region_boundary_conditions.jl +++ b/src/MultiRegion/multi_region_boundary_conditions.jl @@ -102,11 +102,7 @@ end function fill_halo_regions!(c::MultiRegionObject, bcs, indices, loc, mrg::MultiRegionGrid, buffers, args...; fill_boundary_normal_velocities = true, kwargs...) arch = architecture(mrg) @apply_regionally fill_halos!, bcs = multi_region_permute_boundary_conditions(bcs) - - if fill_boundary_normal_velocities - apply_regionally!(fill_open_boundary_regions!, c, bcs, indices, loc, mrg, args...) - end - + # The number of tasks is fixed to 3 (see `multi_region_permute_boundary_conditions`). # When we want to allow asynchronous communication, we will might need to split the halos sides # and the number of tasks might increase. @@ -116,8 +112,13 @@ function fill_halo_regions!(c::MultiRegionObject, bcs, indices, loc, mrg::MultiR fill_halo_side! = getindex(fill_halos!, task) fill_multiregion_send_buffers!(c, buffers, mrg, bcs_side) end - + buff = Reference(buffers.regional_objects) + + if fill_boundary_normal_velocities + apply_regionally!(fill_open_boundary_regions!, c, bcs_side, indices, loc, mrg, args...) + end + apply_regionally!(fill_halo_event!, c, fill_halo_side!, bcs_side, indices, loc, arch, mrg, buff, args...; kwargs...) From f3e382fd2db765305ea724ec0322950bd414494b Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Tue, 25 Jun 2024 22:24:13 +0100 Subject: [PATCH 069/113] I can't use githubs merge conflict thing --- src/TimeSteppers/quasi_adams_bashforth_2.jl | 4 ++-- src/TimeSteppers/runge_kutta_3.jl | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/TimeSteppers/quasi_adams_bashforth_2.jl b/src/TimeSteppers/quasi_adams_bashforth_2.jl index b4fdabda5d..f138398ffd 100644 --- a/src/TimeSteppers/quasi_adams_bashforth_2.jl +++ b/src/TimeSteppers/quasi_adams_bashforth_2.jl @@ -101,9 +101,9 @@ function time_step!(model::AbstractModel{<:QuasiAdamsBashforth2TimeStepper}, Δt # Be paranoid and update state at iteration 0 model.clock.iteration == 0 && update_state!(model, callbacks) - ab2_step!(model, Δt, χ) # full step for tracers, fractional step for velocities. + ab2_step!(model, Δt) # full step for tracers, fractional step for velocities. - tick!(model.clock, Δt) + tick!(model.clock, Δt) model.clock.last_Δt = Δt model.clock.last_stage_Δt = Δt # just one stage diff --git a/src/TimeSteppers/runge_kutta_3.jl b/src/TimeSteppers/runge_kutta_3.jl index ee6f7c5726..c54023e65a 100644 --- a/src/TimeSteppers/runge_kutta_3.jl +++ b/src/TimeSteppers/runge_kutta_3.jl @@ -136,17 +136,17 @@ function time_step!(model::AbstractModel{<:RungeKutta3TimeStepper}, Δt; callbac rk3_substep!(model, Δt, γ³, ζ³) + # This adjustment of the final time-step reduces the accumulation of + # round-off error when Δt is added to model.clock.time. Note that we still use + # third_stage_Δt for the substep, pressure correction, and Lagrangian particles step. + corrected_third_stage_Δt = tⁿ⁺¹ - model.clock.time + tick!(model.clock, third_stage_Δt) model.clock.last_stage_Δt = corrected_third_stage_Δt model.clock.last_Δt = Δt calculate_pressure_correction!(model, third_stage_Δt) pressure_correct_velocities!(model, third_stage_Δt) - - # This adjustment of the final time-step reduces the accumulation of - # round-off error when Δt is added to model.clock.time. Note that we still use - # third_stage_Δt for the substep, pressure correction, and Lagrangian particles step. - corrected_third_stage_Δt = tⁿ⁺¹ - model.clock.time update_state!(model, callbacks; compute_tendencies) step_lagrangian_particles!(model, third_stage_Δt) From 8df05343773b41801e1504fa1d7a3d2810ff952c Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Wed, 26 Jun 2024 01:15:49 +0100 Subject: [PATCH 070/113] renamed distributed halo fill --- src/DistributedComputations/DistributedComputations.jl | 2 +- .../{halo_communication.jl => fill_distributed_halo_regions.jl} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename src/DistributedComputations/{halo_communication.jl => fill_distributed_halo_regions.jl} (100%) diff --git a/src/DistributedComputations/DistributedComputations.jl b/src/DistributedComputations/DistributedComputations.jl index b1e224c607..de3dec6796 100644 --- a/src/DistributedComputations/DistributedComputations.jl +++ b/src/DistributedComputations/DistributedComputations.jl @@ -18,7 +18,7 @@ include("distributed_on_architecture.jl") include("distributed_kernel_launching.jl") include("halo_communication_bcs.jl") include("distributed_fields.jl") -include("halo_communication.jl") +include("fill_distributed_halo_regions.jl") include("distributed_fft_based_poisson_solver.jl") end # module diff --git a/src/DistributedComputations/halo_communication.jl b/src/DistributedComputations/fill_distributed_halo_regions.jl similarity index 100% rename from src/DistributedComputations/halo_communication.jl rename to src/DistributedComputations/fill_distributed_halo_regions.jl From d6515bc88c9b7bb91fea33dc3ae997b460007adc Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Wed, 26 Jun 2024 11:33:17 +0100 Subject: [PATCH 071/113] revert computed field test change --- test/test_computed_field.jl | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/test/test_computed_field.jl b/test/test_computed_field.jl index 7bba5040bf..2f76ceaf23 100644 --- a/test/test_computed_field.jl +++ b/test/test_computed_field.jl @@ -561,11 +561,11 @@ for arch in archs tke_window = Field(tke_ccc, indices=(2:3, 2:3, 2:3)) if (grid isa ImmersedBoundaryGrid) & (arch==GPU()) - @test try compute!(computed_tke); true; catch; false end - @test try compute!(Field(tke)); true; catch; false; end - @test try compute!(tke_window); true; catch; false; end - @test all(interior(computed_tke, 2:3, 2:3, 2:3) .== 9/2) - @test all(interior(tke_window) .== 9/2) + @test_broken try compute!(computed_tke); true; catch; false end + @test_broken try compute!(Field(tke)); true; catch; false; end + @test_broken try compute!(tke_window); true; catch; false; end + @test_broken all(interior(computed_tke, 2:3, 2:3, 2:3) .== 9/2) + @test_broken all(interior(tke_window) .== 9/2) else @test try compute!(computed_tke); true; catch; false end @test try compute!(Field(tke)); true; catch; false; end @@ -581,17 +581,17 @@ for arch in archs tke_x = Field(tke_ccc, indices=(2:3, 2, 2)) if (grid isa ImmersedBoundaryGrid) & (arch==GPU()) - @test try compute!(tke_xy); true; catch; false; end - @test all(interior(tke_xy, 2:3, 2:3, 1) .== 9/2) + @test_broken try compute!(tke_xy); true; catch; false; end + @test_broken all(interior(tke_xy, 2:3, 2:3, 1) .== 9/2) - @test try compute!(tke_xz); true; catch; false; end - @test all(interior(tke_xz) .== 9/2) + @test_broken try compute!(tke_xz); true; catch; false; end + @test_broken all(interior(tke_xz) .== 9/2) - @test try compute!(tke_yz); true; catch; false; end - @test all(interior(tke_yz) .== 9/2) + @test_broken try compute!(tke_yz); true; catch; false; end + @test_broken all(interior(tke_yz) .== 9/2) - @test try compute!(tke_x); true; catch; false; end - @test all(interior(tke_x) .== 9/2) + @test_broken try compute!(tke_x); true; catch; false; end + @test_broken all(interior(tke_x) .== 9/2) else @test try compute!(tke_xy); true; catch; false; end @test all(interior(tke_xy, 2:3, 2:3, 1) .== 9/2) From a443e8c46927d32a1f25603dc451078bf515845f Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Wed, 26 Jun 2024 14:52:47 +0100 Subject: [PATCH 072/113] finally understand distributed bc filling --- .../fill_halo_regions_open.jl | 18 +++++++++++------- .../fill_distributed_halo_regions.jl | 8 ++++---- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/BoundaryConditions/fill_halo_regions_open.jl b/src/BoundaryConditions/fill_halo_regions_open.jl index a5ada919ea..565110ffc2 100644 --- a/src/BoundaryConditions/fill_halo_regions_open.jl +++ b/src/BoundaryConditions/fill_halo_regions_open.jl @@ -18,21 +18,25 @@ end fill_open_boundary_regions!(fields::NTuple, boundary_conditions, indices, loc, grid, args...; kwargs...) = [fill_open_boundary_regions!(field, boundary_conditions[n], indices, loc[n], grid, args...; kwargs...) for (n, field) in enumerate(fields)] -@inline left_velocity_open_boundary_condition(boundary_conditions, loc) = nothing +# for regular halo fills +@inline left_velocity_open_boundary_condition(boundary_condition, loc) = nothing @inline left_velocity_open_boundary_condition(boundary_conditions, loc::Tuple{Face, Center, Center}) = boundary_conditions.west @inline left_velocity_open_boundary_condition(boundary_conditions, loc::Tuple{Center, Face, Center}) = boundary_conditions.south @inline left_velocity_open_boundary_condition(boundary_conditions, loc::Tuple{Center, Center, Face}) = boundary_conditions.bottom -@inline left_velocity_open_boundary_condition(boundary_conditions::Tuple, loc::Tuple{Face, Center, Center}) = @inbounds boundary_conditions[1] -@inline left_velocity_open_boundary_condition(boundary_conditions::Tuple, loc::Tuple{Center, Face, Center}) = @inbounds boundary_conditions[1] -@inline left_velocity_open_boundary_condition(boundary_conditions::Tuple, loc::Tuple{Center, Center, Face}) = @inbounds boundary_conditions[1] @inline right_velocity_open_boundary_condition(boundary_conditions, loc) = nothing @inline right_velocity_open_boundary_condition(boundary_conditions, loc::Tuple{Face, Center, Center}) = boundary_conditions.east @inline right_velocity_open_boundary_condition(boundary_conditions, loc::Tuple{Center, Face, Center}) = boundary_conditions.north @inline right_velocity_open_boundary_condition(boundary_conditions, loc::Tuple{Center, Center, Face}) = boundary_conditions.top -@inline right_velocity_open_boundary_condition(boundary_conditions::Tuple, loc::Tuple{Face, Center, Center}) = @inbounds boundary_conditions[2] -@inline right_velocity_open_boundary_condition(boundary_conditions::Tuple, loc::Tuple{Center, Face, Center}) = @inbounds boundary_conditions[2] -@inline right_velocity_open_boundary_condition(boundary_conditions::Tuple, loc::Tuple{Center, Center, Face}) = @inbounds boundary_conditions[2] + +# for multi region halo fills +@inline left_velocity_open_boundary_condition(boundary_conditions::NTuple{2}, loc::Tuple{Face, Center, Center}) = @inbounds boundary_conditions[1] +@inline left_velocity_open_boundary_condition(boundary_conditions::NTuple{2}, loc::Tuple{Center, Face, Center}) = @inbounds boundary_conditions[1] +@inline left_velocity_open_boundary_condition(boundary_conditions::NTuple{2}, loc::Tuple{Center, Center, Face}) = @inbounds boundary_conditions[1] + +@inline right_velocity_open_boundary_condition(boundary_conditions::NTuple{2}, loc::Tuple{Face, Center, Center}) = @inbounds boundary_conditions[2] +@inline right_velocity_open_boundary_condition(boundary_conditions::NTuple{2}, loc::Tuple{Center, Face, Center}) = @inbounds boundary_conditions[2] +@inline right_velocity_open_boundary_condition(boundary_conditions::NTuple{2}, loc::Tuple{Center, Center, Face}) = @inbounds boundary_conditions[2] @inline fill_open_halo(loc) = _no_fill!, _no_fill! @inline fill_open_halo(loc::Tuple{Face, Center, Center}) = _fill_west_and_east_open_halo!, fill_west_and_east_halo! diff --git a/src/DistributedComputations/fill_distributed_halo_regions.jl b/src/DistributedComputations/fill_distributed_halo_regions.jl index d749e24839..872ffa6331 100644 --- a/src/DistributedComputations/fill_distributed_halo_regions.jl +++ b/src/DistributedComputations/fill_distributed_halo_regions.jl @@ -99,15 +99,15 @@ function fill_halo_regions!(field::DistributedField, args...; kwargs...) end function fill_halo_regions!(c::OffsetArray, bcs, indices, loc, grid::DistributedGrid, buffers, args...; fill_boundary_normal_velocities = true, kwargs...) + if fill_boundary_normal_velocities + fill_open_boundary_regions!(c, bcs, indices, loc, grid, args...; kwargs...) + end + arch = architecture(grid) fill_halos!, bcs = permute_boundary_conditions(bcs) number_of_tasks = length(fill_halos!) - if fill_boundary_normal_velocities - fill_open_boundary_regions!(c, bcs, indices, loc, grid, args...; kwargs...) - end - for task = 1:number_of_tasks fill_halo_event!(c, fill_halos![task], bcs[task], indices, loc, arch, grid, buffers, args...; kwargs...) end From 7afba99244adaeac8069bd4fde9732d4e21564b1 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Wed, 26 Jun 2024 15:26:49 +0100 Subject: [PATCH 073/113] mistake --- src/BoundaryConditions/fill_halo_regions_open.jl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/BoundaryConditions/fill_halo_regions_open.jl b/src/BoundaryConditions/fill_halo_regions_open.jl index 565110ffc2..2eda4fb8e6 100644 --- a/src/BoundaryConditions/fill_halo_regions_open.jl +++ b/src/BoundaryConditions/fill_halo_regions_open.jl @@ -30,13 +30,13 @@ fill_open_boundary_regions!(fields::NTuple, boundary_conditions, indices, loc, g @inline right_velocity_open_boundary_condition(boundary_conditions, loc::Tuple{Center, Center, Face}) = boundary_conditions.top # for multi region halo fills -@inline left_velocity_open_boundary_condition(boundary_conditions::NTuple{2}, loc::Tuple{Face, Center, Center}) = @inbounds boundary_conditions[1] -@inline left_velocity_open_boundary_condition(boundary_conditions::NTuple{2}, loc::Tuple{Center, Face, Center}) = @inbounds boundary_conditions[1] -@inline left_velocity_open_boundary_condition(boundary_conditions::NTuple{2}, loc::Tuple{Center, Center, Face}) = @inbounds boundary_conditions[1] +@inline left_velocity_open_boundary_condition(boundary_conditions::Tuple, loc::Tuple{Face, Center, Center}) = @inbounds boundary_conditions[1] +@inline left_velocity_open_boundary_condition(boundary_conditions::Tuple, loc::Tuple{Center, Face, Center}) = @inbounds boundary_conditions[1] +@inline left_velocity_open_boundary_condition(boundary_conditions::Tuple, loc::Tuple{Center, Center, Face}) = @inbounds boundary_conditions[1] -@inline right_velocity_open_boundary_condition(boundary_conditions::NTuple{2}, loc::Tuple{Face, Center, Center}) = @inbounds boundary_conditions[2] -@inline right_velocity_open_boundary_condition(boundary_conditions::NTuple{2}, loc::Tuple{Center, Face, Center}) = @inbounds boundary_conditions[2] -@inline right_velocity_open_boundary_condition(boundary_conditions::NTuple{2}, loc::Tuple{Center, Center, Face}) = @inbounds boundary_conditions[2] +@inline right_velocity_open_boundary_condition(boundary_conditions::Tuple, loc::Tuple{Face, Center, Center}) = @inbounds boundary_conditions[2] +@inline right_velocity_open_boundary_condition(boundary_conditions::Tuple, loc::Tuple{Center, Face, Center}) = @inbounds boundary_conditions[2] +@inline right_velocity_open_boundary_condition(boundary_conditions::Tuple, loc::Tuple{Center, Center, Face}) = @inbounds boundary_conditions[2] @inline fill_open_halo(loc) = _no_fill!, _no_fill! @inline fill_open_halo(loc::Tuple{Face, Center, Center}) = _fill_west_and_east_open_halo!, fill_west_and_east_halo! From 231658d69cb0d45c0c7b35f3f183e2f1390cc1a4 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Fri, 28 Jun 2024 23:07:04 +0100 Subject: [PATCH 074/113] added oscillating example and added relaxation to flat extrapolation --- ...apolation_open_boundary_matching_scheme.jl | 50 +++++-- validation/open_boundaries/cylinder.jl | 130 ++++++++++++++++ .../open_boundaries/mode_one_internal_wave.jl | 111 -------------- .../open_boundaries/oscillating_flow.jl | 139 ++++++++++++++++++ 4 files changed, 309 insertions(+), 121 deletions(-) create mode 100644 validation/open_boundaries/cylinder.jl delete mode 100644 validation/open_boundaries/mode_one_internal_wave.jl create mode 100644 validation/open_boundaries/oscillating_flow.jl diff --git a/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl b/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl index 3d2687ecc1..a4c4cbceaa 100644 --- a/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl +++ b/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl @@ -18,42 +18,72 @@ f′′(xᵢ₋₁) ≈ (f′(xᵢ) - f′(xᵢ₋₂)) / 2Δx = - f′(xᵢ₋ ∴ f(xᵢ) ≈ f(xᵢ₋₂) + (f(xᵢ₋₁) - f(xᵢ₋₃))/2 + O(Δx²) ``` """ -struct FlatExtrapolation end +struct FlatExtrapolation{FT} + relaxation_timescale :: FT +end const FEOBC = BoundaryCondition{<:Open{<:FlatExtrapolation}} -function FlatExtrapolationOpenBoundaryCondition() - classifcation = Open(FlatExtrapolation()) +function FlatExtrapolationOpenBoundaryCondition(val = nothing; relaxation_timescale = Inf, kwargs...) + classifcation = Open(FlatExtrapolation(relaxation_timescale)) - return BoundaryCondition(classifcation, nothing) + return BoundaryCondition(classifcation, val; kwargs...) end -@inline _fill_west_open_halo!(j, k, grid, c, bc::FEOBC, loc, clock, model_fields) = @inbounds c[0, j, k] = c[2, j, k] + (c[1, j, k] - c[3, j, k]) / 2 +@inline relax(j, k, c, bc, grid, clock, model_fields) = + c + ifelse(isnothing(bc.condition)||isfinite(clock.last_stage_Δt), 0, + (getbc(bc, j, k, grid, clock, model_fields) - c) * min(1, clock.last_stage_Δt / bc.classification.matching_scheme.relaxation_timescale)) + +@inline function _fill_west_open_halo!(j, k, grid, c, bc::FEOBC, loc, clock, model_fields) + unrelaxed = @inbounds c[3, j, k] + (c[2, j, k] - c[4, j, k]) / 2 + + @inbounds c[1, j, k] = relax(j, k, unrelaxed, bc, grid, clock, model_fields) + + return nothing +end @inline function _fill_east_open_halo!(j, k, grid, c, bc::FEOBC, loc, clock, model_fields) i = grid.Nx + 1 - @inbounds c[i, j, k] = c[i - 2, j, k] + (c[i - 1, j, k] - c[i - 3, j, k]) / 2 + unrelaxed = @inbounds c[i - 2, j, k] + (c[i - 1, j, k] - c[i - 3, j, k]) / 2 + + @inbounds c[i, j, k] = relax(j, k, unrelaxed, bc, grid, clock, model_fields) return nothing end -@inline _fill_south_open_halo!(i, k, grid, c, bc::FEOBC, loc, clock, model_fields) = @inbounds c[i, 0, k] = c[i, 2, k] + (c[i, 1, k] - c[i, 3, k]) / 2 +@inline function _fill_south_open_halo!(i, k, grid, c, bc::FEOBC, loc, clock, model_fields) + unrelaxed = c[i, 3, k] + (c[i, 2, k] - c[i, 4, k]) / 2 + + @inbounds c[i, 1, k] = relax(i, k, unrelaxed, bc, grid, clock, model_fields) + + return nothing +end @inline function _fill_north_open_halo!(i, k, grid, c, bc::FEOBC, loc, clock, model_fields) j = grid.Ny + 1 - @inbounds c[i, j, k] = c[i, j - 2, k] + (c[i, j - 1, k] - c[i, j - 3, k]) / 2 + unrelaxed = @inbounds c[i, j - 2, k] + (c[i, j - 1, k] - c[i, j - 3, k]) / 2 + + @inbounds c[i, j, k] = relax(i, k, unrelaxed, bc, grid, clock, model_fields) return nothing end -@inline _fill_bottom_open_halo!(i, j, grid, c, bc::FEOBC, loc, clock, model_fields) = @inbounds c[i, j, 0] = c[i, j, 2] + (c[i, k, 1] - c[i, j, 3]) / 2 +@inline function _fill_bottom_open_halo!(i, j, grid, c, bc::FEOBC, loc, clock, model_fields) + unrelaxed = @inbounds c[i, j, 3] + (c[i, k, 2] - c[i, j, 4]) / 2 + + @inbounds c[i, j, 1] = relax(i, j, unrelaxed, bc, grid, clock, model_fields) + + return nothing +end @inline function _fill_top_open_halo!(i, j, grid, c, bc::FEOBC, loc, clock, model_fields) k = grid.Nz + 1 - @inbounds c[i, j, k] = c[i, j, k - 2] + (c[i, j, k - 1] - c[i, j, k - 3]) / 2 + unrelaxed = @inbounds c[i, j, k - 2] + (c[i, j, k - 1] - c[i, j, k - 3]) / 2 + + @inbounds c[i, j, k] = relax(i, j, unrelaxed, bc, grid, clock, model_fields) return nothing end \ No newline at end of file diff --git a/validation/open_boundaries/cylinder.jl b/validation/open_boundaries/cylinder.jl new file mode 100644 index 0000000000..193f8af873 --- /dev/null +++ b/validation/open_boundaries/cylinder.jl @@ -0,0 +1,130 @@ +# This validation script shows open boundaries working in a simple case where the +# flow remains largely unidirectional and so at one end we have no matching scheme +# but just prescribe the inflow. At the other end we then make no assumptions about +# the flow and use a very simple open boundary condition to permit information to +# exit the domain. If, for example, the flow at the prescribed boundary was reversed +# then the model would likely fail. + +using Oceananigans, Adapt, CairoMakie +using Oceananigans.BoundaryConditions: FlatExtrapolationOpenBoundaryCondition + +import Adapt: adapt_structure + +@kwdef struct Cylinder{FT} + D :: FT = 1. + x₀ :: FT = 0. + y₀ :: FT = 0. +end + +@inline (cylinder::Cylinder)(x, y) = ifelse((x - cylinder.x₀)^2 + (y - cylinder.y₀)^2 < (cylinder.D/2)^2, 1, 0) + +Adapt.adapt_structure(to, cylinder::Cylinder) = cylinder + +architecture = GPU() + +# model parameters +Re = 200 +U = 1 +D = 1. +resolution = D / 40 + +# add extra downstream distance to see if the solution near the cylinder changes +extra_downstream = 0 + +cylinder = Cylinder(; D) + +x = (-5, 5 + extra_downstream) .* D +y = (-5, 5) .* D + +Ny = Int(10 / resolution) +Nx = Ny + Int(extra_downstream / resolution) + +ν = U * D / Re + +closure = ScalarDiffusivity(;ν, κ = ν) + +grid = RectilinearGrid(architecture; topology = (Bounded, Periodic, Flat), size = (Nx, Ny), x, y) + +@inline u(y, t, U) = U * (1 + 0.01 * randn()) + +u_boundaries = FieldBoundaryConditions(east = FlatExtrapolationOpenBoundaryCondition(), + west = OpenBoundaryCondition(u, parameters = U)) + +v_boundaries = FieldBoundaryConditions(east = GradientBoundaryCondition(0), + west = GradientBoundaryCondition(0)) + +Δt = .3 * resolution / U + +u_forcing = Relaxation(; rate = 1 / (2 * Δt), mask = cylinder) +v_forcing = Relaxation(; rate = 1 / (2 * Δt), mask = cylinder) + +model = NonhydrostaticModel(; grid, + closure, + forcing = (u = u_forcing, v = v_forcing), + boundary_conditions = (u = u_boundaries, v = v_boundaries)) + +@info "Constructed model" + +# initial noise to induce turbulance faster +set!(model, u = U, v = (x, y) -> randn() * U * 0.01) + +@info "Set initial conditions" + +simulation = Simulation(model; Δt = Δt, stop_time = 300) + +wizard = TimeStepWizard(cfl = 0.3) + +simulation.callbacks[:wizard] = Callback(wizard, IterationInterval(100)) + +progress(sim) = @info "$(time(sim)) with Δt = $(prettytime(sim.Δt)) in $(prettytime(sim.run_wall_time))" + +simulation.callbacks[:progress] = Callback(progress, IterationInterval(1000)) + +simulation.output_writers[:velocity] = JLD2OutputWriter(model, model.velocities, + overwrite_existing = true, + filename = "cylinder_$(extra_downstream)_Re_$Re.jld2", + schedule = TimeInterval(1), + with_halos = true) + +run!(simulation) + +# load the results + +u_ts = FieldTimeSeries("cylinder_$(extra_downstream)_Re_$Re.jld2", "u") +v_ts = FieldTimeSeries("cylinder_$(extra_downstream)_Re_$Re.jld2", "v") + +u′, v′, w′ = Oceananigans.Fields.VelocityFields(u_ts.grid) + +ζ = Field((@at (Center, Center, Center) ∂x(v′)) - (@at (Center, Center, Center) ∂y(u′))) + +# there is probably a more memory efficient way todo this + +ζ_ts = zeros(size(grid, 1), size(grid, 2), length(u_ts.times)) # u_ts.grid so its always on cpu + +for n in 1:length(u_ts.times) + set!(u′, u_ts[n]) + set!(v′, v_ts[n]) + compute!(ζ) + ζ_ts[:, :, n] = interior(ζ, :, :, 1) +end + +@info "Loaded results" + +# plot the results + +fig = Figure(size = (600, 600)) + +ax = Axis(fig[1, 1], aspect = DataAspect()) + +xc, yc, zc = nodes(ζ) + +n = Observable(1) + +ζ_plt = @lift ζ_ts[:, :, $n] + +contour!(ax, xc, yc, ζ_plt, levels = [-2, 2], colorrange = (-2, 2), colormap = :roma) + +record(fig, "ζ_Re_$Re.mp4", 1:length(u_ts.times), framerate = 5) do i; + n[] = i + i % 10 == 0 && @info "$(n.val) of $(length(u_ts.times))" +end \ No newline at end of file diff --git a/validation/open_boundaries/mode_one_internal_wave.jl b/validation/open_boundaries/mode_one_internal_wave.jl deleted file mode 100644 index cb663c7ae1..0000000000 --- a/validation/open_boundaries/mode_one_internal_wave.jl +++ /dev/null @@ -1,111 +0,0 @@ -using Oceananigans -using Oceananigans.Fields: FunctionField - -# TODO: clean up all the global variables in this script - -h = 10 -k = π/h - -λ = 2π / k - -grid = RectilinearGrid(topology = (Bounded, Flat, Bounded), size = (256, 128), extent = (3*λ, h)) - -xc, yc, zf = nodes(grid, Center(), Center(), Face()) -xf, yc, zc = nodes(grid, Face(), Center(), Center()) - -N² = 0.2 -w₀ = .03 - -ω = @show sqrt(k^2 * N² / (k^2 + (π/h)^2)) - -u(x, z, t) = w₀ * π / (k * h) * cos(π * z / h) * cos(k * x - ω * t) - -w(x, z, t) = w₀ * sin(π * z / h) * sin(k * x - ω * t) - -b(x, z, t) = - N² * w₀ / ω * sin(π * z / h) * cos(k * x - ω * t) + N² * z - -@inline u_east(z, t) = u(xf[grid.Nx], z, t) -@inline u_west(z, t) = u(0, z, t) - -@inline w_east(z, t) = w(xc[grid.Nx], z, t) -@inline w_west(z, t) = w(0, z, t) - -@inline b_east(z, t) = b(xc[grid.Nx], z, t) -@inline b_west(z, t) = b(0, z, t) - -u_boundaries = FieldBoundaryConditions(east = OpenBoundaryCondition(u_east), - west = OpenBoundaryCondition(u_west)) - -w_boundaries = FieldBoundaryConditions(east = OpenBoundaryCondition(w_east), - west = OpenBoundaryCondition(w_west)) - -b_boundaries = FieldBoundaryConditions(east = OpenBoundaryCondition(b_east), - west = OpenBoundaryCondition(b_west)) - -const sponge_thickness = λ/2 - -left_sponge(x) = ((sponge_thickness - x) / sponge_thickness) * (x < sponge_thickness) -right_sponge(x) = ((x - grid.Lx + sponge_thickness) / sponge_thickness) * (x > grid.Lx - sponge_thickness) - -mask(x, z) = left_sponge(x) + right_sponge(x) - -cₚ = ω/k - -cᵣ = 4 * (w₀ * π / (k * h) + cₚ) - -u_forcing = Relaxation(; rate = cᵣ / sponge_thickness, mask, target = u) -w_forcing = Relaxation(; rate = cᵣ / sponge_thickness, mask, target = w) -b_forcing = Relaxation(; rate = cᵣ / sponge_thickness, mask, target = b) - -model = NonhydrostaticModel(; grid, - forcing = (; u = u_forcing, w = w_forcing, b = b_forcing), - boundary_conditions = (; u = u_boundaries, w = w_boundaries, b = b_boundaries), - advection = WENO(; order = 3), - buoyancy = BuoyancyTracer(), - tracers = :b) - -set!(model, u = (x, z) -> u(x, z, 0), w = (x, z) -> w(x, z, 0), b = (x, z) -> b(x, z, 0)) - -simulation = Simulation(model, Δt = 0.1, stop_time = 400) - -simulation.output_writers[:velocities] = JLD2OutputWriter(model, - merge(model.velocities, model.tracers), - filename = "internal_wave.jld2", - schedule = TimeInterval(0.5), - overwrite_existing=true) - -prog(sim) = @info prettytime(time(sim))*" with Δt = "*prettytime(sim.Δt) - -simulation.callbacks[:progress] = Callback(prog, TimeInterval(5)) - -conjure_time_step_wizard!(simulation, cfl=0.5) - -run!(simulation) - -using CairoMakie - -fig = Figure(); - -n = Observable(1) - -u_ts = FieldTimeSeries("internal_wave.jld2", "u"); -w_ts = FieldTimeSeries("internal_wave.jld2", "w"); -b_ts = FieldTimeSeries("internal_wave.jld2", "b"); - -u_plt = @lift interior(u_ts[$n], :, 1, :) -w_plt = @lift interior(w_ts[$n], :, 1, :) -b_plt = @lift interior(b_ts[$n], :, 1, :) .- [N² * z for x in xc, z in zc] - -title = @lift "t = $(u_ts.times[$n])" - -ax = Axis(fig[1, 1], aspect = DataAspect(); title) -ax2 = Axis(fig[2, 1], aspect = DataAspect()) -ax3 = Axis(fig[3, 1], aspect = DataAspect()) - -hm = heatmap!(ax, xf./h, zc./h, u_plt, colorrange = w₀ * π / (k * h).*(-1, 1), colormap = Reverse(:roma)) -hm2 = heatmap!(ax2, xc./h, zf./h, w_plt, colorrange = w₀.*(-1, 1), colormap = Reverse(:roma)) -hm3 = heatmap!(ax3, xc./h, zc./h, b_plt, colorrange = N² * w₀ / ω.*(-1, 1), colormap = Reverse(:roma)) - -record(fig, "internal_wave.mp4", 1:length(u_ts.times); framerate = 20) do i; - n[] = i -end \ No newline at end of file diff --git a/validation/open_boundaries/oscillating_flow.jl b/validation/open_boundaries/oscillating_flow.jl new file mode 100644 index 0000000000..ec318b9927 --- /dev/null +++ b/validation/open_boundaries/oscillating_flow.jl @@ -0,0 +1,139 @@ +# This validation script shows open boundaries working in a simple case where the +# flow remains largely unidirectional and so at one end we have no matching scheme +# but just prescribe the inflow. At the other end we then make no assumptions about +# the flow and use a very simple open boundary condition to permit information to +# exit the domain. If, for example, the flow at the prescribed boundary was reversed +# then the model would likely fail. + +using Oceananigans, Adapt, CairoMakie +using Oceananigans.BoundaryConditions: FlatExtrapolationOpenBoundaryCondition + +import Adapt: adapt_structure + +@kwdef struct Cylinder{FT} + D :: FT = 1. + x₀ :: FT = 0. + y₀ :: FT = 0. +end + +@inline (cylinder::Cylinder)(x, y) = ifelse((x - cylinder.x₀)^2 + (y - cylinder.y₀)^2 < (cylinder.D/2)^2, 1, 0) + +Adapt.adapt_structure(to, cylinder::Cylinder) = cylinder + +architecture = CPU() + +# model parameters +Re = 200 +U = 1 +D = 1. +resolution = D / 10 + +# add extra downstream distance to see if the solution near the cylinder changes +extra_downstream = 0 + +cylinder = Cylinder(; D) + +x = (-5, 5 + extra_downstream) .* D +y = (-5, 5) .* D + +Ny = Int(10 / resolution) +Nx = Ny + Int(extra_downstream / resolution) + +ν = U * D / Re + +closure = ScalarDiffusivity(;ν, κ = ν) + +grid = RectilinearGrid(architecture; topology = (Bounded, Periodic, Flat), size = (Nx, Ny), x, y) + +T = 20 / U + +@inline u(t, p) = p.U * sin(t * 2π / p.T) +@inline u(y, t, p) = u(t, p) + +@inline ∂ₜu(x, y, t, p) = p.U * cos(t * 2π / p.T) * 2π / p.T + +relaxation_timescale = 0.15 + +u_boundaries = FieldBoundaryConditions(east = FlatExtrapolationOpenBoundaryCondition(),#u; relaxation_timescale, parameters = (; U, T)), + west = FlatExtrapolationOpenBoundaryCondition())#u; relaxation_timescale, parameters = (; U, T))) + +v_boundaries = FieldBoundaryConditions(east = GradientBoundaryCondition(0), + west = GradientBoundaryCondition(0)) + +Δt = .3 * resolution / U + +@show Δt + +u_forcing = (Relaxation(; rate = 1 / (2 * Δt), mask = cylinder), Forcing(∂ₜu; parameters = (; U, T))) +v_forcing = Relaxation(; rate = 1 / (2 * Δt), mask = cylinder) + +model = NonhydrostaticModel(; grid, + closure, + forcing = (u = u_forcing, v = v_forcing), + boundary_conditions = (u = u_boundaries, v = v_boundaries)) + +@info "Constructed model" + +# initial noise to induce turbulance faster +set!(model, u = (x, y) -> randn() * U * 0.01, v = (x, y) -> randn() * U * 0.01) + +@info "Set initial conditions" + +simulation = Simulation(model; Δt = Δt, stop_time = 300) + +wizard = TimeStepWizard(cfl = 0.3) + +simulation.callbacks[:wizard] = Callback(wizard, IterationInterval(100)) + +progress(sim) = @info "$(time(sim)) with Δt = $(prettytime(sim.Δt)) in $(prettytime(sim.run_wall_time))" + +simulation.callbacks[:progress] = Callback(progress, IterationInterval(1000)) + +simulation.output_writers[:velocity] = JLD2OutputWriter(model, model.velocities, + overwrite_existing = true, + filename = "oscillating_cylinder_$(extra_downstream)_Re_$Re.jld2", + schedule = TimeInterval(1), + with_halos = true) + +run!(simulation) + +# load the results + +u_ts = FieldTimeSeries("oscillating_cylinder_$(extra_downstream)_Re_$Re.jld2", "u") +v_ts = FieldTimeSeries("oscillating_cylinder_$(extra_downstream)_Re_$Re.jld2", "v") + +u′, v′, w′ = Oceananigans.Fields.VelocityFields(u_ts.grid) + +ζ = Field((@at (Center, Center, Center) ∂x(v′)) - (@at (Center, Center, Center) ∂y(u′))) + +# there is probably a more memory efficient way todo this + +ζ_ts = zeros(size(grid, 1), size(grid, 2), length(u_ts.times)) # u_ts.grid so its always on cpu + +for n in 1:length(u_ts.times) + set!(u′, u_ts[n]) + set!(v′, v_ts[n]) + compute!(ζ) + ζ_ts[:, :, n] = interior(ζ, :, :, 1) +end + +@info "Loaded results" + +# plot the results + +fig = Figure(size = (600, 600)) + +ax = Axis(fig[1, 1], aspect = DataAspect()) + +xc, yc, zc = nodes(ζ) + +n = Observable(1) + +ζ_plt = @lift ζ_ts[:, :, $n] + +contour!(ax, xc, yc, ζ_plt, levels = [-1, 1], colorrange = (-1, 1), colormap = :roma) + +record(fig, "oscillating_ζ_Re_$(Re)_no_exterior.mp4", 1:length(u_ts.times), framerate = 5) do i; + n[] = i + i % 10 == 0 && @info "$(n.val) of $(length(u_ts.times))" +end \ No newline at end of file From 0cdefc7cc4d85fe248c89f9cd513b0033ec4cc9e Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Fri, 28 Jun 2024 23:26:38 +0100 Subject: [PATCH 075/113] fixed --- ...at_extrapolation_open_boundary_matching_scheme.jl | 2 +- validation/open_boundaries/oscillating_flow.jl | 12 +++++------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl b/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl index a4c4cbceaa..a8a2a736ea 100644 --- a/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl +++ b/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl @@ -31,7 +31,7 @@ function FlatExtrapolationOpenBoundaryCondition(val = nothing; relaxation_timesc end @inline relax(j, k, c, bc, grid, clock, model_fields) = - c + ifelse(isnothing(bc.condition)||isfinite(clock.last_stage_Δt), 0, + c + ifelse(isnothing(bc.condition)||!isfinite(clock.last_stage_Δt), 0, (getbc(bc, j, k, grid, clock, model_fields) - c) * min(1, clock.last_stage_Δt / bc.classification.matching_scheme.relaxation_timescale)) @inline function _fill_west_open_halo!(j, k, grid, c, bc::FEOBC, loc, clock, model_fields) diff --git a/validation/open_boundaries/oscillating_flow.jl b/validation/open_boundaries/oscillating_flow.jl index ec318b9927..a19ee70484 100644 --- a/validation/open_boundaries/oscillating_flow.jl +++ b/validation/open_boundaries/oscillating_flow.jl @@ -50,12 +50,10 @@ T = 20 / U @inline u(t, p) = p.U * sin(t * 2π / p.T) @inline u(y, t, p) = u(t, p) -@inline ∂ₜu(x, y, t, p) = p.U * cos(t * 2π / p.T) * 2π / p.T - relaxation_timescale = 0.15 -u_boundaries = FieldBoundaryConditions(east = FlatExtrapolationOpenBoundaryCondition(),#u; relaxation_timescale, parameters = (; U, T)), - west = FlatExtrapolationOpenBoundaryCondition())#u; relaxation_timescale, parameters = (; U, T))) +u_boundaries = FieldBoundaryConditions(east = FlatExtrapolationOpenBoundaryCondition(u; relaxation_timescale, parameters = (; U, T)), + west = FlatExtrapolationOpenBoundaryCondition(u; relaxation_timescale, parameters = (; U, T))) v_boundaries = FieldBoundaryConditions(east = GradientBoundaryCondition(0), west = GradientBoundaryCondition(0)) @@ -64,8 +62,8 @@ v_boundaries = FieldBoundaryConditions(east = GradientBoundaryCondition(0), @show Δt -u_forcing = (Relaxation(; rate = 1 / (2 * Δt), mask = cylinder), Forcing(∂ₜu; parameters = (; U, T))) -v_forcing = Relaxation(; rate = 1 / (2 * Δt), mask = cylinder) +u_forcing = Relaxation(; rate = 1 / (2 * Δt), mask = cylinder) +v_forcing = Relaxation(; rate = 1 / (2 * Δt), mask = cylinder) model = NonhydrostaticModel(; grid, closure, @@ -81,7 +79,7 @@ set!(model, u = (x, y) -> randn() * U * 0.01, v = (x, y) -> randn() * U * 0.01) simulation = Simulation(model; Δt = Δt, stop_time = 300) -wizard = TimeStepWizard(cfl = 0.3) +wizard = TimeStepWizard(cfl = 0.3, max_Δt = Δt) simulation.callbacks[:wizard] = Callback(wizard, IterationInterval(100)) From 102bee6cd72828a4a91c3160295fbbcabf9b087d Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Sat, 29 Jun 2024 14:44:41 +0100 Subject: [PATCH 076/113] added documentation for open boundary conditions --- .../boundary_conditions.md | 100 ++++++++++++++++++ .../numerical_implementation/finite_volume.md | 2 +- .../fill_halo_regions_open.jl | 5 + 3 files changed, 106 insertions(+), 1 deletion(-) diff --git a/docs/src/numerical_implementation/boundary_conditions.md b/docs/src/numerical_implementation/boundary_conditions.md index a3485036e2..e5f2be0b21 100644 --- a/docs/src/numerical_implementation/boundary_conditions.md +++ b/docs/src/numerical_implementation/boundary_conditions.md @@ -123,3 +123,103 @@ boundaries ``\partial \Omega_b``: values of ``c``. Flux boundary conditions are represented by the [`Flux`](@ref) type. + +## Open boundary conditions + +Open boundary conditions directly specify the value of the halo points. Typically this is used +to impose no penitration boundary conditions, i.e. setting wall normal velocity components on +to zero on the boundary. + +The nuance here is that open boundaries behave differently for fields on face points in the +boundary direction due to the [staggere grid](@ref finite_volume). For example, the u-component +of velocity lies on `(Face, Center, Center)` points so for open `west` or `east` boundaries the +point specified by the boundary condition is the point lying on the boundary, where as for a +tracer on `(Center, Center, Center)` points the open boundary condition specifies a point outside +of the domain (hence the difference with `Value` boundary conditions). + +The other important detail is that open (including no-penitration) boundary conditions are the +only conditions used on wall normal velocities when the domain is not periodic. This means that +their value effects the pressure calculation for nonhydrostatic models as it is involved in +calculating the divergence in the boundary adjacent center point (as described in the +[fractional step method](@ref time_stepping) documentation). Usually boundary points are filled +for the predictor velocity (i.e. before the pressure is calculated), and on the corrected field +(i.e. after the pressure correction is applied), but for open boundaries this would result in +the boundary adjacent center point becoming divergent so open boundaries are only filled for the +predictor velocity and stay the same after the pressure correction (so the boundary point is filled +with the final corrected velocity at the predictor step). + +The restriction arrises as the boundary condition is specifying the wall normal velocity, +``\hat{\boldsymbol{n}}\cdot\boldsymbol{u}``, which leads to the pressure boundary condition +```math + \begin{equation} + \label{eq:pressure_boundary_condition} + \Delta t\hat{\boldsymbol{n}}\cdot\boldsymbol{\nabla}p^{n+1}\big |_{\partial\Omega} = \left[\Delta t\hat{\boldsymbol{n}}\cdot\boldsymbol{u}^\star - \hat{\boldsymbol{n}}\cdot\boldsymbol{u}^{n+1}\right], + \end{equation} +``` +implying that there is a pressure gradient across the boundary. Since we solve the pressure poisson +equation (``\nabla^2p^{n+1}=\frac{\boldsymbol{\nabla}\cdot\boldsymbol{u}^\star}{\Delta t}``) +using the method described by [Schumann88](@citet) we have to move inhomogeneus boundary conditions +on the pressure to the right hand side. In order todo this we define a new field ``\phi`` where +```math + \begin{equation} + \label{eq:modified_pressure_field} + \phi = p^{n+1} \quad \text{inside} \quad \Omega \quad \text{but} \quad \boldsymbol{\nabla} \cdot \boldsymbol{\nable} \phi \, \big |_{\partial\Omega} = 0. + \end{equation} +``` +This moves the boundary condition to the right hand side as ``\phi`` becomes +```math + \begin{equation} + \label{eq:modified_pressure_poisson} + \boldsymbol{\nabla}^2\phi^{n+1} = \boldsymbol{\nabla}\cdot\left[\frac{\boldsymbol{u}^\star}{\Delta t} - \delta\left(\boldsymbol{x} - \boldsymbol{x}_\Omega\right)\boldsymbol{\nabla}p\right]. + \end{equation} +``` +Given the boundary condition on pressure given above, we can define a new modified predictor velocity +which is equal to the predictor velocity within the domain but shares boundary conditions with the +corrected field, +```math + \begin{equation} + \label{eq:quasi_predictor_velocity} + \tilde{\boldsymbol{u}}^\star:=\boldsymbol{u}^\star + \delta\left(\boldsymbol{x} - \boldsymbol{x}_\Omega\right)(\boldsymbol{u}^{n+1} - \boldsymbol{u}^\star). + \end{equation} +``` +The modified pressure poisson equation becomes ``\nabla^2p^{n+1}=\frac{\boldsymbol{\nabla}\cdot\tilde{\boldsymbol{u}}^\star}{\Delta t}`` +which can easily be solved. + +Perhaps a more intuative way to consider this is to recall that the corrector step projects ``\boldsymbol{u}^\star`` +to the space of divergenece free velocity by applying +```math + \begin{equation} + \label{eq:pressure_correction_step} + \boldsymbol{u}^{n+1} = \boldsymbol{u}^\star - \Delta t\boldsymbol{\nabla}p^{n+1}, + \end{equation} +``` +but we have changed ``p^{n+1}`` to ``\phi`` and ``\boldsymbol{u}^\star`` to ``\tilde{\boldsymbol{u}}^\star`` +so for ``\boldsymbol{\nabla}\phi \big |_{\partial\Omega} = 0`` the modified predictor velocity must +equal the corrected velocity on the boundary. + +For simple open boundary conditions such as no penitration or a straight forward prescription of +a known velocity at ``t^{n+1}`` this is simple to implement as we just set the boundary condition +on the predictor velocity and don't change it after the correction. But some open boundary methods +calculate the boundary value based on the interior solution. As a simple example, if we wanted to +set the wall normal veloicty gradient to zero at the west boundary then we would set the boundary +point to +```math + \begin{equation} + \label{eq:zero_wall_normal_velocity_gradient} + u^\star_{1jk} \approx u^\star_{3jk} + (u^\star_{2jk} - u^\star_{jk4}) / 2 + \mathcal{O}(\Delta x^2), + \end{equation} +``` +but we then pressure correct the interior so a new ``\mathcal{O}(\Delta t)`` error is introduced as +```math + \begin{equation} + \label{eq:zero_wall_normal_velocity_gradient} + \begin{align} + u^{n+1}_{1jk} &\approx u^{n+1}_{3jk} + (u^{n+1}_{2jk} - u^{n+1}_{jk4}) / 2 + \mathcal{O}(\Delta x^2),\\ + &= u^\star_{1jk} - \Delta t \left(\boldsymbol{\nabla}p^{n+1}_{3jk} + (\boldsymbol{\nabla}p^{n+1}_{2jk} - \boldsymbol{\nabla}p^{n+1}_{4jk}) / 2\right) + \mathcal{O}(\Delta x^2),\\ + &\approx u^\star_{1jk} + \mathcal{O}(\Delta x^2) + \mathcal{O}(\Delta t). + \end{align} + \end{equation} +``` +This is prefered to a divergent interior solution as open boundary conditions (except no penitration) +are typlically already unphysical and only used in an attempt to allow information to enter or exit +the domain. \ No newline at end of file diff --git a/docs/src/numerical_implementation/finite_volume.md b/docs/src/numerical_implementation/finite_volume.md index 091501f60b..9e2378bb43 100644 --- a/docs/src/numerical_implementation/finite_volume.md +++ b/docs/src/numerical_implementation/finite_volume.md @@ -1,4 +1,4 @@ -# Finite volume method on a staggered grid +# [Finite volume method on a staggered grid](@id finite_volume) The `Oceananigans.jl` staggered grid is defined by a rectilinear array of cuboids of horizontal dimensions ``\Delta x_{i, j, k}, \Delta y_{i, j, k}`` and vertical dimension diff --git a/src/BoundaryConditions/fill_halo_regions_open.jl b/src/BoundaryConditions/fill_halo_regions_open.jl index 2eda4fb8e6..fe982357cf 100644 --- a/src/BoundaryConditions/fill_halo_regions_open.jl +++ b/src/BoundaryConditions/fill_halo_regions_open.jl @@ -1,6 +1,11 @@ @inline fill_open_boundary_regions!(field, args...) = fill_open_boundary_regions!(field, field.boundary_conditions, field.indices, instantiated_location(field), field.grid) +""" + fill_open_boundary_regions!(fields, boundary_conditions, indices, loc, grid, args...; kwargs...) + +Fill open boundary halo regions by filling boundary conditions on field faces with `open_fill`. +""" function fill_open_boundary_regions!(field, boundary_conditions, indices, loc, grid, args...; kwargs...) arch = architecture(grid) From d5cd0da65960953a4d78832190b7f7eb2e73f70f Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Mon, 1 Jul 2024 19:04:24 +0100 Subject: [PATCH 077/113] Apply suggestions from code review Co-authored-by: Tomas Chor --- .../numerical_implementation/boundary_conditions.md | 12 ++++++------ ...at_extrapolation_open_boundary_matching_scheme.jl | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/src/numerical_implementation/boundary_conditions.md b/docs/src/numerical_implementation/boundary_conditions.md index e5f2be0b21..290aa51d27 100644 --- a/docs/src/numerical_implementation/boundary_conditions.md +++ b/docs/src/numerical_implementation/boundary_conditions.md @@ -127,11 +127,11 @@ Flux boundary conditions are represented by the [`Flux`](@ref) type. ## Open boundary conditions Open boundary conditions directly specify the value of the halo points. Typically this is used -to impose no penitration boundary conditions, i.e. setting wall normal velocity components on +to impose no penetration boundary conditions, i.e. setting wall normal velocity components on to zero on the boundary. The nuance here is that open boundaries behave differently for fields on face points in the -boundary direction due to the [staggere grid](@ref finite_volume). For example, the u-component +boundary direction due to the [staggered grid](@ref finite_volume). For example, the u-component of velocity lies on `(Face, Center, Center)` points so for open `west` or `east` boundaries the point specified by the boundary condition is the point lying on the boundary, where as for a tracer on `(Center, Center, Center)` points the open boundary condition specifies a point outside @@ -139,7 +139,7 @@ of the domain (hence the difference with `Value` boundary conditions). The other important detail is that open (including no-penitration) boundary conditions are the only conditions used on wall normal velocities when the domain is not periodic. This means that -their value effects the pressure calculation for nonhydrostatic models as it is involved in +their value affects the pressure calculation for nonhydrostatic models as it is involved in calculating the divergence in the boundary adjacent center point (as described in the [fractional step method](@ref time_stepping) documentation). Usually boundary points are filled for the predictor velocity (i.e. before the pressure is calculated), and on the corrected field @@ -159,11 +159,11 @@ The restriction arrises as the boundary condition is specifying the wall normal implying that there is a pressure gradient across the boundary. Since we solve the pressure poisson equation (``\nabla^2p^{n+1}=\frac{\boldsymbol{\nabla}\cdot\boldsymbol{u}^\star}{\Delta t}``) using the method described by [Schumann88](@citet) we have to move inhomogeneus boundary conditions -on the pressure to the right hand side. In order todo this we define a new field ``\phi`` where +on the pressure to the right hand side. In order to do this we define a new field ``\phi`` where ```math \begin{equation} \label{eq:modified_pressure_field} - \phi = p^{n+1} \quad \text{inside} \quad \Omega \quad \text{but} \quad \boldsymbol{\nabla} \cdot \boldsymbol{\nable} \phi \, \big |_{\partial\Omega} = 0. + \phi = p^{n+1} \quad \text{inside} \quad \Omega \quad \text{but} \quad \boldsymbol{\nabla} \cdot \boldsymbol{\nabla} \phi \, \big |_{\partial\Omega} = 0. \end{equation} ``` This moves the boundary condition to the right hand side as ``\phi`` becomes @@ -185,7 +185,7 @@ corrected field, The modified pressure poisson equation becomes ``\nabla^2p^{n+1}=\frac{\boldsymbol{\nabla}\cdot\tilde{\boldsymbol{u}}^\star}{\Delta t}`` which can easily be solved. -Perhaps a more intuative way to consider this is to recall that the corrector step projects ``\boldsymbol{u}^\star`` +Perhaps a more intuitive way to consider this is to recall that the corrector step projects ``\boldsymbol{u}^\star`` to the space of divergenece free velocity by applying ```math \begin{equation} diff --git a/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl b/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl index a8a2a736ea..6d0712ebbe 100644 --- a/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl +++ b/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl @@ -3,7 +3,7 @@ using Oceananigans.Operators: ∂xᶜᶜᶜ """ FlatExtrapolation -Zero gradient perepndicular velocity boundary condition. +Zero gradient perpendicular velocity boundary condition. *Given constant spacing* ```math @@ -30,9 +30,9 @@ function FlatExtrapolationOpenBoundaryCondition(val = nothing; relaxation_timesc return BoundaryCondition(classifcation, val; kwargs...) end -@inline relax(j, k, c, bc, grid, clock, model_fields) = +@inline relax(l, m, c, bc, grid, clock, model_fields) = c + ifelse(isnothing(bc.condition)||!isfinite(clock.last_stage_Δt), 0, - (getbc(bc, j, k, grid, clock, model_fields) - c) * min(1, clock.last_stage_Δt / bc.classification.matching_scheme.relaxation_timescale)) + (getbc(bc, l, m, grid, clock, model_fields) - c) * min(1, clock.last_stage_Δt / bc.classification.matching_scheme.relaxation_timescale)) @inline function _fill_west_open_halo!(j, k, grid, c, bc::FEOBC, loc, clock, model_fields) unrelaxed = @inbounds c[3, j, k] + (c[2, j, k] - c[4, j, k]) / 2 From 16f9a0b3416b61e0267c69b0ba4446ad55078c65 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Mon, 1 Jul 2024 19:07:14 +0100 Subject: [PATCH 078/113] spelling --- docs/src/numerical_implementation/boundary_conditions.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/src/numerical_implementation/boundary_conditions.md b/docs/src/numerical_implementation/boundary_conditions.md index 290aa51d27..587cdff9b7 100644 --- a/docs/src/numerical_implementation/boundary_conditions.md +++ b/docs/src/numerical_implementation/boundary_conditions.md @@ -137,7 +137,7 @@ point specified by the boundary condition is the point lying on the boundary, wh tracer on `(Center, Center, Center)` points the open boundary condition specifies a point outside of the domain (hence the difference with `Value` boundary conditions). -The other important detail is that open (including no-penitration) boundary conditions are the +The other important detail is that open (including no-penetration) boundary conditions are the only conditions used on wall normal velocities when the domain is not periodic. This means that their value affects the pressure calculation for nonhydrostatic models as it is involved in calculating the divergence in the boundary adjacent center point (as described in the @@ -197,7 +197,7 @@ but we have changed ``p^{n+1}`` to ``\phi`` and ``\boldsymbol{u}^\star`` to ``\t so for ``\boldsymbol{\nabla}\phi \big |_{\partial\Omega} = 0`` the modified predictor velocity must equal the corrected velocity on the boundary. -For simple open boundary conditions such as no penitration or a straight forward prescription of +For simple open boundary conditions such as no penetration or a straight forward prescription of a known velocity at ``t^{n+1}`` this is simple to implement as we just set the boundary condition on the predictor velocity and don't change it after the correction. But some open boundary methods calculate the boundary value based on the interior solution. As a simple example, if we wanted to @@ -220,6 +220,6 @@ but we then pressure correct the interior so a new ``\mathcal{O}(\Delta t)`` err \end{align} \end{equation} ``` -This is prefered to a divergent interior solution as open boundary conditions (except no penitration) +This is prefered to a divergent interior solution as open boundary conditions (except no penetration) are typlically already unphysical and only used in an attempt to allow information to enter or exit the domain. \ No newline at end of file From 312b786c1f8e17c85e9b5a04a426e443cd63edcb Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Mon, 1 Jul 2024 19:09:36 +0100 Subject: [PATCH 079/113] Update src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl --- .../flat_extrapolation_open_boundary_matching_scheme.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl b/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl index 6d0712ebbe..d28954011a 100644 --- a/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl +++ b/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl @@ -1,4 +1,3 @@ -using Oceananigans.Operators: ∂xᶜᶜᶜ """ FlatExtrapolation From ab961c42a6e75d334574b232c9999101b025cb6e Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Tue, 2 Jul 2024 17:05:23 +0100 Subject: [PATCH 080/113] made FEOBC work for stretched grids --- ...apolation_open_boundary_matching_scheme.jl | 36 +++++++++++---- .../open_boundaries/oscillating_flow.jl | 45 ++++++++++++++----- 2 files changed, 61 insertions(+), 20 deletions(-) diff --git a/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl b/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl index d28954011a..d261408c16 100644 --- a/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl +++ b/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl @@ -1,10 +1,11 @@ +using Oceananigans.Grids: xspacings, yspacings, zspacings """ FlatExtrapolation Zero gradient perpendicular velocity boundary condition. -*Given constant spacing* +For constant spacing: ```math f′(xᵢ) ≈ f′(xᵢ₋₁) + f′′(xᵢ₋₁)(xᵢ₋₁ - xᵢ) + O(Δx²) = f′(xᵢ₋₁) + f′′(xᵢ₋₁)Δx + O(Δx²), @@ -14,8 +15,11 @@ f′(xᵢ) ≈ (f(xᵢ) - f(xᵢ₋₂)) / 2Δx + O(Δx) = 0 ∴ f(xᵢ) ≈ f(x f′′(xᵢ₋₁) ≈ (f′(xᵢ) - f′(xᵢ₋₂)) / 2Δx = - f′(xᵢ₋₂) / 2Δx ≈ - (f(xᵢ₋₁) - f(xᵢ₋₃)) / (2Δx)² -∴ f(xᵢ) ≈ f(xᵢ₋₂) + (f(xᵢ₋₁) - f(xᵢ₋₃))/2 + O(Δx²) +∴ f(xᵢ) ≈ f(xᵢ₋₂) - (f(xᵢ₋₁) - f(xᵢ₋₃))/2 + O(Δx²) ``` + +When the grid spacing is not constant it works out that the factor of 1/2 changes to +``Δx₋₁/(Δx₋₂ + Δx₋₃)`` instead. """ struct FlatExtrapolation{FT} relaxation_timescale :: FT @@ -33,8 +37,14 @@ end c + ifelse(isnothing(bc.condition)||!isfinite(clock.last_stage_Δt), 0, (getbc(bc, l, m, grid, clock, model_fields) - c) * min(1, clock.last_stage_Δt / bc.classification.matching_scheme.relaxation_timescale)) +@inline spacing_factor(args...) = 1/2 +@inline spacing_factor(Δ::AbstractArray, ::Val{:right}) = @inbounds Δ[end] / (Δ[end-1] + Δ[end-2]) +@inline spacing_factor(Δ::AbstractArray, ::Val{:left}) = @inbounds Δ[1] / (Δ[2] + Δ[3] ) + @inline function _fill_west_open_halo!(j, k, grid, c, bc::FEOBC, loc, clock, model_fields) - unrelaxed = @inbounds c[3, j, k] + (c[2, j, k] - c[4, j, k]) / 2 + Δx = xspacings(grid, Center(), Center(), Center()) + + unrelaxed = @inbounds c[3, j, k] - (c[2, j, k] - c[4, j, k]) * spacing_factor(Δx, Val(:left)) @inbounds c[1, j, k] = relax(j, k, unrelaxed, bc, grid, clock, model_fields) @@ -44,7 +54,9 @@ end @inline function _fill_east_open_halo!(j, k, grid, c, bc::FEOBC, loc, clock, model_fields) i = grid.Nx + 1 - unrelaxed = @inbounds c[i - 2, j, k] + (c[i - 1, j, k] - c[i - 3, j, k]) / 2 + Δx = xspacings(grid, Center(), Center(), Center()) + + unrelaxed = @inbounds c[i - 2, j, k] - (c[i - 1, j, k] - c[i - 3, j, k]) * spacing_factor(Δx, Val(:right)) @inbounds c[i, j, k] = relax(j, k, unrelaxed, bc, grid, clock, model_fields) @@ -52,7 +64,9 @@ end end @inline function _fill_south_open_halo!(i, k, grid, c, bc::FEOBC, loc, clock, model_fields) - unrelaxed = c[i, 3, k] + (c[i, 2, k] - c[i, 4, k]) / 2 + Δy = yspacings(grid, Center(), Center(), Center()) + + unrelaxed = c[i, 3, k] - (c[i, 2, k] - c[i, 4, k]) * spacing_factor(Δy, Val(:left)) @inbounds c[i, 1, k] = relax(i, k, unrelaxed, bc, grid, clock, model_fields) @@ -62,7 +76,9 @@ end @inline function _fill_north_open_halo!(i, k, grid, c, bc::FEOBC, loc, clock, model_fields) j = grid.Ny + 1 - unrelaxed = @inbounds c[i, j - 2, k] + (c[i, j - 1, k] - c[i, j - 3, k]) / 2 + Δy = yspacings(grid, Center(), Center(), Center()) + + unrelaxed = @inbounds c[i, j - 2, k] - (c[i, j - 1, k] - c[i, j - 3, k]) * spacing_factor(Δy, Val(:right)) @inbounds c[i, j, k] = relax(i, k, unrelaxed, bc, grid, clock, model_fields) @@ -70,7 +86,9 @@ end end @inline function _fill_bottom_open_halo!(i, j, grid, c, bc::FEOBC, loc, clock, model_fields) - unrelaxed = @inbounds c[i, j, 3] + (c[i, k, 2] - c[i, j, 4]) / 2 + Δz = zspacings(grid, Center(), Center(), Center()) + + unrelaxed = @inbounds c[i, j, 3] - (c[i, k, 2] - c[i, j, 4]) * spacing_factor(Δz, Val(:left)) @inbounds c[i, j, 1] = relax(i, j, unrelaxed, bc, grid, clock, model_fields) @@ -80,7 +98,9 @@ end @inline function _fill_top_open_halo!(i, j, grid, c, bc::FEOBC, loc, clock, model_fields) k = grid.Nz + 1 - unrelaxed = @inbounds c[i, j, k - 2] + (c[i, j, k - 1] - c[i, j, k - 3]) / 2 + Δz = zspacings(grid, Center(), Center(), Center()) + + unrelaxed = @inbounds c[i, j, k - 2] - (c[i, j, k - 1] - c[i, j, k - 3]) * spacing_factor(Δz, Val(:right)) @inbounds c[i, j, k] = relax(i, j, unrelaxed, bc, grid, clock, model_fields) diff --git a/validation/open_boundaries/oscillating_flow.jl b/validation/open_boundaries/oscillating_flow.jl index a19ee70484..a7679ed181 100644 --- a/validation/open_boundaries/oscillating_flow.jl +++ b/validation/open_boundaries/oscillating_flow.jl @@ -1,9 +1,12 @@ # This validation script shows open boundaries working in a simple case where the -# flow remains largely unidirectional and so at one end we have no matching scheme -# but just prescribe the inflow. At the other end we then make no assumptions about -# the flow and use a very simple open boundary condition to permit information to -# exit the domain. If, for example, the flow at the prescribed boundary was reversed -# then the model would likely fail. +# oscillates sinusoidally so changes sign across two open boundaries. This is similar +# to a more realistic case where we know some arbitary external conditions. +# This necessitates using a combination allowing information to exit the domain, in +# this case by setting the wall normal velocity gradient to zero, as well as forcing +# to the external value in this example by relaxing to it. + +# This case also has a stretched grid to validate the zero wall normal velocity +# gradient matching scheme on a stretched grid. using Oceananigans, Adapt, CairoMakie using Oceananigans.BoundaryConditions: FlatExtrapolationOpenBoundaryCondition @@ -34,16 +37,30 @@ extra_downstream = 0 cylinder = Cylinder(; D) x = (-5, 5 + extra_downstream) .* D -y = (-5, 5) .* D Ny = Int(10 / resolution) -Nx = Ny + Int(extra_downstream / resolution) + +Nx = Int((10 + extra_downstream) / resolution) + +function Δy(j) + if Ny/2 - 2/resolution < j < Ny/2 + 2/resolution + return resolution + elseif j <= Ny/2 - 2/resolution + return resolution * (1 + (Ny/2 - 2/resolution - j) / (Ny/2 - 2/resolution)) + elseif j >= Ny/2 + 2/resolution + return resolution * (1 + (j - Ny/2 - 2/resolution) / (Ny/2 - 2/resolution)) + else + Throw(ArgumentError("$j not in range")) + end +end + +y(j) = sum(Δy.([1:j;])) - sum(Δy.([1:Ny;]))/2 ν = U * D / Re closure = ScalarDiffusivity(;ν, κ = ν) -grid = RectilinearGrid(architecture; topology = (Bounded, Periodic, Flat), size = (Nx, Ny), x, y) +grid = RectilinearGrid(architecture; topology = (Bounded, Bounded, Flat), size = (Nx, Ny), x = y, y = x) T = 20 / U @@ -53,10 +70,14 @@ T = 20 / U relaxation_timescale = 0.15 u_boundaries = FieldBoundaryConditions(east = FlatExtrapolationOpenBoundaryCondition(u; relaxation_timescale, parameters = (; U, T)), - west = FlatExtrapolationOpenBoundaryCondition(u; relaxation_timescale, parameters = (; U, T))) + west = FlatExtrapolationOpenBoundaryCondition(u; relaxation_timescale, parameters = (; U, T)), + south = GradientBoundaryCondition(0), + north = GradientBoundaryCondition(0)) v_boundaries = FieldBoundaryConditions(east = GradientBoundaryCondition(0), - west = GradientBoundaryCondition(0)) + west = GradientBoundaryCondition(0), + south = FlatExtrapolationOpenBoundaryCondition(0; relaxation_timescale), + north = FlatExtrapolationOpenBoundaryCondition(0; relaxation_timescale)) Δt = .3 * resolution / U @@ -121,10 +142,10 @@ end fig = Figure(size = (600, 600)) -ax = Axis(fig[1, 1], aspect = DataAspect()) - xc, yc, zc = nodes(ζ) +ax = Axis(fig[1, 1], aspect = DataAspect(), limits = (minimum(xc), maximum(xc), minimum(yc), maximum(yc))) + n = Observable(1) ζ_plt = @lift ζ_ts[:, :, $n] From 68003d48554b5c6389286b8aee465a7fec0811b4 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Mon, 8 Jul 2024 10:46:51 +0100 Subject: [PATCH 081/113] Apply suggestions from code review Thank you! Co-authored-by: Tomas Chor --- docs/src/numerical_implementation/boundary_conditions.md | 7 ++++--- .../flat_extrapolation_open_boundary_matching_scheme.jl | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/src/numerical_implementation/boundary_conditions.md b/docs/src/numerical_implementation/boundary_conditions.md index 587cdff9b7..ef5b6d08f4 100644 --- a/docs/src/numerical_implementation/boundary_conditions.md +++ b/docs/src/numerical_implementation/boundary_conditions.md @@ -153,7 +153,7 @@ The restriction arrises as the boundary condition is specifying the wall normal ```math \begin{equation} \label{eq:pressure_boundary_condition} - \Delta t\hat{\boldsymbol{n}}\cdot\boldsymbol{\nabla}p^{n+1}\big |_{\partial\Omega} = \left[\Delta t\hat{\boldsymbol{n}}\cdot\boldsymbol{u}^\star - \hat{\boldsymbol{n}}\cdot\boldsymbol{u}^{n+1}\right], + \Delta t \, \hat{\boldsymbol{n}}\cdot\boldsymbol{\nabla}p^{n+1}\big |_{\partial\Omega} = \left[\Delta t \, \hat{\boldsymbol{n}}\cdot\boldsymbol{u}^\star - \hat{\boldsymbol{n}}\cdot\boldsymbol{u}^{n+1}\right], \end{equation} ``` implying that there is a pressure gradient across the boundary. Since we solve the pressure poisson @@ -212,7 +212,6 @@ point to but we then pressure correct the interior so a new ``\mathcal{O}(\Delta t)`` error is introduced as ```math \begin{equation} - \label{eq:zero_wall_normal_velocity_gradient} \begin{align} u^{n+1}_{1jk} &\approx u^{n+1}_{3jk} + (u^{n+1}_{2jk} - u^{n+1}_{jk4}) / 2 + \mathcal{O}(\Delta x^2),\\ &= u^\star_{1jk} - \Delta t \left(\boldsymbol{\nabla}p^{n+1}_{3jk} + (\boldsymbol{\nabla}p^{n+1}_{2jk} - \boldsymbol{\nabla}p^{n+1}_{4jk}) / 2\right) + \mathcal{O}(\Delta x^2),\\ @@ -222,4 +221,6 @@ but we then pressure correct the interior so a new ``\mathcal{O}(\Delta t)`` err ``` This is prefered to a divergent interior solution as open boundary conditions (except no penetration) are typlically already unphysical and only used in an attempt to allow information to enter or exit -the domain. \ No newline at end of file +the domain. + +Open boundary conditions are represented by the [`Open`](@ref) type. \ No newline at end of file diff --git a/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl b/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl index d261408c16..0aa86693d0 100644 --- a/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl +++ b/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl @@ -28,9 +28,9 @@ end const FEOBC = BoundaryCondition{<:Open{<:FlatExtrapolation}} function FlatExtrapolationOpenBoundaryCondition(val = nothing; relaxation_timescale = Inf, kwargs...) - classifcation = Open(FlatExtrapolation(relaxation_timescale)) + classification = Open(FlatExtrapolation(relaxation_timescale)) - return BoundaryCondition(classifcation, val; kwargs...) + return BoundaryCondition(classification, val; kwargs...) end @inline relax(l, m, c, bc, grid, clock, model_fields) = From 99e2c80dfd6767841c34ecdb62c17a9789d90967 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Mon, 8 Jul 2024 10:50:51 +0100 Subject: [PATCH 082/113] Update src/BoundaryConditions/BoundaryConditions.jl --- src/BoundaryConditions/BoundaryConditions.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/BoundaryConditions/BoundaryConditions.jl b/src/BoundaryConditions/BoundaryConditions.jl index a61cf86ac8..2011bd3df4 100644 --- a/src/BoundaryConditions/BoundaryConditions.jl +++ b/src/BoundaryConditions/BoundaryConditions.jl @@ -4,7 +4,6 @@ export BCType, Flux, Gradient, Value, Open, BoundaryCondition, getbc, setbc!, PeriodicBoundaryCondition, OpenBoundaryCondition, NoFluxBoundaryCondition, MultiRegionCommunicationBoundaryCondition, - MeanOutflowOpenBoundaryCondition, FluxBoundaryCondition, ValueBoundaryCondition, GradientBoundaryCondition, DistributedCommunicationBoundaryCondition, validate_boundary_condition_topology, validate_boundary_condition_architecture, FieldBoundaryConditions, From 3b3aca339c807d886fc56b113199b02541e1d149 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Mon, 8 Jul 2024 23:41:00 +0100 Subject: [PATCH 083/113] bump version --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 791aa80534..92dc873a5c 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "Oceananigans" uuid = "9e8cae18-63c1-5223-a75c-80ca9d6e9a09" authors = ["Climate Modeling Alliance and contributors"] -version = "0.91.3" +version = "0.92.0" [deps] Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" From 4a176d7636c15f6353cea9be2eaddf28a473942f Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Wed, 10 Jul 2024 17:12:03 +0100 Subject: [PATCH 084/113] review comments --- src/BoundaryConditions/boundary_condition.jl | 2 +- src/BoundaryConditions/fill_halo_regions.jl | 3 +- .../fill_halo_regions_open.jl | 14 +++--- ...apolation_open_boundary_matching_scheme.jl | 48 +++++++++++++------ .../lagrangian_particle_advection.jl | 2 +- 5 files changed, 46 insertions(+), 23 deletions(-) diff --git a/src/BoundaryConditions/boundary_condition.jl b/src/BoundaryConditions/boundary_condition.jl index 139c676033..0547bf768b 100644 --- a/src/BoundaryConditions/boundary_condition.jl +++ b/src/BoundaryConditions/boundary_condition.jl @@ -82,7 +82,7 @@ const DCBC = BoundaryCondition{<:DistributedCommunication} # More readable BC constructors for the public API. PeriodicBoundaryCondition() = BoundaryCondition(Periodic(), nothing) NoFluxBoundaryCondition() = BoundaryCondition(Flux(), nothing) - ImpenetrableBoundaryCondition() = OpenBoundaryCondition(0) + ImpenetrableBoundaryCondition() = OpenBoundaryCondition(nothing) MultiRegionCommunicationBoundaryCondition() = BoundaryCondition(MultiRegionCommunication(), nothing) DistributedCommunicationBoundaryCondition() = BoundaryCondition(DistributedCommunication(), nothing) diff --git a/src/BoundaryConditions/fill_halo_regions.jl b/src/BoundaryConditions/fill_halo_regions.jl index 3f3b9a399d..44d72c2bd3 100644 --- a/src/BoundaryConditions/fill_halo_regions.jl +++ b/src/BoundaryConditions/fill_halo_regions.jl @@ -45,7 +45,8 @@ end const MaybeTupledData = Union{OffsetArray, NTuple{<:Any, OffsetArray}} "Fill halo regions in ``x``, ``y``, and ``z`` for a given field's data." -function fill_halo_regions!(c::MaybeTupledData, boundary_conditions, indices, loc, grid, args...; fill_boundary_normal_velocities = true, kwargs...) +function fill_halo_regions!(c::MaybeTupledData, boundary_conditions, indices, loc, grid, args...; + fill_boundary_normal_velocities = true, kwargs...) arch = architecture(grid) if fill_boundary_normal_velocities diff --git a/src/BoundaryConditions/fill_halo_regions_open.jl b/src/BoundaryConditions/fill_halo_regions_open.jl index fe982357cf..32c7b24484 100644 --- a/src/BoundaryConditions/fill_halo_regions_open.jl +++ b/src/BoundaryConditions/fill_halo_regions_open.jl @@ -12,8 +12,10 @@ function fill_open_boundary_regions!(field, boundary_conditions, indices, loc, g left_bc = left_velocity_open_boundary_condition(boundary_conditions, loc) right_bc = right_velocity_open_boundary_condition(boundary_conditions, loc) - open_fill, normal_fill = fill_open_halo(loc) - fill_size = fill_halo_size(field, normal_fill, indices, boundary_conditions, loc, grid) + # gets `open_fill`, the function which fills open boundaries at `loc`, as well as `regular_fill` + # which is the function which fills non-open boundaries at `loc` which informs `fill_halo_size` + open_fill, regular_fill = get_open_halo_filling_functions(loc) + fill_size = fill_halo_size(field, regular_fill, indices, boundary_conditions, loc, grid) launch!(arch, grid, fill_size, open_fill, field, left_bc, right_bc, loc, grid, args) @@ -43,10 +45,10 @@ fill_open_boundary_regions!(fields::NTuple, boundary_conditions, indices, loc, g @inline right_velocity_open_boundary_condition(boundary_conditions::Tuple, loc::Tuple{Center, Face, Center}) = @inbounds boundary_conditions[2] @inline right_velocity_open_boundary_condition(boundary_conditions::Tuple, loc::Tuple{Center, Center, Face}) = @inbounds boundary_conditions[2] -@inline fill_open_halo(loc) = _no_fill!, _no_fill! -@inline fill_open_halo(loc::Tuple{Face, Center, Center}) = _fill_west_and_east_open_halo!, fill_west_and_east_halo! -@inline fill_open_halo(loc::Tuple{Center, Face, Center}) = _fill_south_and_north_open_halo!, fill_south_and_north_halo! -@inline fill_open_halo(loc::Tuple{Center, Center, Face}) = _fill_bottom_and_top_open_halo!, fill_bottom_and_top_halo! +@inline get_open_halo_filling_functions(loc) = _no_fill!, _no_fill! +@inline get_open_halo_filling_functions(loc::Tuple{Face, Center, Center}) = _fill_west_and_east_open_halo!, fill_west_and_east_halo! +@inline get_open_halo_filling_functions(loc::Tuple{Center, Face, Center}) = _fill_south_and_north_open_halo!, fill_south_and_north_halo! +@inline get_open_halo_filling_functions(loc::Tuple{Center, Center, Face}) = _fill_bottom_and_top_open_halo!, fill_bottom_and_top_halo! @kernel _no_fill!(args...) = nothing diff --git a/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl b/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl index 0aa86693d0..986bf4cf21 100644 --- a/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl +++ b/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl @@ -5,21 +5,33 @@ using Oceananigans.Grids: xspacings, yspacings, zspacings Zero gradient perpendicular velocity boundary condition. -For constant spacing: +We find the boundary value by Taylor expanding the gradient at the boundary point (`xᵢ`) +to second order: ```math f′(xᵢ) ≈ f′(xᵢ₋₁) + f′′(xᵢ₋₁)(xᵢ₋₁ - xᵢ) + O(Δx²) = f′(xᵢ₋₁) + f′′(xᵢ₋₁)Δx + O(Δx²), - -f′(xᵢ₋₁) ≈ (f(xᵢ) - f(xᵢ₋₂)) / 2Δx - -f′(xᵢ) ≈ (f(xᵢ) - f(xᵢ₋₂)) / 2Δx + O(Δx) = 0 ∴ f(xᵢ) ≈ f(xᵢ₋₂) + O(Δx) - -f′′(xᵢ₋₁) ≈ (f′(xᵢ) - f′(xᵢ₋₂)) / 2Δx = - f′(xᵢ₋₂) / 2Δx ≈ - (f(xᵢ₋₁) - f(xᵢ₋₃)) / (2Δx)² - -∴ f(xᵢ) ≈ f(xᵢ₋₂) - (f(xᵢ₋₁) - f(xᵢ₋₃))/2 + O(Δx²) +``` +where ``Δx=xᵢ₋₁ - xᵢ`` (for simplicity, we will also assume the spacing is constant at +all ``i`` for now). +We can substitute the gradinet at some point ``j`` (``f′(xⱼ)``) with the central +difference approximation: +```math +f′(xⱼ) ≈ (f(xⱼ₊₁) - f(xⱼ₋₁)) / 2Δx, +``` +and the second derivative at some point ``j`` (``f′′(xⱼ)``) can be approximated as: +```math +f′′(xⱼ) ≈ (f′(xⱼ₊₁) - f′(xⱼ₋₁)) / 2Δx = ((f(xⱼ₊₂) - f(xⱼ)) - (f(xⱼ) - f(xⱼ₋₂))) / (2Δx)². +``` +When we then substitute for the boundary adjacent point ``f′′(xᵢ₋₁)`` we know that +``f′(xⱼ₊₁)=f′(xᵢ)=0`` so the Taylor expansion becomes: +```math +f(xᵢ) ≈ f(xᵢ₋₂) - (f(xᵢ₋₁) - f(xᵢ₋₃))/2 + O(Δx²). ``` -When the grid spacing is not constant it works out that the factor of 1/2 changes to -``Δx₋₁/(Δx₋₂ + Δx₋₃)`` instead. +When the grid spacing is not constant the above can be repeated resulting in the factor +of 1/2 changes to ``Δx₋₁/(Δx₋₂ + Δx₋₃)`` instead, i.e.: +```math +f(xᵢ) ≈ f(xᵢ₋₂) - (f(xᵢ₋₁) - f(xᵢ₋₃))Δxᵢ₋₁/(Δxᵢ₋₂ + Δxᵢ₋₃) + O(Δx²) +```. """ struct FlatExtrapolation{FT} relaxation_timescale :: FT @@ -33,10 +45,18 @@ function FlatExtrapolationOpenBoundaryCondition(val = nothing; relaxation_timesc return BoundaryCondition(classification, val; kwargs...) end -@inline relax(l, m, c, bc, grid, clock, model_fields) = - c + ifelse(isnothing(bc.condition)||!isfinite(clock.last_stage_Δt), 0, - (getbc(bc, l, m, grid, clock, model_fields) - c) * min(1, clock.last_stage_Δt / bc.classification.matching_scheme.relaxation_timescale)) +@inline function relax(l, m, c, bc, grid, clock, model_fields) + Δt = clock.last_stage_Δt + τ = bc.classification.matching_scheme.relaxation_timescale + Δt̄ = min(1, Δt / τ) + cₑₓₜ = getbc(bc, l, m, grid, clock, model_fields) + + Δc = ifelse(isnothing(bc.condition)||!isfinite(clock.last_stage_Δt), + 0, (cₑₓₜ - c) * Δt̄) + + return c + Δc +end @inline spacing_factor(args...) = 1/2 @inline spacing_factor(Δ::AbstractArray, ::Val{:right}) = @inbounds Δ[end] / (Δ[end-1] + Δ[end-2]) @inline spacing_factor(Δ::AbstractArray, ::Val{:left}) = @inbounds Δ[1] / (Δ[2] + Δ[3] ) diff --git a/src/Models/LagrangianParticleTracking/lagrangian_particle_advection.jl b/src/Models/LagrangianParticleTracking/lagrangian_particle_advection.jl index 443d144c4e..0d732acb21 100644 --- a/src/Models/LagrangianParticleTracking/lagrangian_particle_advection.jl +++ b/src/Models/LagrangianParticleTracking/lagrangian_particle_advection.jl @@ -78,7 +78,7 @@ Return new position `(x⁺, y⁺, z⁺)` for a particle at current position (x, given `velocities`, time-step `Δt, and coefficient of `restitution`. """ @inline function advect_particle((x, y, z), p, restitution, grid, Δt, velocities) - X = flattened_node((x, y, z), grid) + X = @show flattened_node((x, y, z), grid) # Obtain current particle indices fi, fj, fk = fractional_indices(X, grid, c, c, c) From 762197d7ff49b42e57737889a54d68d5b62202dc Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Wed, 10 Jul 2024 17:13:23 +0100 Subject: [PATCH 085/113] rename \`update_boundary_conditions!\` to \`update_boundary_condition!\` --- .../update_boundary_conditions.jl | 22 +++++++++---------- .../immersed_boundary_condition.jl | 4 ++-- ...te_hydrostatic_free_surface_model_state.jl | 4 ++-- .../update_nonhydrostatic_model_state.jl | 4 ++-- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/BoundaryConditions/update_boundary_conditions.jl b/src/BoundaryConditions/update_boundary_conditions.jl index 53a9497808..a4894b86aa 100644 --- a/src/BoundaryConditions/update_boundary_conditions.jl +++ b/src/BoundaryConditions/update_boundary_conditions.jl @@ -1,16 +1,16 @@ -update_boundary_conditions!(bc::Union{BoundaryCondition, Nothing}, args...) = nothing +update_boundary_condition!(bc::Union{BoundaryCondition, Nothing}, args...) = nothing -function update_boundary_conditions!(bcs::FieldBoundaryConditions, field, model) - update_boundary_conditions!(bcs.west, field, model, Val(:west)) - update_boundary_conditions!(bcs.east, field, model, Val(:east)) - update_boundary_conditions!(bcs.south, field, model, Val(:south)) - update_boundary_conditions!(bcs.north, field, model, Val(:north)) - update_boundary_conditions!(bcs.bottom, field, model, Val(:bottom)) - update_boundary_conditions!(bcs.top, field, model, Val(:top)) - update_boundary_conditions!(bcs.immersed, field, model, Val(:immersed)) +function update_boundary_condition!(bcs::FieldBoundaryConditions, field, model) + update_boundary_condition!(bcs.west, field, model, Val(:west)) + update_boundary_condition!(bcs.east, field, model, Val(:east)) + update_boundary_condition!(bcs.south, field, model, Val(:south)) + update_boundary_condition!(bcs.north, field, model, Val(:north)) + update_boundary_condition!(bcs.bottom, field, model, Val(:bottom)) + update_boundary_condition!(bcs.top, field, model, Val(:top)) + update_boundary_condition!(bcs.immersed, field, model, Val(:immersed)) end -update_boundary_conditions!(fields::Union{NamedTuple, Tuple}, model) = - Tuple(update_boundary_conditions!(get_boundary_conditions(field), field, model) for field in fields) +update_boundary_condition!(fields::Union{NamedTuple, Tuple}, model) = + Tuple(update_boundary_condition!(get_boundary_conditions(field), field, model) for field in fields) get_boundary_conditions(field) = nothing \ No newline at end of file diff --git a/src/ImmersedBoundaries/immersed_boundary_condition.jl b/src/ImmersedBoundaries/immersed_boundary_condition.jl index fb2aa5f22e..423b2cd381 100644 --- a/src/ImmersedBoundaries/immersed_boundary_condition.jl +++ b/src/ImmersedBoundaries/immersed_boundary_condition.jl @@ -4,7 +4,7 @@ using Oceananigans.BoundaryConditions: DefaultBoundaryCondition using Oceananigans.TurbulenceClosures: AbstractScalarDiffusivity, h_diffusivity, z_diffusivity using Oceananigans.Operators: index_left, index_right, Δx, Δy, Δz, div -import Oceananigans.BoundaryConditions: regularize_immersed_boundary_condition, bc_str, update_boundary_conditions! +import Oceananigans.BoundaryConditions: regularize_immersed_boundary_condition, bc_str, update_boundary_condition! import Oceananigans.TurbulenceClosures: immersed_∂ⱼ_τ₁ⱼ, immersed_∂ⱼ_τ₂ⱼ, @@ -273,7 +273,7 @@ Adapt.adapt_structure(to, bc::ImmersedBoundaryCondition) = ImmersedBoundaryCondi Adapt.adapt(to, bc.bottom), Adapt.adapt(to, bc.top)) -update_boundary_conditions!(bc::ImmersedBoundaryCondition, args...) = nothing +update_boundary_condition!(bc::ImmersedBoundaryCondition, args...) = nothing ##### ##### Alternative implementation for immersed flux divergence diff --git a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl index d709e4047f..fe94fbe3fa 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl @@ -3,7 +3,7 @@ using Oceananigans.BoundaryConditions using Oceananigans: UpdateStateCallsite using Oceananigans.Biogeochemistry: update_biogeochemical_state! -using Oceananigans.BoundaryConditions: update_boundary_conditions! +using Oceananigans.BoundaryConditions: update_boundary_condition! using Oceananigans.TurbulenceClosures: compute_diffusivities! using Oceananigans.ImmersedBoundaries: mask_immersed_field!, mask_immersed_field_xy!, inactive_node using Oceananigans.Models: update_model_field_time_series! @@ -35,7 +35,7 @@ function update_state!(model::HydrostaticFreeSurfaceModel, grid, callbacks; comp @apply_regionally update_model_field_time_series!(model, model.clock) # Update the boundary conditions - @apply_regionally update_boundary_conditions!(fields(model), model) + @apply_regionally update_boundary_condition!(fields(model), model) fill_halo_regions!(prognostic_fields(model), model.clock, fields(model); async = true) @apply_regionally replace_horizontal_vector_halos!(model.velocities, model.grid) diff --git a/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl b/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl index 2562f4f9c4..692a9ea618 100644 --- a/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl +++ b/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl @@ -2,7 +2,7 @@ using Oceananigans: UpdateStateCallsite using Oceananigans.Architectures using Oceananigans.BoundaryConditions using Oceananigans.Biogeochemistry: update_biogeochemical_state! -using Oceananigans.BoundaryConditions: update_boundary_conditions! +using Oceananigans.BoundaryConditions: update_boundary_condition! using Oceananigans.TurbulenceClosures: compute_diffusivities! using Oceananigans.Fields: compute! using Oceananigans.ImmersedBoundaries: mask_immersed_field! @@ -28,7 +28,7 @@ function update_state!(model::NonhydrostaticModel, callbacks=[]; compute_tendenc #update_model_field_time_series!(model, model.clock) # Update the boundary conditions - update_boundary_conditions!(fields(model), model) + update_boundary_condition!(fields(model), model) # Fill halos for velocities and tracers fill_halo_regions!(merge(model.velocities, model.tracers), model.clock, fields(model); fill_boundary_normal_velocities = false, async = true) From abb06b75c584afac1299ef12ad787bd5169fbcaa Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Wed, 10 Jul 2024 17:22:14 +0100 Subject: [PATCH 086/113] made `spacing_factor` stuff grid agnostic --- ...apolation_open_boundary_matching_scheme.jl | 55 +++++++++++++------ 1 file changed, 39 insertions(+), 16 deletions(-) diff --git a/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl b/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl index 986bf4cf21..f23ddebb00 100644 --- a/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl +++ b/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl @@ -1,4 +1,4 @@ -using Oceananigans.Grids: xspacings, yspacings, zspacings +using Oceananigans.Grids: xspacing, yspacing, zspacing """ FlatExtrapolation @@ -57,14 +57,17 @@ end return c + Δc end -@inline spacing_factor(args...) = 1/2 -@inline spacing_factor(Δ::AbstractArray, ::Val{:right}) = @inbounds Δ[end] / (Δ[end-1] + Δ[end-2]) -@inline spacing_factor(Δ::AbstractArray, ::Val{:left}) = @inbounds Δ[1] / (Δ[2] + Δ[3] ) + +c = Center() @inline function _fill_west_open_halo!(j, k, grid, c, bc::FEOBC, loc, clock, model_fields) - Δx = xspacings(grid, Center(), Center(), Center()) + Δx₁ = xspacing(1, j, k, grid, c, c, c) + Δx₂ = xspacing(2, j, k, grid, c, c, c) + Δx₃ = xspacing(3, j, k, grid, c, c, c) + + spacing_factor = Δx₁ / (Δx₂ + Δx₃) - unrelaxed = @inbounds c[3, j, k] - (c[2, j, k] - c[4, j, k]) * spacing_factor(Δx, Val(:left)) + unrelaxed = @inbounds c[3, j, k] - (c[2, j, k] - c[4, j, k]) * spacing_factor @inbounds c[1, j, k] = relax(j, k, unrelaxed, bc, grid, clock, model_fields) @@ -74,9 +77,13 @@ end @inline function _fill_east_open_halo!(j, k, grid, c, bc::FEOBC, loc, clock, model_fields) i = grid.Nx + 1 - Δx = xspacings(grid, Center(), Center(), Center()) + Δx₁ = xspacing(i-1, j, k, grid, c, c, c) + Δx₂ = xspacing(i-3, j, k, grid, c, c, c) + Δx₃ = xspacing(i-3, j, k, grid, c, c, c) - unrelaxed = @inbounds c[i - 2, j, k] - (c[i - 1, j, k] - c[i - 3, j, k]) * spacing_factor(Δx, Val(:right)) + spacing_factor = Δx₁ / (Δx₂ + Δx₃) + + unrelaxed = @inbounds c[i - 2, j, k] - (c[i - 1, j, k] - c[i - 3, j, k]) * spacing_factor @inbounds c[i, j, k] = relax(j, k, unrelaxed, bc, grid, clock, model_fields) @@ -84,9 +91,13 @@ end end @inline function _fill_south_open_halo!(i, k, grid, c, bc::FEOBC, loc, clock, model_fields) - Δy = yspacings(grid, Center(), Center(), Center()) + Δy₁ = yspacing(i, 1, k, grid, c, c, c) + Δy₂ = yspacing(i, 2, k, grid, c, c, c) + Δy₃ = yspacing(i, 3, k, grid, c, c, c) + + spacing_factor = Δy₁ / (Δy₂ + Δy₃) - unrelaxed = c[i, 3, k] - (c[i, 2, k] - c[i, 4, k]) * spacing_factor(Δy, Val(:left)) + unrelaxed = c[i, 3, k] - (c[i, 2, k] - c[i, 4, k]) * spacing_factor @inbounds c[i, 1, k] = relax(i, k, unrelaxed, bc, grid, clock, model_fields) @@ -96,9 +107,13 @@ end @inline function _fill_north_open_halo!(i, k, grid, c, bc::FEOBC, loc, clock, model_fields) j = grid.Ny + 1 - Δy = yspacings(grid, Center(), Center(), Center()) + Δy₁ = yspacing(i, j-1, k, grid, c, c, c) + Δy₂ = yspacing(i, j-3, k, grid, c, c, c) + Δy₃ = yspacing(i, j-3, k, grid, c, c, c) - unrelaxed = @inbounds c[i, j - 2, k] - (c[i, j - 1, k] - c[i, j - 3, k]) * spacing_factor(Δy, Val(:right)) + spacing_factor = Δy₁ / (Δy₂ + Δy₃) + + unrelaxed = @inbounds c[i, j - 2, k] - (c[i, j - 1, k] - c[i, j - 3, k]) * spacing_factor @inbounds c[i, j, k] = relax(i, k, unrelaxed, bc, grid, clock, model_fields) @@ -106,9 +121,13 @@ end end @inline function _fill_bottom_open_halo!(i, j, grid, c, bc::FEOBC, loc, clock, model_fields) - Δz = zspacings(grid, Center(), Center(), Center()) + Δz₁ = zspacing(i, j, 1, grid, c, c, c) + Δz₂ = zspacing(i, j, 2, grid, c, c, c) + Δz₃ = zspacing(i, j, 3, grid, c, c, c) + + spacing_factor = Δz₁ / (Δz₂ + Δz₃) - unrelaxed = @inbounds c[i, j, 3] - (c[i, k, 2] - c[i, j, 4]) * spacing_factor(Δz, Val(:left)) + unrelaxed = @inbounds c[i, j, 3] - (c[i, k, 2] - c[i, j, 4]) * spacing_factor @inbounds c[i, j, 1] = relax(i, j, unrelaxed, bc, grid, clock, model_fields) @@ -118,9 +137,13 @@ end @inline function _fill_top_open_halo!(i, j, grid, c, bc::FEOBC, loc, clock, model_fields) k = grid.Nz + 1 - Δz = zspacings(grid, Center(), Center(), Center()) + Δz₁ = zspacing(i, j, k-1, grid, c, c, c) + Δz₂ = zspacing(i, j, k-3, grid, c, c, c) + Δz₃ = zspacing(i, j, k-3, grid, c, c, c) + + spacing_factor = Δz₁ / (Δz₂ + Δz₃) - unrelaxed = @inbounds c[i, j, k - 2] - (c[i, j, k - 1] - c[i, j, k - 3]) * spacing_factor(Δz, Val(:right)) + unrelaxed = @inbounds c[i, j, k - 2] - (c[i, j, k - 1] - c[i, j, k - 3]) * spacing_factor @inbounds c[i, j, k] = relax(i, j, unrelaxed, bc, grid, clock, model_fields) From 589fb605388163cf3e48ff0c47b6e20a8d48e46d Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Wed, 10 Jul 2024 17:25:35 +0100 Subject: [PATCH 087/113] fix --- ...apolation_open_boundary_matching_scheme.jl | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl b/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl index f23ddebb00..21077d5cee 100644 --- a/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl +++ b/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl @@ -58,12 +58,12 @@ end return c + Δc end -c = Center() +const C = Center() @inline function _fill_west_open_halo!(j, k, grid, c, bc::FEOBC, loc, clock, model_fields) - Δx₁ = xspacing(1, j, k, grid, c, c, c) - Δx₂ = xspacing(2, j, k, grid, c, c, c) - Δx₃ = xspacing(3, j, k, grid, c, c, c) + Δx₁ = xspacing(1, j, k, grid, C, C, C) + Δx₂ = xspacing(2, j, k, grid, C, C, C) + Δx₃ = xspacing(3, j, k, grid, C, C, C) spacing_factor = Δx₁ / (Δx₂ + Δx₃) @@ -77,9 +77,9 @@ end @inline function _fill_east_open_halo!(j, k, grid, c, bc::FEOBC, loc, clock, model_fields) i = grid.Nx + 1 - Δx₁ = xspacing(i-1, j, k, grid, c, c, c) - Δx₂ = xspacing(i-3, j, k, grid, c, c, c) - Δx₃ = xspacing(i-3, j, k, grid, c, c, c) + Δx₁ = xspacing(i-1, j, k, grid, C, C, C) + Δx₂ = xspacing(i-3, j, k, grid, C, C, C) + Δx₃ = xspacing(i-3, j, k, grid, C, C, C) spacing_factor = Δx₁ / (Δx₂ + Δx₃) @@ -91,9 +91,9 @@ end end @inline function _fill_south_open_halo!(i, k, grid, c, bc::FEOBC, loc, clock, model_fields) - Δy₁ = yspacing(i, 1, k, grid, c, c, c) - Δy₂ = yspacing(i, 2, k, grid, c, c, c) - Δy₃ = yspacing(i, 3, k, grid, c, c, c) + Δy₁ = yspacing(i, 1, k, grid, C, C, C) + Δy₂ = yspacing(i, 2, k, grid, C, C, C) + Δy₃ = yspacing(i, 3, k, grid, C, C, C) spacing_factor = Δy₁ / (Δy₂ + Δy₃) @@ -107,9 +107,9 @@ end @inline function _fill_north_open_halo!(i, k, grid, c, bc::FEOBC, loc, clock, model_fields) j = grid.Ny + 1 - Δy₁ = yspacing(i, j-1, k, grid, c, c, c) - Δy₂ = yspacing(i, j-3, k, grid, c, c, c) - Δy₃ = yspacing(i, j-3, k, grid, c, c, c) + Δy₁ = yspacing(i, j-1, k, grid, C, C, C) + Δy₂ = yspacing(i, j-3, k, grid, C, C, C) + Δy₃ = yspacing(i, j-3, k, grid, C, C, C) spacing_factor = Δy₁ / (Δy₂ + Δy₃) @@ -121,9 +121,9 @@ end end @inline function _fill_bottom_open_halo!(i, j, grid, c, bc::FEOBC, loc, clock, model_fields) - Δz₁ = zspacing(i, j, 1, grid, c, c, c) - Δz₂ = zspacing(i, j, 2, grid, c, c, c) - Δz₃ = zspacing(i, j, 3, grid, c, c, c) + Δz₁ = zspacing(i, j, 1, grid, C, C, C) + Δz₂ = zspacing(i, j, 2, grid, C, C, C) + Δz₃ = zspacing(i, j, 3, grid, C, C, C) spacing_factor = Δz₁ / (Δz₂ + Δz₃) @@ -137,9 +137,9 @@ end @inline function _fill_top_open_halo!(i, j, grid, c, bc::FEOBC, loc, clock, model_fields) k = grid.Nz + 1 - Δz₁ = zspacing(i, j, k-1, grid, c, c, c) - Δz₂ = zspacing(i, j, k-3, grid, c, c, c) - Δz₃ = zspacing(i, j, k-3, grid, c, c, c) + Δz₁ = zspacing(i, j, k-1, grid, C, C, C) + Δz₂ = zspacing(i, j, k-3, grid, C, C, C) + Δz₃ = zspacing(i, j, k-3, grid, C, C, C) spacing_factor = Δz₁ / (Δz₂ + Δz₃) From 1787e1dd9cacf9db4f8e5d99b949beb7f483c764 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Wed, 10 Jul 2024 17:26:51 +0100 Subject: [PATCH 088/113] renamed \`unrelaxed\` to \`gradient_free_c\` --- ...apolation_open_boundary_matching_scheme.jl | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl b/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl index 21077d5cee..7d617d5bcc 100644 --- a/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl +++ b/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl @@ -67,9 +67,9 @@ const C = Center() spacing_factor = Δx₁ / (Δx₂ + Δx₃) - unrelaxed = @inbounds c[3, j, k] - (c[2, j, k] - c[4, j, k]) * spacing_factor + gradient_free_c = @inbounds c[3, j, k] - (c[2, j, k] - c[4, j, k]) * spacing_factor - @inbounds c[1, j, k] = relax(j, k, unrelaxed, bc, grid, clock, model_fields) + @inbounds c[1, j, k] = relax(j, k, gradient_free_c, bc, grid, clock, model_fields) return nothing end @@ -83,9 +83,9 @@ end spacing_factor = Δx₁ / (Δx₂ + Δx₃) - unrelaxed = @inbounds c[i - 2, j, k] - (c[i - 1, j, k] - c[i - 3, j, k]) * spacing_factor + gradient_free_c = @inbounds c[i - 2, j, k] - (c[i - 1, j, k] - c[i - 3, j, k]) * spacing_factor - @inbounds c[i, j, k] = relax(j, k, unrelaxed, bc, grid, clock, model_fields) + @inbounds c[i, j, k] = relax(j, k, gradient_free_c, bc, grid, clock, model_fields) return nothing end @@ -97,9 +97,9 @@ end spacing_factor = Δy₁ / (Δy₂ + Δy₃) - unrelaxed = c[i, 3, k] - (c[i, 2, k] - c[i, 4, k]) * spacing_factor + gradient_free_c = c[i, 3, k] - (c[i, 2, k] - c[i, 4, k]) * spacing_factor - @inbounds c[i, 1, k] = relax(i, k, unrelaxed, bc, grid, clock, model_fields) + @inbounds c[i, 1, k] = relax(i, k, gradient_free_c, bc, grid, clock, model_fields) return nothing end @@ -113,9 +113,9 @@ end spacing_factor = Δy₁ / (Δy₂ + Δy₃) - unrelaxed = @inbounds c[i, j - 2, k] - (c[i, j - 1, k] - c[i, j - 3, k]) * spacing_factor + gradient_free_c = @inbounds c[i, j - 2, k] - (c[i, j - 1, k] - c[i, j - 3, k]) * spacing_factor - @inbounds c[i, j, k] = relax(i, k, unrelaxed, bc, grid, clock, model_fields) + @inbounds c[i, j, k] = relax(i, k, gradient_free_c, bc, grid, clock, model_fields) return nothing end @@ -127,9 +127,9 @@ end spacing_factor = Δz₁ / (Δz₂ + Δz₃) - unrelaxed = @inbounds c[i, j, 3] - (c[i, k, 2] - c[i, j, 4]) * spacing_factor + gradient_free_c = @inbounds c[i, j, 3] - (c[i, k, 2] - c[i, j, 4]) * spacing_factor - @inbounds c[i, j, 1] = relax(i, j, unrelaxed, bc, grid, clock, model_fields) + @inbounds c[i, j, 1] = relax(i, j, gradient_free_c, bc, grid, clock, model_fields) return nothing end @@ -143,9 +143,9 @@ end spacing_factor = Δz₁ / (Δz₂ + Δz₃) - unrelaxed = @inbounds c[i, j, k - 2] - (c[i, j, k - 1] - c[i, j, k - 3]) * spacing_factor + gradient_free_c = @inbounds c[i, j, k - 2] - (c[i, j, k - 1] - c[i, j, k - 3]) * spacing_factor - @inbounds c[i, j, k] = relax(i, j, unrelaxed, bc, grid, clock, model_fields) + @inbounds c[i, j, k] = relax(i, j, gradient_free_c, bc, grid, clock, model_fields) return nothing end \ No newline at end of file From a8b563a70211cc2a8b1ecfa2a243897bf72ebbcf Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Wed, 10 Jul 2024 17:27:24 +0100 Subject: [PATCH 089/113] fix formatting in validation scripts --- validation/open_boundaries/cylinder.jl | 6 +++--- validation/open_boundaries/oscillating_flow.jl | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/validation/open_boundaries/cylinder.jl b/validation/open_boundaries/cylinder.jl index 193f8af873..83d14d1c98 100644 --- a/validation/open_boundaries/cylinder.jl +++ b/validation/open_boundaries/cylinder.jl @@ -11,9 +11,9 @@ using Oceananigans.BoundaryConditions: FlatExtrapolationOpenBoundaryCondition import Adapt: adapt_structure @kwdef struct Cylinder{FT} - D :: FT = 1. - x₀ :: FT = 0. - y₀ :: FT = 0. + D :: FT = 1.0 + x₀ :: FT = 0.0 + y₀ :: FT = 0.0 end @inline (cylinder::Cylinder)(x, y) = ifelse((x - cylinder.x₀)^2 + (y - cylinder.y₀)^2 < (cylinder.D/2)^2, 1, 0) diff --git a/validation/open_boundaries/oscillating_flow.jl b/validation/open_boundaries/oscillating_flow.jl index a7679ed181..389a935380 100644 --- a/validation/open_boundaries/oscillating_flow.jl +++ b/validation/open_boundaries/oscillating_flow.jl @@ -14,9 +14,9 @@ using Oceananigans.BoundaryConditions: FlatExtrapolationOpenBoundaryCondition import Adapt: adapt_structure @kwdef struct Cylinder{FT} - D :: FT = 1. - x₀ :: FT = 0. - y₀ :: FT = 0. + D :: FT = 1.0 + x₀ :: FT = 0.0 + y₀ :: FT = 0.0 end @inline (cylinder::Cylinder)(x, y) = ifelse((x - cylinder.x₀)^2 + (y - cylinder.y₀)^2 < (cylinder.D/2)^2, 1, 0) From ca7ad37c926a5c29b014ce71690beb803e835d9f Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Wed, 10 Jul 2024 17:28:49 +0100 Subject: [PATCH 090/113] removed redundant adapts --- validation/open_boundaries/cylinder.jl | 5 +---- validation/open_boundaries/oscillating_flow.jl | 6 +----- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/validation/open_boundaries/cylinder.jl b/validation/open_boundaries/cylinder.jl index 83d14d1c98..993a41c4c1 100644 --- a/validation/open_boundaries/cylinder.jl +++ b/validation/open_boundaries/cylinder.jl @@ -5,10 +5,9 @@ # exit the domain. If, for example, the flow at the prescribed boundary was reversed # then the model would likely fail. -using Oceananigans, Adapt, CairoMakie +using Oceananigans, CairoMakie using Oceananigans.BoundaryConditions: FlatExtrapolationOpenBoundaryCondition -import Adapt: adapt_structure @kwdef struct Cylinder{FT} D :: FT = 1.0 @@ -18,8 +17,6 @@ end @inline (cylinder::Cylinder)(x, y) = ifelse((x - cylinder.x₀)^2 + (y - cylinder.y₀)^2 < (cylinder.D/2)^2, 1, 0) -Adapt.adapt_structure(to, cylinder::Cylinder) = cylinder - architecture = GPU() # model parameters diff --git a/validation/open_boundaries/oscillating_flow.jl b/validation/open_boundaries/oscillating_flow.jl index 389a935380..6bf48337ea 100644 --- a/validation/open_boundaries/oscillating_flow.jl +++ b/validation/open_boundaries/oscillating_flow.jl @@ -8,11 +8,9 @@ # This case also has a stretched grid to validate the zero wall normal velocity # gradient matching scheme on a stretched grid. -using Oceananigans, Adapt, CairoMakie +using Oceananigans, CairoMakie using Oceananigans.BoundaryConditions: FlatExtrapolationOpenBoundaryCondition -import Adapt: adapt_structure - @kwdef struct Cylinder{FT} D :: FT = 1.0 x₀ :: FT = 0.0 @@ -21,8 +19,6 @@ end @inline (cylinder::Cylinder)(x, y) = ifelse((x - cylinder.x₀)^2 + (y - cylinder.y₀)^2 < (cylinder.D/2)^2, 1, 0) -Adapt.adapt_structure(to, cylinder::Cylinder) = cylinder - architecture = CPU() # model parameters From ff0a93fa412bc7138c9461ea2312d013ef8f621b Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Wed, 10 Jul 2024 17:29:47 +0100 Subject: [PATCH 091/113] formatting in update model --- .../NonhydrostaticModels/update_nonhydrostatic_model_state.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl b/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl index 692a9ea618..2478333548 100644 --- a/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl +++ b/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl @@ -31,7 +31,8 @@ function update_state!(model::NonhydrostaticModel, callbacks=[]; compute_tendenc update_boundary_condition!(fields(model), model) # Fill halos for velocities and tracers - fill_halo_regions!(merge(model.velocities, model.tracers), model.clock, fields(model); fill_boundary_normal_velocities = false, async = true) + fill_halo_regions!(merge(model.velocities, model.tracers), model.clock, fields(model); + fill_boundary_normal_velocities = false, async = true) # Compute auxiliary fields for aux_field in model.auxiliary_fields From c3c136f09551321ede9a92b50d54c96640820b82 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Wed, 10 Jul 2024 17:30:12 +0100 Subject: [PATCH 092/113] removed commenting out of \`update_model_field_time_series!\` --- .../NonhydrostaticModels/update_nonhydrostatic_model_state.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl b/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl index 2478333548..671253be84 100644 --- a/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl +++ b/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl @@ -25,7 +25,7 @@ function update_state!(model::NonhydrostaticModel, callbacks=[]; compute_tendenc end # Update all FieldTimeSeries used in the model - #update_model_field_time_series!(model, model.clock) + update_model_field_time_series!(model, model.clock) # Update the boundary conditions update_boundary_condition!(fields(model), model) From 7e587a1e46de3b4b9de96ae4a181feccaa98b516 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Wed, 10 Jul 2024 17:35:58 +0100 Subject: [PATCH 093/113] replaced redundant \`get_boundary_conditions\` --- src/BoundaryConditions/update_boundary_conditions.jl | 6 +++--- src/Fields/field.jl | 6 ++---- src/Oceananigans.jl | 1 + 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/BoundaryConditions/update_boundary_conditions.jl b/src/BoundaryConditions/update_boundary_conditions.jl index a4894b86aa..270eade4e6 100644 --- a/src/BoundaryConditions/update_boundary_conditions.jl +++ b/src/BoundaryConditions/update_boundary_conditions.jl @@ -1,3 +1,5 @@ +using Oceananigans: boundary_conditions + update_boundary_condition!(bc::Union{BoundaryCondition, Nothing}, args...) = nothing function update_boundary_condition!(bcs::FieldBoundaryConditions, field, model) @@ -11,6 +13,4 @@ function update_boundary_condition!(bcs::FieldBoundaryConditions, field, model) end update_boundary_condition!(fields::Union{NamedTuple, Tuple}, model) = - Tuple(update_boundary_condition!(get_boundary_conditions(field), field, model) for field in fields) - -get_boundary_conditions(field) = nothing \ No newline at end of file + Tuple(update_boundary_condition!(boundary_conditions(field), field, model) for field in fields) \ No newline at end of file diff --git a/src/Fields/field.jl b/src/Fields/field.jl index f507be709a..484057f8a6 100644 --- a/src/Fields/field.jl +++ b/src/Fields/field.jl @@ -6,8 +6,9 @@ using Adapt using KernelAbstractions: @kernel, @index using Base: @propagate_inbounds +import Oceananigans: boundary_conditions import Oceananigans.Architectures: on_architecture -import Oceananigans.BoundaryConditions: fill_halo_regions!, getbc, get_boundary_conditions +import Oceananigans.BoundaryConditions: fill_halo_regions!, getbc import Statistics: norm, mean, mean! import Base: == @@ -780,6 +781,3 @@ function fill_halo_regions!(field::Field, args...; kwargs...) return nothing end - -# return a fields boundary conditions -get_boundary_conditions(field::Field) = field.boundary_conditions \ No newline at end of file diff --git a/src/Oceananigans.jl b/src/Oceananigans.jl index b61fd69c3b..af9d1c5ed0 100644 --- a/src/Oceananigans.jl +++ b/src/Oceananigans.jl @@ -188,6 +188,7 @@ function tupleit end function fields end function prognostic_fields end function tracer_tendency_kernel_function end +function boundary_conditions end ##### ##### Include all the submodules From 35e31b79e91f0d580f250ed4d00cc3bcc98bbaeb Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Mon, 15 Jul 2024 14:14:03 +0100 Subject: [PATCH 094/113] Apply suggestions from code review Thank you @tomchor Co-authored-by: Tomas Chor --- .../flat_extrapolation_open_boundary_matching_scheme.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl b/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl index 7d617d5bcc..9b9573485f 100644 --- a/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl +++ b/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl @@ -78,7 +78,7 @@ end i = grid.Nx + 1 Δx₁ = xspacing(i-1, j, k, grid, C, C, C) - Δx₂ = xspacing(i-3, j, k, grid, C, C, C) + Δx₂ = xspacing(i-2, j, k, grid, C, C, C) Δx₃ = xspacing(i-3, j, k, grid, C, C, C) spacing_factor = Δx₁ / (Δx₂ + Δx₃) @@ -108,7 +108,7 @@ end j = grid.Ny + 1 Δy₁ = yspacing(i, j-1, k, grid, C, C, C) - Δy₂ = yspacing(i, j-3, k, grid, C, C, C) + Δy₂ = yspacing(i, j-2, k, grid, C, C, C) Δy₃ = yspacing(i, j-3, k, grid, C, C, C) spacing_factor = Δy₁ / (Δy₂ + Δy₃) @@ -138,7 +138,7 @@ end k = grid.Nz + 1 Δz₁ = zspacing(i, j, k-1, grid, C, C, C) - Δz₂ = zspacing(i, j, k-3, grid, C, C, C) + Δz₂ = zspacing(i, j, k-2, grid, C, C, C) Δz₃ = zspacing(i, j, k-3, grid, C, C, C) spacing_factor = Δz₁ / (Δz₂ + Δz₃) From ecdc3724950d53d10d5ab01868e3f9eb9b76f555 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Tue, 23 Jul 2024 17:27:37 +0100 Subject: [PATCH 095/113] fix boundary conditions doctest --- docs/src/model_setup/boundary_conditions.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/model_setup/boundary_conditions.md b/docs/src/model_setup/boundary_conditions.md index 3c5447e3c2..48310aa9be 100644 --- a/docs/src/model_setup/boundary_conditions.md +++ b/docs/src/model_setup/boundary_conditions.md @@ -92,8 +92,8 @@ julia> model.velocities.v.boundary_conditions Oceananigans.FieldBoundaryConditions, with boundary conditions ├── west: PeriodicBoundaryCondition ├── east: PeriodicBoundaryCondition -├── south: OpenBoundaryCondition{Nothing}: 0.0 -├── north: OpenBoundaryCondition{Nothing}: 0.0 +├── south: OpenBoundaryCondition{Nothing}: Nothing +├── north: OpenBoundaryCondition{Nothing}: Nothing ├── bottom: ValueBoundaryCondition: 0.0 ├── top: FluxBoundaryCondition: Nothing └── immersed: FluxBoundaryCondition: Nothing From c3c76bff7c3566d276550f6a3bc7f2ff2beb72ff Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Tue, 23 Jul 2024 17:33:05 +0100 Subject: [PATCH 096/113] clarified fallbacks for open boundary conditions --- .../fill_halo_regions_open.jl | 56 ++++++++----------- 1 file changed, 24 insertions(+), 32 deletions(-) diff --git a/src/BoundaryConditions/fill_halo_regions_open.jl b/src/BoundaryConditions/fill_halo_regions_open.jl index 32c7b24484..6954993a7d 100644 --- a/src/BoundaryConditions/fill_halo_regions_open.jl +++ b/src/BoundaryConditions/fill_halo_regions_open.jl @@ -27,28 +27,28 @@ fill_open_boundary_regions!(fields::NTuple, boundary_conditions, indices, loc, g # for regular halo fills @inline left_velocity_open_boundary_condition(boundary_condition, loc) = nothing -@inline left_velocity_open_boundary_condition(boundary_conditions, loc::Tuple{Face, Center, Center}) = boundary_conditions.west -@inline left_velocity_open_boundary_condition(boundary_conditions, loc::Tuple{Center, Face, Center}) = boundary_conditions.south -@inline left_velocity_open_boundary_condition(boundary_conditions, loc::Tuple{Center, Center, Face}) = boundary_conditions.bottom +@inline left_velocity_open_boundary_condition(boundary_conditions, ::Tuple{Face, Center, Center}) = boundary_conditions.west +@inline left_velocity_open_boundary_condition(boundary_conditions, ::Tuple{Center, Face, Center}) = boundary_conditions.south +@inline left_velocity_open_boundary_condition(boundary_conditions, ::Tuple{Center, Center, Face}) = boundary_conditions.bottom @inline right_velocity_open_boundary_condition(boundary_conditions, loc) = nothing -@inline right_velocity_open_boundary_condition(boundary_conditions, loc::Tuple{Face, Center, Center}) = boundary_conditions.east -@inline right_velocity_open_boundary_condition(boundary_conditions, loc::Tuple{Center, Face, Center}) = boundary_conditions.north -@inline right_velocity_open_boundary_condition(boundary_conditions, loc::Tuple{Center, Center, Face}) = boundary_conditions.top +@inline right_velocity_open_boundary_condition(boundary_conditions, ::Tuple{Face, Center, Center}) = boundary_conditions.east +@inline right_velocity_open_boundary_condition(boundary_conditions, ::Tuple{Center, Face, Center}) = boundary_conditions.north +@inline right_velocity_open_boundary_condition(boundary_conditions, ::Tuple{Center, Center, Face}) = boundary_conditions.top # for multi region halo fills -@inline left_velocity_open_boundary_condition(boundary_conditions::Tuple, loc::Tuple{Face, Center, Center}) = @inbounds boundary_conditions[1] -@inline left_velocity_open_boundary_condition(boundary_conditions::Tuple, loc::Tuple{Center, Face, Center}) = @inbounds boundary_conditions[1] -@inline left_velocity_open_boundary_condition(boundary_conditions::Tuple, loc::Tuple{Center, Center, Face}) = @inbounds boundary_conditions[1] +@inline left_velocity_open_boundary_condition(boundary_conditions::Tuple, ::Tuple{Face, Center, Center}) = @inbounds boundary_conditions[1] +@inline left_velocity_open_boundary_condition(boundary_conditions::Tuple, ::Tuple{Center, Face, Center}) = @inbounds boundary_conditions[1] +@inline left_velocity_open_boundary_condition(boundary_conditions::Tuple, ::Tuple{Center, Center, Face}) = @inbounds boundary_conditions[1] -@inline right_velocity_open_boundary_condition(boundary_conditions::Tuple, loc::Tuple{Face, Center, Center}) = @inbounds boundary_conditions[2] -@inline right_velocity_open_boundary_condition(boundary_conditions::Tuple, loc::Tuple{Center, Face, Center}) = @inbounds boundary_conditions[2] -@inline right_velocity_open_boundary_condition(boundary_conditions::Tuple, loc::Tuple{Center, Center, Face}) = @inbounds boundary_conditions[2] +@inline right_velocity_open_boundary_condition(boundary_conditions::Tuple, ::Tuple{Face, Center, Center}) = @inbounds boundary_conditions[2] +@inline right_velocity_open_boundary_condition(boundary_conditions::Tuple, ::Tuple{Center, Face, Center}) = @inbounds boundary_conditions[2] +@inline right_velocity_open_boundary_condition(boundary_conditions::Tuple, ::Tuple{Center, Center, Face}) = @inbounds boundary_conditions[2] @inline get_open_halo_filling_functions(loc) = _no_fill!, _no_fill! -@inline get_open_halo_filling_functions(loc::Tuple{Face, Center, Center}) = _fill_west_and_east_open_halo!, fill_west_and_east_halo! -@inline get_open_halo_filling_functions(loc::Tuple{Center, Face, Center}) = _fill_south_and_north_open_halo!, fill_south_and_north_halo! -@inline get_open_halo_filling_functions(loc::Tuple{Center, Center, Face}) = _fill_bottom_and_top_open_halo!, fill_bottom_and_top_halo! +@inline get_open_halo_filling_functions(::Tuple{Face, Center, Center}) = _fill_west_and_east_open_halo!, fill_west_and_east_halo! +@inline get_open_halo_filling_functions(::Tuple{Center, Face, Center}) = _fill_south_and_north_open_halo!, fill_south_and_north_halo! +@inline get_open_halo_filling_functions(::Tuple{Center, Center, Face}) = _fill_bottom_and_top_open_halo!, fill_bottom_and_top_halo! @kernel _no_fill!(args...) = nothing @@ -72,7 +72,7 @@ end _fill_top_open_halo!(i, j, grid, c, top_bc, loc, args...) end -# fallback for non-open boundary conditions +# Generic fallback @inline _fill_west_open_halo!(j, k, grid, c, bc, loc, args...) = nothing @inline _fill_east_open_halo!(j, k, grid, c, bc, loc, args...) = nothing @@ -81,16 +81,16 @@ end @inline _fill_bottom_open_halo!(i, j, grid, c, bc, loc, args...) = nothing @inline _fill_top_open_halo!(i, j, grid, c, bc, loc, args...) = nothing -# and don't do anything on the non-open boundary fill call +# Open boundary condition fallback -@inline _fill_west_halo!(j, k, grid, c, bc, loc, args...) = nothing -@inline _fill_east_halo!(j, k, grid, c, bc, loc, args...) = nothing -@inline _fill_south_halo!(i, k, grid, c, bc, loc, args...) = nothing -@inline _fill_north_halo!(i, k, grid, c, bc, loc, args...) = nothing -@inline _fill_bottom_halo!(i, j, grid, c, bc, loc, args...) = nothing -@inline _fill_top_halo!(i, j, grid, c, bc, loc, args...) = nothing +@inline _fill_west_open_halo!(j, k, grid, c, bc::OBC, loc, args...) = @inbounds c[1, j, k] = getbc(bc, j, k, grid, args...) +@inline _fill_east_open_halo!(j, k, grid, c, bc::OBC, loc, args...) = @inbounds c[grid.Nx + 1, j, k] = getbc(bc, j, k, grid, args...) +@inline _fill_south_open_halo!(i, k, grid, c, bc::OBC, loc, args...) = @inbounds c[i, 1, k] = getbc(bc, i, k, grid, args...) +@inline _fill_north_open_halo!(i, k, grid, c, bc::OBC, loc, args...) = @inbounds c[i, grid.Ny + 1, k] = getbc(bc, i, k, grid, args...) +@inline _fill_bottom_open_halo!(i, j, grid, c, bc::OBC, loc, args...) = @inbounds c[i, j, 1] = getbc(bc, i, j, grid, args...) +@inline _fill_top_open_halo!(i, j, grid, c, bc::OBC, loc, args...) = @inbounds c[i, j, grid.Nz + 1] = getbc(bc, i, j, grid, args...) -# except when its a center field (so presumably are tracers) +# Open boundary condition fallback for `CenterField` @inline _fill_west_halo!(j, k, grid, c, bc::OBC, loc::Tuple{Center, Center, Center}, args...) = @inbounds c[1, j, k] = getbc(bc, j, k, grid, args...) @inline _fill_east_halo!(j, k, grid, c, bc::OBC, loc::Tuple{Center, Center, Center}, args...) = @inbounds c[grid.Nx + 1, j, k] = getbc(bc, j, k, grid, args...) @@ -99,11 +99,3 @@ end @inline _fill_bottom_halo!(i, j, grid, c, bc::OBC, loc::Tuple{Center, Center, Center}, args...) = @inbounds c[i, j, 1] = getbc(bc, i, j, grid, args...) @inline _fill_top_halo!(i, j, grid, c, bc::OBC, loc::Tuple{Center, Center, Center}, args...) = @inbounds c[i, j, grid.Nz + 1] = getbc(bc, i, j, grid, args...) -# generic for open boundary conditions - -@inline _fill_west_open_halo!(j, k, grid, c, bc::OBC, loc, args...) = @inbounds c[1, j, k] = getbc(bc, j, k, grid, args...) -@inline _fill_east_open_halo!(j, k, grid, c, bc::OBC, loc, args...) = @inbounds c[grid.Nx + 1, j, k] = getbc(bc, j, k, grid, args...) -@inline _fill_south_open_halo!(i, k, grid, c, bc::OBC, loc, args...) = @inbounds c[i, 1, k] = getbc(bc, i, k, grid, args...) -@inline _fill_north_open_halo!(i, k, grid, c, bc::OBC, loc, args...) = @inbounds c[i, grid.Ny + 1, k] = getbc(bc, i, k, grid, args...) -@inline _fill_bottom_open_halo!(i, j, grid, c, bc::OBC, loc, args...) = @inbounds c[i, j, 1] = getbc(bc, i, j, grid, args...) -@inline _fill_top_open_halo!(i, j, grid, c, bc::OBC, loc, args...) = @inbounds c[i, j, grid.Nz + 1] = getbc(bc, i, j, grid, args...) From db7ab1190ca7df95405d8d16be7bc196bc818304 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Tue, 23 Jul 2024 17:41:10 +0100 Subject: [PATCH 097/113] Apply suggestions from code review Co-authored-by: Gregory L. Wagner --- src/BoundaryConditions/boundary_condition.jl | 2 +- src/BoundaryConditions/boundary_condition_classifications.jl | 2 +- .../flat_extrapolation_open_boundary_matching_scheme.jl | 5 +++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/BoundaryConditions/boundary_condition.jl b/src/BoundaryConditions/boundary_condition.jl index 0547bf768b..2e67f4d24e 100644 --- a/src/BoundaryConditions/boundary_condition.jl +++ b/src/BoundaryConditions/boundary_condition.jl @@ -82,7 +82,7 @@ const DCBC = BoundaryCondition{<:DistributedCommunication} # More readable BC constructors for the public API. PeriodicBoundaryCondition() = BoundaryCondition(Periodic(), nothing) NoFluxBoundaryCondition() = BoundaryCondition(Flux(), nothing) - ImpenetrableBoundaryCondition() = OpenBoundaryCondition(nothing) + ImpenetrableBoundaryCondition() = BoundaryCondition(Open(), nothing) MultiRegionCommunicationBoundaryCondition() = BoundaryCondition(MultiRegionCommunication(), nothing) DistributedCommunicationBoundaryCondition() = BoundaryCondition(DistributedCommunication(), nothing) diff --git a/src/BoundaryConditions/boundary_condition_classifications.jl b/src/BoundaryConditions/boundary_condition_classifications.jl index 2966eff2f3..234e4f25eb 100644 --- a/src/BoundaryConditions/boundary_condition_classifications.jl +++ b/src/BoundaryConditions/boundary_condition_classifications.jl @@ -67,7 +67,7 @@ Open() = Open(nothing) (open::Open)() = open -Adapt.adapt_structure(to, bc::Open) = Open(adapt(to, bc.matching_scheme)) +Adapt.adapt_structure(to, open::Open) = Open(adapt(to, open.matching_scheme)) """ struct MultiRegionCommunication <: AbstractBoundaryConditionClassification diff --git a/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl b/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl index 9b9573485f..a69feb2809 100644 --- a/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl +++ b/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl @@ -52,8 +52,9 @@ end Δt̄ = min(1, Δt / τ) cₑₓₜ = getbc(bc, l, m, grid, clock, model_fields) - Δc = ifelse(isnothing(bc.condition)||!isfinite(clock.last_stage_Δt), - 0, (cₑₓₜ - c) * Δt̄) + Δc = (cₑₓₜ - c) * Δt̄ + not_relaxing = isnothing(bc.condition) | !isfinite(clock.last_stage_Δt) + Δc = ifelse(not_relaxing, zero(c), Δc) return c + Δc end From 6fab8dfa46a6c462e5c0c175fcf9223c34f2a976 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Wed, 24 Jul 2024 10:09:04 +0100 Subject: [PATCH 098/113] turns out we did need this fallback --- .../fill_halo_regions_open.jl | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/BoundaryConditions/fill_halo_regions_open.jl b/src/BoundaryConditions/fill_halo_regions_open.jl index 6954993a7d..432e7a61a0 100644 --- a/src/BoundaryConditions/fill_halo_regions_open.jl +++ b/src/BoundaryConditions/fill_halo_regions_open.jl @@ -92,10 +92,18 @@ end # Open boundary condition fallback for `CenterField` -@inline _fill_west_halo!(j, k, grid, c, bc::OBC, loc::Tuple{Center, Center, Center}, args...) = @inbounds c[1, j, k] = getbc(bc, j, k, grid, args...) -@inline _fill_east_halo!(j, k, grid, c, bc::OBC, loc::Tuple{Center, Center, Center}, args...) = @inbounds c[grid.Nx + 1, j, k] = getbc(bc, j, k, grid, args...) -@inline _fill_south_halo!(i, k, grid, c, bc::OBC, loc::Tuple{Center, Center, Center}, args...) = @inbounds c[i, 1, k] = getbc(bc, i, k, grid, args...) -@inline _fill_north_halo!(i, k, grid, c, bc::OBC, loc::Tuple{Center, Center, Center}, args...) = @inbounds c[i, grid.Ny + 1, k] = getbc(bc, i, k, grid, args...) -@inline _fill_bottom_halo!(i, j, grid, c, bc::OBC, loc::Tuple{Center, Center, Center}, args...) = @inbounds c[i, j, 1] = getbc(bc, i, j, grid, args...) -@inline _fill_top_halo!(i, j, grid, c, bc::OBC, loc::Tuple{Center, Center, Center}, args...) = @inbounds c[i, j, grid.Nz + 1] = getbc(bc, i, j, grid, args...) - +@inline _fill_west_halo!(j, k, grid, c, bc::OBC, ::Tuple{Center, Center, Center}, args...) = @inbounds c[1, j, k] = getbc(bc, j, k, grid, args...) +@inline _fill_east_halo!(j, k, grid, c, bc::OBC, ::Tuple{Center, Center, Center}, args...) = @inbounds c[grid.Nx + 1, j, k] = getbc(bc, j, k, grid, args...) +@inline _fill_south_halo!(i, k, grid, c, bc::OBC, ::Tuple{Center, Center, Center}, args...) = @inbounds c[i, 1, k] = getbc(bc, i, k, grid, args...) +@inline _fill_north_halo!(i, k, grid, c, bc::OBC, ::Tuple{Center, Center, Center}, args...) = @inbounds c[i, grid.Ny + 1, k] = getbc(bc, i, k, grid, args...) +@inline _fill_bottom_halo!(i, j, grid, c, bc::OBC, ::Tuple{Center, Center, Center}, args...) = @inbounds c[i, j, 1] = getbc(bc, i, j, grid, args...) +@inline _fill_top_halo!(i, j, grid, c, bc::OBC, ::Tuple{Center, Center, Center}, args...) = @inbounds c[i, j, grid.Nz + 1] = getbc(bc, i, j, grid, args...) + +# Regular boundary fill for non-`CenterField` open boundaries + +@inline _fill_west_halo!(j, k, grid, c, bc::OBC, loc, args...) = nothing +@inline _fill_east_halo!(j, k, grid, c, bc::OBC, loc, args...) = nothing +@inline _fill_south_halo!(i, k, grid, c, bc::OBC, loc, args...) = nothing +@inline _fill_north_halo!(i, k, grid, c, bc::OBC, loc, args...) = nothing +@inline _fill_bottom_halo!(i, j, grid, c, bc::OBC, loc, args...) = nothing +@inline _fill_top_halo!(i, j, grid, c, bc::OBC, loc, args...) = nothing From fa50db4ba8fc2d5a8a0d9b72f99e04ecefb3b539 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Thu, 25 Jul 2024 17:20:56 +0100 Subject: [PATCH 099/113] changed `C` to `c` and `c` to `a` --- ...apolation_open_boundary_matching_scheme.jl | 84 +++++++++---------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl b/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl index a69feb2809..c35408dfba 100644 --- a/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl +++ b/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl @@ -45,108 +45,108 @@ function FlatExtrapolationOpenBoundaryCondition(val = nothing; relaxation_timesc return BoundaryCondition(classification, val; kwargs...) end -@inline function relax(l, m, c, bc, grid, clock, model_fields) +@inline function relax(l, m, a, bc, grid, clock, model_fields) Δt = clock.last_stage_Δt τ = bc.classification.matching_scheme.relaxation_timescale Δt̄ = min(1, Δt / τ) - cₑₓₜ = getbc(bc, l, m, grid, clock, model_fields) + aₑₓₜ = getbc(bc, l, m, grid, clock, model_fields) - Δc = (cₑₓₜ - c) * Δt̄ + Δa = (aₑₓₜ - a) * Δt̄ not_relaxing = isnothing(bc.condition) | !isfinite(clock.last_stage_Δt) - Δc = ifelse(not_relaxing, zero(c), Δc) + Δa = ifelse(not_relaxing, zero(a), Δc) - return c + Δc + return a + Δa end -const C = Center() +const c = Center() -@inline function _fill_west_open_halo!(j, k, grid, c, bc::FEOBC, loc, clock, model_fields) - Δx₁ = xspacing(1, j, k, grid, C, C, C) - Δx₂ = xspacing(2, j, k, grid, C, C, C) - Δx₃ = xspacing(3, j, k, grid, C, C, C) +@inline function _fill_west_open_halo!(j, k, grid, a, bc::FEOBC, loc, clock, model_fields) + Δx₁ = xspacing(1, j, k, grid, c, c, c) + Δx₂ = xspacing(2, j, k, grid, c, c, c) + Δx₃ = xspacing(3, j, k, grid, c, c, c) spacing_factor = Δx₁ / (Δx₂ + Δx₃) - gradient_free_c = @inbounds c[3, j, k] - (c[2, j, k] - c[4, j, k]) * spacing_factor + gradient_free_c = @inbounds a[3, j, k] - (a[2, j, k] - a[4, j, k]) * spacing_factor - @inbounds c[1, j, k] = relax(j, k, gradient_free_c, bc, grid, clock, model_fields) + @inbounds a[1, j, k] = relax(j, k, gradient_free_c, bc, grid, clock, model_fields) return nothing end -@inline function _fill_east_open_halo!(j, k, grid, c, bc::FEOBC, loc, clock, model_fields) +@inline function _fill_east_open_halo!(j, k, grid, a, bc::FEOBC, loc, clock, model_fields) i = grid.Nx + 1 - Δx₁ = xspacing(i-1, j, k, grid, C, C, C) - Δx₂ = xspacing(i-2, j, k, grid, C, C, C) - Δx₃ = xspacing(i-3, j, k, grid, C, C, C) + Δx₁ = xspacing(i-1, j, k, grid, c, c, c) + Δx₂ = xspacing(i-2, j, k, grid, c, c, c) + Δx₃ = xspacing(i-3, j, k, grid, c, c, c) spacing_factor = Δx₁ / (Δx₂ + Δx₃) - gradient_free_c = @inbounds c[i - 2, j, k] - (c[i - 1, j, k] - c[i - 3, j, k]) * spacing_factor + gradient_free_c = @inbounds a[i - 2, j, k] - (a[i - 1, j, k] - a[i - 3, j, k]) * spacing_factor - @inbounds c[i, j, k] = relax(j, k, gradient_free_c, bc, grid, clock, model_fields) + @inbounds a[i, j, k] = relax(j, k, gradient_free_c, bc, grid, clock, model_fields) return nothing end -@inline function _fill_south_open_halo!(i, k, grid, c, bc::FEOBC, loc, clock, model_fields) - Δy₁ = yspacing(i, 1, k, grid, C, C, C) - Δy₂ = yspacing(i, 2, k, grid, C, C, C) - Δy₃ = yspacing(i, 3, k, grid, C, C, C) +@inline function _fill_south_open_halo!(i, k, grid, a, bc::FEOBC, loc, clock, model_fields) + Δy₁ = yspacing(i, 1, k, grid, c, c, c) + Δy₂ = yspacing(i, 2, k, grid, c, c, c) + Δy₃ = yspacing(i, 3, k, grid, c, c, c) spacing_factor = Δy₁ / (Δy₂ + Δy₃) - gradient_free_c = c[i, 3, k] - (c[i, 2, k] - c[i, 4, k]) * spacing_factor + gradient_free_c = a[i, 3, k] - (a[i, 2, k] - a[i, 4, k]) * spacing_factor - @inbounds c[i, 1, k] = relax(i, k, gradient_free_c, bc, grid, clock, model_fields) + @inbounds a[i, 1, k] = relax(i, k, gradient_free_c, bc, grid, clock, model_fields) return nothing end -@inline function _fill_north_open_halo!(i, k, grid, c, bc::FEOBC, loc, clock, model_fields) +@inline function _fill_north_open_halo!(i, k, grid, a, bc::FEOBC, loc, clock, model_fields) j = grid.Ny + 1 - Δy₁ = yspacing(i, j-1, k, grid, C, C, C) - Δy₂ = yspacing(i, j-2, k, grid, C, C, C) - Δy₃ = yspacing(i, j-3, k, grid, C, C, C) + Δy₁ = yspacing(i, j-1, k, grid, c, c, c) + Δy₂ = yspacing(i, j-2, k, grid, c, c, c) + Δy₃ = yspacing(i, j-3, k, grid, c, c, c) spacing_factor = Δy₁ / (Δy₂ + Δy₃) - gradient_free_c = @inbounds c[i, j - 2, k] - (c[i, j - 1, k] - c[i, j - 3, k]) * spacing_factor + gradient_free_c = @inbounds a[i, j - 2, k] - (a[i, j - 1, k] - a[i, j - 3, k]) * spacing_factor - @inbounds c[i, j, k] = relax(i, k, gradient_free_c, bc, grid, clock, model_fields) + @inbounds a[i, j, k] = relax(i, k, gradient_free_c, bc, grid, clock, model_fields) return nothing end -@inline function _fill_bottom_open_halo!(i, j, grid, c, bc::FEOBC, loc, clock, model_fields) - Δz₁ = zspacing(i, j, 1, grid, C, C, C) - Δz₂ = zspacing(i, j, 2, grid, C, C, C) - Δz₃ = zspacing(i, j, 3, grid, C, C, C) +@inline function _fill_bottom_open_halo!(i, j, grid, a, bc::FEOBC, loc, clock, model_fields) + Δz₁ = zspacing(i, j, 1, grid, c, c, c) + Δz₂ = zspacing(i, j, 2, grid, c, c, c) + Δz₃ = zspacing(i, j, 3, grid, c, c, c) spacing_factor = Δz₁ / (Δz₂ + Δz₃) - gradient_free_c = @inbounds c[i, j, 3] - (c[i, k, 2] - c[i, j, 4]) * spacing_factor + gradient_free_c = @inbounds a[i, j, 3] - (a[i, k, 2] - a[i, j, 4]) * spacing_factor - @inbounds c[i, j, 1] = relax(i, j, gradient_free_c, bc, grid, clock, model_fields) + @inbounds a[i, j, 1] = relax(i, j, gradient_free_c, bc, grid, clock, model_fields) return nothing end -@inline function _fill_top_open_halo!(i, j, grid, c, bc::FEOBC, loc, clock, model_fields) +@inline function _fill_top_open_halo!(i, j, grid, a, bc::FEOBC, loc, clock, model_fields) k = grid.Nz + 1 - Δz₁ = zspacing(i, j, k-1, grid, C, C, C) - Δz₂ = zspacing(i, j, k-2, grid, C, C, C) - Δz₃ = zspacing(i, j, k-3, grid, C, C, C) + Δz₁ = zspacing(i, j, k-1, grid, c, c, c) + Δz₂ = zspacing(i, j, k-2, grid, c, c, c) + Δz₃ = zspacing(i, j, k-3, grid, c, c, c) spacing_factor = Δz₁ / (Δz₂ + Δz₃) - gradient_free_c = @inbounds c[i, j, k - 2] - (c[i, j, k - 1] - c[i, j, k - 3]) * spacing_factor + gradient_free_c = @inbounds a[i, j, k - 2] - (a[i, j, k - 1] - a[i, j, k - 3]) * spacing_factor - @inbounds c[i, j, k] = relax(i, j, gradient_free_c, bc, grid, clock, model_fields) + @inbounds a[i, j, k] = relax(i, j, gradient_free_c, bc, grid, clock, model_fields) return nothing end \ No newline at end of file From 137b2decf6238ea7914116e9e690dcf2383eb6d1 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Thu, 25 Jul 2024 17:51:07 +0100 Subject: [PATCH 100/113] =?UTF-8?q?changed=20`a`=20to=20`=CF=95`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...apolation_open_boundary_matching_scheme.jl | 44 +++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl b/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl index c35408dfba..07057d88b6 100644 --- a/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl +++ b/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl @@ -45,37 +45,37 @@ function FlatExtrapolationOpenBoundaryCondition(val = nothing; relaxation_timesc return BoundaryCondition(classification, val; kwargs...) end -@inline function relax(l, m, a, bc, grid, clock, model_fields) +@inline function relax(l, m, ϕ, bc, grid, clock, model_fields) Δt = clock.last_stage_Δt τ = bc.classification.matching_scheme.relaxation_timescale Δt̄ = min(1, Δt / τ) aₑₓₜ = getbc(bc, l, m, grid, clock, model_fields) - Δa = (aₑₓₜ - a) * Δt̄ + Δa = (aₑₓₜ - ϕ) * Δt̄ not_relaxing = isnothing(bc.condition) | !isfinite(clock.last_stage_Δt) - Δa = ifelse(not_relaxing, zero(a), Δc) + Δa = ifelse(not_relaxing, zero(ϕ), Δc) - return a + Δa + return ϕ + Δa end const c = Center() -@inline function _fill_west_open_halo!(j, k, grid, a, bc::FEOBC, loc, clock, model_fields) +@inline function _fill_west_open_halo!(j, k, grid, ϕ, bc::FEOBC, loc, clock, model_fields) Δx₁ = xspacing(1, j, k, grid, c, c, c) Δx₂ = xspacing(2, j, k, grid, c, c, c) Δx₃ = xspacing(3, j, k, grid, c, c, c) spacing_factor = Δx₁ / (Δx₂ + Δx₃) - gradient_free_c = @inbounds a[3, j, k] - (a[2, j, k] - a[4, j, k]) * spacing_factor + gradient_free_c = @inbounds ϕ[3, j, k] - (ϕ[2, j, k] - ϕ[4, j, k]) * spacing_factor - @inbounds a[1, j, k] = relax(j, k, gradient_free_c, bc, grid, clock, model_fields) + @inbounds ϕ[1, j, k] = relax(j, k, gradient_free_c, bc, grid, clock, model_fields) return nothing end -@inline function _fill_east_open_halo!(j, k, grid, a, bc::FEOBC, loc, clock, model_fields) +@inline function _fill_east_open_halo!(j, k, grid, ϕ, bc::FEOBC, loc, clock, model_fields) i = grid.Nx + 1 Δx₁ = xspacing(i-1, j, k, grid, c, c, c) @@ -84,28 +84,28 @@ end spacing_factor = Δx₁ / (Δx₂ + Δx₃) - gradient_free_c = @inbounds a[i - 2, j, k] - (a[i - 1, j, k] - a[i - 3, j, k]) * spacing_factor + gradient_free_c = @inbounds ϕ[i - 2, j, k] - (ϕ[i - 1, j, k] - ϕ[i - 3, j, k]) * spacing_factor - @inbounds a[i, j, k] = relax(j, k, gradient_free_c, bc, grid, clock, model_fields) + @inbounds ϕ[i, j, k] = relax(j, k, gradient_free_c, bc, grid, clock, model_fields) return nothing end -@inline function _fill_south_open_halo!(i, k, grid, a, bc::FEOBC, loc, clock, model_fields) +@inline function _fill_south_open_halo!(i, k, grid, ϕ, bc::FEOBC, loc, clock, model_fields) Δy₁ = yspacing(i, 1, k, grid, c, c, c) Δy₂ = yspacing(i, 2, k, grid, c, c, c) Δy₃ = yspacing(i, 3, k, grid, c, c, c) spacing_factor = Δy₁ / (Δy₂ + Δy₃) - gradient_free_c = a[i, 3, k] - (a[i, 2, k] - a[i, 4, k]) * spacing_factor + gradient_free_c = ϕ[i, 3, k] - (ϕ[i, 2, k] - ϕ[i, 4, k]) * spacing_factor - @inbounds a[i, 1, k] = relax(i, k, gradient_free_c, bc, grid, clock, model_fields) + @inbounds ϕ[i, 1, k] = relax(i, k, gradient_free_c, bc, grid, clock, model_fields) return nothing end -@inline function _fill_north_open_halo!(i, k, grid, a, bc::FEOBC, loc, clock, model_fields) +@inline function _fill_north_open_halo!(i, k, grid, ϕ, bc::FEOBC, loc, clock, model_fields) j = grid.Ny + 1 Δy₁ = yspacing(i, j-1, k, grid, c, c, c) @@ -114,28 +114,28 @@ end spacing_factor = Δy₁ / (Δy₂ + Δy₃) - gradient_free_c = @inbounds a[i, j - 2, k] - (a[i, j - 1, k] - a[i, j - 3, k]) * spacing_factor + gradient_free_c = @inbounds ϕ[i, j - 2, k] - (ϕ[i, j - 1, k] - ϕ[i, j - 3, k]) * spacing_factor - @inbounds a[i, j, k] = relax(i, k, gradient_free_c, bc, grid, clock, model_fields) + @inbounds ϕ[i, j, k] = relax(i, k, gradient_free_c, bc, grid, clock, model_fields) return nothing end -@inline function _fill_bottom_open_halo!(i, j, grid, a, bc::FEOBC, loc, clock, model_fields) +@inline function _fill_bottom_open_halo!(i, j, grid, ϕ, bc::FEOBC, loc, clock, model_fields) Δz₁ = zspacing(i, j, 1, grid, c, c, c) Δz₂ = zspacing(i, j, 2, grid, c, c, c) Δz₃ = zspacing(i, j, 3, grid, c, c, c) spacing_factor = Δz₁ / (Δz₂ + Δz₃) - gradient_free_c = @inbounds a[i, j, 3] - (a[i, k, 2] - a[i, j, 4]) * spacing_factor + gradient_free_c = @inbounds ϕ[i, j, 3] - (ϕ[i, k, 2] - ϕ[i, j, 4]) * spacing_factor - @inbounds a[i, j, 1] = relax(i, j, gradient_free_c, bc, grid, clock, model_fields) + @inbounds ϕ[i, j, 1] = relax(i, j, gradient_free_c, bc, grid, clock, model_fields) return nothing end -@inline function _fill_top_open_halo!(i, j, grid, a, bc::FEOBC, loc, clock, model_fields) +@inline function _fill_top_open_halo!(i, j, grid, ϕ, bc::FEOBC, loc, clock, model_fields) k = grid.Nz + 1 Δz₁ = zspacing(i, j, k-1, grid, c, c, c) @@ -144,9 +144,9 @@ end spacing_factor = Δz₁ / (Δz₂ + Δz₃) - gradient_free_c = @inbounds a[i, j, k - 2] - (a[i, j, k - 1] - a[i, j, k - 3]) * spacing_factor + gradient_free_c = @inbounds ϕ[i, j, k - 2] - (ϕ[i, j, k - 1] - ϕ[i, j, k - 3]) * spacing_factor - @inbounds a[i, j, k] = relax(i, j, gradient_free_c, bc, grid, clock, model_fields) + @inbounds ϕ[i, j, k] = relax(i, j, gradient_free_c, bc, grid, clock, model_fields) return nothing end \ No newline at end of file From d9bb081bf525d2e6b611e3f1a7e481fd3f998ea6 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Thu, 25 Jul 2024 17:52:24 +0100 Subject: [PATCH 101/113] changed argument order for `update_boundary_condition!` --- .../update_boundary_conditions.jl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/BoundaryConditions/update_boundary_conditions.jl b/src/BoundaryConditions/update_boundary_conditions.jl index 270eade4e6..7023145f10 100644 --- a/src/BoundaryConditions/update_boundary_conditions.jl +++ b/src/BoundaryConditions/update_boundary_conditions.jl @@ -3,13 +3,13 @@ using Oceananigans: boundary_conditions update_boundary_condition!(bc::Union{BoundaryCondition, Nothing}, args...) = nothing function update_boundary_condition!(bcs::FieldBoundaryConditions, field, model) - update_boundary_condition!(bcs.west, field, model, Val(:west)) - update_boundary_condition!(bcs.east, field, model, Val(:east)) - update_boundary_condition!(bcs.south, field, model, Val(:south)) - update_boundary_condition!(bcs.north, field, model, Val(:north)) - update_boundary_condition!(bcs.bottom, field, model, Val(:bottom)) - update_boundary_condition!(bcs.top, field, model, Val(:top)) - update_boundary_condition!(bcs.immersed, field, model, Val(:immersed)) + update_boundary_condition!(bcs.west, Val(:west), field, model) + update_boundary_condition!(bcs.east, Val(:east), field, model) + update_boundary_condition!(bcs.south, Val(:south), field, model) + update_boundary_condition!(bcs.north, Val(:north), field, model) + update_boundary_condition!(bcs.bottom, Val(:bottom), field, model) + update_boundary_condition!(bcs.top, Val(:top), field, model) + update_boundary_condition!(bcs.immersed, Val(:immersed), field, model) end update_boundary_condition!(fields::Union{NamedTuple, Tuple}, model) = From 0720bd8835cda2b64b3fe024b71bf35d46a376a5 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Thu, 25 Jul 2024 18:12:55 +0100 Subject: [PATCH 102/113] failure to find and replace --- ...apolation_open_boundary_matching_scheme.jl | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl b/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl index 07057d88b6..d4fc8ae5fd 100644 --- a/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl +++ b/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl @@ -68,9 +68,9 @@ const c = Center() spacing_factor = Δx₁ / (Δx₂ + Δx₃) - gradient_free_c = @inbounds ϕ[3, j, k] - (ϕ[2, j, k] - ϕ[4, j, k]) * spacing_factor + gradient_free_ϕ = @inbounds ϕ[3, j, k] - (ϕ[2, j, k] - ϕ[4, j, k]) * spacing_factor - @inbounds ϕ[1, j, k] = relax(j, k, gradient_free_c, bc, grid, clock, model_fields) + @inbounds ϕ[1, j, k] = relax(j, k, gradient_free_ϕ, bc, grid, clock, model_fields) return nothing end @@ -84,9 +84,9 @@ end spacing_factor = Δx₁ / (Δx₂ + Δx₃) - gradient_free_c = @inbounds ϕ[i - 2, j, k] - (ϕ[i - 1, j, k] - ϕ[i - 3, j, k]) * spacing_factor + gradient_free_ϕ = @inbounds ϕ[i - 2, j, k] - (ϕ[i - 1, j, k] - ϕ[i - 3, j, k]) * spacing_factor - @inbounds ϕ[i, j, k] = relax(j, k, gradient_free_c, bc, grid, clock, model_fields) + @inbounds ϕ[i, j, k] = relax(j, k, gradient_free_ϕ, bc, grid, clock, model_fields) return nothing end @@ -98,9 +98,9 @@ end spacing_factor = Δy₁ / (Δy₂ + Δy₃) - gradient_free_c = ϕ[i, 3, k] - (ϕ[i, 2, k] - ϕ[i, 4, k]) * spacing_factor + gradient_free_ϕ = ϕ[i, 3, k] - (ϕ[i, 2, k] - ϕ[i, 4, k]) * spacing_factor - @inbounds ϕ[i, 1, k] = relax(i, k, gradient_free_c, bc, grid, clock, model_fields) + @inbounds ϕ[i, 1, k] = relax(i, k, gradient_free_ϕ, bc, grid, clock, model_fields) return nothing end @@ -114,9 +114,9 @@ end spacing_factor = Δy₁ / (Δy₂ + Δy₃) - gradient_free_c = @inbounds ϕ[i, j - 2, k] - (ϕ[i, j - 1, k] - ϕ[i, j - 3, k]) * spacing_factor + gradient_free_ϕ = @inbounds ϕ[i, j - 2, k] - (ϕ[i, j - 1, k] - ϕ[i, j - 3, k]) * spacing_factor - @inbounds ϕ[i, j, k] = relax(i, k, gradient_free_c, bc, grid, clock, model_fields) + @inbounds ϕ[i, j, k] = relax(i, k, gradient_free_ϕ, bc, grid, clock, model_fields) return nothing end @@ -128,9 +128,9 @@ end spacing_factor = Δz₁ / (Δz₂ + Δz₃) - gradient_free_c = @inbounds ϕ[i, j, 3] - (ϕ[i, k, 2] - ϕ[i, j, 4]) * spacing_factor + gradient_free_ϕ = @inbounds ϕ[i, j, 3] - (ϕ[i, k, 2] - ϕ[i, j, 4]) * spacing_factor - @inbounds ϕ[i, j, 1] = relax(i, j, gradient_free_c, bc, grid, clock, model_fields) + @inbounds ϕ[i, j, 1] = relax(i, j, gradient_free_ϕ, bc, grid, clock, model_fields) return nothing end @@ -144,9 +144,9 @@ end spacing_factor = Δz₁ / (Δz₂ + Δz₃) - gradient_free_c = @inbounds ϕ[i, j, k - 2] - (ϕ[i, j, k - 1] - ϕ[i, j, k - 3]) * spacing_factor + gradient_free_ϕ = @inbounds ϕ[i, j, k - 2] - (ϕ[i, j, k - 1] - ϕ[i, j, k - 3]) * spacing_factor - @inbounds ϕ[i, j, k] = relax(i, j, gradient_free_c, bc, grid, clock, model_fields) + @inbounds ϕ[i, j, k] = relax(i, j, gradient_free_ϕ, bc, grid, clock, model_fields) return nothing end \ No newline at end of file From cd218e1e3e1d3cb9af5bbfa44a4445e9dd8f58c1 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Thu, 25 Jul 2024 18:13:44 +0100 Subject: [PATCH 103/113] failure to find and replace more --- .../flat_extrapolation_open_boundary_matching_scheme.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl b/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl index d4fc8ae5fd..4ad2bd87fb 100644 --- a/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl +++ b/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl @@ -50,13 +50,13 @@ end τ = bc.classification.matching_scheme.relaxation_timescale Δt̄ = min(1, Δt / τ) - aₑₓₜ = getbc(bc, l, m, grid, clock, model_fields) + ϕₑₓₜ = getbc(bc, l, m, grid, clock, model_fields) - Δa = (aₑₓₜ - ϕ) * Δt̄ + Δϕ = (ϕₑₓₜ - ϕ) * Δt̄ not_relaxing = isnothing(bc.condition) | !isfinite(clock.last_stage_Δt) - Δa = ifelse(not_relaxing, zero(ϕ), Δc) + Δϕ = ifelse(not_relaxing, zero(ϕ), Δc) - return ϕ + Δa + return ϕ + Δϕ end const c = Center() From 310440dbfc8e5283398217091dc08e1ebde17efc Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Fri, 26 Jul 2024 12:20:30 +0100 Subject: [PATCH 104/113] Update lagrangian_particle_advection.jl --- .../LagrangianParticleTracking/lagrangian_particle_advection.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Models/LagrangianParticleTracking/lagrangian_particle_advection.jl b/src/Models/LagrangianParticleTracking/lagrangian_particle_advection.jl index 0d732acb21..443d144c4e 100644 --- a/src/Models/LagrangianParticleTracking/lagrangian_particle_advection.jl +++ b/src/Models/LagrangianParticleTracking/lagrangian_particle_advection.jl @@ -78,7 +78,7 @@ Return new position `(x⁺, y⁺, z⁺)` for a particle at current position (x, given `velocities`, time-step `Δt, and coefficient of `restitution`. """ @inline function advect_particle((x, y, z), p, restitution, grid, Δt, velocities) - X = @show flattened_node((x, y, z), grid) + X = flattened_node((x, y, z), grid) # Obtain current particle indices fi, fj, fk = fractional_indices(X, grid, c, c, c) From 775d1544610d5af6f602254ec5dff3dec7f18480 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Fri, 26 Jul 2024 20:49:34 +0100 Subject: [PATCH 105/113] changed argument order in `relax` --- ..._extrapolation_open_boundary_matching_scheme.jl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl b/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl index 4ad2bd87fb..2b2c9005c5 100644 --- a/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl +++ b/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl @@ -45,7 +45,7 @@ function FlatExtrapolationOpenBoundaryCondition(val = nothing; relaxation_timesc return BoundaryCondition(classification, val; kwargs...) end -@inline function relax(l, m, ϕ, bc, grid, clock, model_fields) +@inline function relax(l, m, grid, ϕ, bc, clock, model_fields) Δt = clock.last_stage_Δt τ = bc.classification.matching_scheme.relaxation_timescale @@ -70,7 +70,7 @@ const c = Center() gradient_free_ϕ = @inbounds ϕ[3, j, k] - (ϕ[2, j, k] - ϕ[4, j, k]) * spacing_factor - @inbounds ϕ[1, j, k] = relax(j, k, gradient_free_ϕ, bc, grid, clock, model_fields) + @inbounds ϕ[1, j, k] = relax(j, k, grid, gradient_free_ϕ, bc, clock, model_fields) return nothing end @@ -86,7 +86,7 @@ end gradient_free_ϕ = @inbounds ϕ[i - 2, j, k] - (ϕ[i - 1, j, k] - ϕ[i - 3, j, k]) * spacing_factor - @inbounds ϕ[i, j, k] = relax(j, k, gradient_free_ϕ, bc, grid, clock, model_fields) + @inbounds ϕ[i, j, k] = relax(j, k, grid, gradient_free_ϕ, bc, clock, model_fields) return nothing end @@ -100,7 +100,7 @@ end gradient_free_ϕ = ϕ[i, 3, k] - (ϕ[i, 2, k] - ϕ[i, 4, k]) * spacing_factor - @inbounds ϕ[i, 1, k] = relax(i, k, gradient_free_ϕ, bc, grid, clock, model_fields) + @inbounds ϕ[i, 1, k] = relax(i, k, grid, gradient_free_ϕ, bc, clock, model_fields) return nothing end @@ -116,7 +116,7 @@ end gradient_free_ϕ = @inbounds ϕ[i, j - 2, k] - (ϕ[i, j - 1, k] - ϕ[i, j - 3, k]) * spacing_factor - @inbounds ϕ[i, j, k] = relax(i, k, gradient_free_ϕ, bc, grid, clock, model_fields) + @inbounds ϕ[i, j, k] = relax(i, k, grid, gradient_free_ϕ, bc, clock, model_fields) return nothing end @@ -130,7 +130,7 @@ end gradient_free_ϕ = @inbounds ϕ[i, j, 3] - (ϕ[i, k, 2] - ϕ[i, j, 4]) * spacing_factor - @inbounds ϕ[i, j, 1] = relax(i, j, gradient_free_ϕ, bc, grid, clock, model_fields) + @inbounds ϕ[i, j, 1] = relax(i, j, grid, gradient_free_ϕ, bc, clock, model_fields) return nothing end @@ -146,7 +146,7 @@ end gradient_free_ϕ = @inbounds ϕ[i, j, k - 2] - (ϕ[i, j, k - 1] - ϕ[i, j, k - 3]) * spacing_factor - @inbounds ϕ[i, j, k] = relax(i, j, gradient_free_ϕ, bc, grid, clock, model_fields) + @inbounds ϕ[i, j, k] = relax(i, j, grid, gradient_free_ϕ, bc, clock, model_fields) return nothing end \ No newline at end of file From cc3667b1d9857ea13e06ea402c3af14aa4dac2c9 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Tue, 30 Jul 2024 21:11:23 +0100 Subject: [PATCH 106/113] Attempts to fix enzyme tests --- src/BoundaryConditions/field_boundary_conditions.jl | 3 ++- src/BoundaryConditions/update_boundary_conditions.jl | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/BoundaryConditions/field_boundary_conditions.jl b/src/BoundaryConditions/field_boundary_conditions.jl index 3651cc6835..0c2fa3e122 100644 --- a/src/BoundaryConditions/field_boundary_conditions.jl +++ b/src/BoundaryConditions/field_boundary_conditions.jl @@ -57,7 +57,8 @@ function FieldBoundaryConditions(indices::Tuple, west, east, south, north, botto end FieldBoundaryConditions(indices::Tuple, bcs::FieldBoundaryConditions) = - FieldBoundaryConditions(indices, (getproperty(bcs, side) for side in fieldnames(FieldBoundaryConditions))...) + FieldBoundaryConditions(indices, (getproperty(bcs, side) for side in propertynames(bcs))...) + window_boundary_conditions(::Colon, left, right) = left, right window_boundary_conditions(::UnitRange, left, right) = nothing, nothing diff --git a/src/BoundaryConditions/update_boundary_conditions.jl b/src/BoundaryConditions/update_boundary_conditions.jl index 7023145f10..8f0426c12f 100644 --- a/src/BoundaryConditions/update_boundary_conditions.jl +++ b/src/BoundaryConditions/update_boundary_conditions.jl @@ -1,6 +1,6 @@ using Oceananigans: boundary_conditions -update_boundary_condition!(bc::Union{BoundaryCondition, Nothing}, args...) = nothing +update_boundary_condition!(bcs, args...) = nothing function update_boundary_condition!(bcs::FieldBoundaryConditions, field, model) update_boundary_condition!(bcs.west, Val(:west), field, model) From 99e09f0a8e777bbe7fc7cf68482bce62d015e166 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Thu, 1 Aug 2024 15:15:25 -0400 Subject: [PATCH 107/113] Apply suggestions from code review Co-authored-by: Gregory L. Wagner --- .../update_boundary_conditions.jl | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/BoundaryConditions/update_boundary_conditions.jl b/src/BoundaryConditions/update_boundary_conditions.jl index 7023145f10..ff1a2e7fbf 100644 --- a/src/BoundaryConditions/update_boundary_conditions.jl +++ b/src/BoundaryConditions/update_boundary_conditions.jl @@ -1,6 +1,6 @@ using Oceananigans: boundary_conditions -update_boundary_condition!(bc::Union{BoundaryCondition, Nothing}, args...) = nothing +@inline update_boundary_condition!(bc::Union{BoundaryCondition, Nothing}, args...) = nothing function update_boundary_condition!(bcs::FieldBoundaryConditions, field, model) update_boundary_condition!(bcs.west, Val(:west), field, model) @@ -10,7 +10,16 @@ function update_boundary_condition!(bcs::FieldBoundaryConditions, field, model) update_boundary_condition!(bcs.bottom, Val(:bottom), field, model) update_boundary_condition!(bcs.top, Val(:top), field, model) update_boundary_condition!(bcs.immersed, Val(:immersed), field, model) + return nothing end -update_boundary_condition!(fields::Union{NamedTuple, Tuple}, model) = - Tuple(update_boundary_condition!(boundary_conditions(field), field, model) for field in fields) \ No newline at end of file +update_boundary_condition!(fields::NamedTuple, model) = update_boundary_condition!(values(fields), model) + +function update_boundary_condition!(fields::Tuple, model) + N = length(fields) + ntuple(Val(N)) do n + field = fields[n] + bcs = boundary_conditions(field) + update_boundary_condition!(bcs, field, model) + end +end \ No newline at end of file From c1d051cad507b16bbea07117f270e5904463821b Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Thu, 1 Aug 2024 15:15:45 -0400 Subject: [PATCH 108/113] Update update_boundary_conditions.jl --- src/BoundaryConditions/update_boundary_conditions.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/BoundaryConditions/update_boundary_conditions.jl b/src/BoundaryConditions/update_boundary_conditions.jl index ff1a2e7fbf..d9cfe74838 100644 --- a/src/BoundaryConditions/update_boundary_conditions.jl +++ b/src/BoundaryConditions/update_boundary_conditions.jl @@ -22,4 +22,6 @@ function update_boundary_condition!(fields::Tuple, model) bcs = boundary_conditions(field) update_boundary_condition!(bcs, field, model) end -end \ No newline at end of file + + return nothing +end From cdd0a221751dcacf761da667e43f4bb747ce590a Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Thu, 1 Aug 2024 15:20:38 -0400 Subject: [PATCH 109/113] modify `boundary_conditions` --- src/Fields/field.jl | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/Fields/field.jl b/src/Fields/field.jl index 484057f8a6..7df9b83296 100644 --- a/src/Fields/field.jl +++ b/src/Fields/field.jl @@ -350,13 +350,8 @@ Base.view(f::Field, i, j) = view(f, i, j, :) boundary_conditions(not_field) = nothing -function boundary_conditions(f::Field) - if f.indices === default_indices(3) # default boundary conditions - return f.boundary_conditions - else # filter boundary conditions in windowed directions - return FieldBoundaryConditions(f.indices, f.boundary_conditions) - end -end +@inline boundary_conditions(f::Field) = f.boundary_conditions +@inline boundary_conditions(w::WindowedField) = FieldBoundaryConditions(w.indices, w.boundary_conditions) immersed_boundary_condition(f::Field) = f.boundary_conditions.immersed data(field::Field) = field.data From 91bdd79dbc0e73a52a6c192c3e7d06b04a22de33 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Sat, 3 Aug 2024 11:34:58 -0400 Subject: [PATCH 110/113] Rearranged regular fills for OBCs --- .../fill_halo_regions_open.jl | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/BoundaryConditions/fill_halo_regions_open.jl b/src/BoundaryConditions/fill_halo_regions_open.jl index 432e7a61a0..8c7a3a79e0 100644 --- a/src/BoundaryConditions/fill_halo_regions_open.jl +++ b/src/BoundaryConditions/fill_halo_regions_open.jl @@ -90,20 +90,20 @@ end @inline _fill_bottom_open_halo!(i, j, grid, c, bc::OBC, loc, args...) = @inbounds c[i, j, 1] = getbc(bc, i, j, grid, args...) @inline _fill_top_open_halo!(i, j, grid, c, bc::OBC, loc, args...) = @inbounds c[i, j, grid.Nz + 1] = getbc(bc, i, j, grid, args...) -# Open boundary condition fallback for `CenterField` - -@inline _fill_west_halo!(j, k, grid, c, bc::OBC, ::Tuple{Center, Center, Center}, args...) = @inbounds c[1, j, k] = getbc(bc, j, k, grid, args...) -@inline _fill_east_halo!(j, k, grid, c, bc::OBC, ::Tuple{Center, Center, Center}, args...) = @inbounds c[grid.Nx + 1, j, k] = getbc(bc, j, k, grid, args...) -@inline _fill_south_halo!(i, k, grid, c, bc::OBC, ::Tuple{Center, Center, Center}, args...) = @inbounds c[i, 1, k] = getbc(bc, i, k, grid, args...) -@inline _fill_north_halo!(i, k, grid, c, bc::OBC, ::Tuple{Center, Center, Center}, args...) = @inbounds c[i, grid.Ny + 1, k] = getbc(bc, i, k, grid, args...) -@inline _fill_bottom_halo!(i, j, grid, c, bc::OBC, ::Tuple{Center, Center, Center}, args...) = @inbounds c[i, j, 1] = getbc(bc, i, j, grid, args...) -@inline _fill_top_halo!(i, j, grid, c, bc::OBC, ::Tuple{Center, Center, Center}, args...) = @inbounds c[i, j, grid.Nz + 1] = getbc(bc, i, j, grid, args...) - -# Regular boundary fill for non-`CenterField` open boundaries - -@inline _fill_west_halo!(j, k, grid, c, bc::OBC, loc, args...) = nothing -@inline _fill_east_halo!(j, k, grid, c, bc::OBC, loc, args...) = nothing -@inline _fill_south_halo!(i, k, grid, c, bc::OBC, loc, args...) = nothing -@inline _fill_north_halo!(i, k, grid, c, bc::OBC, loc, args...) = nothing -@inline _fill_bottom_halo!(i, j, grid, c, bc::OBC, loc, args...) = nothing -@inline _fill_top_halo!(i, j, grid, c, bc::OBC, loc, args...) = nothing +# Regular boundary fill defaults + +@inline _fill_west_halo!(j, k, grid, c, bc::OBC, loc, args...) = _fill_west_open_halo!(j, k, grid, c, bc, loc, args...) +@inline _fill_east_halo!(j, k, grid, c, bc::OBC, loc, args...) = _fill_east_open_halo!(j, k, grid, c, bc, loc, args...) +@inline _fill_south_halo!(i, k, grid, c, bc::OBC, loc, args...) = _fill_south_open_halo!(j, k, grid, c, bc, loc, args...) +@inline _fill_north_halo!(i, k, grid, c, bc::OBC, loc, args...) = _fill_north_open_halo!(j, k, grid, c, bc, loc, args...) +@inline _fill_bottom_halo!(i, j, grid, c, bc::OBC, loc, args...) = _fill_bottom_open_halo!(j, k, grid, c, bc, loc, args...) +@inline _fill_top_halo!(i, j, grid, c, bc::OBC, loc, args...) = _fill_top_open_halo!(j, k, grid, c, bc, loc, args...) + +# Regular boundary fill for wall normal velocities + +@inline _fill_west_halo!(j, k, grid, c, bc::OBC, ::Tuple{Face, <:Any, <:Any}, args...) = nothing +@inline _fill_east_halo!(j, k, grid, c, bc::OBC, ::Tuple{Face, <:Any, <:Any}, args...) = nothing +@inline _fill_south_halo!(i, k, grid, c, bc::OBC, ::Tuple{<:Any, Face, <:Any}, args...) = nothing +@inline _fill_north_halo!(i, k, grid, c, bc::OBC, ::Tuple{<:Any, Face, <:Any}, args...) = nothing +@inline _fill_bottom_halo!(i, j, grid, c, bc::OBC, ::Tuple{<:Any, <:Any, Face}, args...) = nothing +@inline _fill_top_halo!(i, j, grid, c, bc::OBC, ::Tuple{<:Any, <:Any, Face}, args...) = nothing From 5e9c99cab913c880c477a1a07345b6744da61e00 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Sun, 4 Aug 2024 11:20:24 -0400 Subject: [PATCH 111/113] Update fill_halo_regions_open.jl Co-authored-by: Tomas Chor --- src/BoundaryConditions/fill_halo_regions_open.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/BoundaryConditions/fill_halo_regions_open.jl b/src/BoundaryConditions/fill_halo_regions_open.jl index 8c7a3a79e0..152ebd29b7 100644 --- a/src/BoundaryConditions/fill_halo_regions_open.jl +++ b/src/BoundaryConditions/fill_halo_regions_open.jl @@ -94,10 +94,10 @@ end @inline _fill_west_halo!(j, k, grid, c, bc::OBC, loc, args...) = _fill_west_open_halo!(j, k, grid, c, bc, loc, args...) @inline _fill_east_halo!(j, k, grid, c, bc::OBC, loc, args...) = _fill_east_open_halo!(j, k, grid, c, bc, loc, args...) -@inline _fill_south_halo!(i, k, grid, c, bc::OBC, loc, args...) = _fill_south_open_halo!(j, k, grid, c, bc, loc, args...) -@inline _fill_north_halo!(i, k, grid, c, bc::OBC, loc, args...) = _fill_north_open_halo!(j, k, grid, c, bc, loc, args...) -@inline _fill_bottom_halo!(i, j, grid, c, bc::OBC, loc, args...) = _fill_bottom_open_halo!(j, k, grid, c, bc, loc, args...) -@inline _fill_top_halo!(i, j, grid, c, bc::OBC, loc, args...) = _fill_top_open_halo!(j, k, grid, c, bc, loc, args...) +@inline _fill_south_halo!(i, k, grid, c, bc::OBC, loc, args...) = _fill_south_open_halo!(i, k, grid, c, bc, loc, args...) +@inline _fill_north_halo!(i, k, grid, c, bc::OBC, loc, args...) = _fill_north_open_halo!(i, k, grid, c, bc, loc, args...) +@inline _fill_bottom_halo!(i, j, grid, c, bc::OBC, loc, args...) = _fill_bottom_open_halo!(i, j, grid, c, bc, loc, args...) +@inline _fill_top_halo!(i, j, grid, c, bc::OBC, loc, args...) = _fill_top_open_halo!(i, j, grid, c, bc, loc, args...) # Regular boundary fill for wall normal velocities From 87ac61dd7e73beb5cbd17a35f3c203b780b5bd38 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Tue, 6 Aug 2024 10:11:56 -0400 Subject: [PATCH 112/113] Apply suggestions from code review Co-authored-by: Simone Silvestri Co-authored-by: Tomas Chor --- .../flat_extrapolation_open_boundary_matching_scheme.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl b/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl index 2b2c9005c5..a5bc6435c7 100644 --- a/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl +++ b/src/BoundaryConditions/flat_extrapolation_open_boundary_matching_scheme.jl @@ -12,7 +12,7 @@ f′(xᵢ) ≈ f′(xᵢ₋₁) + f′′(xᵢ₋₁)(xᵢ₋₁ - xᵢ) + O(Δx ``` where ``Δx=xᵢ₋₁ - xᵢ`` (for simplicity, we will also assume the spacing is constant at all ``i`` for now). -We can substitute the gradinet at some point ``j`` (``f′(xⱼ)``) with the central +We can substitute the gradient at some point ``j`` (``f′(xⱼ)``) with the central difference approximation: ```math f′(xⱼ) ≈ (f(xⱼ₊₁) - f(xⱼ₋₁)) / 2Δx, @@ -54,7 +54,7 @@ end Δϕ = (ϕₑₓₜ - ϕ) * Δt̄ not_relaxing = isnothing(bc.condition) | !isfinite(clock.last_stage_Δt) - Δϕ = ifelse(not_relaxing, zero(ϕ), Δc) + Δϕ = ifelse(not_relaxing, zero(ϕ), Δϕ) return ϕ + Δϕ end From ef4d2fc0f77528fdd9f554e11ff3bb2f932543c2 Mon Sep 17 00:00:00 2001 From: Jago Strong-Wright Date: Wed, 7 Aug 2024 14:31:32 -0400 Subject: [PATCH 113/113] correct version number --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 7853ec0c98..aaafb96e45 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "Oceananigans" uuid = "9e8cae18-63c1-5223-a75c-80ca9d6e9a09" authors = ["Climate Modeling Alliance and contributors"] -version = "0.92.0" +version = "0.91.7" [deps] Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e"