diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 5392dd1f42..d38e01c31c 100755 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -289,6 +289,16 @@ steps: key: unit_sphere command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/sphere.jl" + - label: "Unit: sphere gpu" + key: unit_sphere_gpu + command: + - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + - "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/sphere.jl" + env: + CLIMACOMMS_DEVICE: "CUDA" + agents: + slurm_gpus: 1 + - label: "Unit: terrain warp" key: unit_terrain_warp command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/terrain_warp.jl" @@ -556,6 +566,16 @@ steps: - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/finitedifference/unit_column.jl" - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/finitedifference/convergence_column.jl" + - label: "Unit: column gpu" + key: unit_column_gpu + command: + - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/finitedifference/unit_column.jl" + env: + CLIMACOMMS_DEVICE: "CUDA" + agents: + slurm_gpus: 1 + - label: "Unit: wfact" key: unit_wfact command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/finitedifference/wfact.jl" @@ -1190,6 +1210,16 @@ steps: key: perf_FD command: "julia --color=yes --project=.buildkite test/Operators/finitedifference/opt.jl" + - label: "Perf: FD operators gpu" + key: perf_FD_gpu + command: + - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/finitedifference/opt.jl" + env: + CLIMACOMMS_DEVICE: "CUDA" + agents: + slurm_gpus: 1 + # TODO: combine this with FD operators above - label: "Perf: FD operators from the wild" key: perf_FD_ops_examples diff --git a/test/Operators/finitedifference/opt.jl b/test/Operators/finitedifference/opt.jl index 33d2cc6f68..fc9dc33cc8 100644 --- a/test/Operators/finitedifference/opt.jl +++ b/test/Operators/finitedifference/opt.jl @@ -1,3 +1,8 @@ +#= +julia --check-bounds=yes --project +julia --project +using Revise; include(joinpath("test", "Operators", "finitedifference", "opt.jl")) +=# using Test using JET @@ -209,6 +214,7 @@ end @static if @isdefined(var"@test_opt") @testset "Scalar Field FiniteDifferenceSpaces optimizations" begin device = ClimaComms.device() + is_gpu = device isa ClimaComms.CUDADevice for FT in (Float64,) domain = Domains.IntervalDomain( Geometry.ZPoint{FT}(0.0), @@ -231,74 +237,89 @@ end filter(@nospecialize f) = f !== Base.mapreduce_empty # face space operators - @test_opt function_filter = filter sum(ones(FT, face_space)) - @test_opt function_filter = filter sum(sin.(faces)) + @test_opt broken = is_gpu function_filter = filter sum( + ones(FT, face_space), + ) + @test_opt broken = is_gpu function_filter = filter sum(sin.(faces)) - @test_opt opt_InterpolateF2C(faces) - @test_opt opt_WeightedInterpolateF2C(face_values, faces) + @test_opt broken = is_gpu opt_InterpolateF2C(faces) + @test_opt broken = is_gpu opt_WeightedInterpolateF2C( + face_values, + faces, + ) - @test_opt opt_LeftBiasedF2C(faces) - @test_opt opt_RightBiasedF2C(faces) + @test_opt broken = is_gpu opt_LeftBiasedF2C(faces) + @test_opt broken = is_gpu opt_RightBiasedF2C(faces) - # @test_opt opt_AdvectionF2F(face_velocities, faces) + @test_opt broken = is_gpu opt_AdvectionF2F(face_velocities, faces) - @test_opt opt_FluxCorrectionF2F_Extrapolate( + @test_opt broken = is_gpu opt_FluxCorrectionF2F_Extrapolate( center_velocities, faces, ) - @test_opt opt_GradientF2C(faces) - @test_opt opt_DivergenceF2C(faces) + @test_opt broken = is_gpu opt_GradientF2C(faces) + @test_opt broken = is_gpu opt_DivergenceF2C(faces) - @test_opt opt_SetBoundary_SetValue(faces) + @test_opt broken = is_gpu opt_SetBoundary_SetValue(faces) # center space operators - @test_opt function_filter = filter sum(ones(FT, center_space)) - @test_opt function_filter = filter sum(sin.(centers)) + @test_opt broken = is_gpu function_filter = filter sum( + ones(FT, center_space), + ) + @test_opt broken = is_gpu function_filter = filter sum( + sin.(centers), + ) - @test_opt opt_InterpolateC2F_SetValue(centers) - @test_opt opt_InterpolateC2F_SetGradient(centers) - @test_opt opt_InterpolateC2F_Extrapolate(centers) + @test_opt broken = is_gpu opt_InterpolateC2F_SetValue(centers) + @test_opt broken = is_gpu opt_InterpolateC2F_SetGradient(centers) + @test_opt broken = is_gpu opt_InterpolateC2F_Extrapolate(centers) - @test_opt opt_WeightedInterpolateC2F_SetValue( + @test_opt broken = is_gpu opt_WeightedInterpolateC2F_SetValue( center_values, centers, ) - @test_opt opt_WeightedInterpolateC2F_SetGradient( + @test_opt broken = is_gpu opt_WeightedInterpolateC2F_SetGradient( center_values, centers, ) - @test_opt opt_WeightedInterpolateC2F_Extrapolate( + @test_opt broken = is_gpu opt_WeightedInterpolateC2F_Extrapolate( center_values, centers, ) - @test_opt opt_LeftBiasedC2F(centers) - @test_opt opt_RightBiasedC2F(centers) + @test_opt broken = is_gpu opt_LeftBiasedC2F(centers) + @test_opt broken = is_gpu opt_RightBiasedC2F(centers) - @test_opt opt_UpwindBiasedProductC2F_SetValue( + @test_opt broken = is_gpu opt_UpwindBiasedProductC2F_SetValue( face_velocities, centers, ) - @test_opt opt_UpwindBiasedProductC2F_Extrapolate( + @test_opt broken = is_gpu opt_UpwindBiasedProductC2F_Extrapolate( face_velocities, centers, ) - @test_opt opt_AdvectionC2C_SetValue(face_velocities, centers) - @test_opt opt_AdvectionC2C_Extrapolate(face_velocities, centers) + @test_opt broken = is_gpu opt_AdvectionC2C_SetValue( + face_velocities, + centers, + ) + @test_opt broken = is_gpu opt_AdvectionC2C_Extrapolate( + face_velocities, + centers, + ) - @test_opt opt_FluxCorrectionC2C_Extrapolate( + @test_opt broken = is_gpu opt_FluxCorrectionC2C_Extrapolate( face_velocities, centers, ) - @test_opt opt_GradientC2F_SetValue(centers) - @test_opt opt_GradientC2F_SetGradient(centers) + @test_opt broken = is_gpu opt_GradientC2F_SetValue(centers) + @test_opt broken = is_gpu opt_GradientC2F_SetGradient(centers) - @test_opt opt_DivergenceC2F_SetValue(centers) - @test_opt opt_DivergenceC2F_SetDivergence(centers) - @test_opt opt_CurlC2F_SetValue(centers) + @test_opt broken = is_gpu opt_DivergenceC2F_SetValue(centers) + @test_opt broken = is_gpu opt_DivergenceC2F_SetDivergence(centers) + @test_opt broken = is_gpu opt_CurlC2F_SetValue(centers) end end end diff --git a/test/Operators/finitedifference/unit_column.jl b/test/Operators/finitedifference/unit_column.jl index bb4b13bf4f..985b1406f9 100644 --- a/test/Operators/finitedifference/unit_column.jl +++ b/test/Operators/finitedifference/unit_column.jl @@ -1,3 +1,8 @@ +#= +julia --check-bounds=yes --project +julia --project +using Revise; include(joinpath("test", "Operators", "finitedifference", "unit_column.jl")) +=# using Test using StaticArrays, IntervalSets, LinearAlgebra @@ -26,7 +31,9 @@ device = ClimaComms.device() face_space = Spaces.FaceFiniteDifferenceSpace(center_space) @test sum(ones(FT, center_space)) ≈ pi - @test sum(ones(FT, face_space)) ≈ pi + ClimaComms.allowscalar(device) do + @test sum(ones(FT, face_space)) ≈ pi + end centers = getproperty(Fields.coordinate_field(center_space), :z) @test sum(sin.(centers)) ≈ FT(2.0) atol = 1e-2 @@ -102,20 +109,22 @@ end @test sum(sinz_f) ≈ FT(0.0) atol = 1e-2 ∇ᶜ = Operators.GradientF2C() - ∂sin = Geometry.WVector.(∇ᶜ.(sinz_f)) - @test ∂sin ≈ Geometry.WVector.(cosz_c) atol = 1e-2 + @test begin + ∂sin = Geometry.WVector.(∇ᶜ.(sinz_f)) + @test ∂sin ≈ Geometry.WVector.(cosz_c) atol = 1e-2 - divᶜ = Operators.DivergenceF2C() - ∂sin = divᶜ.(Geometry.WVector.(sinz_f)) - @test ∂sin ≈ cosz_c atol = 1e-2 + divᶜ = Operators.DivergenceF2C() + ∂sin = divᶜ.(Geometry.WVector.(sinz_f)) + @test ∂sin ≈ cosz_c atol = 1e-2 - ∇ᶠ = Operators.GradientC2F() - ∂cos = Geometry.WVector.(∇ᶠ.(cosz_c)) - @test ∂cos ≈ Geometry.WVector.(.-sinz_f) atol = 1e-1 + ∇ᶠ = Operators.GradientC2F() + ∂cos = Geometry.WVector.(∇ᶠ.(cosz_c)) + @test ∂cos ≈ Geometry.WVector.(.-sinz_f) atol = 1e-1 - ∇ᶠ = Operators.GradientC2F() - ∂cos = Geometry.WVector.(∇ᶠ.(cosz_c)) - @test ∂cos ≈ Geometry.WVector.(.-sinz_f) atol = 1e-2 + ∇ᶠ = Operators.GradientC2F() + ∂cos = Geometry.WVector.(∇ᶠ.(cosz_c)) + @test ∂cos ≈ Geometry.WVector.(.-sinz_f) atol = 1e-2 + end broken = device isa ClimaComms.CUDADevice # test that broadcasting into incorrect field space throws an error empty_centers = zeros(FT, center_space) @@ -269,32 +278,32 @@ end cy = cfield.y fy = ffield.y - cyp = parent(cy) - fyp = parent(fy) + cyp = Array(parent(cy)) + fyp = Array(parent(fy)) # C2F biased operators LBC2F = Operators.LeftBiasedC2F(; bottom = Operators.SetValue(10)) @. cy = cos(zc) @. fy = LBC2F(cy) fy_ref = [FT(10), [cyp[i] for i in 1:length(cyp)]...] - @test all(fy_ref .== fyp) + @test all(fy_ref .== fyp) broken = device isa ClimaComms.CUDADevice RBC2F = Operators.RightBiasedC2F(; top = Operators.SetValue(10)) @. cy = cos(zc) @. fy = RBC2F(cy) fy_ref = [[cyp[i] for i in 1:length(cyp)]..., FT(10)] - @test all(fy_ref .== fyp) + @test all(fy_ref .== fyp) broken = device isa ClimaComms.CUDADevice # F2C biased operators LBF2C = Operators.LeftBiasedF2C(; bottom = Operators.SetValue(10)) @. cy = cos(zc) @. cy = LBF2C(fy) cy_ref = [i == 1 ? FT(10) : fyp[i] for i in 1:length(cyp)] - @test all(cy_ref .== cyp) + @test all(cy_ref .== cyp) broken = device isa ClimaComms.CUDADevice RBF2C = Operators.RightBiasedF2C(; top = Operators.SetValue(10)) @. cy = cos(zc) @. cy = RBF2C(fy) cy_ref = [i == length(cyp) ? FT(10) : fyp[i + 1] for i in 1:length(cyp)] - @test all(cy_ref .== cyp) + @test all(cy_ref .== cyp) broken = device isa ClimaComms.CUDADevice end