From 1127287bfecbd6f37ef006b1c03cc877179537cd Mon Sep 17 00:00:00 2001 From: Charles Kawczynski Date: Fri, 19 Jan 2024 11:25:42 -0500 Subject: [PATCH 1/3] Update deps and julia version --- .buildkite/pipeline.yml | 4 +-- .github/workflows/CodeCov.yml | 6 ++-- .github/workflows/DocCleanUp.yml | 2 +- .github/workflows/JuliaFormatter.yml | 6 ++-- .github/workflows/ci.yml | 3 +- .github/workflows/docs.yml | 4 +-- .github/workflows/invalidations.yml | 4 +-- Project.toml | 2 +- docs/Project.toml | 1 - gpuenv/Project.toml | 9 ++---- perf/Project.toml | 1 - perf/common.jl | 3 +- perf/common_micro_bm.jl | 1 - test/Project.toml | 4 +-- test/relations.jl | 43 ++++++++++++++-------------- test/runtests_gpu.jl | 31 +++++++++++--------- 16 files changed, 60 insertions(+), 64 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 7a4119b0..7f944870 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,10 +1,10 @@ agents: queue: central slurm_mem: 8G - modules: julia/1.9.3 cuda/12.2 + modules: julia/1.10.0 cuda/12.2 env: - JULIA_VERSION: "1.9.3" + JULIA_VERSION: "1.10.0" OPENBLAS_NUM_THREADS: 1 CUDA_VERSION: "12.2" OMPI_MCA_opal_warn_on_missing_libcuda: 0 diff --git a/.github/workflows/CodeCov.yml b/.github/workflows/CodeCov.yml index 33767e39..62b5dde6 100644 --- a/.github/workflows/CodeCov.yml +++ b/.github/workflows/CodeCov.yml @@ -13,12 +13,12 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2.2.0 + - uses: actions/checkout@v4 - name: Set up Julia uses: julia-actions/setup-julia@latest with: - version: 1.9 + version: '1.10' - name: Test with coverage env: @@ -36,7 +36,7 @@ jobs: if: success() - name: Submit coverage - uses: codecov/codecov-action@v1.0.7 + uses: codecov/codecov-action@v3 with: token: ${{secrets.CODECOV_TOKEN}} if: success() diff --git a/.github/workflows/DocCleanUp.yml b/.github/workflows/DocCleanUp.yml index f3d42b6b..41ac294d 100644 --- a/.github/workflows/DocCleanUp.yml +++ b/.github/workflows/DocCleanUp.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout gh-pages branch - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: ref: gh-pages diff --git a/.github/workflows/JuliaFormatter.yml b/.github/workflows/JuliaFormatter.yml index 924c7d69..e84ee690 100644 --- a/.github/workflows/JuliaFormatter.yml +++ b/.github/workflows/JuliaFormatter.yml @@ -18,9 +18,9 @@ jobs: with: access_token: ${{ github.token }} - - uses: actions/checkout@v2.2.0 + - uses: actions/checkout@v4 - - uses: dorny/paths-filter@v2.9.1 + - uses: dorny/paths-filter@v2 id: filter with: filters: | @@ -30,7 +30,7 @@ jobs: - uses: julia-actions/setup-julia@latest if: steps.filter.outputs.julia_file_change == 'true' with: - version: 1.9 + version: '1.10' - name: Apply JuliaFormatter if: steps.filter.outputs.julia_file_change == 'true' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index feceb51e..0d414e18 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,6 +19,7 @@ jobs: - '1.7' - '1.8' - '1.9' + - '1.10' os: - ubuntu-latest - macOS-latest @@ -26,7 +27,7 @@ jobs: arch: - x64 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: julia-actions/setup-julia@v1 with: version: ${{ matrix.version }} diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 45b1d8dd..b804950f 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -12,10 +12,10 @@ jobs: docbuild: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: julia-actions/setup-julia@latest with: - version: 1.9 + version: '1.10' - name: Install dependencies run: julia --project=docs/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate()' - name: Build and deploy diff --git a/.github/workflows/invalidations.yml b/.github/workflows/invalidations.yml index 9aa4742c..881ebecd 100644 --- a/.github/workflows/invalidations.yml +++ b/.github/workflows/invalidations.yml @@ -19,12 +19,12 @@ jobs: - uses: julia-actions/setup-julia@v1 with: version: '1' - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: julia-actions/julia-buildpkg@v1 - uses: julia-actions/julia-invalidations@v1 id: invs_pr - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: ${{ github.event.repository.default_branch }} - uses: julia-actions/julia-buildpkg@v1 diff --git a/Project.toml b/Project.toml index 34390007..a9f08262 100644 --- a/Project.toml +++ b/Project.toml @@ -19,7 +19,7 @@ CreateParametersExt = "CLIMAParameters" [compat] DocStringExtensions = "0.8.1, 0.9" -KernelAbstractions = "0.7, 0.8, 0.9" +KernelAbstractions = "0.9" Random = "1" RootSolvers = "0.4" julia = "1.6" diff --git a/docs/Project.toml b/docs/Project.toml index 93171eed..bbb4ad94 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -8,7 +8,6 @@ KernelAbstractions = "63c18a36-062a-441e-b654-da1e3ab1ce7c" Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" RootSolvers = "7181ea78-2dcb-4de3-ab41-2b8ab5a31e74" Thermodynamics = "b60c26fb-14c3-4610-9d3e-2d17fe7ff00c" -UnPack = "3a884ed6-31ef-47d7-9d2a-63182c4928ed" [compat] julia = "1.7" diff --git a/gpuenv/Project.toml b/gpuenv/Project.toml index 85789179..13287bb8 100644 --- a/gpuenv/Project.toml +++ b/gpuenv/Project.toml @@ -1,19 +1,16 @@ [deps] CLIMAParameters = "6eacf6c3-8458-43b9-ae03-caf5306d3d53" CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" -CUDAKernels = "72cfdca4-0801-4ab0-bf6a-d52aa10adc57" KernelAbstractions = "63c18a36-062a-441e-b654-da1e3ab1ce7c" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" RootSolvers = "7181ea78-2dcb-4de3-ab41-2b8ab5a31e74" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" -UnPack = "3a884ed6-31ef-47d7-9d2a-63182c4928ed" +Thermodynamics = "b60c26fb-14c3-4610-9d3e-2d17fe7ff00c" [compat] CLIMAParameters = "0.8" -CUDA = "3.5" -CUDAKernels = "0.3, 0.4" -KernelAbstractions = "0.7, 0.8" +CUDA = "3.5, 4, 5" +KernelAbstractions = "0.9" RootSolvers = "0.4" -UnPack = "1.0" julia = "1.7" diff --git a/perf/Project.toml b/perf/Project.toml index c061fac6..8e9b4b8f 100644 --- a/perf/Project.toml +++ b/perf/Project.toml @@ -13,4 +13,3 @@ RootSolvers = "7181ea78-2dcb-4de3-ab41-2b8ab5a31e74" StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" Thermodynamics = "b60c26fb-14c3-4610-9d3e-2d17fe7ff00c" -UnPack = "3a884ed6-31ef-47d7-9d2a-63182c4928ed" diff --git a/perf/common.jl b/perf/common.jl index 00b1397a..459b999c 100644 --- a/perf/common.jl +++ b/perf/common.jl @@ -1,6 +1,5 @@ using Test -import UnPack import BenchmarkTools import RootSolvers as RS @@ -13,7 +12,7 @@ const param_set = TP.ThermodynamicsParameters(FT) ArrayType = Array{FT} profiles = TD.TestedProfiles.PhaseEquilProfiles(param_set, ArrayType) -UnPack.@unpack e_int, T, ρ, p, θ_liq_ice, q_tot = profiles +(; e_int, T, ρ, p, θ_liq_ice, q_tot) = profiles function thermo_state_ρeq() ts = TD.PhaseEquil_ρeq.(param_set, ρ, e_int, q_tot) diff --git a/perf/common_micro_bm.jl b/perf/common_micro_bm.jl index c9572858..654a3cee 100644 --- a/perf/common_micro_bm.jl +++ b/perf/common_micro_bm.jl @@ -3,7 +3,6 @@ import PrettyTables import OrderedCollections using JET using Test -import UnPack import BenchmarkTools import Thermodynamics as TD diff --git a/test/Project.toml b/test/Project.toml index 239e4606..d889c0f3 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -16,11 +16,9 @@ ReportMetrics = "c1654acf-408b-4272-96ce-66c258df8a6c" RootSolvers = "7181ea78-2dcb-4de3-ab41-2b8ab5a31e74" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" Thermodynamics = "b60c26fb-14c3-4610-9d3e-2d17fe7ff00c" -UnPack = "3a884ed6-31ef-47d7-9d2a-63182c4928ed" [compat] CLIMAParameters = "0.8" -KernelAbstractions = "0.7, 0.8" +KernelAbstractions = "0.9" RootSolvers = "0.4" -UnPack = "1.0" julia = "1.7" diff --git a/test/relations.jl b/test/relations.jl index 452c40b9..353e80ec 100644 --- a/test/relations.jl +++ b/test/relations.jl @@ -1,6 +1,5 @@ using Test -using UnPack using Random import RootSolvers as RS using LinearAlgebra @@ -69,8 +68,8 @@ compare_moisture(param_set, ts::PhaseNonEquil, q_pt::PhasePartition) = all(( _kappa_d = FT(TP.kappa_d(param_set)) profiles = TestedProfiles.PhaseEquilProfiles(param_set, ArrayType) - @unpack T, p, e_int, ρ, θ_liq_ice, phase_type = profiles - @unpack q_tot, q_liq, q_ice, q_pt, RH, e_kin, e_pot = profiles + (; T, p, e_int, ρ, θ_liq_ice, phase_type) = profiles + (; q_tot, q_liq, q_ice, q_pt, RH, e_kin, e_pot) = profiles # Test state for thermodynamic consistency (with ideal gas law) T_idgl = TD.air_temperature_from_ideal_gas_law.(param_set, p, ρ, q_pt) @@ -503,8 +502,8 @@ end FT = eltype(ArrayType) param_set = TP.ThermodynamicsParameters(FT) profiles = TestedProfiles.PhaseEquilProfiles(param_set, ArrayType) - @unpack T, p, e_int, ρ, θ_liq_ice, phase_type = profiles - @unpack q_tot, q_liq, q_ice, q_pt, RH, e_kin, e_pot = profiles + (; T, p, e_int, ρ, θ_liq_ice, phase_type) = profiles + (; q_tot, q_liq, q_ice, q_pt, RH, e_kin, e_pot) = profiles RH_sat_mask = or.(RH .> 1, RH .≈ 1) RH_unsat_mask = .!or.(RH .> 1, RH .≈ 1) @@ -879,8 +878,8 @@ end FT = eltype(ArrayType) param_set = TP.ThermodynamicsParameters(FT) profiles = TestedProfiles.PhaseEquilProfiles(param_set, ArrayType) - @unpack T, p, e_int, ρ, θ_liq_ice, phase_type = profiles - @unpack q_tot, q_liq, q_ice, q_pt, RH, e_kin, e_pot = profiles + (; T, p, e_int, ρ, θ_liq_ice, phase_type) = profiles + (; q_tot, q_liq, q_ice, q_pt, RH, e_kin, e_pot) = profiles @test_throws ErrorException TD.saturation_adjustment.( RS.NewtonsMethod, @@ -989,8 +988,8 @@ end param_set = TP.ThermodynamicsParameters(FT) profiles = TestedProfiles.PhaseDryProfiles(param_set, ArrayType) - @unpack T, p, e_int, h, ρ, θ_liq_ice, phase_type = profiles - @unpack q_tot, q_liq, q_ice, q_pt, RH, e_kin, e_pot = profiles + (; T, p, e_int, h, ρ, θ_liq_ice, phase_type) = profiles + (; q_tot, q_liq, q_ice, q_pt, RH, e_kin, e_pot) = profiles # PhaseDry ts = PhaseDry.(param_set, e_int, ρ) @@ -1051,8 +1050,8 @@ end profiles = TestedProfiles.PhaseEquilProfiles(param_set, ArrayType) - @unpack T, p, e_int, h, ρ, θ_liq_ice, phase_type = profiles - @unpack q_tot, q_liq, q_ice, q_pt, RH, e_kin, e_pot = profiles + (; T, p, e_int, h, ρ, θ_liq_ice, phase_type) = profiles + (; q_tot, q_liq, q_ice, q_pt, RH, e_kin, e_pot) = profiles # PhaseEquil ts = @@ -1266,8 +1265,8 @@ end @test all(compare_moisture.(param_set, ts, q_pt)) profiles = TestedProfiles.PhaseEquilProfiles(param_set, ArrayType) - @unpack T, p, e_int, ρ, θ_liq_ice, phase_type = profiles - @unpack q_tot, q_liq, q_ice, q_pt, RH, e_kin, e_pot = profiles + (; T, p, e_int, ρ, θ_liq_ice, phase_type) = profiles + (; q_tot, q_liq, q_ice, q_pt, RH, e_kin, e_pot) = profiles # Test that relative humidity is 1 for saturated conditions q_sat = q_vap_saturation.(param_set, T, ρ, Ref(phase_type)) @@ -1349,8 +1348,8 @@ end param_set = TP.ThermodynamicsParameters(FT) profiles = TestedProfiles.PhaseDryProfiles(param_set, ArrayType) - @unpack T, p, e_int, ρ, θ_liq_ice, phase_type = profiles - @unpack q_tot, q_liq, q_ice, q_pt, RH, e_kin, e_pot = profiles + (; T, p, e_int, ρ, θ_liq_ice, phase_type) = profiles + (; q_tot, q_liq, q_ice, q_pt, RH, e_kin, e_pot) = profiles θ_dry = dry_pottemp.(param_set, T, ρ) ts_dry = PhaseDry.(param_set, e_int, ρ) @@ -1360,8 +1359,8 @@ end ts_dry_pθ = PhaseDry_pθ.(param_set, p, θ_dry) profiles = TestedProfiles.PhaseEquilProfiles(param_set, ArrayType) - @unpack T, p, e_int, ρ, θ_liq_ice, phase_type = profiles - @unpack q_tot, q_liq, q_ice, q_pt, RH, e_kin, e_pot = profiles + (; T, p, e_int, ρ, θ_liq_ice, phase_type) = profiles + (; q_tot, q_liq, q_ice, q_pt, RH, e_kin, e_pot) = profiles ρu = FT[1.0, 2.0, 3.0] @test typeof.(internal_energy.(ρ, ρ .* e_int, Ref(ρu), e_pot)) == @@ -1489,8 +1488,8 @@ end FT = eltype(ArrayType) param_set = TP.ThermodynamicsParameters(FT) profiles = TestedProfiles.PhaseEquilProfiles(param_set, ArrayType) - @unpack T, p, e_int, ρ, θ_liq_ice, phase_type = profiles - @unpack q_tot, q_liq, q_ice, q_pt, RH, e_kin, e_pot = profiles + (; T, p, e_int, ρ, θ_liq_ice, phase_type) = profiles + (; q_tot, q_liq, q_ice, q_pt, RH, e_kin, e_pot) = profiles # PhasePartition test is noisy, so do this only once: ts_dry = PhaseDry(param_set, first(e_int), first(ρ)) @@ -1657,7 +1656,7 @@ end FT = eltype(ArrayType) param_set = TP.ThermodynamicsParameters(FT) profiles = TestedProfiles.PhaseEquilProfiles(param_set, ArrayType) - @unpack T, q_pt, z, phase_type = profiles + (; T, q_pt, z, phase_type) = profiles @test all(z .≈ (nt.z for nt in profiles)) @test all(T .≈ (nt.T for nt in profiles)) @test all(getproperty.(q_pt, :tot) .≈ (nt.q_pt.tot for nt in profiles)) @@ -1677,7 +1676,7 @@ end FT = eltype(ArrayType) param_set = TP.ThermodynamicsParameters(FT) profiles = TestedProfiles.PhaseEquilProfiles(param_set, ArrayType) - @unpack p, ρ, e_int, h, θ_liq_ice, q_tot, T, phase_type = profiles + (; p, ρ, e_int, h, θ_liq_ice, q_tot, T, phase_type) = profiles T_guess = T .+ (FT(0.2) .* randn(FT, length(T))) args = (q_tot, 40, FT(1e-1)) ts = @@ -1714,7 +1713,7 @@ TD.solution_type() = RS.VerboseSolution() FT = eltype(ArrayType) param_set = TP.ThermodynamicsParameters(FT) profiles = TestedProfiles.PhaseEquilProfiles(param_set, ArrayType) - @unpack ρ, e_int, q_tot = profiles + (; ρ, e_int, q_tot) = profiles ts = PhaseEquil_ρeq.(param_set, ρ, e_int, q_tot) data = TD.DataCollection.get_data() TD.DataCollection.print_summary(data) diff --git a/test/runtests_gpu.jl b/test/runtests_gpu.jl index 72960a62..65c25cd7 100644 --- a/test/runtests_gpu.jl +++ b/test/runtests_gpu.jl @@ -1,9 +1,12 @@ +#= +using Revise +push!(ARGS, "CuArray") +include("test/runtests_gpu.jl") +=# using Test using KernelAbstractions import KernelAbstractions as KA -import CUDAKernels as CK -import UnPack using Random using LinearAlgebra import RootSolvers as RS @@ -16,10 +19,8 @@ if get(ARGS, 1, "Array") == "CuArray" import CUDA ArrayType = CUDA.CuArray CUDA.allowscalar(false) - device(::Type{T}) where {T <: CUDA.CuArray} = CK.CUDADevice() else ArrayType = Array - device(::Type{T}) where {T <: Array} = CK.CPU() end const param_set_Float64 = TP.ThermodynamicsParameters(Float64) @@ -37,7 +38,7 @@ parameter_set(::Type{Float32}) = param_set_Float32 p, q_tot, ) where {FT} - i = @index(Group, Linear) + i = @index(Global) @inbounds begin param_set = parameter_set(FT) @@ -87,7 +88,6 @@ convert_profile_set(ps::TD.TestedProfiles.ProfileSet, ArrayType, slice) = @testset "Thermodynamics - kernels" begin FT = Float32 param_set = parameter_set(FT) - dev = device(ArrayType) profiles = TD.TestedProfiles.PhaseEquilProfiles(param_set, Array) slice = Colon() profiles = convert_profile_set(profiles, ArrayType, slice) @@ -97,17 +97,22 @@ convert_profile_set(ps::TD.TestedProfiles.ProfileSet, ArrayType, slice) = d_dst = ArrayType(Array{FT}(undef, 2, n_profiles)) fill!(d_dst, 0) - UnPack.@unpack e_int, ρ, p, q_tot = profiles + (; e_int, ρ, p, q_tot) = profiles - work_groups = (1,) ndrange = (n_profiles,) - kernel! = test_thermo_kernel!(dev, work_groups) + backend = KA.get_backend(d_dst) + kernel! = test_thermo_kernel!(backend) event = kernel!(param_set, d_dst, e_int, ρ, p, q_tot, ndrange = ndrange) - wait(dev, event) + KA.synchronize(backend) - ts_correct = - TD.PhaseEquil_ρeq.(param_set, Array(ρ), Array(e_int), Array(q_tot)) - @test all(Array(d_dst)[1, :] .≈ TD.air_temperature.(param_set, ts_correct)) + ts_cpu = + TD.PhaseEquil_ρeq.( + param_set, + Array{FT}(ρ), + Array{FT}(e_int), + Array{FT}(q_tot), + ) + @test all(Array(d_dst)[1, :] .≈ TD.air_temperature.(param_set, ts_cpu)) ts_correct = TD.PhaseEquil_ρpq.( From 6996afd598db96200c5ae7a080670846e836a83a Mon Sep 17 00:00:00 2001 From: Charles Kawczynski Date: Fri, 19 Jan 2024 12:00:42 -0500 Subject: [PATCH 2/3] Drop testing on 1.6 --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0d414e18..b0ac4dfa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,6 @@ jobs: fail-fast: false matrix: version: - - '1.6' - '1.7' - '1.8' - '1.9' From 5f1bd592b2cc3d8110471b4f9f60b5e664d84600 Mon Sep 17 00:00:00 2001 From: Charles Kawczynski Date: Fri, 19 Jan 2024 14:34:19 -0500 Subject: [PATCH 3/3] Temporarily disable GFlops tests --- .buildkite/pipeline.yml | 12 ++++++------ perf/Project.toml | 1 - 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 7f944870..68dbe7cb 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -58,12 +58,12 @@ steps: slurm_ntasks: 1 timeout_in_minutes: 60 - - label: ":mag::rocket: Constructor flop measurements" - command: - - "julia --project=perf perf/flops.jl" - agents: - slurm_ntasks: 1 - timeout_in_minutes: 60 + # - label: ":mag::rocket: Constructor flop measurements" + # command: + # - "julia --project=perf perf/flops.jl" + # agents: + # slurm_ntasks: 1 + # timeout_in_minutes: 60 - label: ":mag::rocket: JET tests" command: diff --git a/perf/Project.toml b/perf/Project.toml index 8e9b4b8f..0cceb92b 100644 --- a/perf/Project.toml +++ b/perf/Project.toml @@ -1,7 +1,6 @@ [deps] BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" CLIMAParameters = "6eacf6c3-8458-43b9-ae03-caf5306d3d53" -GFlops = "2ea8233c-34d4-5acc-88b4-02f326385bcc" JET = "c3a54625-cd67-489e-a8e7-0a5a0ff4e31b" OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"