Debugging

CliMA · Jan 6, 2025 · 13aafaa · 13aafaa
1 parent 6057d99
commit 13aafaa
Show file tree

Hide file tree

Showing 4 changed files with 116 additions and 8 deletions.
diff --git a/.buildkite/Manifest.toml b/.buildkite/Manifest.toml
@@ -1,6 +1,6 @@
 # This file is machine-generated - editing it directly is not advised
 
-julia_version = "1.10.6"
+julia_version = "1.10.7"
 manifest_format = "2.0"
 project_hash = "fc780a0dc833e209a503e8cdbd13bf068bf6b381"
 
@@ -264,9 +264,11 @@ weakdeps = ["CUDA", "MPI"]
 
 [[deps.ClimaCore]]
 deps = ["Adapt", "BandedMatrices", "BlockArrays", "ClimaComms", "CubedSphere", "DataStructures", "ForwardDiff", "GaussQuadrature", "GilbertCurves", "HDF5", "InteractiveUtils", "IntervalSets", "KrylovKit", "LinearAlgebra", "MultiBroadcastFusion", "NVTX", "PkgVersion", "RecursiveArrayTools", "RootSolvers", "SparseArrays", "StaticArrays", "Statistics", "Unrolled"]
-git-tree-sha1 = "7632911d7b09fb3e067984626b07b3eb1113e55e"
+git-tree-sha1 = "eefcc6efa9398af0e83cbe65402f49f0aef304f8"
+repo-rev = "ck/debugging_hooks"
+repo-url = "https://github.com/CliMA/ClimaCore.jl.git"
 uuid = "d414da3d-4745-48bb-8d80-42e94e092884"
-version = "0.14.20"
+version = "0.14.22"
 weakdeps = ["CUDA", "Krylov"]
 
     [deps.ClimaCore.extensions]

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -170,105 +170,173 @@ steps:
 
       - label: "Convergence: SSP22Heuns"
         command: "julia --color=yes --check-bounds=yes --project=.buildkite docs/src/dev/report_gen_alg.jl --alg SSP22Heuns"
+        artifact_paths:
+          - "output/SSP22Heuns/*"
 
       - label: "Convergence: SSP33ShuOsher"
         command: "julia --color=yes --check-bounds=yes --project=.buildkite docs/src/dev/report_gen_alg.jl --alg SSP33ShuOsher"
+        artifact_paths:
+          - "output/SSP33ShuOsher/*"
 
       - label: "Convergence: RK4"
         command: "julia --color=yes --check-bounds=yes --project=.buildkite docs/src/dev/report_gen_alg.jl --alg RK4"
+        artifact_paths:
+          - "output/RK4/*"
 
       - label: "Convergence: ARK2GKC"
         command: "julia --color=yes --check-bounds=yes --project=.buildkite docs/src/dev/report_gen_alg.jl --alg ARK2GKC"
+        artifact_paths:
+          - "output/ARK2GKC/*"
 
       - label: "Convergence: ARS111"
         command: "julia --color=yes --check-bounds=yes --project=.buildkite docs/src/dev/report_gen_alg.jl --alg ARS111"
+        artifact_paths:
+          - "output/ARS111/*"
 
       - label: "Convergence: ARS121"
         command: "julia --color=yes --check-bounds=yes --project=.buildkite docs/src/dev/report_gen_alg.jl --alg ARS121"
+        artifact_paths:
+          - "output/ARS121/*"
 
       - label: "Convergence: ARS122"
         command: "julia --color=yes --check-bounds=yes --project=.buildkite docs/src/dev/report_gen_alg.jl --alg ARS122"
+        artifact_paths:
+          - "output/ARS122/*"
 
       - label: "Convergence: ARS222"
         command: "julia --color=yes --check-bounds=yes --project=.buildkite docs/src/dev/report_gen_alg.jl --alg ARS222"
+        artifact_paths:
+          - "output/ARS222/*"
 
       - label: "Convergence: ARS232"
         command: "julia --color=yes --check-bounds=yes --project=.buildkite docs/src/dev/report_gen_alg.jl --alg ARS232"
+        artifact_paths:
+          - "output/ARS232/*"
 
       - label: "Convergence: ARS233"
         command: "julia --color=yes --check-bounds=yes --project=.buildkite docs/src/dev/report_gen_alg.jl --alg ARS233"
+        artifact_paths:
+          - "output/ARS233/*"
 
       - label: "Convergence: ARS343"
         command: "julia --color=yes --check-bounds=yes --project=.buildkite docs/src/dev/report_gen_alg.jl --alg ARS343"
+        artifact_paths:
+          - "output/ARS343/*"
 
       - label: "Convergence: ARS443"
         command: "julia --color=yes --check-bounds=yes --project=.buildkite docs/src/dev/report_gen_alg.jl --alg ARS443"
+        artifact_paths:
+          - "output/ARS443/*"
 
       - label: "Convergence: SSP222"
         command: "julia --color=yes --check-bounds=yes --project=.buildkite docs/src/dev/report_gen_alg.jl --alg SSP222"
+        artifact_paths:
+          - "output/SSP222/*"
 
       - label: "Convergence: SSP322"
         command: "julia --color=yes --check-bounds=yes --project=.buildkite docs/src/dev/report_gen_alg.jl --alg SSP322"
+        artifact_paths:
+          - "output/SSP322/*"
 
       - label: "Convergence: SSP332"
         command: "julia --color=yes --check-bounds=yes --project=.buildkite docs/src/dev/report_gen_alg.jl --alg SSP332"
+        artifact_paths:
+          - "output/SSP332/*"
 
       - label: "Convergence: SSP333"
         command: "julia --color=yes --check-bounds=yes --project=.buildkite docs/src/dev/report_gen_alg.jl --alg SSP333"
+        artifact_paths:
+          - "output/SSP333/*"
 
       - label: "Convergence: SSP433"
         command: "julia --color=yes --check-bounds=yes --project=.buildkite docs/src/dev/report_gen_alg.jl --alg SSP433"
+        artifact_paths:
+          - "output/SSP433/*"
 
       - label: "Convergence: DBM453"
         command: "julia --color=yes --check-bounds=yes --project=.buildkite docs/src/dev/report_gen_alg.jl --alg DBM453"
+        artifact_paths:
+          - "output/DBM453/*"
 
       - label: "Convergence: HOMMEM1"
         command: "julia --color=yes --check-bounds=yes --project=.buildkite docs/src/dev/report_gen_alg.jl --alg HOMMEM1"
+        artifact_paths:
+          - "output/HOMMEM1/*"
 
       - label: "Convergence: IMKG232a"
         command: "julia --color=yes --check-bounds=yes --project=.buildkite docs/src/dev/report_gen_alg.jl --alg IMKG232a"
+        artifact_paths:
+          - "output/IMKG232a/*"
 
       - label: "Convergence: IMKG232b"
         command: "julia --color=yes --check-bounds=yes --project=.buildkite docs/src/dev/report_gen_alg.jl --alg IMKG232b"
+        artifact_paths:
+          - "output/IMKG232b/*"
 
       - label: "Convergence: IMKG242a"
         command: "julia --color=yes --check-bounds=yes --project=.buildkite docs/src/dev/report_gen_alg.jl --alg IMKG242a"
+        artifact_paths:
+          - "output/IMKG242a/*"
 
       - label: "Convergence: IMKG242b"
         command: "julia --color=yes --check-bounds=yes --project=.buildkite docs/src/dev/report_gen_alg.jl --alg IMKG242b"
+        artifact_paths:
+          - "output/IMKG242b/*"
 
       - label: "Convergence: IMKG243a"
         command: "julia --color=yes --check-bounds=yes --project=.buildkite docs/src/dev/report_gen_alg.jl --alg IMKG243a"
+        artifact_paths:
+          - "output/IMKG243a/*"
 
       - label: "Convergence: IMKG252a"
         command: "julia --color=yes --check-bounds=yes --project=.buildkite docs/src/dev/report_gen_alg.jl --alg IMKG252a"
+        artifact_paths:
+          - "output/IMKG252a/*"
 
       - label: "Convergence: IMKG252b"
         command: "julia --color=yes --check-bounds=yes --project=.buildkite docs/src/dev/report_gen_alg.jl --alg IMKG252b"
+        artifact_paths:
+          - "output/IMKG252b/*"
 
       - label: "Convergence: IMKG253a"
         command: "julia --color=yes --check-bounds=yes --project=.buildkite docs/src/dev/report_gen_alg.jl --alg IMKG253a"
+        artifact_paths:
+          - "output/IMKG253a/*"
 
       - label: "Convergence: IMKG253b"
         command: "julia --color=yes --check-bounds=yes --project=.buildkite docs/src/dev/report_gen_alg.jl --alg IMKG253b"
+        artifact_paths:
+          - "output/IMKG253b/*"
 
       - label: "Convergence: IMKG254a"
         command: "julia --color=yes --check-bounds=yes --project=.buildkite docs/src/dev/report_gen_alg.jl --alg IMKG254a"
+        artifact_paths:
+          - "output/IMKG254a/*"
 
       - label: "Convergence: IMKG254b"
         command: "julia --color=yes --check-bounds=yes --project=.buildkite docs/src/dev/report_gen_alg.jl --alg IMKG254b"
+        artifact_paths:
+          - "output/IMKG254b/*"
 
       - label: "Convergence: IMKG254c"
         command: "julia --color=yes --check-bounds=yes --project=.buildkite docs/src/dev/report_gen_alg.jl --alg IMKG254c"
+        artifact_paths:
+          - "output/IMKG254c/*"
 
       - label: "Convergence: IMKG342a"
         command: "julia --color=yes --check-bounds=yes --project=.buildkite docs/src/dev/report_gen_alg.jl --alg IMKG342a"
+        artifact_paths:
+          - "output/IMKG342a/*"
 
       - label: "Convergence: IMKG343a"
         command: "julia --color=yes --check-bounds=yes --project=.buildkite docs/src/dev/report_gen_alg.jl --alg IMKG343a"
+        artifact_paths:
+          - "output/IMKG343a/*"
 
       - label: "Convergence: SSPKnoth"
         command: "julia --color=yes --check-bounds=yes --project=.buildkite docs/src/dev/report_gen_alg.jl --alg SSPKnoth"
+        artifact_paths:
+          - "output/SSPKnoth/*"
 
   - group: "Summaries"
     key: summaries

diff --git a/docs/src/dev/report_gen_alg.jl b/docs/src/dev/report_gen_alg.jl
@@ -62,13 +62,51 @@ include(joinpath(@__DIR__, "compute_convergence.jl"))
 include(joinpath(pkgdir(ClimaTimeSteppers), "test", "problems.jl"))
 import ClimaTimeSteppers as CTS
 all_subtypes(::Type{T}) where {T} = isabstracttype(T) ? vcat(all_subtypes.(subtypes(T))...) : [T]
+_global_parsed_args = parse_commandline()
+_global_alg_name = getproperty(CTS, Symbol(_global_parsed_args["alg"]))()
+
+import ClimaCore
+ClimaCore.DataLayouts.call_post_op_callback() = true
+function ClimaCore.DataLayouts.post_op_callback(data)
+    try
+        if any(isnan, parent(data))
+            @show count(isnan, parent(data))
+            @show length(parent(data))
+            @show size(parent(data))
+            file = "output/$_global_alg_name/data_with_nans.dat"
+            open(file, "w") do io
+                println(io, parent(data))
+            end
+            error("Found NaNs!")
+        end
+    catch e
+        @show typeof(data)
+        @show data
+        println("Caught exception")
+        throw(e)
+    end
+end
+function ClimaCore.DataLayouts.post_op_callback(x::ClimaCore.Fields.FieldVector)
+    try
+        if any(pn -> any(isnan, parent(getproperty(x, pn))), propertynames(x))
+            # mkpath(repro_dir)
+            error("Found NaNs!")
+        end
+    catch e
+        @show typeof(data)
+        @show data
+        println("Caught exception")
+        throw(e)
+    end
+end
 
 let # Convergence
-    parsed_args = parse_commandline()
-    alg_name = getproperty(CTS, Symbol(parsed_args["alg"]))()
     # NOTE: Some imperfections in the convergence order for SSPKnoth are to be
     # expected because we are not using the exact Jacobian
 
+    parsed_args = parse_commandline()
+    alg_name = getproperty(CTS, Symbol(parsed_args["alg"]))()
+
     export_convergence_results(alg_name, ark_analytic_nonlin_test_cts(Float64), 200)
     export_convergence_results(alg_name, ark_analytic_sys_test_cts(Float64), 400)
 

diff --git a/test/problems.jl b/test/problems.jl
@@ -510,7 +510,7 @@ function climacore_2Dheat_test_cts(::Type{FT}) where {FT}
         dss_tendency && Spaces.weighted_dss!(tendency.u)
         _FT = Spaces.undertype(axes(state.u))
         set_boundaries!(tendency.u, _FT(0))
-        if t ≤ t_end*0.01
+        if t ≤ t_end * 0.01
             e0 = extrema(state.u)
             e1 = extrema(divgradu)
             @show e0, e1, count(isnan, parent(state.u)), length(parent(state.u))
@@ -519,12 +519,12 @@ function climacore_2Dheat_test_cts(::Type{FT}) where {FT}
     end
 
     function dss!(state, _, t)
-        if t ≤ t_end*0.01
+        if t ≤ t_end * 0.01
             e0_dss = extrema(state.u)
             @show e0_dss, count(isnan, parent(state.u)), length(parent(state.u))
         end
         dss_tendency || Spaces.weighted_dss!(state.u)
-        if t ≤ t_end*0.01
+        if t ≤ t_end * 0.01
             e1_dss = extrema(state.u)
             @show e1_dss, count(isnan, parent(state.u)), length(parent(state.u))
         end