Merge branch 'sb-jmc/cubed-sphere-aquaplanet' of github.com:CliMA/Oce…

…ananigans.jl into sb-jmc/cubed-sphere-aquaplanet
CliMA · Oct 16, 2024 · a62459d · a62459d
2 parents f517507 + d87710b
commit a62459d
Show file tree

Hide file tree

Showing 271 changed files with 12,906 additions and 5,979 deletions.
diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml
@@ -3,34 +3,26 @@ agents:
   slurm_mem: 8G
   modules: climacommon/2024_05_27
 
-
 env:
   JULIA_LOAD_PATH: "${JULIA_LOAD_PATH}:${BUILDKITE_BUILD_CHECKOUT_PATH}/.buildkite/distributed"
   OPENBLAS_NUM_THREADS: 1
+  JULIA_PKG_SERVER_REGISTRY_PREFERENCE: eager
+  JULIA_NUM_PRECOMPILE_TASKS: 8
+  JULIA_NUM_THREADS: 8
   OMPI_MCA_opal_warn_on_missing_libcuda: 0
 
 steps:
   - label: "initialize"
     key: "init_central"
+    env:
+      TEST_GROUP: "init"
     command:
       - echo "--- Instantiate project"
-      - "julia --project -e 'using Pkg; Pkg.instantiate(;verbose=true); Pkg.precompile(;strict=true)'"
-      # force the initialization of the CUDA runtime as it is lazily loaded by default
-      - "julia --project -e 'using CUDA; CUDA.precompile_runtime(); CUDA.versioninfo()'"
-      - "julia --project -e 'using MPI; MPI.versioninfo()'"
-      # Download artifacts by running an empty testgroup and thereby executing /test/runtests.jl
       - "julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'"
-
-      - echo "--- Instantiate status"
-      - "julia --project -e 'using Pkg; Pkg.status()'"
-
     agents:
       slurm_mem: 120G
       slurm_gpus: 1
       slurm_cpus_per_task: 8
-    env:
-      JULIA_NUM_PRECOMPILE_TASKS: 8
-      TEST_GROUP: "init"
 
   - wait
 
@@ -54,6 +46,11 @@ steps:
       slurm_mem: 120G
       slurm_ntasks: 4
       slurm_gpus_per_task: 1
+    retry:
+      automatic:
+        - exit_status: 1      
+          limit: 1
+
 
   - label: "🦾 cpu distributed solvers tests"
     key: "distributed_solvers_cpu"
@@ -65,6 +62,21 @@ steps:
       slurm_mem: 120G
       slurm_ntasks: 4
 
+  - label: "🛸 gpu distributed solvers tests"
+    key: "distributed_solvers_gpu"
+    env:
+      TEST_GROUP: "distributed_solvers"
+    commands:
+      - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'"
+    agents:
+      slurm_mem: 120G
+      slurm_ntasks: 4
+      slurm_gpus_per_task: 1
+    retry:
+      automatic:
+        - exit_status: 1      
+          limit: 1
+
   - label: "🤺 cpu distributed hydrostatic model tests"
     key: "distributed_hydrostatic_model_cpu"
     env:
@@ -75,7 +87,7 @@ steps:
       slurm_mem: 120G
       slurm_ntasks: 4
 
-  - label: "🕺 gpu distributed hydrostatic model tests"
+  - label: "🦏 gpu distributed hydrostatic model tests"
     key: "distributed_hydrostatic_model_gpu"
     env:
       TEST_GROUP: "distributed_hydrostatic_model"
@@ -85,11 +97,41 @@ steps:
       slurm_mem: 120G
       slurm_ntasks: 4
       slurm_gpus_per_task: 1
-
+    retry:
+      automatic:
+        - exit_status: 1      
+          limit: 1
+
+  - label: "🦍 cpu distributed nonhydrostatic regression"
+    key: "distributed_nonhydrostatic_regression_cpu"
+    env:
+      TEST_GROUP: "distributed_nonhydrostatic_regression"
+    commands:
+      - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'"
+    agents:
+      slurm_mem: 120G
+      slurm_ntasks: 4
+
+  - label: "🕺 gpu distributed nonhydrostatic regression"
+    key: "distributed_nonhydrostatic_regression_gpu"
+    env:
+      TEST_GROUP: "distributed_nonhydrostatic_regression"
+    commands:
+      - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'"
+    agents:
+      slurm_mem: 120G
+      slurm_ntasks: 4
+      slurm_gpus_per_task: 1
+    retry:
+      automatic:
+        - exit_status: 1      
+          limit: 1
+
   - wait
 
   - label: ":chart_with_downwards_trend: build history"
     command:
       - "build_history staging"
     artifact_paths:
       - "build_history.html"
+