Update radiuss-shared-ci to new release (with radiuss-spack-configs) (L…

…LNL#509)

* Update radiuss-shared-ci to new release (with radiuss-spack-configs)

* Fix

.gitlab-ci.yml

@@ -1,5 +1,5 @@
 ###############################################################################
-# Copyright (c) 2022, Lawrence Livermore National Security, LLC and RADIUSS
+# Copyright (c) 2022-23, Lawrence Livermore National Security, LLC and RADIUSS
 # project contributors. See the COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (MIT)
@@ -11,64 +11,76 @@
 # at Lawrence Livermore National Laboratory (LLNL).
 # This entire pipeline is LLNL-specific
 #
-# Important note: This file is a template provided by
-# llnl/radiuss-shared-ci. It should not require any change from the project to
-# get started but could feature project-specific stages.
+# Important note: This file is a template provided by llnl/radiuss-shared-ci.
+# Remains to set variable values, change the reference to the radiuss-shared-ci
+# repo, opt-in and out optional features. The project can then extend it with
+# additional stages.
 #
-# Instead, each project should provide:
+# In addition, each project should copy over and complete:
 # - .gitlab/custom-jobs-and-variables.yml
 # - .gitlab/subscribed-pipelines.yml
-# - .gitlab/${MACHINE}-build-and-test-extra.yml
+#
+# The jobs should be specified in a file local to the project,
+# - .gitlab/jobs/${CI_MACHINE}.yml
+# or generated (see LLNL/Umpire for an example).
 ###############################################################################
 
 # We define the following GitLab pipeline variables:
 variables:
-# Required information about GitHub repository
-  GITHUB_PROJECT_NAME: "Caliper"
-  GITHUB_PROJECT_ORG: "LLNL"
-# Use an LLNL service user to run CI. This prevents from running pipelines as
-# an actual user.
+##### LC GITLAB CONFIGURATION
+# Use the umdev LLNL service user to run CI. This prevents from running
+# pipelines as an actual user.
   LLNL_SERVICE_USER: ""
 # Use the service user workspace. Solves permission issues, stores everything
 # at the same location whoever triggers a pipeline.
 #  CUSTOM_CI_BUILDS_DIR: ""
 # Tells Gitlab to recursively update the submodules when cloning the project.
   GIT_SUBMODULE_STRATEGY: recursive
+
+##### PROJECT VARIABLES
 # We build the projects in the CI clone directory.
-# TODO: add a clean-up mechanism
+# Used in script/gitlab/build_and_test.sh script.
+# TODO: add a clean-up mechanism.
   BUILD_ROOT: ${CI_PROJECT_DIR}
+
+##### SHARED_CI CONFIGURATION
+# Required information about GitHub repository
+  GITHUB_PROJECT_NAME: "Caliper"
+  GITHUB_PROJECT_ORG: "LLNL"
 # Set the build-and-test command.
-  BUILD_AND_TEST_CMD: "./scripts/gitlab/build-and-test.sh"
-# Override the list of branch that will skip the "draft PR test".
-# Add protected branches here. Defaults to "develop main master".
+  JOB_CMD: "./scripts/gitlab/build-and-test.sh"
+# Override the pattern describing branches that will skip the "draft PR filter
+# test".  Add protected branches here. See default value in
+# preliminary-ignore-draft-pr.yml.
   ALWAYS_RUN_PATTERN: "^develop$|^master$|^v[0-9.]*$|^releases/$"
 
-# We organize the build-and-test stage in sub-pipelines. Each sub-pipeline
+# We organize the build-and-test stage with sub-pipelines. Each sub-pipeline
 # corresponds to a test batch on a given machine.
 
 # High level stages
 stages:
-  - machine-checks
+  - prerequisites
   - build-and-test
 
-# Template for jobs triggering a build-and-test sub-pipelines:
+# Template for jobs triggering a build-and-test sub-pipeline:
 .build-and-test:
   stage: build-and-test
   trigger:
     include:
       - local: '.gitlab/custom-jobs-and-variables.yml'
       - project: 'radiuss/radiuss-shared-ci'
-        ref: v2023.08.0
-        file: '${CI_MACHINE}-build-and-test.yml'
-      - local: '.gitlab/${CI_MACHINE}-build-and-test-extra.yml'
+        ref: 'v2023.09.0'
+        file: 'pipelines/${CI_MACHINE}.yml'
+      - artifact: '${CI_MACHINE}-jobs.yml'
+        job: 'generate-job-lists'
     strategy: depend
     forward:
       pipeline_variables: true
 
 include:
   # [Optional] checks preliminary to running the actual CI test
   - project: 'radiuss/radiuss-shared-ci'
-    ref: v2023.08.0
-    file: 'preliminary-ignore-draft-pr.yml'
+    ref: 'v2023.09.0'
+    file: 'utilities/preliminary-ignore-draft-pr.yml'
   # pipelines subscribed by the project
   - local: '.gitlab/subscribed-pipelines.yml'

.gitlab/custom-jobs-and-variables.yml

@@ -1,5 +1,5 @@
 ###############################################################################
-# Copyright (c) 2022, Lawrence Livermore National Security, LLC and RADIUSS
+# Copyright (c) 2022-23, Lawrence Livermore National Security, LLC and RADIUSS
 # project contributors. See the COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (MIT)
@@ -51,7 +51,6 @@ variables:
 # Project specific deps for lassen
   PROJECT_LASSEN_DEPS: "^adiak ~mpi"
 
-
 # Configuration shared by build and test jobs specific to this project.
 # Not all configuration can be shared. Here projects can fine tune the
 # CI behavior.

.gitlab/subscribed-pipelines.yml

@@ -1,5 +1,5 @@
 ###############################################################################
-# Copyright (c) 2022, Lawrence Livermore National Security, LLC and RADIUSS
+# Copyright (c) 2022-23, Lawrence Livermore National Security, LLC and RADIUSS
 # project contributors. See the COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (MIT)
@@ -8,7 +8,7 @@
 # The template job to test whether a machine is up.
 # Expects CI_MACHINE defined to machine name.
 .machine-check:
-  stage: machine-checks
+  stage: prerequisites
   tags: [shell, oslic]
   variables:
     GIT_STRATEGY: none
@@ -29,6 +29,25 @@
 # Comment the jobs for machines you don’t need.
 ###
 
+# One job to generate the job list for all the subpipelines
+generate-job-lists:
+  stage: prerequisites
+  tags: [shell, oslic]
+  variables:
+    RADIUSS_JOBS_PATH: "scripts/radiuss-spack-configs/gitlab/radiuss-jobs"
+    LOCAL_JOBS_PATH: ".gitlab/jobs"
+  script:
+    - cat ${RADIUSS_JOBS_PATH}/ruby.yml ${LOCAL_JOBS_PATH}/ruby.yml > ruby-jobs.yml
+    - cat ${RADIUSS_JOBS_PATH}/lassen.yml ${LOCAL_JOBS_PATH}/lassen.yml > lassen-jobs.yml
+    - cat ${RADIUSS_JOBS_PATH}/corona.yml ${LOCAL_JOBS_PATH}/corona.yml > corona-jobs.yml
+    - cat ${RADIUSS_JOBS_PATH}/tioga.yml ${LOCAL_JOBS_PATH}/tioga.yml > tioga-jobs.yml
+  artifacts:
+    paths:
+      - ruby-jobs.yml
+      - lassen-jobs.yml
+      - corona-jobs.yml
+      - tioga-jobs.yml
+
 # RUBY
 ruby-up-check:
   variables:
@@ -38,20 +57,20 @@ ruby-up-check:
 ruby-build-and-test:
   variables:
     CI_MACHINE: "ruby"
-  needs: [ruby-up-check]
+  needs: [ruby-up-check, generate-job-lists]
   extends: [.build-and-test]
 
 # Quartz it very similar to ruby, let’s deactivate it for now
 ## QUARTZ
 #quartz-up-check:
 #  variables:
-#    CI_MACHINE: "QUARTZ"
+#    CI_MACHINE: "quartz"
 #  extends: [.machine-check]
 #
 #quartz-build-and-test:
 #  variables:
 #    CI_MACHINE: "quartz"
-#  needs: [quartz-up-check]
+#  needs: [quartz-up-check, generate-job-lists]
 #  extends: [.build-and-test]
 
 # CORONA
@@ -63,7 +82,7 @@ corona-up-check:
 corona-build-and-test:
   variables:
     CI_MACHINE: "corona"
-  needs: [corona-up-check]
+  needs: [corona-up-check, generate-job-lists]
   extends: [.build-and-test]
 
 # TIOGA
@@ -75,7 +94,7 @@ tioga-up-check:
 tioga-build-and-test:
   variables:
     CI_MACHINE: "tioga"
-  needs: [tioga-up-check]
+  needs: [tioga-up-check, generate-job-lists]
   extends: [.build-and-test]
 
 # LASSEN
@@ -87,6 +106,6 @@ lassen-up-check:
 lassen-build-and-test:
   variables:
     CI_MACHINE: "lassen"
-  needs: [lassen-up-check]
+  needs: [lassen-up-check, generate-job-lists]
   extends: [.build-and-test]