[kie-issues#777] Allow restricting jenkins agent labels... (#1165)

This change should allow restricting on which nodes pipeline can run. If environment variable 'FAULTY_NODES' is set or inherited from parent folders, nodes listed here will be added to the label expression to avoid them. This way we can have a single source (KIE folder in jenkins) when we can exclude faulty nodes before Apache admins remove the node, which is sometimes lengthy procedure.
apache · Jan 31, 2024 · 3ca94f0 · 3ca94f0
1 parent ecac201
commit 3ca94f0
Show file tree

Hide file tree

Showing 21 changed files with 71 additions and 20 deletions.
diff --git a/.ci/jenkins/Jenkinsfile.build-kogito-ci-image b/.ci/jenkins/Jenkinsfile.build-kogito-ci-image
@@ -1,12 +1,14 @@
 import org.jenkinsci.plugins.workflow.libs.Library
 
 pipeline {
-    agent {
-        label 'ubuntu'
-    }
     libraries {
         lib("jenkins-pipeline-shared-libraries@${BRANCH_NAME}")
     }
+
+    agent {
+        label util.getLabel('ubuntu')
+    }
+
     environment {
         DOCKER_CONFIG = "${WORKSPACE}/.docker"
 

diff --git a/.ci/jenkins/Jenkinsfile.build-operator-node b/.ci/jenkins/Jenkinsfile.build-operator-node
@@ -4,7 +4,7 @@ import org.jenkinsci.plugins.workflow.libs.Library
 
 pipeline {
     agent {
-        label 'ubuntu'
+        label util.getLabel('ubuntu')
     }
 
     options {

diff --git a/.ci/jenkins/Jenkinsfile.nightly b/.ci/jenkins/Jenkinsfile.nightly
@@ -19,7 +19,7 @@ UNSTABLE_STAGES = [:]
 // Should be multibranch pipeline
 pipeline {
     agent {
-        label 'ubuntu'
+        label util.getLabel('ubuntu')
     }
 
     options {

diff --git a/.ci/jenkins/Jenkinsfile.nightly.cloud b/.ci/jenkins/Jenkinsfile.nightly.cloud
@@ -25,7 +25,7 @@ promoteImageParamsPrefix = 'PROMOTE_IMAGE'
 // Should be multibranch pipeline
 pipeline {
     agent {
-        label 'ubuntu'
+        label util.getLabel('ubuntu')
     }
 
     options {

diff --git a/.ci/jenkins/Jenkinsfile.release b/.ci/jenkins/Jenkinsfile.release
@@ -19,7 +19,7 @@ releaseProperties = [:]
 
 pipeline {
     agent {
-        label 'ubuntu'
+        label util.getLabel('ubuntu')
     }
 
     // parameters {

diff --git a/.ci/jenkins/Jenkinsfile.release.cloud b/.ci/jenkins/Jenkinsfile.release.cloud
@@ -26,7 +26,7 @@ promoteImageParamsPrefix = 'PROMOTE_IMAGE'
 
 pipeline {
     agent {
-        label 'ubuntu'
+        label util.getLabel('ubuntu')
     }
 
     // parameters {

diff --git a/.ci/jenkins/Jenkinsfile.setup-branch b/.ci/jenkins/Jenkinsfile.setup-branch
@@ -12,7 +12,7 @@ UNSTABLE_STAGES = [:]
 
 pipeline {
     agent {
-        label 'ubuntu'
+        label util.getLabel('ubuntu')
     }
 
     options {

diff --git a/.ci/jenkins/Jenkinsfile.setup-branch.cloud b/.ci/jenkins/Jenkinsfile.setup-branch.cloud
@@ -16,7 +16,7 @@ examplesImagesParamsPrefix = 'EXAMPLES_IMAGES'
 
 pipeline {
     agent {
-        label 'ubuntu'
+        label util.getLabel('ubuntu')
     }
 
     options {

diff --git a/.ci/jenkins/Jenkinsfile.tools.update-jenkins-dependencies b/.ci/jenkins/Jenkinsfile.tools.update-jenkins-dependencies
@@ -10,7 +10,7 @@ branchCreated = false
 
 pipeline {
     agent {
-        label 'ubuntu'
+        label util.getLabel('ubuntu')
     }
 
     options {

diff --git a/.ci/jenkins/tests/src/test/groovy/org/kie/jenkins/JenkinsfileNightly.groovy b/.ci/jenkins/tests/src/test/groovy/org/kie/jenkins/JenkinsfileNightly.groovy
@@ -48,6 +48,7 @@ class TestJenkinsfileNightly extends SingleFileDeclarativePipelineTest {
         addEnvVar('STAGE_NAME', 'STAGE_NAME')
         addEnvVar('BUILD_NUMBER', 'BUILD_NUMBER')
         addEnvVar('BUILD_URL', 'BUILD_URL')
+        addEnvVar('FAULTY_NODES', 'faultyNode99')
 
         helper.addShMock('date -u "+%Y-%m-%d"', 'date', 0)
 
@@ -58,6 +59,7 @@ class TestJenkinsfileNightly extends SingleFileDeclarativePipelineTest {
         mockSharedLibVarsCall('githubscm', 'resolveRepository', { repo, author, branch, ignoreErrors, credsId -> registerTestCallstack('githubscm.resolveRepository', "${repo}, ${author}, ${branch}, ${ignoreErrors}, ${credsId}") })
         mockSharedLibVarsCall('githubscm', 'createBranch', { branch -> registerTestCallstack('githubscm.createBranch', "${branch}") })
         mockSharedLibVarsCall('githubscm', 'pushObject', { remote, branch, credsId -> registerTestCallstack('githubscm.pushObject', "${remote}, ${branch}, ${credsId}") })
+        mockSharedLibVarsCall('util', 'getLabel', { label -> return label})
 
     }
     @Test

diff --git a/README.md b/README.md
@@ -14,7 +14,11 @@ This repository contains some of the pipelines of Kogito project.
       - [GitHub Action checks](#github-action-checks)
     - [Sonar cloud](#sonar-cloud)
 - [Configuration of pipelines](#configuration-of-pipelines)
+  - [Source Code](#source-code)
+    - [Pipelines](#pipelines)
+    - [Shared Pipeline libraries](#shared-pipeline-libraries)
   - [Jenkins](#jenkins)
+    - [Jenkins KIE folder](#jenkins-kie-folder)
     - [Jenkins jobs generation](#jenkins-jobs-generation)
   - [Zulip notifications](#zulip-notifications)
     - [Format](#format)
@@ -132,9 +136,35 @@ NOTE: test coverage analysis is executed only by **Jenkins PR simple build&test*
 
 # Configuration of pipelines
 
+## Source Code
+*Note: Creating separate readme.md documenting how-tos and best practices for implementing pipelines might be useful*
+
+### Pipelines
+In this repository two types of pipelines can be found:
+- **Kogito pipelines** (obviously) - located in the [.ci/jenkins](./.ci/jenkins) folder
+- **Seed jobs library** - see [Jenkins documentation](../docs/jenkins.md)
+
+### Shared pipeline libraries
+Apart from these pipelines, the `jenkins-pipeline-shared-libraries` are also stored in this repository. Functions and classes contained in these libraries can be freely used in all pipelines located under [KIE Jenkins folder](https://ci-builds.apache.org/job/KIE). Just include correct import and annotation in your Jenkinsfile:
+```
+import org.jenkinsci.plugins.workflow.libs.Library
+
+@Library('jenkins-pipeline-shared-libraries')_
+```
+For more details please see our [Jenkins pipelines shared libraries documentation](./jenkins-pipeline-shared-libraries/README.md) and [Jenkins.io documentation](https://www.jenkins.io/doc/book/pipeline/shared-libraries/)
+
 ## Jenkins
 
-All pipelines can be found in [kogito Jenkins folder](https://eng-jenkins-csb-business-automation.apps.ocp-c1.prod.psi.redhat.com/job/KIE/job/kogito).
+### Jenkins KIE folder
+
+All KIE jobs (pipelines) can be found  in [KIE Jenkins folder](https://ci-builds.apache.org/job/KIE)<br />
+For this folder and all its descendants there is several useful things set at this folder level:<br />
+- **Pipeline library** - accessible in pipelines under name `jenkins-pipeline-shared-libraries` it gives access to some useful functions used throughout various KIE pipelines. More details can be found in our [Jenkins pipeline shared libraries documentation](./jenkins-pipeline-shared-libraries/README.md) and in the [previous chapter](#source-code)
+- **Environment Variables** - Environment variables set here are inherited by all the folders and jobs located in the [KIE folder](https://ci-builds.apache.org/job/KIE) tree in Jenkins. However, they can be overridden or extended. You can modify the variables by clicking `Configure` in the left menu (if you have necessary permissions). Currently present Environment Variables are:
+  - **FAULTY_NODES** - Comma separated list of Jenkins execution nodes that are faulty in some way and cause KIE jobs to fail. This variable is expected by the *pipeline-library* function `getLabel(String label)`, which extends desired `label` by expression that ensures avoiding these faulty nodes. This way we can increase durability of KIE automation by the time the Apache CI team fixes the issue with faulty node. 
+
+
+All pipelines from this repository can be found in [kogito Jenkins folder](https://ci-builds.apache.org/job/KIE/job/kogito/).
 
 ### Jenkins jobs generation
 

diff --git a/dsl/seed/jenkinsfiles/Jenkinsfile.buildchain b/dsl/seed/jenkinsfiles/Jenkinsfile.buildchain
@@ -49,6 +49,7 @@ pipeline {
     agent {
         docker { 
             image env.AGENT_DOCKER_BUILDER_IMAGE
+            label util.getLabel(agentLabel)
             args env.AGENT_DOCKER_BUILDER_ARGS
         }
     }

diff --git a/dsl/seed/jenkinsfiles/Jenkinsfile.environment.integration-branch b/dsl/seed/jenkinsfiles/Jenkinsfile.environment.integration-branch
@@ -4,7 +4,7 @@ import org.jenkinsci.plugins.workflow.libs.Library
 
 pipeline {
     agent {
-        label 'ubuntu'
+        label util.getLabel('ubuntu')
     }
 
     environment {

diff --git a/dsl/seed/jenkinsfiles/Jenkinsfile.release.prepare b/dsl/seed/jenkinsfiles/Jenkinsfile.release.prepare
@@ -9,7 +9,7 @@ repositoriesToBranch = []
 
 pipeline {
     agent {
-        label 'ubuntu'
+        label util.getLabel('ubuntu')
     }
 
     environment {

diff --git a/dsl/seed/jenkinsfiles/Jenkinsfile.remove.branches b/dsl/seed/jenkinsfiles/Jenkinsfile.remove.branches
@@ -6,7 +6,7 @@ import org.kie.jenkins.MavenCommand
 pipeline {
 
     agent {
-        label 'ubuntu'
+        label util.getLabel('ubuntu')
     }
 
     options{

diff --git a/dsl/seed/jenkinsfiles/Jenkinsfile.seed.branch b/dsl/seed/jenkinsfiles/Jenkinsfile.seed.branch
@@ -28,7 +28,7 @@ SeedJobUtils.createSeedJobTrigger(
 
 pipeline {
     agent {
-        label "${env.AGENT_LABEL}"
+        label util.getLabel(env.AGENT_LABEL)
     }
 
     tools {

diff --git a/dsl/seed/jenkinsfiles/Jenkinsfile.seed.main b/dsl/seed/jenkinsfiles/Jenkinsfile.seed.main
@@ -11,7 +11,7 @@ seedConfig = [:]
 
 pipeline {
     agent {
-        label "${env.AGENT_LABEL}"
+        label util.getLabel(env.AGENT_LABEL)
     }
 
     options {

diff --git a/dsl/seed/jenkinsfiles/Jenkinsfile.seed.trigger b/dsl/seed/jenkinsfiles/Jenkinsfile.seed.trigger
@@ -4,7 +4,7 @@ import org.jenkinsci.plugins.workflow.libs.Library
 
 pipeline {
     agent {
-        label "${env.AGENT_LABEL}"
+        label util.getLabel(env.AGENT_LABEL)
     }
 
     options {

diff --git a/dsl/seed/jenkinsfiles/Jenkinsfile.tools.toggle-triggers b/dsl/seed/jenkinsfiles/Jenkinsfile.tools.toggle-triggers
@@ -5,7 +5,7 @@ import org.kie.jenkins.MavenCommand
 
 pipeline {
     agent {
-        label 'ubuntu'
+        label util.getLabel('ubuntu')
     }
 
     options {

diff --git a/dsl/seed/jenkinsfiles/Jenkinsfile.update-quarkus-version b/dsl/seed/jenkinsfiles/Jenkinsfile.update-quarkus-version
@@ -7,7 +7,7 @@ failingRepos = []
 
 pipeline {
     agent {
-        label 'ubuntu'
+        label util.getLabel('ubuntu')
     }
 
     environment {

diff --git a/jenkins-pipeline-shared-libraries/vars/util.groovy b/jenkins-pipeline-shared-libraries/vars/util.groovy
@@ -555,3 +555,19 @@ void waitForDocker() {
     sleep(10) // give it some ahead time not to invoke docker exec immediately after container start
     sh 'wait-for-docker.sh' // script in kogito-ci-build image itself put in /usr/local/bin
 }
+
+/**
+ * Method to wrap original label and exclude nodes that are marked as faulty in some of the parent folders.
+ * This environment variable is inherited down the folder tree and available in jobs.
+ * @param label Node label to be used
+ * @return Node label extended with an expression ensuring to exclude nodes marked as faulty.
+ */
+String getLabel(String label) {
+    String faultyNodesString = env.FAULTY_NODES
+    if((faultyNodesString == null) || faultyNodesString.isEmpty()) {
+        return label
+    }
+    String[] faultyNodes = faultyNodesString.split(',')
+    String result = "(${label}) && !(${String.join(' || ', faultyNodes)})"
+    return result.toString()
+}