From a491806cd90e0aed1d896e0ccdd52cdb59ee2b7b Mon Sep 17 00:00:00 2001 From: Tristan Radisson Date: Tue, 29 Aug 2023 10:58:09 +0200 Subject: [PATCH 01/25] Quarkus 3: Nightly drools should launch kie-jpmml (#1059) * Quarkus 3: Nightly drools should launch kie-jpmml * Update project-dependencies.yaml * Update pull-request-config.yaml --- .ci/jenkins/dsl/jobs.groovy | 2 +- .ci/project-dependencies.yaml | 4 ++++ .ci/pull-request-config.yaml | 8 ++++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.ci/jenkins/dsl/jobs.groovy b/.ci/jenkins/dsl/jobs.groovy index 232595444..379f1911e 100644 --- a/.ci/jenkins/dsl/jobs.groovy +++ b/.ci/jenkins/dsl/jobs.groovy @@ -196,7 +196,7 @@ void setupQuarkus3NightlyJob() { jobParams.env.put('ADDITIONAL_TIMEOUT', '180') jobParams.env.put('BUILD_ENVIRONMENT_OPTIONS_CURRENT', 'rewrite push_changes') jobParams.env.put('INTEGRATION_BRANCH_CURRENT', '9.x') - jobParams.env.put('LAUNCH_DOWNSTREAM_JOBS', 'kogito-runtimes.integration') + jobParams.env.put('LAUNCH_DOWNSTREAM_JOBS', 'kie-jpmml-integration.integration,kogito-runtimes.integration') jobParams.parametersValues.put('SKIP_TESTS', true) jobParams.parametersValues.put('SKIP_INTEGRATION_TESTS', true) return jobParams diff --git a/.ci/project-dependencies.yaml b/.ci/project-dependencies.yaml index 89a3bb8f1..76673f8b4 100644 --- a/.ci/project-dependencies.yaml +++ b/.ci/project-dependencies.yaml @@ -59,3 +59,7 @@ dependencies: exclude: - kiegroup/kogito-apps - kiegroup/kogito-runtimes + - project: kiegroup/kie-jpmml-integration + dependencies: + - project: kiegroup/drools + # no mapping needed diff --git a/.ci/pull-request-config.yaml b/.ci/pull-request-config.yaml index b43215f79..c65b6cd42 100644 --- a/.ci/pull-request-config.yaml +++ b/.ci/pull-request-config.yaml @@ -64,3 +64,11 @@ build: mvn dependency:tree -f ${{ env.KOGITO_EXAMPLES_SUBFOLDER_POM }}pom.xml clean ${{ env.MVN_CMD }} ${{ env.BUILD_MVN_OPTS }} ${{ env.BUILD_MVN_OPTS_CURRENT }} ${{ env.KOGITO_EXAMPLES_BUILD_MVN_OPTS }} upstream: | mvn dependency:tree clean install -DskipTests -DskipITs ${{ env.BUILD_MVN_OPTS }} ${{ env.BUILD_MVN_OPTS_UPSTREAM }} ${{ env.KOGITO_EXAMPLES_BUILD_MVN_OPTS_UPSTREAM }} + + - project: kiegroup/kie-jpmml-integration + build-command: + current: | + export MVN_CMD=`bash -c "if [ '${{ env.ENABLE_DEPLOY }}' = 'true' ]; then printf 'deploy ${{ env.DEPLOY_MVN_OPTS }} ${{ env.KIE_JPMML_INTEGRATION_DEPLOY_MVN_OPTS }}'; else printf 'install'; fi"` + mvn clean ${{ env.MVN_CMD }} ${{ env.BUILD_MVN_OPTS }} ${{ env.BUILD_MVN_OPTS_CURRENT }} ${{ env.KIE_JPMML_INTEGRATION_BUILD_MVN_OPTS }} + upstream: | + mvn clean install -DskipTests ${{ env.BUILD_MVN_OPTS }} ${{ env.BUILD_MVN_OPTS_UPSTREAM }} ${{ env.KIE_JPMML_INTEGRATION_BUILD_MVN_OPTS_UPSTREAM }} From 3273e24e4c6a801787816c2d976a9fd2b8ac6b7c Mon Sep 17 00:00:00 2001 From: Tristan Radisson Date: Fri, 1 Sep 2023 10:28:21 +0200 Subject: [PATCH 02/25] Build Chain: Exclude mapping drools/kie-jpmml-integration (#1062) --- .ci/project-dependencies.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.ci/project-dependencies.yaml b/.ci/project-dependencies.yaml index 76673f8b4..3ccedbab8 100644 --- a/.ci/project-dependencies.yaml +++ b/.ci/project-dependencies.yaml @@ -10,6 +10,8 @@ dependencies: default: - source: (\d*)\.(.*) targetExpression: "process.env.GITHUB_BASE_REF.replace(/(\\d*)\\.(.*)/g, (m, n1, n2) => `${+n1+7}.${n2}`)" + exclude: + - kiegroup/kie-jpmml-integration - project: kiegroup/kogito-runtimes dependencies: From 7b0d12f2ac770f718fe80d50bda22017d5bb2430 Mon Sep 17 00:00:00 2001 From: Jenkins CI Date: Mon, 4 Sep 2023 10:21:52 -0400 Subject: [PATCH 03/25] Add release branch 1.44.x after cut-off --- .ci/jenkins/config/main.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.ci/jenkins/config/main.yaml b/.ci/jenkins/config/main.yaml index ce8ece56f..3bfe3b389 100644 --- a/.ci/jenkins/config/main.yaml +++ b/.ci/jenkins/config/main.yaml @@ -67,6 +67,9 @@ git: branch: seed-kogito-1.43.x config_file: path: dsl/config/branch.yaml + - name: 1.44.x + seed: + branch: seed-kogito-1.44.x seed: config_file: git: From 0bf9b674b4a7ac2927526d8a9b8329afd918592b Mon Sep 17 00:00:00 2001 From: Roberto Oliveira Date: Mon, 4 Sep 2023 17:06:17 +0200 Subject: [PATCH 04/25] update cypress to version 12.17.0 (#1064) --- .ci/jenkins/Jenkinsfile.prod.nightly | 2 +- dsl/seed/jenkinsfiles/Jenkinsfile.buildchain | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/jenkins/Jenkinsfile.prod.nightly b/.ci/jenkins/Jenkinsfile.prod.nightly index 917cec3fb..faa0d6b89 100644 --- a/.ci/jenkins/Jenkinsfile.prod.nightly +++ b/.ci/jenkins/Jenkinsfile.prod.nightly @@ -32,7 +32,7 @@ pipeline { env.DROOLS_PRODUCT_VERSION = "${DROOLS_PRODUCT_VERSION ?: parseVersionFromPom('kiegroup/drools', getDroolsBranch())}" // Needed by Kogito Apps - setupCypressEnv('9.7.0') + setupCypressEnv('12.17.0') } } } diff --git a/dsl/seed/jenkinsfiles/Jenkinsfile.buildchain b/dsl/seed/jenkinsfiles/Jenkinsfile.buildchain index 79889115e..2a02a1f6b 100644 --- a/dsl/seed/jenkinsfiles/Jenkinsfile.buildchain +++ b/dsl/seed/jenkinsfiles/Jenkinsfile.buildchain @@ -75,7 +75,7 @@ pipeline { currentBuild.displayName = params.DISPLAY_NAME ?: env.GIT_BRANCH_NAME ?: currentBuild.displayName // Needed by Kogito Apps - setupCypressEnv('9.7.0') + setupCypressEnv('12.17.0') } } } From 7d47222eb00347594ed1e0c31f4fa4770c4ceeb2 Mon Sep 17 00:00:00 2001 From: Tristan Radisson Date: Tue, 5 Sep 2023 14:49:54 +0200 Subject: [PATCH 05/25] Kogito 2: Fix setup-branch (#1067) --- .ci/jenkins/Jenkinsfile.setup-branch | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.ci/jenkins/Jenkinsfile.setup-branch b/.ci/jenkins/Jenkinsfile.setup-branch index 3388cb097..dcd73b62d 100644 --- a/.ci/jenkins/Jenkinsfile.setup-branch +++ b/.ci/jenkins/Jenkinsfile.setup-branch @@ -91,6 +91,9 @@ pipeline { } stage('Init Docs') { + when { + expression { return isMainStream() } + } steps { script { def buildParams = getDefaultBuildParams() @@ -251,3 +254,7 @@ String getVersionFromReleaseBranch(String releaseBranch, int microVersion = 0, S error 'Cannot parse given branch as a release branch, aka [M].[m].x ...' } } + +boolean isMainStream() { + return env.STREAM == 'main' +} From f5830c9af9b25c1f28644cd6f65b94a67e9c05e3 Mon Sep 17 00:00:00 2001 From: Jenkins CI Date: Thu, 7 Sep 2023 04:04:32 -0400 Subject: [PATCH 06/25] [main] DSL: Remove branch 1.42.x --- .ci/jenkins/config/main.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.ci/jenkins/config/main.yaml b/.ci/jenkins/config/main.yaml index 3bfe3b389..d1299c5a2 100644 --- a/.ci/jenkins/config/main.yaml +++ b/.ci/jenkins/config/main.yaml @@ -57,11 +57,6 @@ git: branch: seed-kogito-1.40.x config_file: path: dsl/config/branch.yaml - - name: 1.42.x - seed: - branch: seed-kogito-1.42.x - config_file: - path: dsl/config/branch.yaml - name: 1.43.x seed: branch: seed-kogito-1.43.x From 79e5c53cacde6bf3914c4db6b3e528775f85d3f5 Mon Sep 17 00:00:00 2001 From: Jenkins CI Date: Thu, 7 Sep 2023 04:12:00 -0400 Subject: [PATCH 07/25] [main] DSL: Remove branch 1.43.x --- .ci/jenkins/config/main.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.ci/jenkins/config/main.yaml b/.ci/jenkins/config/main.yaml index d1299c5a2..9e7ff461c 100644 --- a/.ci/jenkins/config/main.yaml +++ b/.ci/jenkins/config/main.yaml @@ -57,11 +57,6 @@ git: branch: seed-kogito-1.40.x config_file: path: dsl/config/branch.yaml - - name: 1.43.x - seed: - branch: seed-kogito-1.43.x - config_file: - path: dsl/config/branch.yaml - name: 1.44.x seed: branch: seed-kogito-1.44.x From 3e8290f53dcc4b49c4a1c93f613c51a3c74ec3f5 Mon Sep 17 00:00:00 2001 From: Jan Stastny Date: Tue, 19 Sep 2023 20:21:20 +0200 Subject: [PATCH 08/25] kie-issues#574 Initial ASF Jenkins CI Setup (#1090) Co-authored-by: jstastny-cz --- .ci/actions/build-chain/action.yml | 2 +- .ci/actions/dsl-tests/action.yml | 2 +- .ci/actions/os-preparation/action.yaml | 2 +- .ci/chain-status-info.md | 2 +- .ci/jenkins/Jenkinsfile.build-kogito-ci-image | 97 +++ .ci/jenkins/Jenkinsfile.build-operator-node | 16 +- .ci/jenkins/Jenkinsfile.nightly | 4 +- .ci/jenkins/Jenkinsfile.nightly.cloud | 6 +- .../Jenkinsfile.nightly.quarkus-platform | 25 +- .ci/jenkins/Jenkinsfile.prod.nightly | 269 ------- .ci/jenkins/Jenkinsfile.release | 6 +- .ci/jenkins/Jenkinsfile.release.cloud | 6 +- .ci/jenkins/Jenkinsfile.setup-branch | 4 +- .ci/jenkins/Jenkinsfile.setup-branch.cloud | 2 +- .../Jenkinsfile.tools.clean-nightly-images | 7 +- .../Jenkinsfile.tools.clean-old-namespaces | 7 +- .ci/jenkins/Jenkinsfile.tools.create-issue | 10 +- ...kinsfile.tools.update-jenkins-dependencies | 6 +- .ci/jenkins/config/branch.yaml | 119 ++-- .ci/jenkins/config/main.yaml | 60 +- .ci/jenkins/dsl/jobs.groovy | 64 +- .ci/nightly-build-config.yaml | 61 -- .ci/nightly-project-dependencies.yaml | 82 --- .ci/project-dependencies.yaml | 38 +- .ci/pull-request-config.yaml | 22 +- .github/workflows/generate_status_page.yaml | 4 +- .../workflows/generate_status_page_data.yaml | 4 +- .github/workflows/jenkins-dsl-downstream.yml | 20 +- .github/workflows/jenkins-tests.yml | 4 +- README.md | 14 +- apache-nodes/Dockerfile.kogito-ci-build | 114 +++ docs/jenkins.md | 14 +- docs/nightly pipeline.md | 24 +- docs/release pipeline.md | 28 +- dsl/scripts/pr_check.groovy | 135 ++++ dsl/seed/build.gradle | 1 + dsl/seed/gradle.properties | 1 + dsl/seed/jenkinsfiles/Jenkinsfile.buildchain | 221 +++--- ...Jenkinsfile.environment.integration-branch | 2 +- .../jenkinsfiles/Jenkinsfile.prod.prepare | 664 ------------------ .../jenkinsfiles/Jenkinsfile.release.prepare | 6 +- .../jenkinsfiles/Jenkinsfile.remove.branches | 2 +- dsl/seed/jenkinsfiles/Jenkinsfile.seed.branch | 12 +- dsl/seed/jenkinsfiles/Jenkinsfile.seed.main | 6 +- .../jenkinsfiles/Jenkinsfile.seed.trigger | 2 +- .../Jenkinsfile.tools.toggle-triggers | 2 +- .../Jenkinsfile.update-dependency-version | 16 +- .../Jenkinsfile.update-quarkus-platform | 14 +- .../Jenkinsfile.update-quarkus-version | 8 +- dsl/seed/jobs/root_jobs.groovy | 60 -- dsl/seed/jobs/seed_job_branch.groovy | 4 +- dsl/seed/jobs/seed_job_main.groovy | 10 +- dsl/seed/scripts/seed_test.sh | 12 +- .../jenkins/jobdsl/KogitoJobTemplate.groovy | 102 ++- .../kie/jenkins/jobdsl/KogitoJobUtils.groovy | 15 +- .../org/kie/jenkins/jobdsl/Utils.groovy | 30 +- .../jobdsl/utils/JobParamsUtils.groovy | 13 +- .../jenkins/jobdsl/utils/SeedJobUtils.groovy | 5 +- .../jenkins/jobdsl/utils/VersionUtils.groovy | 8 +- .../kie/jenkins/jobdsl/JobScriptsSpec.groovy | 1 - tools/update-kie7-version.sh | 4 +- tools/update-quarkus-versions.sh | 4 +- 62 files changed, 864 insertions(+), 1641 deletions(-) create mode 100644 .ci/jenkins/Jenkinsfile.build-kogito-ci-image delete mode 100644 .ci/jenkins/Jenkinsfile.prod.nightly delete mode 100644 .ci/nightly-build-config.yaml delete mode 100644 .ci/nightly-project-dependencies.yaml create mode 100644 apache-nodes/Dockerfile.kogito-ci-build create mode 100644 dsl/scripts/pr_check.groovy delete mode 100644 dsl/seed/jenkinsfiles/Jenkinsfile.prod.prepare diff --git a/.ci/actions/build-chain/action.yml b/.ci/actions/build-chain/action.yml index bd171e795..83a78dab4 100644 --- a/.ci/actions/build-chain/action.yml +++ b/.ci/actions/build-chain/action.yml @@ -7,7 +7,7 @@ inputs: definition-file: description: 'The `definition-file` input for the build-chain' required: false - default: 'https://raw.githubusercontent.com/${GROUP:kiegroup}/kogito-pipelines/${BRANCH:main}/.ci/pull-request-config.yaml' + default: 'https://raw.githubusercontent.com/${GROUP:apache}/incubator-kie-kogito-pipelines/${BRANCH:main}/.ci/pull-request-config.yaml' flow-type: description: "the flow to execute, it can be 'cross_pr' (or 'pull-request' which is deprecated), 'full_downstream' (or 'full-downstream' which is deprecated), 'single_pr' (or 'single' which is deprecated) or 'branch'" default: "cross_pr" diff --git a/.ci/actions/dsl-tests/action.yml b/.ci/actions/dsl-tests/action.yml index 0f4a330a5..c693a8eb0 100644 --- a/.ci/actions/dsl-tests/action.yml +++ b/.ci/actions/dsl-tests/action.yml @@ -46,7 +46,7 @@ runs: - name: Checkout Seed repo uses: actions/checkout@v3 with: - repository: kiegroup/kogito-pipelines + repository: apache/incubator-kie-kogito-pipelines ref: main path: seed_repo diff --git a/.ci/actions/os-preparation/action.yaml b/.ci/actions/os-preparation/action.yaml index c9d1d8a24..af8004d76 100644 --- a/.ci/actions/os-preparation/action.yaml +++ b/.ci/actions/os-preparation/action.yaml @@ -5,7 +5,7 @@ inputs: repository: description: "The repository containing the matrix-os.json os configuration" required: false - default: "kiegroup/kogito-pipelines" + default: "apache/incubator-kie-kogito-pipelines" branch: description: "The repository branch from which the matrix configuration must be taken" required: false diff --git a/.ci/chain-status-info.md b/.ci/chain-status-info.md index 33cedd822..fe3a3be33 100644 --- a/.ci/chain-status-info.md +++ b/.ci/chain-status-info.md @@ -1,5 +1,5 @@ # Kiegroup Kogito organization repositories CI Status -This project is based on [chain-status](https://github.com/kiegroup/chain-status) and information generated thanks to [build-chain-configuration-reader](https://github.com/kiegroup/build-chain-configuration-reader) using [kogito-pipelines definition file](https://github.com/kiegroup/kogito-pipelines/blob/main/.ci/pull-request-config.yaml). +This project is based on [chain-status](https://github.com/kiegroup/chain-status) and information generated thanks to [build-chain-configuration-reader](https://github.com/kiegroup/build-chain-configuration-reader) using [kogito-pipelines definition file](https://github.com/apache/incubator-kie-kogito-pipelines/blob/main/.ci/pull-request-config.yaml). Due to the retrieved information requires a `GITHUB_TOKEN` and github API has a limitation it is better to not collect the information on every request from this webpage is made. So this webpage does not really requires an application service and all the required data is stored in github pages. This information is normally retrieved, treated and stored using [chain-status/action tool](https://github.com/kiegroup/chain-status/tree/main/packages/action) which is exposed as a Github Action tool, so we can customize the job execution frequency as we need. diff --git a/.ci/jenkins/Jenkinsfile.build-kogito-ci-image b/.ci/jenkins/Jenkinsfile.build-kogito-ci-image new file mode 100644 index 000000000..fba0135fd --- /dev/null +++ b/.ci/jenkins/Jenkinsfile.build-kogito-ci-image @@ -0,0 +1,97 @@ +import org.jenkinsci.plugins.workflow.libs.Library + +@Library('jenkins-pipeline-shared-libraries')_ + +pipeline { + agent { + label 'ubuntu' + } + + environment { + DOCKER_CONFIG = "${WORKSPACE}/.docker" + + IMAGE_NAME = 'quay.io/kiegroup/kogito-ci-build' + IMAGE_TAG = "${BRANCH_NAME}-build-${BUILD_NUMBER}" + IMAGE_NAME_TAG = "${env.IMAGE_NAME}:${env.IMAGE_TAG}" + } + + options { + timeout(time: 60, unit: 'MINUTES') + } + + stages { + stage('Build image') { + steps { + script { + sh 'printenv' + + currentBuild.displayName = env.IMAGE_TAG + + cloud.loginContainerRegistry('quay.io', 'quay_kiegroup_registry_token') + + dir('kogito-pipelines') { + deleteDir() + + checkout scm + + sh """ + docker build --build-arg SDKMAN_JAVA=11.0.20-tem -t ${env.IMAGE_NAME_TAG} -f apache-nodes/Dockerfile.kogito-ci-build . + docker tag ${env.IMAGE_NAME_TAG} ${env.IMAGE_NAME}:${GIT_COMMIT} + docker tag ${env.IMAGE_NAME_TAG} ${env.IMAGE_NAME}:${BRANCH_NAME}-latest + """ + + sh """ + docker --config ${DOCKER_CONFIG} push ${env.IMAGE_NAME_TAG} + docker --config ${DOCKER_CONFIG} push ${env.IMAGE_NAME}:${GIT_COMMIT} + docker --config ${DOCKER_CONFIG} push ${env.IMAGE_NAME}:${BRANCH_NAME}-latest + """ + } + } + } + post { + always { + script { + sh "rm -rf ${DOCKER_CONFIG}" + sh 'docker logout quay.io' + } + } + } + } + stage('Test built image') { + agent { + docker { + image env.IMAGE_NAME_TAG + args '-v /var/run/docker.sock:/var/run/docker.sock --group-add docker --group-add input --group-add render' + } + } + steps { + echo 'Debug basics' + sh ''' + locale + printenv + ''' + + echo 'Test tools' + sh ''' + java -version + mvn --version + python --version + pip --version + node --version + npm --version + go version + s2i version + gh version + kubectl version --client + oc version --client + ''' + + echo 'Test docker' + sh ''' + docker info + docker run hello-world + ''' + } + } + } +} diff --git a/.ci/jenkins/Jenkinsfile.build-operator-node b/.ci/jenkins/Jenkinsfile.build-operator-node index 116500afb..fba8ec3e8 100644 --- a/.ci/jenkins/Jenkinsfile.build-operator-node +++ b/.ci/jenkins/Jenkinsfile.build-operator-node @@ -4,7 +4,7 @@ import org.jenkinsci.plugins.workflow.libs.Library pipeline { agent { - label 'kie-rhel8-priority && !built-in' + label 'ubuntu' } options { @@ -20,7 +20,7 @@ pipeline { steps { script { deleteDir() - checkout(githubscm.resolveRepository('kogito-pipelines', 'kiegroup', 'main', false)) + checkout(githubscm.resolveRepository('incubator-kogito-pipelines', 'apache', 'main', false)) } } } @@ -28,7 +28,7 @@ pipeline { steps { script { dir('kogito-operator-jenkins-node') { - sh 'cekit -v build podman' + sh 'cekit -v build docker' } } } @@ -43,17 +43,17 @@ pipeline { echo "Got version for tagging -> ${version}" sh """ - podman tag kogito-operator-jenkins-node:latest quay.io/kiegroup/kogito-operator-jenkins-node:${version} - podman tag kogito-operator-jenkins-node:latest quay.io/kiegroup/kogito-operator-jenkins-node:latest + docker tag kogito-operator-jenkins-node:latest quay.io/kiegroup/kogito-operator-jenkins-node:${version} + docker tag kogito-operator-jenkins-node:latest quay.io/kiegroup/kogito-operator-jenkins-node:latest """ withCredentials([usernamePassword(credentialsId: 'nightly_kogito', usernameVariable: 'REGISTRY_USER', passwordVariable: 'REGISTRY_PWD')]) { - sh "podman login --tls-verify=false -u ${REGISTRY_USER} -p ${REGISTRY_PWD} quay.io" + sh "docker login -u ${REGISTRY_USER} -p ${REGISTRY_PWD} quay.io" } sh """ - podman push quay.io/kiegroup/kogito-operator-jenkins-node:${version} - podman push quay.io/kiegroup/kogito-operator-jenkins-node:latest + docker push quay.io/kiegroup/kogito-operator-jenkins-node:${version} + docker push quay.io/kiegroup/kogito-operator-jenkins-node:latest """ } } diff --git a/.ci/jenkins/Jenkinsfile.nightly b/.ci/jenkins/Jenkinsfile.nightly index 3a41a7aaf..e8880a1de 100644 --- a/.ci/jenkins/Jenkinsfile.nightly +++ b/.ci/jenkins/Jenkinsfile.nightly @@ -19,7 +19,7 @@ UNSTABLE_STAGES = [:] // Should be multibranch pipeline pipeline { agent { - label 'kie-rhel8 && !built-in' + label 'ubuntu' } options { @@ -118,7 +118,7 @@ pipeline { // Update examples nightly branch dir('kogito-examples') { deleteDir() - checkout(githubscm.resolveRepository('kogito-examples', getGitAuthor(), getBuildBranch(), false, getGitAuthorCredsID())) + checkout(githubscm.resolveRepository('incubator-kie-kogito-examples', getGitAuthor(), getBuildBranch(), false, getGitAuthorCredsID())) String nightlyBranch = "nightly-${getBuildBranch()}" githubscm.createBranch(nightlyBranch) githubscm.pushObject('origin', nightlyBranch, getGitAuthorCredsID()) diff --git a/.ci/jenkins/Jenkinsfile.nightly.cloud b/.ci/jenkins/Jenkinsfile.nightly.cloud index 65015d054..fc6e572ea 100644 --- a/.ci/jenkins/Jenkinsfile.nightly.cloud +++ b/.ci/jenkins/Jenkinsfile.nightly.cloud @@ -25,7 +25,7 @@ promoteImageParamsPrefix = 'PROMOTE_IMAGE' // Should be multibranch pipeline pipeline { agent { - label 'kie-rhel8 && !built-in' + label 'ubuntu' } options { @@ -266,11 +266,11 @@ void addSkipIntegrationTestsParam(buildParams) { void addAppsParam(buildParams) { addStringParam(buildParams, 'APPS_REF', "${getBuildBranch()}") - addStringParam(buildParams, 'APPS_URI', "https://github.com/${getGitAuthor()}/kogito-apps") + addStringParam(buildParams, 'APPS_URI', "https://github.com/${getGitAuthor()}/incubator-kie-kogito-apps") } void addExamplesParam(buildParams) { - addStringParam(buildParams, 'EXAMPLES_URI', "https://github.com/${getGitAuthor()}/kogito-examples") + addStringParam(buildParams, 'EXAMPLES_URI', "https://github.com/${getGitAuthor()}/incubator-kie-kogito-examples") addStringParam(buildParams, 'EXAMPLES_REF', "nightly-${getBuildBranch()}") } diff --git a/.ci/jenkins/Jenkinsfile.nightly.quarkus-platform b/.ci/jenkins/Jenkinsfile.nightly.quarkus-platform index 7485e292d..6797aedd6 100644 --- a/.ci/jenkins/Jenkinsfile.nightly.quarkus-platform +++ b/.ci/jenkins/Jenkinsfile.nightly.quarkus-platform @@ -6,7 +6,10 @@ import org.kie.jenkins.MavenSettingsUtils pipeline { agent { - label 'rhel8 && docker && !built-in' + docker { + image env.AGENT_DOCKER_BUILDER_IMAGE + args env.AGENT_DOCKER_BUILDER_ARGS + } } options { @@ -14,11 +17,6 @@ pipeline { timeout(time: 240, unit: 'MINUTES') } - tools { - maven "${BUILD_MAVEN_TOOL}" - jdk "${BUILD_JDK_TOOL}" - } - environment { // Static env is defined into .jenkins/dsl/jobs.groovy file KOGITO_CI_EMAIL_TO = credentials("${JENKINS_EMAIL_CREDS_ID}") @@ -44,27 +42,27 @@ pipeline { script { dir('drools') { deleteDir() - checkout(githubscm.resolveRepository('drools', getGitAuthor(), getTargetBranch(7), false)) + checkout(githubscm.resolveRepository('incubator-kie-drools', getGitAuthor(), getTargetBranch(7), false)) } dir('kogito-runtimes') { deleteDir() - checkout(githubscm.resolveRepository('kogito-runtimes', getGitAuthor(), getGitBranch(), false)) + checkout(githubscm.resolveRepository('incubator-kie-kogito-runtimes', getGitAuthor(), getGitBranch(), false)) } dir('kogito-apps') { deleteDir() - checkout(githubscm.resolveRepository('kogito-apps', getGitAuthor(), getGitBranch(), false)) + checkout(githubscm.resolveRepository('incubator-kie-kogito-apps', getGitAuthor(), getGitBranch(), false)) } dir('optaplanner') { deleteDir() // Get current corresponding branch and if not working, latest tag String opBranch = getTargetBranch(7) try { - checkout(githubscm.resolveRepository('optaplanner', getGitAuthor(), opBranch, false)) + checkout(githubscm.resolveRepository('incubator-kie-optaplanner', getGitAuthor(), opBranch, false)) } catch(err) { echo "Error checking out ${opBranch}. Fallback to latest tag." String latestTag = githubscm.getLatestTag("${opBranch.split("\\.")[0]}\\.") echo "Got latest tag = ${latestTag}" - checkout(githubscm.resolveRepository('optaplanner', getGitAuthor(), latestTag, false)) + checkout(githubscm.resolveRepository('incubator-kie-optaplanner', getGitAuthor(), latestTag, false)) } } } @@ -156,8 +154,9 @@ pipeline { .run('deploy') // Upload to specific repository with credentials - String mavenDeployRepositoryZipUrl = "${QUARKUS_PLATFORM_NEXUS_URL.replaceAll('/content/', '/service/local/').replaceFirst('/*$', '')}/content-compressed" - maven.uploadLocalArtifacts("${QUARKUS_PLATFORM_NEXUS_CREDS}", getMavenDeployLocalDir(), mavenDeployRepositoryZipUrl) + // Removed deployment to reposiory as no repository defined for now ... + // String mavenDeployRepositoryZipUrl = "${QUARKUS_PLATFORM_NEXUS_URL.replaceAll('/content/', '/service/local/').replaceFirst('/*$', '')}/content-compressed" + // maven.uploadLocalArtifacts("${QUARKUS_PLATFORM_NEXUS_CREDS}", getMavenDeployLocalDir(), mavenDeployRepositoryZipUrl) } } } diff --git a/.ci/jenkins/Jenkinsfile.prod.nightly b/.ci/jenkins/Jenkinsfile.prod.nightly deleted file mode 100644 index faa0d6b89..000000000 --- a/.ci/jenkins/Jenkinsfile.prod.nightly +++ /dev/null @@ -1,269 +0,0 @@ -@Library('jenkins-pipeline-shared-libraries')_ - -import org.kie.jenkins.MavenCommand - -pipeline { - agent { - label 'kie-rhel8 && docker && kie-mem24g && !built-in' - } - tools { - maven 'kie-maven-3.8.7' - jdk 'kie-jdk11' - nodejs 'nodejs-16.2.0' - } - parameters { - string(description: 'The UMB message version', name: 'UMB_VERSION', defaultValue: 'main') - string(description: 'The product version, if not provided the kogito-runtimes main branch one will be use', name: 'PRODUCT_VERSION') - string(description: 'The drools product version, if not provided the drools main branch one will be used', name: 'DROOLS_PRODUCT_VERSION') - string(description: 'The config repository branch', name: 'CONFIG_BRANCH', defaultValue: 'master') - } - options { - buildDiscarder logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10') - } - environment { - DATE_TIME_SUFFIX_FORMAT = 'yyyyMMddHH' - } - stages { - stage('Initialize') { - steps { - script { - // Fetch versions from pom if not provided - env.PRODUCT_VERSION = "${PRODUCT_VERSION ?: parseVersionFromPom('kiegroup/kogito-runtimes', getBuildBranch())}" - env.DROOLS_PRODUCT_VERSION = "${DROOLS_PRODUCT_VERSION ?: parseVersionFromPom('kiegroup/drools', getDroolsBranch())}" - - // Needed by Kogito Apps - setupCypressEnv('12.17.0') - } - } - } - stage('Install build-chain tool') { - steps { - script { - println "[INFO] Installing build-chain tool" - sh "npm install -g @kie/build-chain-action${env.NPM_REGISTRY_URL ? " -registry=${NPM_REGISTRY_URL}" : ''}" - - sh "npm list -g | grep build-chain" - } - } - } - stage('Clone build configuration repo') { - steps { - script { - def currentBranch = getBuildConfigBranch() - println "Checking out ${env.BUILD_CONFIGURATION_REPO_URL}:${currentBranch} into build_config folder" - sh "git clone -b ${currentBranch} --single-branch ${env.BUILD_CONFIGURATION_REPO_URL} build_config" - - // export PME parameters and build scripts from nightly build config - def buildConfigAdditionalVariables = [ - 'productVersion': env.PRODUCT_VERSION, - 'droolsProductVersion': env.DROOLS_PRODUCT_VERSION - ] - pmebuild.parseBuildConfig("$WORKSPACE/build_config/openshift-serverless-logic/nightly", buildConfigAdditionalVariables) - - def PME_BUILD_VARIABLES = env.PME_BUILD_VARIABLES.split(';').collect{ it.split('=')}.inject([:]) {map, item -> map << [(item.length == 2 ? item[0] : null): (item.length == 2 ? item[1] : null)]} - - // export Quarkus community versions - env.QUARKUS_VERSION_COMMUNITY = PME_BUILD_VARIABLES['quarkusVersionCommunity'] - env.QUARKUS_VERSION = PME_BUILD_VARIABLES['quarkusVersion'].replaceAll("\\{\\{.*\\}\\}", env.QUARKUS_VERSION_COMMUNITY) - env.QUARKUS_PLATFORM_VERSION = PME_BUILD_VARIABLES['quarkusPlatformVersion'].replaceAll("\\{\\{.*\\}\\}", env.QUARKUS_VERSION_COMMUNITY) - - // export images registries - env.IMAGE_TAG_VERSION = PME_BUILD_VARIABLES['imageTagVersion'] - env.DATA_INDEX_IMAGE_REGISTRY = PME_BUILD_VARIABLES['dataIndexImageRegistry'].replaceAll("\\{\\{.*\\}\\}", env.IMAGE_TAG_VERSION) - env.SWF_BUILDER_IMAGE_REGISTRY = PME_BUILD_VARIABLES['swfBuilderImageRegistry'].replaceAll("\\{\\{.*\\}\\}", env.IMAGE_TAG_VERSION) - env.SWF_DEVMODE_IMAGE_REGISTRY = PME_BUILD_VARIABLES['swfDevModeImageRegistry'].replaceAll("\\{\\{.*\\}\\}", env.IMAGE_TAG_VERSION) - env.OPERATOR_IMAGE_REGISTRY = PME_BUILD_VARIABLES['operatorImageRegistry'].replaceAll("\\{\\{.*\\}\\}", env.IMAGE_TAG_VERSION) - - sh 'printenv' - } - } - } - stage('Build Nightly projects') { - steps { - script { - def pipelineHelper = new PipelineHelper(this) - pipelineHelper.retry( - { - withCredentials([string(credentialsId: "kie-ci1-token", variable: 'GITHUB_TOKEN')]) { - def SETTINGS_XML_ID = '5d9884a1-178a-4d67-a3ac-9735d2df2cef' - def buildBranch = getBuildBranch() - def definitionFile = "https://raw.githubusercontent.com/\\\${GROUP:kiegroup}/kogito-pipelines/${getDefinitionFileBranch()}/.ci/nightly-build-config.yaml" - - configFileProvider([configFile(fileId: "${SETTINGS_XML_ID}", variable: 'PME_MAVEN_SETTINGS_XML')]) { - sh "build-chain build branch --token=${GITHUB_TOKEN} -f ${definitionFile} -b ${buildBranch} -o bc -p kiegroup/kogito-runtimes --fullProjectDependencyTree --skipParallelCheckout" - } - } - }, 2, 480*60) - } - } - } - stage('Upload maven repository') { - steps { - script { - echo "[INFO] Start uploading ${env.WORKSPACE}/deployDirectory" - if(fileExists("${env.WORKSPACE}/deployDirectory")){ - def destDir = "${env.WORKSPACE}/deployDirectoryFinal/openshift-serverless-logic-${PRODUCT_VERSION}-maven-repository/maven-repository/" - sh "mkdir -p ${destDir}" - sh "mv ${env.WORKSPACE}/deployDirectory/* ${destDir}" - dir("${env.WORKSPACE}/deployDirectoryFinal") { - sh "zip -r maven-repository-${env.DATE_TIME_SUFFIX} ." - def relativeFile = "rhoss/rhoss-logic-${PRODUCT_VERSION}.nightly" - def folder="${env.RCM_GUEST_FOLDER}/${relativeFile}" - - util.withKerberos('rhba-prod-keytab') { - sh "ssh 'rhba-prod@${env.RCM_HOST}' 'mkdir -p ${folder}'" - sh "rsync -rlp --info=progress2 maven-repository-${env.DATE_TIME_SUFFIX}.zip rhba-prod@${env.RCM_HOST}:staging/${relativeFile}" - } - } - } - } - } - } - stage ('Extract Git Information') { - steps { - script { - def projectFolders = sh(returnStdout: true, script: "ls ${env.WORKSPACE}/bc").trim().split("\n") - for (f in projectFolders) { - dir("${env.WORKSPACE}/bc/${f}") { - def projectName = f.replace("_", "/") - // extract git infos - util.storeGitInformation(projectName) - env.ALREADY_BUILT_PROJECTS = "${env.ALREADY_BUILT_PROJECTS ?: ''}${projectName};" - - def sanitizedProjectName = f.replaceAll('/', '_').replaceAll('-', '_') - def buildScript = env["PME_BUILD_SCRIPT_${sanitizedProjectName}"] - def pomPath = maven.getProjectPomFromBuildCmd(buildScript) - env["VERSION_${f}"] = maven.mvnGetVersionProperty(new MavenCommand(this).withProperty('productized'), 'project.version', pomPath) - } - } - - sh 'printenv' - } - } - } - stage ('Send UMB Message to QE.') { - steps { - script { - echo '[INFO] Sending OPENSHIFT SERVERLESS LOGIC UMB message to QE.' - - def mavenRepositoryFileUrl = "${env.STAGING_SERVER_URL}/rhoss/rhoss-logic-${PRODUCT_VERSION}.nightly/maven-repository-${env.DATE_TIME_SUFFIX}.zip" - def topic = "VirtualTopic.qe.ci.ba.openshift-serverless-logic.${env.UMB_VERSION}.nightly.trigger" - def eventType = "openshift-serverless-logic-${env.UMB_VERSION}-nightly-qe-trigger" - def messageBody = getMessageBody( - mavenRepositoryFileUrl, - env.ALREADY_BUILT_PROJECTS, - ['serverlesslogic': env['VERSION_kiegroup_kogito-runtimes'], 'serverlesslogic-rhba': env['VERSION_kiegroup_kogito-runtimes'], 'drools': env.VERSION_kiegroup_drools, 'platform.quarkus.bom': env.QUARKUS_PLATFORM_VERSION, 'quarkus.bom': env.QUARKUS_VERSION], - ['data-index-ephemeral': env.DATA_INDEX_IMAGE_REGISTRY, 'swf-builder': env.SWF_BUILDER_IMAGE_REGISTRY, 'swf-devmode': env.SWF_DEVMODE_IMAGE_REGISTRY, 'operator': env.OPERATOR_IMAGE_REGISTRY], - gitHashesToCollection(env.GIT_INFORMATION_HASHES) - ) - echo "[INFO] Message Body: ${messageBody}" - echo "[INFO] Topic: ${topic}" - echo "[INFO] Event Type: ${eventType}" - build job: env.SEND_UMB_MESSAGE_JOB_PATH, parameters: [ - [$class: 'StringParameterValue', name: 'MESSAGE_BODY', value: messageBody], - [$class: 'StringParameterValue', name: 'TOPIC', value: topic], - [$class: 'StringParameterValue', name: 'EVENT_TYPE', value: eventType] - ] - echo '[SUCCESS] Message was successfully sent.' - } - } - } - } - post { - failure { - emailext body: 'OPENSHIFT SERVERLESS LOGIC ${PRODUCT_VERSION}:nightly-build #${BUILD_NUMBER} was: ' + "${currentBuild.currentResult}" + '\n' + - 'Please look here: ${BUILD_URL} \n' + - ' \n' + - '${BUILD_LOG, maxLines=750}', subject: 'OPENSHIFT SERVERLESS LOGIC ${PRODUCT_VERSION}:nightly-build #${BUILD_NUMBER}: ' + "${currentBuild.currentResult}", to: 'kie-jenkins-builds@redhat.com' - } - unstable { - emailext body: 'OPENSHIFT SERVERLESS LOGIC ${PRODUCT_VERSION}:nightly-build #${BUILD_NUMBER} was: ' + "${currentBuild.currentResult}" + '\n' + - 'Please look here: ${BUILD_URL} \n' + - ' \n' + - 'Failed tests: ${BUILD_URL}/testReport \n' + - ' \n' + - '${BUILD_LOG, maxLines=750}', subject: 'OPENSHIFT SERVERLESS LOGIC ${PRODUCT_VERSION}:nightly-build #${BUILD_NUMBER}: ' + "${currentBuild.currentResult}", to: 'kie-jenkins-builds@redhat.com' - } - fixed { - emailext body: 'OPENSHIFT SERVERLESS LOGIC ${PRODUCT_VERSION}:nightly-build #${BUILD_NUMBER} was: ' + "${currentBuild.currentResult}" + '\n' + - 'Please look here: ${BUILD_URL}', subject: 'OPENSHIFT SERVERLESS LOGIC ${PRODUCT_VERSION}:nightly-build #${BUILD_NUMBER}: ' + "${currentBuild.currentResult}", to: 'kie-jenkins-builds@redhat.com' - } - always { - archiveArtifacts artifacts: '**/*.maven.log', fingerprint: false, defaultExcludes: true, caseSensitive: true, allowEmptyArchive: true - - echo 'Generating JUnit report...' - junit allowEmptyResults: true, healthScaleFactor: 1.0, testResults: '**/target/*-reports/TEST-*.xml' - - echo 'Archiving logs...' - archiveArtifacts excludes: '**/target/checkstyle.log', artifacts: '**/*.maven.log,**/target/*.log', fingerprint: false, defaultExcludes: true, caseSensitive: true, allowEmptyArchive: true - } - cleanup { - cleanWs() - } - } -} - -def getMessageBody(String mavenRepositoryFileUrl, String alreadyBuiltProjects, Map versions, Map imagesRegistries, Map scmHashes) { - def alreadyBuiltProjectsArray = (alreadyBuiltProjects ?: '').split(";") - return """ -{ - "maven_repository_file_url": "${mavenRepositoryFileUrl}", - "version": ${new groovy.json.JsonBuilder(versions).toString()}, - "image": ${new groovy.json.JsonBuilder(imagesRegistries).toString()}, - "scm_hash": ${new groovy.json.JsonBuilder(scmHashes).toString()}, - "built_projects": ${new groovy.json.JsonBuilder(alreadyBuiltProjectsArray).toString()} -}""" -} - -def setupCypressEnv(String cypressVersion) { - if (env.CYPRESS_BINARY_URL) { - env.CYPRESS_INSTALL_BINARY = "${CYPRESS_BINARY_URL}/cypress-${cypressVersion}.zip" - } -} - -def gitHashesToCollection(String gitInformationHashes) { - return gitInformationHashes.replaceAll(/([\w\d\-\_\.]*\/)([\w\d\-\_\.]*)/,'$2').split(';').findAll { it.split('=').size() }.collectEntries{ [it.split('=')[0], it.split('=')[1]] } -} - -String getBuildConfigBranch() { - return env.CONFIG_BRANCH ?: env.DEFAULT_CONFIG_BRANCH ?: env.BRANCH_NAME ?: env.GIT_BRANCH -} - -String getBuildBranch() { - // Fallback to main if none exist - return env.CHANGE_BRANCH ?: env.BRANCH_NAME ?: env.GIT_BRANCH ?: 'main' -} - -String getDroolsBranch() { - return calculateKieRepoBranch(getBuildBranch()) -} - -String getDefinitionFileBranch() { - // Fallback to getBuildBranch if DEFINITION_FILE_BRANCH not set - return env.DEFINITION_FILE_BRANCH ?: getBuildBranch() -} - -String calculateKieRepoBranch(String branch) { - /* The Drools/OptaPlanner major version is shifted by 7 from the Kogito major version: - Kogito 1.x.y -> Drools 8.x.y. - Kogito 1.x.y -> OptaPlanner 8.x.y. */ - int majorVersionShift = 7 - String [] branchSplit = branch.split("\\.") - if (branchSplit.length == 3) { - Integer optaplannerMajorVersion = Integer.parseInt(branchSplit[0]) + majorVersionShift - return "${optaplannerMajorVersion}.${branchSplit[1]}.${branchSplit[2]}" - } else { - return branch - } -} - -// Parse version from main branch of the given project -// * project: in the form of owner/repository -def parseVersionFromPom(String project, String branch) { - def pomFilename = "${project.replaceAll("/", "_")}_pom.xml" - def pomPath = "${env.WORKSPACE}/${pomFilename}" - - sh "curl https://raw.githubusercontent.com/${project}/${branch}/pom.xml -o ${pomPath}" - def pom = readMavenPom file: pomPath - return pom.getVersion().replaceAll('-SNAPSHOT', '') -} diff --git a/.ci/jenkins/Jenkinsfile.release b/.ci/jenkins/Jenkinsfile.release index cb4d2cb95..2eccc43de 100644 --- a/.ci/jenkins/Jenkinsfile.release +++ b/.ci/jenkins/Jenkinsfile.release @@ -19,7 +19,7 @@ releaseProperties = [:] pipeline { agent { - label 'kie-rhel8 && !built-in' + label 'ubuntu' } // parameters { @@ -50,8 +50,8 @@ pipeline { sendNotification("Release Pipeline has started...\nKogito version = ${getKogitoVersion()}\n=> ${env.BUILD_URL}") // Safety measure to not publish to main JBoss - if (getGitAuthor() != 'kiegroup' && !getArtifactsRepositoryParam()) { - sendNotification("Git Author is different from `kiegroup` and no `ARTIFACTS_REPOSITORY` parameter has been provided. Are you sure you want to continue ? => ${env.BUILD_URL}input") + if (getGitAuthor() != 'apache' && !getArtifactsRepositoryParam()) { + sendNotification("Git Author is different from `apache` and no `ARTIFACTS_REPOSITORY` parameter has been provided. Are you sure you want to continue ? => ${env.BUILD_URL}input") input message: 'Should the pipeline continue with no `ARTIFACTS_REPOSITORY` defined ?', ok: 'Yes' } } diff --git a/.ci/jenkins/Jenkinsfile.release.cloud b/.ci/jenkins/Jenkinsfile.release.cloud index e6df13c4f..7019f1491 100644 --- a/.ci/jenkins/Jenkinsfile.release.cloud +++ b/.ci/jenkins/Jenkinsfile.release.cloud @@ -26,7 +26,7 @@ promoteImageParamsPrefix = 'PROMOTE_IMAGE' pipeline { agent { - label 'kie-rhel8 && !built-in' + label 'ubuntu' } // parameters { @@ -72,9 +72,9 @@ pipeline { stage('Setup configuration for testing') { steps { script { - env.EXAMPLES_URI = params.EXAMPLES_URI ?: (getReleaseProperty('examples.uri') ?: "https://github.com/${getGitAuthor()}/kogito-examples") + env.EXAMPLES_URI = params.EXAMPLES_URI ?: (getReleaseProperty('examples.uri') ?: "https://github.com/${getGitAuthor()}/incubator-kie-kogito-examples") env.EXAMPLES_REF = params.EXAMPLES_REF ?: (getReleaseProperty('examples.ref') ?: getKogitoVersion()) - env.APPS_URI = params.APPS_URI ?: (getReleaseProperty('apps.uri') ?: "https://github.com/${getGitAuthor()}/kogito-apps") + env.APPS_URI = params.APPS_URI ?: (getReleaseProperty('apps.uri') ?: "https://github.com/${getGitAuthor()}/incubator-kie-kogito-apps") env.APPS_REF = params.APPS_REF ?: (getReleaseProperty('apps.ref') ?: getKogitoVersion()) echo "Got examples uri ${env.EXAMPLES_URI}" diff --git a/.ci/jenkins/Jenkinsfile.setup-branch b/.ci/jenkins/Jenkinsfile.setup-branch index dcd73b62d..79a59f027 100644 --- a/.ci/jenkins/Jenkinsfile.setup-branch +++ b/.ci/jenkins/Jenkinsfile.setup-branch @@ -12,7 +12,7 @@ UNSTABLE_STAGES = [:] pipeline { agent { - label 'kie-rhel8 && !built-in' + label 'ubuntu' } options { @@ -75,7 +75,7 @@ pipeline { // Update examples nightly branch dir('kogito-examples') { deleteDir() - checkout(githubscm.resolveRepository('kogito-examples', getGitAuthor(), getBuildBranch(), false)) + checkout(githubscm.resolveRepository('incubator-kie-kogito-examples', getGitAuthor(), getBuildBranch(), false)) String nightlyBranch = "nightly-${getBuildBranch()}" githubscm.createBranch(nightlyBranch) githubscm.pushObject('origin', nightlyBranch, getGitAuthorCredsID()) diff --git a/.ci/jenkins/Jenkinsfile.setup-branch.cloud b/.ci/jenkins/Jenkinsfile.setup-branch.cloud index 2f4aa42b2..f03831de0 100644 --- a/.ci/jenkins/Jenkinsfile.setup-branch.cloud +++ b/.ci/jenkins/Jenkinsfile.setup-branch.cloud @@ -16,7 +16,7 @@ examplesImagesParamsPrefix = 'EXAMPLES_IMAGES' pipeline { agent { - label 'kie-rhel8 && !built-in' + label 'ubuntu' } options { diff --git a/.ci/jenkins/Jenkinsfile.tools.clean-nightly-images b/.ci/jenkins/Jenkinsfile.tools.clean-nightly-images index dfc8d941f..c0f4f0c31 100644 --- a/.ci/jenkins/Jenkinsfile.tools.clean-nightly-images +++ b/.ci/jenkins/Jenkinsfile.tools.clean-nightly-images @@ -1,5 +1,10 @@ pipeline { - agent { label 'kogito-cloud && !built-in' } + agent { + docker { + image env.AGENT_DOCKER_BUILDER_IMAGE + args env.AGENT_DOCKER_BUILDER_ARGS + } + } options { timeout(time: 120, unit: 'MINUTES') } diff --git a/.ci/jenkins/Jenkinsfile.tools.clean-old-namespaces b/.ci/jenkins/Jenkinsfile.tools.clean-old-namespaces index 729e77cc7..5826d2ff0 100644 --- a/.ci/jenkins/Jenkinsfile.tools.clean-old-namespaces +++ b/.ci/jenkins/Jenkinsfile.tools.clean-old-namespaces @@ -1,5 +1,10 @@ pipeline { - agent { label 'kogito-cloud && !built-in' } + agent { + docker { + image 'quay.io/kiegroup/kogito-ci-build:latest' // TODO to set in branch config + args '-v /var/run/docker.sock:/var/run/docker.sock --group-add docker --group-add input --group-add render' + } + } options { timeout(time: 120, unit: 'MINUTES') } diff --git a/.ci/jenkins/Jenkinsfile.tools.create-issue b/.ci/jenkins/Jenkinsfile.tools.create-issue index def95ed84..3d384027e 100644 --- a/.ci/jenkins/Jenkinsfile.tools.create-issue +++ b/.ci/jenkins/Jenkinsfile.tools.create-issue @@ -4,17 +4,15 @@ import org.jenkinsci.plugins.workflow.libs.Library pipeline { agent { - label 'rhel8 && !built-in' + docker { + image env.AGENT_DOCKER_BUILDER_IMAGE + args env.AGENT_DOCKER_BUILDER_ARGS + } } - options { timeout(time: 60, unit: 'MINUTES') } - // environment { - // Some generated env is also defined into ./dsl/jobs.groovy file - // } - stages { stage('Checkout repo') { steps { diff --git a/.ci/jenkins/Jenkinsfile.tools.update-jenkins-dependencies b/.ci/jenkins/Jenkinsfile.tools.update-jenkins-dependencies index 49195f668..f9004c581 100644 --- a/.ci/jenkins/Jenkinsfile.tools.update-jenkins-dependencies +++ b/.ci/jenkins/Jenkinsfile.tools.update-jenkins-dependencies @@ -2,11 +2,15 @@ import org.jenkinsci.plugins.workflow.libs.Library @Library('jenkins-pipeline-shared-libraries')_ +// This Jenkinsfile has not been tested on Apache Jenkins yet +// and will likely fail due to restrictions on Groovy sandbox methods +// See `findPlugin` and `getJenkinsVersion` methods + branchCreated = false pipeline { agent { - label 'kie-rhel8 && !built-in' + label 'ubuntu' } options { diff --git a/.ci/jenkins/config/branch.yaml b/.ci/jenkins/config/branch.yaml index 8d907eb9f..d1509a9fa 100644 --- a/.ci/jenkins/config/branch.yaml +++ b/.ci/jenkins/config/branch.yaml @@ -6,18 +6,6 @@ environments: ADDITIONAL_TIMEOUT: 720 ids: - native - native-lts: - env_vars: - NATIVE: true - BUILD_MVN_OPTS: -Dproductized -Ddata-index-ephemeral.image=quay.io/kiegroup/kogito-data-index-ephemeral - BUILD_MVN_OPTS_CURRENT: -Dnative -Dquarkus.native.container-build=true - QUARKUS_BRANCH: '2.13' - ADDITIONAL_TIMEOUT: 720 - BUILD_ENVIRONMENT: quarkus-lts - ids: - - native - - prod - - lts quarkus-main: enabled: false env_vars: @@ -32,21 +20,12 @@ environments: quarkus-3: pull_request_default_check: false env_vars: - BUILD_MAVEN_TOOL: kie-maven-3.9.1 - BUILD_JDK_TOOL: kie-jdk17 + BUILD_MAVEN_TOOL: maven_3.9.3 + BUILD_JDK_TOOL: jdk_17_latest BUILD_MVN_OPTS: -Denforcer.skip ids: - quarkus - quarkus3 - quarkus-lts: - pull_request_default_check: false - env_vars: - BUILD_MVN_OPTS: -Dproductized -Ddata-index-ephemeral.image=quay.io/kiegroup/kogito-data-index-ephemeral - QUARKUS_BRANCH: '2.13' - ids: - - quarkus - - lts - - prod sonarcloud: auto_generation: false env_vars: @@ -58,35 +37,38 @@ environments: auto_generation: false ids: - ecosystem -productized_branch: true disable: - triggers: false + triggers: true # TODO to set back + deploy: true repositories: -- name: kogito-pipelines -- name: kogito-runtimes -- name: kogito-apps -- name: kogito-examples -- name: kogito-images -- name: kogito-operator -- name: kogito-serverless-operator -- name: kie-tools -- name: kogito-docs -- name: kie-docs +- name: incubator-kie-kogito-pipelines + job_display_name: kogito-pipelines +- name: incubator-kie-kogito-runtimes + job_display_name: kogito-runtimes +- name: incubator-kie-kogito-apps + job_display_name: kogito-apps +- name: incubator-kie-kogito-examples + job_display_name: kogito-examples +- name: incubator-kie-kogito-images + job_display_name: kogito-images +- name: incubator-kie-kogito-operator + job_display_name: kogito-operator +- name: incubator-kie-kogito-serverless-operator + job_display_name: kogito-serverless-operator +- name: incubator-kie-kogito-docs + job_display_name: kogito-docs +# - name: incubator-kie-tools // Commented as this one will be treated at another time +# job_display_name: kie-tools +- name: incubator-kie-docs + job_display_name: kie-docs branch: main-kogito -productized_repositories: -- name: kogito-pipelines -- name: kogito-runtimes -- name: kogito-apps -- name: kogito-examples -- name: kogito-images -- name: kogito-serverless-operator -- name: kogito-operator -- name: kogito-docs git: author: - name: kiegroup - credentials_id: kie-ci3 - token_credentials_id: kie-ci3-token + name: apache + # Taken from https://ci-builds.apache.org/credentials/ + # Need to be verified + credentials_id: 399061d0-5ab5-4142-a186-a52081fef742 + token_credentials_id: ci-builds fork_author: name: kie-ci credentials_id: kie-ci @@ -98,34 +80,41 @@ git: jenkins_config_path: .ci/jenkins buildchain_config: git: - repository: kogito-pipelines + repository: incubator-kie-kogito-pipelines file_path: .ci/pull-request-config.yaml maven: - settings_file_id: kogito_release_settings + settings_file_id: kie-release-settings nexus: - release_url: https://repository.jboss.org/nexus - release_repository: jboss-releases-repository - staging_profile_url: https://repository.jboss.org/nexus/content/groups/kogito-public/ - staging_profile_id: 1976dd1c18392a - build_promotion_profile_id: ea49ccd6f174 + release_url: TO_DEFINE + release_repository: TO_DEFINE + staging_profile_url: TO_DEFINE + staging_profile_id: TO_DEFINE + build_promotion_profile_id: TO_DEFINE artifacts_repository: '' artifacts_upload_repository: - url: https://repository.jboss.org/nexus/content/repositories/snapshots/ - creds_id: kogito-snapshots-upload + url: TO_DEFINE + creds_id: TO_DEFINE quarkus_platform_repository: - url: https://repository.jboss.org/nexus/content/repositories/kogito-internal-repository/ - creds_id: kogito-internal-nexus-creds + url: TO_DEFINE + creds_id: TO_DEFINE cloud: image: - registry_credentials_nightly: nightly_kogito - registry_credentials_release: release_kogito + registry_credentials: tradisso_registry_token # TODO set to `kogito-quay-token` registry: quay.io - namespace: kiegroup + namespace: tradisso # TODO set to `kiegroup` after testing latest_git_branch: main jenkins: - email_creds_id: KOGITO_CI_EMAIL_TO + email_creds_id: KOGITO_CI_NOTIFICATION_EMAILS + agent: + docker: + builder: + # At some point, this image will need to be changed when a release branch is created + # but we need to make sure the image exists first ... simple tag before setting up the branch ? + # See https://github.com/kiegroup/kie-issues/issues/551 + image: quay.io/kiegroup/kogito-ci-build:main-latest + args: -v /var/run/docker.sock:/var/run/docker.sock --group-add docker --group-add input --group-add render default_tools: - jdk: kie-jdk11 - maven: kie-maven-3.8.7 - sonar_jdk: kie-jdk17 + jdk: jdk_11_latest + maven: maven_3.8.6 + sonar_jdk: jdk_17_latest jobs_definition_file: .ci/jenkins/dsl/jobs.groovy diff --git a/.ci/jenkins/config/main.yaml b/.ci/jenkins/config/main.yaml index 9e7ff461c..dab62f910 100644 --- a/.ci/jenkins/config/main.yaml +++ b/.ci/jenkins/config/main.yaml @@ -7,68 +7,30 @@ ecosystem: default_main_version: 2.0.0-SNAPSHOT regexs: - kogito.* - - name: kie-tools - ignore_release: true - ignore_productization: true - regexs: - - kie-tools.* + - incubator-kie-kogito.* + # - name: kie-tools + # ignore_release: true + # regexs: + # - kie-tools.* + # - incubator-kie-tools.* - name: kie-docs ignore_release: true - ignore_productization: true regexs: - kie-docs.* + - incubator-kie-docs.* git: branches: - name: main main_branch: true - seed: - branch: main - - name: 1.13.x - old_folder_structure: true - seed: - branch: 1.13.x - config_file: - path: dsl/seed/config/branch.yaml - jenkinsfile: dsl/seed/jobs/Jenkinsfile.seed.branch - - name: 1.13.x-blue - old_folder_structure: true - seed: - branch: 1.13.x-blue - config_file: - path: dsl/seed/config/branch.yaml - jenkinsfile: dsl/seed/jobs/Jenkinsfile.seed.branch - - name: 1.29.x - seed: - branch: seed-kogito-1.29.x - config_file: - path: dsl/config/branch.yaml - - name: 1.38.x-prod - seed: - branch: seed-kogito-1.38.x - config_file: - path: dsl/config/branch.yaml - - name: 1.40.x - seed: - branch: seed-kogito-1.40.x - config_file: - path: dsl/config/branch.yaml - - name: 1.40.x-prod - seed: - branch: seed-kogito-1.40.x - config_file: - path: dsl/config/branch.yaml - - name: 1.44.x - seed: - branch: seed-kogito-1.44.x seed: config_file: git: - repository: kogito-pipelines + repository: incubator-kie-kogito-pipelines author: - name: kiegroup - credentials_id: kie-ci3 + name: apache + credentials_id: ASF_Cloudbees_Jenkins_ci-builds branch: main path: .ci/jenkins/config/branch.yaml jenkinsfile: dsl/seed/jenkinsfiles/Jenkinsfile.seed.branch jenkins: - email_creds_id: KOGITO_CI_EMAIL_TO + email_creds_id: KOGITO_CI_NOTIFICATION_EMAILS diff --git a/.ci/jenkins/dsl/jobs.groovy b/.ci/jenkins/dsl/jobs.groovy index 379f1911e..1935d46c1 100644 --- a/.ci/jenkins/dsl/jobs.groovy +++ b/.ci/jenkins/dsl/jobs.groovy @@ -5,7 +5,7 @@ import org.kie.jenkins.jobdsl.utils.VersionUtils import org.kie.jenkins.jobdsl.KogitoJobUtils import org.kie.jenkins.jobdsl.Utils -JENKINSFILE_PATH = '.ci/jenkins' +jenkins_path = '.ci/jenkins' boolean isMainStream() { return Utils.getStream(this) == 'main' @@ -57,27 +57,31 @@ KogitoJobUtils.createEnvironmentIntegrationBranchNightlyJob(this, 'quarkus-3', [ setupReleaseArtifactsJob() setupReleaseCloudJob() +Utils.isMainBranch(this) && KogitoJobTemplate.createBranchMultibranchPipelineJob(this, 'kogito-ci-build-image', "${jenkins_path}/Jenkinsfile.build-kogito-ci-image") + ///////////////////////////////////////////////////////////////// // Methods ///////////////////////////////////////////////////////////////// void setupCleanOldNamespacesToolsJob() { - def jobParams = JobParamsUtils.getBasicJobParams(this, 'kogito-clean-old-namespaces', JobType.TOOLS, "${JENKINSFILE_PATH}/Jenkinsfile.tools.clean-old-namespaces") + def jobParams = JobParamsUtils.getBasicJobParams(this, 'kogito-clean-old-namespaces', JobType.TOOLS, "${jenkins_path}/Jenkinsfile.tools.clean-old-namespaces") jobParams.triggers = [ cron : '@midnight' ] KogitoJobTemplate.createPipelineJob(this, jobParams) } void setupCleanOldNightlyImagesToolsJob() { - jobParams = JobParamsUtils.getBasicJobParams(this, 'kogito-clean-old-nightly-images', JobType.TOOLS, "${JENKINSFILE_PATH}/Jenkinsfile.tools.clean-nightly-images") + jobParams = JobParamsUtils.getBasicJobParams(this, 'kogito-clean-old-nightly-images', JobType.TOOLS, "${jenkins_path}/Jenkinsfile.tools.clean-nightly-images") jobParams.triggers = [ cron : 'H 8 * * *' ] + JobParamsUtils.setupJobParamsAgentDockerBuilderImageConfiguration(this, jobParams) KogitoJobTemplate.createPipelineJob(this, jobParams) } void setupCreateIssueToolsJob() { - jobParams = JobParamsUtils.getBasicJobParams(this, 'kogito-create-issue', JobType.TOOLS, "${JENKINSFILE_PATH}/Jenkinsfile.tools.create-issue") + jobParams = JobParamsUtils.getBasicJobParams(this, 'kogito-create-issue', JobType.TOOLS, "${jenkins_path}/Jenkinsfile.tools.create-issue") jobParams.env.putAll([ GITHUB_CLI_PATH: '/opt/tools/gh-cli/bin/gh', ]) + JobParamsUtils.setupJobParamsAgentDockerBuilderImageConfiguration(this, jobParams) KogitoJobTemplate.createPipelineJob(this, jobParams)?.with { parameters { stringParam('AUTHOR', '', 'Git author') @@ -90,10 +94,9 @@ void setupCreateIssueToolsJob() { } void setupUpdateJenkinsDependenciesJob() { - jobParams = JobParamsUtils.getBasicJobParams(this, 'jenkins-update-framework-deps', JobType.TOOLS, "${JENKINSFILE_PATH}/Jenkinsfile.tools.update-jenkins-dependencies", 'Nightly check of Jenkins dependencies from framework against current version of Jenkins') + jobParams = JobParamsUtils.getBasicJobParams(this, 'jenkins-update-framework-deps', JobType.TOOLS, "${jenkins_path}/Jenkinsfile.tools.update-jenkins-dependencies", 'Nightly check of Jenkins dependencies from framework against current version of Jenkins') jobParams.triggers = [cron : '@midnight'] jobParams.env.putAll([ - REPO_NAME: 'kogito-pipelines', JENKINS_EMAIL_CREDS_ID: "${JENKINS_EMAIL_CREDS_ID}", BUILD_BRANCH_NAME: "${GIT_BRANCH}", @@ -104,7 +107,7 @@ void setupUpdateJenkinsDependenciesJob() { } void createSetupBranchJob() { - def jobParams = JobParamsUtils.getBasicJobParams(this, '0-setup-branch', JobType.SETUP_BRANCH, "${JENKINSFILE_PATH}/Jenkinsfile.setup-branch", 'Kogito Setup Branch for Artifacts') + def jobParams = JobParamsUtils.getBasicJobParams(this, '0-setup-branch', JobType.SETUP_BRANCH, "${jenkins_path}/Jenkinsfile.setup-branch", 'Kogito Setup Branch for Artifacts') jobParams.env.putAll([ JENKINS_EMAIL_CREDS_ID: "${JENKINS_EMAIL_CREDS_ID}", @@ -123,7 +126,7 @@ void createSetupBranchJob() { } void createSetupBranchCloudJob() { - def jobParams = JobParamsUtils.getBasicJobParams(this, '0-setup-branch-cloud', JobType.SETUP_BRANCH, "${JENKINSFILE_PATH}/Jenkinsfile.setup-branch.cloud", 'Kogito Setup Branch for Cloud') + def jobParams = JobParamsUtils.getBasicJobParams(this, '0-setup-branch-cloud', JobType.SETUP_BRANCH, "${jenkins_path}/Jenkinsfile.setup-branch.cloud", 'Kogito Setup Branch for Cloud') jobParams.env.putAll([ JENKINS_EMAIL_CREDS_ID: "${JENKINS_EMAIL_CREDS_ID}", @@ -140,7 +143,7 @@ void createSetupBranchCloudJob() { } void setupNightlyJob() { - def jobParams = JobParamsUtils.getBasicJobParams(this, '0-kogito-nightly', JobType.NIGHTLY, "${JENKINSFILE_PATH}/Jenkinsfile.nightly", 'Kogito Nightly') + def jobParams = JobParamsUtils.getBasicJobParams(this, '0-kogito-nightly', JobType.NIGHTLY, "${jenkins_path}/Jenkinsfile.nightly", 'Kogito Nightly') jobParams.triggers = [cron : isMainStream () ? '@midnight' : 'H 4 * * *'] jobParams.env.putAll([ JENKINS_EMAIL_CREDS_ID: "${JENKINS_EMAIL_CREDS_ID}", @@ -158,7 +161,7 @@ void setupNightlyJob() { } void setupNightlyCloudJob() { - def jobParams = JobParamsUtils.getBasicJobParams(this, '0-kogito-nightly-cloud', JobType.NIGHTLY, "${JENKINSFILE_PATH}/Jenkinsfile.nightly.cloud", 'Kogito Nightly') + def jobParams = JobParamsUtils.getBasicJobParams(this, '0-kogito-nightly-cloud', JobType.NIGHTLY, "${jenkins_path}/Jenkinsfile.nightly.cloud", 'Kogito Nightly') jobParams.env.putAll([ JENKINS_EMAIL_CREDS_ID: "${JENKINS_EMAIL_CREDS_ID}", @@ -166,7 +169,7 @@ void setupNightlyCloudJob() { GIT_AUTHOR: "${GIT_AUTHOR_NAME}", GIT_AUTHOR_CREDS_ID: "${GIT_AUTHOR_CREDENTIALS_ID}", - IMAGE_REGISTRY_CREDENTIALS: "${CLOUD_IMAGE_REGISTRY_CREDENTIALS_NIGHTLY}", + IMAGE_REGISTRY_CREDENTIALS: "${CLOUD_IMAGE_REGISTRY_CREDENTIALS}", IMAGE_REGISTRY: "${CLOUD_IMAGE_REGISTRY}", IMAGE_NAMESPACE: "${CLOUD_IMAGE_NAMESPACE}", BRANCH_FOR_LATEST: "${CLOUD_IMAGE_LATEST_GIT_BRANCH}", @@ -191,32 +194,33 @@ void setupQuarkus3NightlyJob() { // Tests are done on 9.x/2.x branch => Create 2.x branch on Kogito // Need to split as Drools and Kogito end up in different integration branches KogitoJobUtils.createNightlyBuildChainIntegrationJob(this, 'quarkus-3', 'drools', true) { script -> - def jobParams = JobParamsUtils.getDefaultJobParams(script, 'drools') + def jobParams = JobParamsUtils.getDefaultJobParams(script, 'incubator-kie-drools') jobParams.git.branch = VersionUtils.getProjectTargetBranch('drools', Utils.getGitBranch(this), Utils.getRepoName(this)) jobParams.env.put('ADDITIONAL_TIMEOUT', '180') jobParams.env.put('BUILD_ENVIRONMENT_OPTIONS_CURRENT', 'rewrite push_changes') jobParams.env.put('INTEGRATION_BRANCH_CURRENT', '9.x') - jobParams.env.put('LAUNCH_DOWNSTREAM_JOBS', 'kie-jpmml-integration.integration,kogito-runtimes.integration') - jobParams.parametersValues.put('SKIP_TESTS', true) - jobParams.parametersValues.put('SKIP_INTEGRATION_TESTS', true) - return jobParams - } - KogitoJobUtils.createBuildChainIntegrationJob(this, 'quarkus-3', 'kie-jpmml-integration', true) { script -> - def jobParams = JobParamsUtils.getDefaultJobParams(script, 'kie-jpmml-integration') - jobParams.env.put('ADDITIONAL_TIMEOUT', '180') - jobParams.env.put('BUILD_ENVIRONMENT_OPTIONS_CURRENT', 'rewrite push_changes') - jobParams.env.put('INTEGRATION_BRANCH_CURRENT', '9.x') + // jobParams.env.put('LAUNCH_DOWNSTREAM_JOBS', 'kie-jpmml-integration.integration,kogito-runtimes.integration') + jobParams.env.put('LAUNCH_DOWNSTREAM_JOBS', 'kogito-runtimes.integration') jobParams.parametersValues.put('SKIP_TESTS', true) jobParams.parametersValues.put('SKIP_INTEGRATION_TESTS', true) return jobParams } + // Commented as not migrated to Apache + // KogitoJobUtils.createBuildChainIntegrationJob(this, 'quarkus-3', 'kie-jpmml-integration', true) { script -> + // def jobParams = JobParamsUtils.getDefaultJobParams(script, 'incubator-kie-jpmml-integration') + // jobParams.env.put('ADDITIONAL_TIMEOUT', '180') + // jobParams.env.put('BUILD_ENVIRONMENT_OPTIONS_CURRENT', 'rewrite push_changes') + // jobParams.env.put('INTEGRATION_BRANCH_CURRENT', '9.x') + // jobParams.parametersValues.put('SKIP_TESTS', true) + // jobParams.parametersValues.put('SKIP_INTEGRATION_TESTS', true) + // return jobParams + // } KogitoJobUtils.createBuildChainIntegrationJob(this, 'quarkus-3', 'kogito-runtimes', true) { script -> - def jobParams = JobParamsUtils.getDefaultJobParams(script, 'kogito-runtimes') + def jobParams = JobParamsUtils.getDefaultJobParams(script, 'incubator-kie-kogito-runtimes') jobParams.env.put('ADDITIONAL_TIMEOUT', '720') jobParams.env.put('BUILD_ENVIRONMENT_OPTIONS_CURRENT', 'rewrite push_changes') jobParams.env.put('INTEGRATION_BRANCH_CURRENT', '2.x') jobParams.env.put('BUILDCHAIN_FULL_BRANCH_DOWNSTREAM_BUILD', 'true') - jobParams.env.put('NODE_OPTIONS', '--max_old_space_size=4096') jobParams.parametersValues.put('SKIP_TESTS', true) jobParams.parametersValues.put('SKIP_INTEGRATION_TESTS', true) return jobParams @@ -224,8 +228,8 @@ void setupQuarkus3NightlyJob() { } void setupQuarkusPlatformJob(JobType jobType) { - def jobParams = JobParamsUtils.getBasicJobParams(this, 'quarkus-platform.deploy', jobType, "${JENKINSFILE_PATH}/Jenkinsfile.nightly.quarkus-platform", 'Kogito Quarkus platform job') - JobParamsUtils.setupJobParamsDefaultMavenConfiguration(this, jobParams) + def jobParams = JobParamsUtils.getBasicJobParams(this, 'quarkus-platform.deploy', jobType, "${jenkins_path}/Jenkinsfile.nightly.quarkus-platform", 'Kogito Quarkus platform job') + JobParamsUtils.setupJobParamsAgentDockerBuilderImageConfiguration(this, jobParams) jobParams.env.putAll([ JENKINS_EMAIL_CREDS_ID: "${JENKINS_EMAIL_CREDS_ID}", @@ -239,7 +243,7 @@ void setupQuarkusPlatformJob(JobType jobType) { } void setupReleaseArtifactsJob() { - def jobParams = JobParamsUtils.getBasicJobParams(this, '0-kogito-release', JobType.RELEASE, "${JENKINSFILE_PATH}/Jenkinsfile.release", 'Kogito Artifacts Release') + def jobParams = JobParamsUtils.getBasicJobParams(this, '0-kogito-release', JobType.RELEASE, "${jenkins_path}/Jenkinsfile.release", 'Kogito Artifacts Release') jobParams.env.putAll([ JENKINS_EMAIL_CREDS_ID: "${JENKINS_EMAIL_CREDS_ID}", @@ -265,14 +269,14 @@ void setupReleaseArtifactsJob() { } void setupReleaseCloudJob() { - def jobParams = JobParamsUtils.getBasicJobParams(this, '0-kogito-release-cloud', JobType.RELEASE, "${JENKINSFILE_PATH}/Jenkinsfile.release.cloud", 'Kogito Cloud Release') + def jobParams = JobParamsUtils.getBasicJobParams(this, '0-kogito-release-cloud', JobType.RELEASE, "${jenkins_path}/Jenkinsfile.release.cloud", 'Kogito Cloud Release') jobParams.env.putAll([ JENKINS_EMAIL_CREDS_ID: "${JENKINS_EMAIL_CREDS_ID}", GIT_BRANCH_NAME: "${GIT_BRANCH}", GIT_AUTHOR: "${GIT_AUTHOR_NAME}", - IMAGE_REGISTRY_CREDENTIALS: "${CLOUD_IMAGE_REGISTRY_CREDENTIALS_RELEASE}", + IMAGE_REGISTRY_CREDENTIALS: "${CLOUD_IMAGE_REGISTRY_CREDENTIALS}", IMAGE_REGISTRY: "${CLOUD_IMAGE_REGISTRY}", IMAGE_NAMESPACE: "${CLOUD_IMAGE_NAMESPACE}", BRANCH_FOR_LATEST: "${CLOUD_IMAGE_LATEST_GIT_BRANCH}", @@ -306,7 +310,7 @@ void setupReleaseCloudJob() { } void setupBuildOperatorNode() { - def jobParams = JobParamsUtils.getBasicJobParams(this, 'build-operator-node', JobType.TOOLS, "${JENKINSFILE_PATH}/Jenkinsfile.build-operator-node") + def jobParams = JobParamsUtils.getBasicJobParams(this, 'build-operator-node', JobType.TOOLS, "${jenkins_path}/Jenkinsfile.build-operator-node") KogitoJobTemplate.createPipelineJob(this, jobParams) } diff --git a/.ci/nightly-build-config.yaml b/.ci/nightly-build-config.yaml deleted file mode 100644 index 9613fbb85..000000000 --- a/.ci/nightly-build-config.yaml +++ /dev/null @@ -1,61 +0,0 @@ -version: "2.1" - -dependencies: ./nightly-project-dependencies.yaml - -pre: | - export PME_CMD="java -jar ${{ env.PME_CLI_PATH }} -s ${{ env.PME_MAVEN_SETTINGS_XML }} -DallowConfigFilePrecedence=true -DprojectSrcSkip=false" - echo "PME_CMD=${{ env.PME_CMD }}" - export BUILD_MVN_OPTS="${{ env.BUILD_MVN_OPTS }} -s ${{ env.PME_MAVEN_SETTINGS_XML }}" - echo "BUILD_MVN_OPTS=${{ env.BUILD_MVN_OPTS }}" - export ALIGN_QUARKUS="mvn versions:compare-dependencies -DremotePom=io.quarkus:quarkus-bom:${{ env.QUARKUS_VERSION_COMMUNITY }} -DupdatePropertyVersions=true -DupdateDependencies=true -DgenerateBackupPoms=false -Dproductized -s ${{ env.PME_MAVEN_SETTINGS_XML }}" - echo "ALIGN_QUARKUS=${{ env.ALIGN_QUARKUS }}" - -default: - build-command: - current: mvn clean install -Dproductized=true -DskipTests=true ${{ env.BUILD_MVN_OPTS }} - upstream: mvn clean install -Dquickly -Dproductized=true -DskipTests=true ${{ env.BUILD_MVN_OPTS }} - downstream: mvn clean install -Dquickly -Dproductized=true -DskipTests=true ${{ env.BUILD_MVN_OPTS }} - after: - current: | - docker system prune -f - -build: - - project: kiegroup/drools - build-command: - upstream: | - ${{ env.PME_CMD }} ${{ env.PME_ALIGNMENT_PARAMS_kiegroup_drools }} - bash -c "set -o pipefail ; ${{ env.ALIGN_QUARKUS }} -pl :drools-build-parent | tee ${{ env.WORKSPACE }}/drools.quarkus-alignment.maven.log" - bash -c "set -o pipefail ; ${{ env.PME_BUILD_SCRIPT_kiegroup_drools }} ${{ env.BUILD_MVN_OPTS }} | tee ${{ env.WORKSPACE }}/drools.maven.log" - - - project: kiegroup/kogito-runtimes - build-command: - current: | - ${{ env.PME_CMD }} ${{ env.PME_ALIGNMENT_PARAMS_kiegroup_kogito_runtimes }} - bash -c "set -o pipefail ; ${{ env.ALIGN_QUARKUS }} -pl :kogito-dependencies-bom | tee ${{ env.WORKSPACE }}/kogito_runtimes.quarkus-alignment.maven.log" - bash -c "set -o pipefail ; ${{ env.PME_BUILD_SCRIPT_kiegroup_kogito_runtimes }} ${{ env.BUILD_MVN_OPTS }} | tee ${{ env.WORKSPACE }}/kogito_runtimes.maven.log" - - - project: kiegroup/kogito-apps - build-command: - downstream: | - ${{ env.PME_CMD }} ${{ env.PME_ALIGNMENT_PARAMS_kiegroup_kogito_apps }} - bash -c "set -o pipefail ; ${{ env.ALIGN_QUARKUS }} -pl :kogito-apps-build-parent | tee ${{ env.WORKSPACE }}/kogito_apps.quarkus-alignment.maven.log" - bash -c "set -o pipefail ; ${{ env.PME_BUILD_SCRIPT_kiegroup_kogito_apps }} ${{ env.BUILD_MVN_OPTS }} | tee ${{ env.WORKSPACE }}/kogito_apps.maven.log" - - - project: kiegroup/kogito-examples - build-command: - downstream: | - ${{ env.PME_CMD }} ${{ env.PME_ALIGNMENT_PARAMS_kiegroup_kogito_examples }} - bash -c "set -o pipefail ; ${{ env.ALIGN_QUARKUS }} | tee ${{ env.WORKSPACE }}/kogito_examples.quarkus-alignment.maven.log" - bash -c "set -o pipefail ; ${{ env.PME_BUILD_SCRIPT_kiegroup_kogito_examples }} ${{ env.BUILD_MVN_OPTS }} | tee ${{ env.WORKSPACE }}/kogito_examples.maven.log" - - - project: kiegroup/kogito-images - build-command: - downstream: | - export MAVEN_OPTIONS="-s ${{ env.PME_MAVEN_SETTINGS_XML }}" - echo "MAVEN_OPTIONS=${{ env.MAVEN_OPTIONS }}" - export NIGHTLY="true" - echo "NIGHTLY=${{ env.NIGHTLY }}" - export NIGHTLY_DEPLOY_FOLDER="${{ env.WORKSPACE }}/deployDirectory" - echo "NIGHTLY_DEPLOY_FOLDER=${{ env.NIGHTLY_DEPLOY_FOLDER }}" - ${{ env.PME_CMD }} ${{ env.PME_ALIGNMENT_PARAMS_kiegroup_kogito_images }} - bash -c "set -o pipefail ; ${{ env.PME_BUILD_SCRIPT_kiegroup_kogito_images }} ${{ env.BUILD_MVN_OPTS }} | tee ${{ env.WORKSPACE }}/kogito_images.maven.log" \ No newline at end of file diff --git a/.ci/nightly-project-dependencies.yaml b/.ci/nightly-project-dependencies.yaml deleted file mode 100644 index 93da3ce4d..000000000 --- a/.ci/nightly-project-dependencies.yaml +++ /dev/null @@ -1,82 +0,0 @@ -version: "2.1" - -dependencies: - - project: kiegroup/drools - mapping: - dependencies: - default: - - source: (\d*)\.(.*)\.(.*) - targetExpression: "process.env.GITHUB_BASE_REF.replace(/(\\d*)\\.(.*)\\.(.*)/g, (m, n1, n2, n3) => `${+n1-7}.${n2}.${n3}`)" - dependant: - default: - - source: (\d*)\.(.*)\.(.*) - targetExpression: "process.env.GITHUB_BASE_REF.replace(/(\\d*)\\.(.*)\\.(.*)/g, (m, n1, n2, n3) => `${+n1+7}.${n2}.${n3}`)" - - - project: kiegroup/kogito-runtimes - dependencies: - - project: kiegroup/drools - mapping: - dependencies: - default: - - source: (\d*)\.(.*)\.(.*) - targetExpression: "process.env.GITHUB_BASE_REF.replace(/(\\d*)\\.(.*)\\.(.*)/g, (m, n1, n2, n3) => `${+n1+7}.${n2}.${n3}`)" - dependant: - default: - - source: (\d*)\.(.*)\.(.*) - targetExpression: "process.env.GITHUB_BASE_REF.replace(/(\\d*)\\.(.*)\\.(.*)/g, (m, n1, n2, n3) => `${+n1-7}.${n2}.${n3}`)" - exclude: - - kiegroup/kogito-examples - - kiegroup/kogito-apps - - kiegroup/kogito-images - - - project: kiegroup/kogito-apps - dependencies: - - project: kiegroup/kogito-runtimes - mapping: - dependencies: - default: - - source: (\d*)\.(.*)\.(.*) - targetExpression: "process.env.GITHUB_BASE_REF.replace(/(\\d*)\\.(.*)\\.(.*)/g, (m, n1, n2, n3) => `${+n1+7}.${n2}.${n3}`)" - dependant: - default: - - source: (\d*)\.(.*)\.(.*) - targetExpression: "process.env.GITHUB_BASE_REF.replace(/(\\d*)\\.(.*)\\.(.*)/g, (m, n1, n2, n3) => `${+n1-7}.${n2}.${n3}`)" - exclude: - - kiegroup/kogito-examples - - kiegroup/kogito-runtimes - - kiegroup/kogito-images - - - project: kiegroup/kogito-examples - dependencies: - - project: kiegroup/kogito-runtimes - - project: kiegroup/kogito-apps - mapping: - dependencies: - default: - - source: (\d*)\.(.*)\.(.*) - targetExpression: "process.env.GITHUB_BASE_REF.replace(/(\\d*)\\.(.*)\\.(.*)/g, (m, n1, n2, n3) => `${+n1+7}.${n2}.${n3}`)" - dependant: - default: - - source: (\d*)\.(.*)\.(.*) - targetExpression: "process.env.GITHUB_BASE_REF.replace(/(\\d*)\\.(.*)\\.(.*)/g, (m, n1, n2, n3) => `${+n1-7}.${n2}.${n3}`)" - exclude: - - kiegroup/kogito-apps - - kiegroup/kogito-runtimes - - kiegroup/kogito-images - - - project: kiegroup/kogito-images - dependencies: - - project: kiegroup/kogito-examples - mapping: - dependencies: - default: - - source: (\d*)\.(.*)\.(.*) - targetExpression: "process.env.GITHUB_BASE_REF.replace(/(\\d*)\\.(.*)\\.(.*)/g, (m, n1, n2, n3) => `${+n1+7}.${n2}.${n3}`)" - dependant: - default: - - source: (\d*)\.(.*)\.(.*) - targetExpression: "process.env.GITHUB_BASE_REF.replace(/(\\d*)\\.(.*)\\.(.*)/g, (m, n1, n2, n3) => `${+n1-7}.${n2}.${n3}`)" - exclude: - - kiegroup/kogito-runtimes - - kiegroup/kogito-apps - - kiegroup/kogito-examples \ No newline at end of file diff --git a/.ci/project-dependencies.yaml b/.ci/project-dependencies.yaml index 3ccedbab8..7599df54b 100644 --- a/.ci/project-dependencies.yaml +++ b/.ci/project-dependencies.yaml @@ -1,6 +1,6 @@ version: "2.1" dependencies: - - project: kiegroup/drools + - project: apache/incubator-kie-drools mapping: dependencies: default: @@ -10,12 +10,12 @@ dependencies: default: - source: (\d*)\.(.*) targetExpression: "process.env.GITHUB_BASE_REF.replace(/(\\d*)\\.(.*)/g, (m, n1, n2) => `${+n1+7}.${n2}`)" - exclude: - - kiegroup/kie-jpmml-integration + # exclude: + # - kiegroup/kie-jpmml-integration - - project: kiegroup/kogito-runtimes + - project: apache/incubator-kie-kogito-runtimes dependencies: - - project: kiegroup/drools + - project: apache/incubator-kie-drools mapping: dependencies: default: @@ -26,12 +26,12 @@ dependencies: - source: (\d*)\.(.*) targetExpression: "process.env.GITHUB_BASE_REF.replace(/(\\d*)\\.(.*)/g, (m, n1, n2) => `${+n1-7}.${n2}`)" exclude: - - kiegroup/kogito-examples - - kiegroup/kogito-apps + - apache/incubator-kie-kogito-examples + - apache/incubator-kie-kogito-apps - - project: kiegroup/kogito-apps + - project: apache/incubator-kie-kogito-apps dependencies: - - project: kiegroup/kogito-runtimes + - project: apache/incubator-kie-kogito-runtimes mapping: dependencies: default: @@ -42,13 +42,13 @@ dependencies: - source: (\d*)\.(.*) targetExpression: "process.env.GITHUB_BASE_REF.replace(/(\\d*)\\.(.*)/g, (m, n1, n2) => `${+n1-7}.${n2}`)" exclude: - - kiegroup/kogito-examples - - kiegroup/kogito-runtimes + - apache/incubator-kie-kogito-examples + - apache/incubator-kie-kogito-runtimes - - project: kiegroup/kogito-examples + - project: apache/incubator-kie-kogito-examples dependencies: - - project: kiegroup/kogito-runtimes - - project: kiegroup/kogito-apps + - project: apache/incubator-kie-kogito-runtimes + - project: apache/incubator-kie-kogito-apps mapping: dependencies: default: @@ -59,9 +59,9 @@ dependencies: - source: (\d*)\.(.*) targetExpression: "process.env.GITHUB_BASE_REF.replace(/(\\d*)\\.(.*)/g, (m, n1, n2) => `${+n1-7}.${n2}`)" exclude: - - kiegroup/kogito-apps - - kiegroup/kogito-runtimes - - project: kiegroup/kie-jpmml-integration - dependencies: - - project: kiegroup/drools + - apache/incubator-kie-kogito-apps + - apache/incubator-kie-kogito-runtimes + # - project: kiegroup/kie-jpmml-integration + # dependencies: + # - project: apache/incubator-kie-drools # no mapping needed diff --git a/.ci/pull-request-config.yaml b/.ci/pull-request-config.yaml index c65b6cd42..f63835e8f 100644 --- a/.ci/pull-request-config.yaml +++ b/.ci/pull-request-config.yaml @@ -25,7 +25,7 @@ default: mvn dependency:tree clean install -Dquickly ${{ env.BUILD_MVN_OPTS }} ${{ env.BUILD_MVN_OPTS_UPSTREAM }} build: - - project: kiegroup/drools + - project: apache/incubator-kie-drools build-command: current: | export MVN_CMD=`bash -c "if [ '${{ env.ENABLE_DEPLOY }}' = 'true' ]; then printf 'deploy ${{ env.DEPLOY_MVN_OPTS }} ${{ env.DROOLS_DEPLOY_MVN_OPTS }}'; else printf 'install'; fi"` @@ -33,7 +33,7 @@ build: upstream: | mvn dependency:tree clean install -Dquickly ${{ env.BUILD_MVN_OPTS }} ${{ env.BUILD_MVN_OPTS_UPSTREAM }} ${{ env.DROOLS_BUILD_MVN_OPTS_UPSTREAM }} - - project: kiegroup/kogito-runtimes + - project: apache/incubator-kie-kogito-runtimes build-command: current: | export MVN_CMD=`bash -c "if [ '${{ env.ENABLE_DEPLOY }}' = 'true' ]; then printf 'deploy ${{ env.DEPLOY_MVN_OPTS }} ${{ env.KOGITO_RUNTIMES_DEPLOY_MVN_OPTS }}'; else printf 'install'; fi"` @@ -41,7 +41,7 @@ build: upstream: | mvn dependency:tree clean install -Dquickly -Dfull ${{ env.BUILD_MVN_OPTS }} ${{ env.BUILD_MVN_OPTS_UPSTREAM }} ${{ env.KOGITO_RUNTIMES_BUILD_MVN_OPTS_UPSTREAM }} - - project: kiegroup/kogito-apps + - project: apache/incubator-kie-kogito-apps build-command: current: | export MVN_CMD=`bash -c "if [ '${{ env.ENABLE_DEPLOY }}' = 'true' ]; then printf 'deploy ${{ env.DEPLOY_MVN_OPTS }} ${{ env.KOGITO_APPS_DEPLOY_MVN_OPTS }}'; else printf 'install'; fi"` @@ -54,7 +54,7 @@ build: **/cypress/screenshots/** **/cypress/videos/** - - project: kiegroup/kogito-examples + - project: apache/incubator-kie-kogito-examples build-command: # First install the main pom # Then build the required submodule pom @@ -65,10 +65,10 @@ build: upstream: | mvn dependency:tree clean install -DskipTests -DskipITs ${{ env.BUILD_MVN_OPTS }} ${{ env.BUILD_MVN_OPTS_UPSTREAM }} ${{ env.KOGITO_EXAMPLES_BUILD_MVN_OPTS_UPSTREAM }} - - project: kiegroup/kie-jpmml-integration - build-command: - current: | - export MVN_CMD=`bash -c "if [ '${{ env.ENABLE_DEPLOY }}' = 'true' ]; then printf 'deploy ${{ env.DEPLOY_MVN_OPTS }} ${{ env.KIE_JPMML_INTEGRATION_DEPLOY_MVN_OPTS }}'; else printf 'install'; fi"` - mvn clean ${{ env.MVN_CMD }} ${{ env.BUILD_MVN_OPTS }} ${{ env.BUILD_MVN_OPTS_CURRENT }} ${{ env.KIE_JPMML_INTEGRATION_BUILD_MVN_OPTS }} - upstream: | - mvn clean install -DskipTests ${{ env.BUILD_MVN_OPTS }} ${{ env.BUILD_MVN_OPTS_UPSTREAM }} ${{ env.KIE_JPMML_INTEGRATION_BUILD_MVN_OPTS_UPSTREAM }} + # - project: kiegroup/kie-jpmml-integration + # build-command: + # current: | + # export MVN_CMD=`bash -c "if [ '${{ env.ENABLE_DEPLOY }}' = 'true' ]; then printf 'deploy ${{ env.DEPLOY_MVN_OPTS }} ${{ env.KIE_JPMML_INTEGRATION_DEPLOY_MVN_OPTS }}'; else printf 'install'; fi"` + # mvn clean ${{ env.MVN_CMD }} ${{ env.BUILD_MVN_OPTS }} ${{ env.BUILD_MVN_OPTS_CURRENT }} ${{ env.KIE_JPMML_INTEGRATION_BUILD_MVN_OPTS }} + # upstream: | + # mvn clean install -DskipTests ${ env.BUILD_MVN_OPTS }} ${{ env.BUILD_MVN_OPTS_UPSTREAM }} ${{ env.KIE_JPMML_INTEGRATION_BUILD_MVN_OPTS_UPSTREAM }} diff --git a/.github/workflows/generate_status_page.yaml b/.github/workflows/generate_status_page.yaml index f6cef9f1e..008392828 100644 --- a/.github/workflows/generate_status_page.yaml +++ b/.github/workflows/generate_status_page.yaml @@ -5,7 +5,7 @@ on: workflow_dispatch jobs: generate-status-page: - if: github.repository_owner == 'kiegroup' + if: github.repository_owner == 'apache' concurrency: group: generate-status-page cancel-in-progress: true @@ -15,5 +15,5 @@ jobs: - name: Generate status page uses: kiegroup/chain-status/.ci/actions/generate-app@main with: - info-md-url: "https://raw.githubusercontent.com/kiegroup/kogito-pipelines/main/.ci/chain-status-info.md" + info-md-url: "https://raw.githubusercontent.com/apache/incubator-kie-kogito-pipelines/main/.ci/chain-status-info.md" github-token: "${{ secrets.GITHUB_TOKEN }}" diff --git a/.github/workflows/generate_status_page_data.yaml b/.github/workflows/generate_status_page_data.yaml index c313cad67..293f7deb1 100644 --- a/.github/workflows/generate_status_page_data.yaml +++ b/.github/workflows/generate_status_page_data.yaml @@ -7,7 +7,7 @@ on: jobs: generate-status-page-data: - if: github.repository_owner == 'kiegroup' + if: github.repository_owner == 'apache' concurrency: group: generate-status-page-data cancel-in-progress: true @@ -32,7 +32,7 @@ jobs: with: projects: "${{ steps.retrieve_active_repos.outputs.result }}" title: Kogito Status - subtitle: Kiegroup Kogito organization repositories CI status + subtitle: Apache Kogito organization repositories CI status base-branch-filter: "${{ steps.retrieve_active_branches.outputs.result }}" branches: 1.13.x,1.13.x-blue created-by: GitHub Action diff --git a/.github/workflows/jenkins-dsl-downstream.yml b/.github/workflows/jenkins-dsl-downstream.yml index 30e2618bc..509c834dd 100644 --- a/.github/workflows/jenkins-dsl-downstream.yml +++ b/.github/workflows/jenkins-dsl-downstream.yml @@ -37,7 +37,7 @@ jobs: - name: Checkout Main Config repo from main branch uses: actions/checkout@v3 with: - repository: kiegroup/kogito-pipelines + repository: apache/incubator-kie-kogito-pipelines ref: main path: main_config_repo @@ -156,9 +156,9 @@ jobs: with: project: kogito repository: ${{ matrix.repository }} - main-config-file-repo: kiegroup/kogito-pipelines + main-config-file-repo: apache/incubator-kie-kogito-pipelines main-config-file-path: .ci/jenkins/config/main.yaml - branch-config-file-repo: kiegroup/kogito-pipelines + branch-config-file-repo: apache/incubator-kie-kogito-pipelines base-branch: ${{ env.BASE_BRANCH }} ############################################################################# @@ -193,7 +193,7 @@ jobs: - name: Checkout Main Config repo from main branch uses: actions/checkout@v3 with: - repository: kiegroup/optaplanner + repository: apache/incubator-kie-optaplanner ref: main path: main_config_repo @@ -311,7 +311,7 @@ jobs: echo 'BASE_BRANCH=${{ github.base_ref }}' >> $GITHUB_ENV - name: Setup correct base branch for optaplanner-quickstarts - if: github.base_ref == 'main' && matrix.repository == 'optaplanner-quickstarts' + if: github.base_ref == 'main' && matrix.repository == 'incubator-kie-optaplanner-quickstarts' run: | echo 'BASE_BRANCH=development' >> $GITHUB_ENV @@ -320,9 +320,9 @@ jobs: with: project: optaplanner repository: ${{ matrix.repository }} - main-config-file-repo: kiegroup/optaplanner + main-config-file-repo: apache/incubator-kie-optaplanner main-config-file-path: .ci/jenkins/config/main.yaml - branch-config-file-repo: kiegroup/optaplanner + branch-config-file-repo: apache/incubator-kie-optaplanner base-branch: ${{ env.BASE_BRANCH }} ############################################################################# @@ -357,7 +357,7 @@ jobs: - name: Checkout Main Config repo from main branch uses: actions/checkout@v3 with: - repository: kiegroup/drools + repository: apache/incubator-kie-drools ref: main path: main_config_repo @@ -479,7 +479,7 @@ jobs: with: project: drools repository: ${{ matrix.repository }} - main-config-file-repo: kiegroup/drools + main-config-file-repo: apache/incubator-kie-drools main-config-file-path: .ci/jenkins/config/main.yaml - branch-config-file-repo: kiegroup/drools + branch-config-file-repo: apache/incubator-kie-drools base-branch: ${{ env.BASE_BRANCH }} diff --git a/.github/workflows/jenkins-tests.yml b/.github/workflows/jenkins-tests.yml index e774d6957..7624e4394 100644 --- a/.github/workflows/jenkins-tests.yml +++ b/.github/workflows/jenkins-tests.yml @@ -49,7 +49,7 @@ jobs: - name: DSL tests uses: kiegroup/kie-ci/.ci/actions/dsl-tests@main with: - main-config-file-repo: kiegroup/kogito-pipelines + main-config-file-repo: apache/incubator-kie-kogito-pipelines main-config-file-path: .ci/jenkins/config/main.yaml - branch-config-file-repo: kiegroup/kogito-pipelines + branch-config-file-repo: apache/incubator-kie-kogito-pipelines \ No newline at end of file diff --git a/README.md b/README.md index a917bff3f..d11695de4 100644 --- a/README.md +++ b/README.md @@ -24,12 +24,12 @@ This repository contains some of the pipelines of Kogito project. Apart from this repository, pipelines are also concerning those repositories: -* [kogito-runtimes](https://github.com/kiegroup/kogito-runtimes) -* [kogito-apps](https://github.com/kiegroup/kogito-apps) -* [kogito-examples](https://github.com/kiegroup/kogito-examples) -* [kogito-images](https://github.com/kiegroup/kogito-images) -* [kogito-operator](https://github.com/kiegroup/kogito-operator) -* [kie-tools](https://github.com/kiegroup/kie-tools) +* [kogito-runtimes](https://github.com/apache/incubator-kie-kogito-runtimes) +* [kogito-apps](https://github.com/apache/incubator-kie-kogito-apps) +* [kogito-examples](https://github.com/apache/incubator-kie-kogito-examples) +* [kogito-images](https://github.com/apache/incubator-kie-kogito-images) +* [kogito-operator](https://github.com/apache/incubator-kie-kogito-operator) +* [kie-tools](https://github.com/apache/incubator-kie-tools) # The different Kogito pipelines @@ -122,7 +122,7 @@ Jenkins PR checks are of 3 different types: Each repository has a different yaml files in `.github/workflows` folder to configure the workflow. -We are additionally using [`composite actions`](https://docs.github.com/en/actions/creating-actions/creating-a-composite-action) to centralized most common steps used by the different Kogito repositories' jobs. You can check the different kind of composite actions we have available at [`.ci/actions` folder](https://github.com/kiegroup/kogito-pipelines/tree/main/.ci/actions). +We are additionally using [`composite actions`](https://docs.github.com/en/actions/creating-actions/creating-a-composite-action) to centralized most common steps used by the different Kogito repositories' jobs. You can check the different kind of composite actions we have available at [`.ci/actions` folder](https://github.com/apache/incubator-kie-kogito-pipelines/tree/main/.ci/actions). After the build, test results are parsed and logged using the [`action-surefire-report`](https://github.com/ScaCap/action-surefire-report) action. diff --git a/apache-nodes/Dockerfile.kogito-ci-build b/apache-nodes/Dockerfile.kogito-ci-build new file mode 100644 index 000000000..d5cafc03b --- /dev/null +++ b/apache-nodes/Dockerfile.kogito-ci-build @@ -0,0 +1,114 @@ +FROM registry.access.redhat.com/ubi9/ubi:9.2 + +SHELL ["/bin/bash", "-c"] + +ARG SDKMAN_JAVA="17.0.8-tem" +ARG SDKMAN_MAVEN="3.8.8" +ARG PYTHON_MAJOR_VERSION="3" +ARG PYTHON_MAJOR_MINOR_VERSION="3.11" + +# set locale to C.UTF-8 +ENV LANG='C.UTF-8' + +RUN dnf -y update && dnf install -y \ +# skdman deps (BEGIN) +git \ +findutils \ +tar \ +gzip \ +unzip \ +zip \ +# skdman deps (END) +# python3 (BEGIN) +python${PYTHON_MAJOR_MINOR_VERSION} \ +python${PYTHON_MAJOR_MINOR_VERSION}-devel \ +python${PYTHON_MAJOR_MINOR_VERSION}-pip \ +python${PYTHON_MAJOR_VERSION}-gssapi \ +krb5-devel \ +gcc \ +# python3 (END) +# system (BEGIN) +shadow-utils \ +sudo \ +wget \ +which \ +# system (END) +&& dnf clean all + +RUN sudo alternatives --install /usr/local/bin/python python $(which python${PYTHON_MAJOR_MINOR_VERSION}) 1 && \ + sudo alternatives --install /usr/local/bin/pip pip $(which pip${PYTHON_MAJOR_MINOR_VERSION}) 1 + +RUN groupadd -g 910 nonrootuser && useradd -u 910 -g 910 nonrootuser && \ + echo "nonrootuser ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers + +# Docker +RUN dnf -y update && dnf install -y yum-utils device-mapper-persistent-data lvm2 && \ + dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo && \ + dnf remove podman buildah && \ + dnf install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin --nobest && \ + dnf clean all + +USER nonrootuser + +RUN curl -s "https://get.sdkman.io" | bash && \ + source "$HOME/.sdkman/bin/sdkman-init.sh" && \ + sdk install java ${SDKMAN_JAVA} && \ + sudo alternatives --install /usr/local/bin/java java $(which java) 1 && \ + sdk install maven ${SDKMAN_MAVEN} && \ + sudo alternatives --install /usr/local/bin/mvn mvn $(which mvn) 1 && \ + sdk flush + +RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.5/install.sh | bash && \ + source ~/.bashrc && nvm install "v16.20.0" && \ + sudo alternatives --install /usr/local/bin/node node $(which node) 1 && \ + sudo alternatives --install /usr/local/bin/npm npm $(which npm) 1 + +RUN wget https://go.dev/dl/go1.19.10.linux-amd64.tar.gz -P /tmp && \ + sudo mkdir /opt/golang && \ + sudo tar -C /opt/golang -xzf /tmp/go*.tar.gz && \ + sudo chown -R nonrootuser:nonrootuser /opt/golang/go && \ + sudo chmod -R 755 /opt/golang/go && \ + echo 'export GOPATH=${HOME}/go' | sudo tee /etc/profile.d/go.sh && \ + echo "source /etc/profile.d/go.sh" >> $HOME/.bashrc && \ + rm -rf /tmp/go* && \ + sudo alternatives --install /usr/local/bin/go go /opt/golang/go/bin/go 1 + +# Cekit +RUN pip3.11 install cekit==4.8.0 behave lxml docker docker-squash elementPath pyyaml ruamel.yaml python-dateutil Jinja2 pykwalify colorlog click +RUN wget https://github.com/openshift/source-to-image/releases/download/v1.3.8/source-to-image-v1.3.8-980ca195-linux-amd64.tar.gz -P /tmp && \ + tmp_dir=$(mktemp -d) && \ + tar -C ${tmp_dir} -xzvf /tmp/source-to-image-v1.3.8-980ca195-linux-amd64.tar.gz && \ + ls ${tmp_dir} && \ + sudo mv ${tmp_dir}/s2i /usr/local/bin/ && \ + rm -rf ${tmp_dir} /tmp/source-to-image/* + +# gh cli +RUN sudo dnf config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo && \ + sudo dnf install -y gh + +# Install kubectl +RUN wget https://dl.k8s.io/release/$(curl -Ls https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl -P /tmp && \ + chmod +x /tmp/kubectl && \ + sudo mv /tmp/kubectl /usr/local/bin/kubectl + +# Install oc client +RUN wget https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest-4.13/openshift-client-linux.tar.gz -P /tmp && \ + tmp_dir=$(mktemp -d) && \ + tar -C ${tmp_dir} -xvf /tmp/openshift-client-linux.tar.gz && \ + sudo mv ${tmp_dir}/oc /usr/local/bin && \ + rm -rf ${tmp_dir} /tmp/openshift-client* + +ENV HOME="/home/nonrootuser/" + +ENV JAVA_HOME="/home/nonrootuser/.sdkman/candidates/java/current/" +ENV MAVEN_HOME="/home/nonrootuser/.sdkman/candidates/maven/current/" +ENV MAVEN_OPTS="-Xms1024m -Xmx12g" +ENV NODE_HOME="/home/nonrootuser/.nvm/versions/node/v16.20.0" +ENV NODE_OPTIONS="--max_old_space_size=4096" +ENV GOPATH="/home/nonrootuser/go" +ENV GOROOT="/opt/golang/go" + +ENV CONTAINER_ENGINE="docker" +ENV CONTAINER_ENGINE_TLS_OPTIONS="" + +WORKDIR /project/directory \ No newline at end of file diff --git a/docs/jenkins.md b/docs/jenkins.md index 05c1c6697..89767f6f9 100644 --- a/docs/jenkins.md +++ b/docs/jenkins.md @@ -142,8 +142,6 @@ environments: ids: - List of ids identifying the environment -productized_branch: true - # Used to force the disabling of triggers # Useful when a branch is no more maintained but you still want to keep job history disable: @@ -185,10 +183,6 @@ maven: build_promotion_profile_id: 1966c60aff37d artifacts_repository: http://nexus3-tradisso-nexus.apps.kogito-cloud.hosted.psi.rdu2.redhat.com/repository/kogito-test/ #artifacts_repository: '' - pr_checks: - repository: - url: https://bxms-qe.rhev-ci-vms.eng.rdu2.redhat.com:8443/nexus/content/repositories/kogito-runtimes-pr-full-testing/ - creds_id: unpacks-zip-on-qa-nexus cloud: image: registry_credentials_nightly: tradisso_registry_token @@ -199,8 +193,8 @@ cloud: jenkins: email_creds_id: KOGITO_CI_EMAIL_TO_PERSO default_tools: - jdk: kie-jdk11 - maven: kie-maven-3.8.7 + jdk: jdk_11_latest + maven: maven_3.8.6 ``` @@ -259,7 +253,7 @@ Then, you can also add a small `.ci/jenkins/dsl/test.sh` to test your groovy scr file=$(mktemp) # For more usage of the script, use ./test.sh -h -curl -o ${file} https://raw.githubusercontent.com/kiegroup/kogito-pipelines/main/dsl/seed/scripts/seed_test.sh +curl -o ${file} https://raw.githubusercontent.com/apache/incubator-kie-kogito-pipelines/main/dsl/seed/scripts/seed_test.sh chmod u+x ${file} ${file} $@ ``` @@ -292,7 +286,7 @@ If you don't have access to the Kogito `0-seed-job`, you can also create one bas 3) Copy the create the `0-seed-job` and point the parameters of the job to the main config file you created. Depending on what you changed and need to test, you may have different parameters to update: * Setup `SEED_CONFIG_FILE_*` parameters correctly to the author/branch you pushed your test modifications. This is always required. - * Change the `SEED_*` parameters only if you did some changes in the Kogito Pipelines `dsl/seed` folder. Else you can use the default, aka `kiegroup/main`. + * Change the `SEED_*` parameters only if you did some changes in the Kogito Pipelines `dsl/seed` folder. Else you can use the default, aka `apache/incubator-kie-main`. 4) Launch the `0-seed-job` and it should create all the needed jobs **Again, please make sure that you setup your own configuration !** diff --git a/docs/nightly pipeline.md b/docs/nightly pipeline.md index 1337390f3..487f54625 100644 --- a/docs/nightly pipeline.md +++ b/docs/nightly pipeline.md @@ -17,18 +17,18 @@ Those jobs should be present at the same level as the nightly and/or release job Here is the list of jobs and link to Jenkinsfiles: -* [kogito-runtimes-deploy](https://github.com/kiegroup/kogito-runtimes/blob/main/Jenkinsfile.deploy) -* [kogito-runtimes-promote](https://github.com/kiegroup/kogito-runtimes/blob/main/Jenkinsfile.promote) -* [kogito-apps-deploy](https://github.com/kiegroup/kogito-apps/blob/main/Jenkinsfile.deploy) -* [kogito-apps-promote](https://github.com/kiegroup/kogito-apps/blob/main/Jenkinsfile.promote) -* [kogito-examples-deploy](https://github.com/kiegroup/kogito-examples/blob/main/Jenkinsfile.deploy) -* [kogito-examples-promote](https://github.com/kiegroup/kogito-examples/blob/main/Jenkinsfile.promote) -* [kogito-images-deploy](https://github.com/kiegroup/kogito-images/blob/main/Jenkinsfile.deploy) -* [kogito-images-promote](https://github.com/kiegroup/kogito-images/blob/main/Jenkinsfile.promote) -* [kogito-examples-images-deploy](https://github.com/kiegroup/kogito-operator/blob/main/Jenkinsfile.examples-images.deploy) -* [kogito-examples-images-promote](https://github.com/kiegroup/kogito-operator/blob/main/Jenkinsfile.examples-images.promote) -* [kogito-operator-deploy](https://github.com/kiegroup/kogito-operator/blob/main/Jenkinsfile.deploy) -* [kogito-operator-promote](https://github.com/kiegroup/kogito-operator/blob/main/Jenkinsfile.promote) +* [kogito-runtimes-deploy](https://github.com/apache/incubator-kie-kogito-runtimes/blob/main/Jenkinsfile.deploy) +* [kogito-runtimes-promote](https://github.com/apache/incubator-kie-kogito-runtimes/blob/main/Jenkinsfile.promote) +* [kogito-apps-deploy](https://github.com/apache/incubator-kie-kogito-apps/blob/main/Jenkinsfile.deploy) +* [kogito-apps-promote](https://github.com/apache/incubator-kie-kogito-apps/blob/main/Jenkinsfile.promote) +* [kogito-examples-deploy](https://github.com/apache/incubator-kie-kogito-examples/blob/main/Jenkinsfile.deploy) +* [kogito-examples-promote](https://github.com/apache/incubator-kie-kogito-examples/blob/main/Jenkinsfile.promote) +* [kogito-images-deploy](https://github.com/apache/incubator-kie-kogito-images/blob/main/Jenkinsfile.deploy) +* [kogito-images-promote](https://github.com/apache/incubator-kie-kogito-images/blob/main/Jenkinsfile.promote) +* [kogito-examples-images-deploy](https://github.com/apache/incubator-kie-kogito-operator/blob/main/Jenkinsfile.examples-images.deploy) +* [kogito-examples-images-promote](https://github.com/apache/incubator-kie-kogito-operator/blob/main/Jenkinsfile.examples-images.promote) +* [kogito-operator-deploy](https://github.com/apache/incubator-kie-kogito-operator/blob/main/Jenkinsfile.deploy) +* [kogito-operator-promote](https://github.com/apache/incubator-kie-kogito-operator/blob/main/Jenkinsfile.promote) ## Nightly pipeline Architecture diff --git a/docs/release pipeline.md b/docs/release pipeline.md index a4174a0c6..8f1a1d79f 100644 --- a/docs/release pipeline.md +++ b/docs/release pipeline.md @@ -38,18 +38,18 @@ Those jobs should be present at the same level as the nightly and/or release job Here is the list of jobs and link to Jenkinsfiles: -* [kogito-runtimes-deploy](https://github.com/kiegroup/kogito-runtimes/blob/main/Jenkinsfile.deploy) -* [kogito-runtimes-promote](https://github.com/kiegroup/kogito-runtimes/blob/main/Jenkinsfile.promote) -* [kogito-apps-deploy](https://github.com/kiegroup/kogito-apps/blob/main/Jenkinsfile.deploy) -* [kogito-apps-promote](https://github.com/kiegroup/kogito-apps/blob/main/Jenkinsfile.promote) -* [kogito-examples-deploy](https://github.com/kiegroup/kogito-examples/blob/main/Jenkinsfile.deploy) -* [kogito-examples-promote](https://github.com/kiegroup/kogito-examples/blob/main/Jenkinsfile.promote) -* [kogito-images-deploy](https://github.com/kiegroup/kogito-images/blob/main/Jenkinsfile.deploy) -* [kogito-images-promote](https://github.com/kiegroup/kogito-images/blob/main/Jenkinsfile.promote) -* [kogito-examples-images-deploy](https://github.com/kiegroup/kogito-operator/blob/main/Jenkinsfile.examples-images.deploy) -* [kogito-examples-images-promote](https://github.com/kiegroup/kogito-operator/blob/main/Jenkinsfile.examples-images.promote) -* [kogito-operator-deploy](https://github.com/kiegroup/kogito-operator/blob/main/Jenkinsfile.deploy) -* [kogito-operator-promote](https://github.com/kiegroup/kogito-operator/blob/main/Jenkinsfile.promote) +* [kogito-runtimes-deploy](https://github.com/apache/incubator-kie-kogito-runtimes/blob/main/Jenkinsfile.deploy) +* [kogito-runtimes-promote](https://github.com/apache/incubator-kie-kogito-runtimes/blob/main/Jenkinsfile.promote) +* [kogito-apps-deploy](https://github.com/apache/incubator-kie-kogito-apps/blob/main/Jenkinsfile.deploy) +* [kogito-apps-promote](https://github.com/apache/incubator-kie-kogito-apps/blob/main/Jenkinsfile.promote) +* [kogito-examples-deploy](https://github.com/apache/incubator-kie-kogito-examples/blob/main/Jenkinsfile.deploy) +* [kogito-examples-promote](https://github.com/apache/incubator-kie-kogito-examples/blob/main/Jenkinsfile.promote) +* [kogito-images-deploy](https://github.com/apache/incubator-kie-kogito-images/blob/main/Jenkinsfile.deploy) +* [kogito-images-promote](https://github.com/apache/incubator-kie-kogito-images/blob/main/Jenkinsfile.promote) +* [kogito-examples-images-deploy](https://github.com/apache/incubator-kie-kogito-operator/blob/main/Jenkinsfile.examples-images.deploy) +* [kogito-examples-images-promote](https://github.com/apache/incubator-kie-kogito-operator/blob/main/Jenkinsfile.examples-images.promote) +* [kogito-operator-deploy](https://github.com/apache/incubator-kie-kogito-operator/blob/main/Jenkinsfile.deploy) +* [kogito-operator-promote](https://github.com/apache/incubator-kie-kogito-operator/blob/main/Jenkinsfile.promote) ## Release process @@ -159,7 +159,7 @@ Once the Release Pipeline is finished, there are some actions to be done: ##### Operator Crd/Csv files -Once the operator's release has been done, it created new csv and crd files under `deploy/olm-catalog/kogito-operator/{VERSION}` on the release branch in [kogito-operator](https://github.com/kiegroup/kogito-operator). +Once the operator's release has been done, it created new csv and crd files under `deploy/olm-catalog/kogito-operator/{VERSION}` on the release branch in [kogito-operator](https://github.com/apache/incubator-kie-kogito-operator). You will need to create, with those files, new OperatorHub PRs (one for Openshift and one for Kubernetes) or asked someone from Cloud part to do it. If there is any change to be done due to PRs, do it on the release branch. @@ -189,7 +189,7 @@ To deploy only artifacts (without Cloud part), you should start the release pipe To deploy only the Cloud part, you should start the release pipeline with those parameters: -* EXAMPLES_URI => point to `kogito-examples` repository (https://github.com/kiegroup/kogito-examples) +* EXAMPLES_URI => point to `kogito-examples` repository (https://github.com/apache/incubator-kie-kogito-examples) * EXAMPLES_REF => point to current release branch * SKIP_ARTIFACTS_DEPLOY => checked * SKIP_ARTIFACTS_PROMOTE => checked diff --git a/dsl/scripts/pr_check.groovy b/dsl/scripts/pr_check.groovy new file mode 100644 index 000000000..ea74d7ab9 --- /dev/null +++ b/dsl/scripts/pr_check.groovy @@ -0,0 +1,135 @@ +// TODO Docker image and args could be passed as env or anything ? +dockerGroups = [ + 'docker', + 'input', + 'render', +] +dockerArgs = [ + '-v /var/run/docker.sock:/var/run/docker.sock', +] + dockerGroups.collect { group -> "--group-add ${group}" } + +void launch() { + String builderImage = 'quay.io/kiegroup/kogito-ci-build:latest' + sh "docker rmi -f ${builderImage} || true" // Remove before launching + + try { + launchInDocker(builderImage) + } finally { + sh "docker rmi -f ${builderImage} || true" + } +} + +void launchInDocker(String builderImage) { + docker.image(builderImage).inside(dockerArgs.join(' ')) { + // Debug. To be removed in the future + sh "printenv" + sh 'ls -last /var/run/docker.sock' + try { + launchStages() + } finally { + echo "Got build result ${currentBuild.currentResult}" + if (currentBuild.currentResult != 'SUCCESS') { + // TODO ci token as env ? + pullrequest.postComment(util.getMarkdownTestSummary('PR', getReproducer(true), "${BUILD_URL}", 'GITHUB'), "kie-ci3-token") + } + } + } +} + +void launchStages() { + stage('Initialize') { + sh 'printenv > env_props' + archiveArtifacts artifacts: 'env_props' + } + stage('Install build-chain tool') { + println '[INFO] Getting build-chain version from composite action file' + def buildChainVersion = buildChain.getBuildChainVersionFromCompositeActionFile() + if ([null, 'null'].contains(buildChainVersion)) { + def errorMessage = "[ERROR] The build-chain version can't be recovered. Please contact administrator" + println errorMessage + error(errorMessage) + } + println "[INFO] build-chain version recovered '${buildChainVersion}'" + sh "npm install -g @kie/build-chain-action@${buildChainVersion}${env.NPM_REGISTRY_URL ? " -registry=${NPM_REGISTRY_URL}" : ''}" + sh 'npm list -g | grep build-chain' + sh 'sudo alternatives --install /usr/local/bin/build-chain build-chain ${NODE_HOME}/bin/build-chain 1' + sh 'build-chain || true' + } + stage('Build projects') { + configFileProvider([configFile(fileId: 'kie-pr-settings', variable: 'MAVEN_SETTINGS_FILE')]) { // TODO as env ? + withCredentials([string(credentialsId: 'kie-ci3-token', variable: 'GITHUB_TOKEN')]) { // TODO as env ? + env.BUILD_MVN_OPTS = "${env.BUILD_MVN_OPTS ?: ''} -s ${MAVEN_SETTINGS_FILE} -Dmaven.wagon.http.ssl.insecure=true -Dmaven.test.failure.ignore=true" + echo "BUILD_MVN_OPTS = ${BUILD_MVN_OPTS}" + + try { + sh getBuildChainCommandline() + } catch (err) { + echo 'Error running the build-chain ...' + util.archiveConsoleLog('', 300) + throw err + } finally { + // Remove `node_modules` to avoid heap space issues with junit command thereafter + // Related to https://github.com/jenkinsci/junit-plugin/issues/478 and https://github.com/jenkinsci/junit-plugin/issues/467 + sh 'find . -type d -name node_modules -exec rm -rf {} \\; || true' + + junit(testResults: '**/junit.xml, **/target/surefire-reports/**/*.xml, **/target/failsafe-reports/**/*.xml, **/target/invoker-reports/**/*.xml', allowEmptyResults: true) + archiveArtifacts(artifacts: '**/cypress/screenshots/**,**/cypress/videos/**', fingerprint: false, allowEmptyArchive: true) + } + } + } + } + stage('Sonar analysis') { + if (isEnableSonarCloudAnalysis()) { + dir(getProjectFolder()) { + maven.runMavenWithSettingsSonar(settingsXmlId, "-e -nsu validate -Psonarcloud-analysis -Denforcer.skip=true ${env.SONARCLOUD_ANALYSIS_MVN_OPTS ?: ''}", 'SONARCLOUD_TOKEN', 'sonar_analysis.maven.log') + } + } + } +} + +String getBuildChainCommandline() { + // Those can be overriden in Jenkinsfiles + String buildChainProject = env.BUILDCHAIN_PROJECT ?: CHANGE_REPO + String buildChainConfigRepo = env.BUILDCHAIN_CONFIG_REPO ?: 'incubator-kie-kogito-pipelines' + String buildChainConfigBranch = env.BUILDCHAIN_CONFIG_BRANCH ?: '\${BRANCH:main}' + String buildChainConfigGitAuthor = env.BUILDCHAIN_CONFIG_AUTHOR ?: '\${AUTHOR:apache}' + String buildChainConfigDefinitionFilePath = env.BUILDCHAIN_CONFIG_FILE_PATH ?: '.ci/pull-request-config.yaml' + + List buildChainAdditionalArguments = [ + "-p ${buildChainProject}", + "-u ${CHANGE_URL}", // Provided by source branch plugin + ] + // TODO remove debug option + return "build-chain build full_downstream ${env.GITHUB_TOKEN ? "--token ${GITHUB_TOKEN} " : ''} -f 'https://raw.githubusercontent.com/${buildChainConfigGitAuthor}/${buildChainConfigRepo}/${buildChainConfigBranch}/${buildChainConfigDefinitionFilePath}' -o 'bc' ${buildChainAdditionalArguments.join(' ')} --skipParallelCheckout" +} + +boolean isEnableSonarCloudAnalysis() { + return env.ENABLE_SONARCLOUD ? env.ENABLE_SONARCLOUD.toBoolean() : false +} + +String getReproducer(boolean isGH = false) { + String reproducer = """ +${env.QUARKUS_BRANCH ? "export QUARKUS_BRANCH=${env.QUARKUS_BRANCH}" : ''} +${env.BUILD_MVN_OPTS_CURRENT ? "export BUILD_MVN_OPTS_CURRENT=${env.BUILD_MVN_OPTS_CURRENT}" : ''} +${getBuildChainCommandline()} + +NOTE: To install the build-chain tool, please refer to https://github.com/kiegroup/github-action-build-chain#local-execution +""" + + if(isGH) { + return """ +
+Reproducer +${reproducer} +
+""" + } else { + return """ +```spoiler Reproducer +${reproducer} +``` +""" + } +} + +return this diff --git a/dsl/seed/build.gradle b/dsl/seed/build.gradle index 95062f11a..3659748e7 100644 --- a/dsl/seed/build.gradle +++ b/dsl/seed/build.gradle @@ -61,6 +61,7 @@ dependencies { testPlugins "com.sonyericsson.hudson.plugins.rebuild:rebuild:${version_jenkins_plugins_rebuild}" testPlugins "org.jenkins-ci.main:maven-plugin:${version_jenkins_plugins_maven_plugin}" testPlugins "org.jenkins-ci.plugins:ansicolor:${version_jenkins_plugins_ansicolor}" + testPlugins "org.jenkins-ci.plugins:github-branch-source:${version_jenkins_plugins_github_branch_source}" testPlugins "org.jenkins-ci.plugins:build-timeout:${version_jenkins_plugins_build_timeout}" testPlugins "org.jenkins-ci.plugins:config-file-provider:${version_jenkins_plugins_config_file_provider}" testPlugins "org.jenkins-ci.plugins:credentials:${version_jenkins_plugins_credentials}" diff --git a/dsl/seed/gradle.properties b/dsl/seed/gradle.properties index 80ab2ff13..3d96d45ba 100644 --- a/dsl/seed/gradle.properties +++ b/dsl/seed/gradle.properties @@ -23,6 +23,7 @@ version_jenkins_test_harness=1873.va_ed9d52f2b_42 # can be found in ${JENKINS_URL}/systemInfo version_jenkins_plugins_ansicolor=1.0.2 version_jenkins_plugins_branch_api=2.1122.v09cb_8ea_8a_724 +version_jenkins_plugins_github_branch_source=1732.v3f1889a_c475b_ version_jenkins_plugins_build_timeout=1.31 version_jenkins_plugins_cloudbees_folder=6.815.v0dd5a_cb_40e0e version_jenkins_plugins_config_file_provider=938.ve2b_8a_591c596 diff --git a/dsl/seed/jenkinsfiles/Jenkinsfile.buildchain b/dsl/seed/jenkinsfiles/Jenkinsfile.buildchain index 2a02a1f6b..dc658eac9 100644 --- a/dsl/seed/jenkinsfiles/Jenkinsfile.buildchain +++ b/dsl/seed/jenkinsfiles/Jenkinsfile.buildchain @@ -3,8 +3,8 @@ import org.kie.jenkins.MavenCommand import org.kie.jenkins.MavenSettingsUtils -agentLabel = "${env.ADDITIONAL_LABEL?.trim() ?: 'kie-rhel8 && docker && kie-mem16g'} && !built-in" -timeoutValue = env.ADDITIONAL_TIMEOUT?.trim() ?: '180' +agentLabel = "${env.ADDITIONAL_LABEL?.trim() ?: 'ubuntu'} && !built-in" +timeoutValue = env.ADDITIONAL_TIMEOUT?.trim() ?: '360' jdkTool = env.BUILD_JDK_TOOL sonarJdkTool = env.SONAR_JDK_TOOL mavenTool = env.BUILD_MAVEN_TOOL @@ -16,15 +16,15 @@ mavenDeployLocalDir = env.MAVEN_DEPLOY_LOCAL_DIR ?: '' buildChainType = env.BUILDCHAIN_TYPE?.trim() ?: 'cross_pr' buildChainProject = env.BUILDCHAIN_PROJECT?.trim() -defaultSettingsXmlId = isPRBuildChainType() || isFDBBuildChainType() ? 'kogito_pr_settings' : 'kogito_release_settings' +defaultSettingsXmlId = isPRBuildChainType() || isFDBBuildChainType() ? 'kie-pr-settings' : 'kie-release-settings' settingsXmlId = env.MAVEN_SETTINGS_FILE_ID ?: defaultSettingsXmlId enableSonarCloudAnalysis = env.ENABLE_SONARCLOUD ? env.ENABLE_SONARCLOUD.toBoolean() : false downstreamBuild = env.DOWNSTREAM_BUILD ? env.DOWNSTREAM_BUILD.toBoolean() : false -buildChainConfigRepo = env.BUILDCHAIN_CONFIG_REPO ?: 'kogito-pipelines' +buildChainConfigRepo = env.BUILDCHAIN_CONFIG_REPO ?: 'incubator-kie-kogito-pipelines' buildChainConfigBranch = env.BUILDCHAIN_CONFIG_BRANCH ?: '\${BRANCH:main}' -buildChainConfigGitAuthor = env.BUILDCHAIN_CONFIG_AUTHOR ?: '\${AUTHOR:kiegroup}' +buildChainConfigGitAuthor = env.BUILDCHAIN_CONFIG_AUTHOR ?: '\${AUTHOR:apache}' buildChainConfigDefinitionFilePath = env.BUILDCHAIN_CONFIG_FILE_PATH ?: '.ci/pull-request-config.yaml' buildChainAction = isFDBBuildChainType() ? 'full_downstream' : buildChainType buildChainAdditionalArguments = buildChainProject ? [ "-p ${buildChainProject}" ] : [] @@ -47,25 +47,22 @@ notificationJobName = env.NOTIFICATION_JOB_NAME ?: 'PR' pipeline { agent { - label agentLabel - } - tools { - nodejs 'nodejs-16.2.0' + docker { + image env.AGENT_DOCKER_BUILDER_IMAGE + args env.AGENT_DOCKER_BUILDER_ARGS + } } options { timestamps() timeout(time: timeoutValue, unit: 'MINUTES') } - environment { - FIREFOX_FOLDER = '/opt/tools/firefox-60esr' - MAVEN_OPTS = '-Xms1024m -Xmx12g' - } stages { stage('Initialize') { steps { script { sh 'printenv > env_props' archiveArtifacts artifacts: 'env_props' + sh 'ls -last /var/run/docker.sock' if (mavenDeployArtifacts) { assert mavenDeployRepository : 'Please provide the maven deploy repository' @@ -73,114 +70,91 @@ pipeline { } currentBuild.displayName = params.DISPLAY_NAME ?: env.GIT_BRANCH_NAME ?: currentBuild.displayName - - // Needed by Kogito Apps - setupCypressEnv('12.17.0') - } - } - } - stage('check space before build') { - steps { - script { - try { - util.spaceLeft() - } catch (err) { - echo "Error when checking the space on node ... ${err}" - } } } } - stage('Install build-chain tool') { - steps { - script { - println '[INFO] Getting build-chain version from composite action file' - def buildChainVersion = buildChain.getBuildChainVersionFromCompositeActionFile() - if ([null, 'null'].contains(buildChainVersion)) { - def errorMessage = "[ERROR] The build-chain version can't be recovered. Please contact administrator" - println errorMessage - error(errorMessage) + stage('Perform build steps') { + stages { + stage('Install build-chain tool') { + steps { + script { + println '[INFO] Getting build-chain version from composite action file' + def buildChainVersion = buildChain.getBuildChainVersionFromCompositeActionFile() + if ([null, 'null'].contains(buildChainVersion)) { + def errorMessage = "[ERROR] The build-chain version can't be recovered. Please contact administrator" + println errorMessage + error(errorMessage) + } + println "[INFO] build-chain version recovered '${buildChainVersion}'" + sh "npm install -g @kie/build-chain-action@${buildChainVersion}${env.NPM_REGISTRY_URL ? " -registry=${NPM_REGISTRY_URL}" : ''}" + + sh "npm list -g | grep build-chain" + sh 'sudo alternatives --install /usr/local/bin/build-chain build-chain ${NODE_HOME}/bin/build-chain 1' + sh 'build-chain || true' + } } - println "[INFO] build-chain version recovered '${buildChainVersion}'" - sh "npm install -g @kie/build-chain-action@${buildChainVersion}${env.NPM_REGISTRY_URL ? " -registry=${NPM_REGISTRY_URL}" : ''}" - - sh "npm list -g | grep build-chain" } - } - } - stage('Build projects') { - tools { - jdk jdkTool - maven mavenTool - } - steps { - script { - env.LD_LIBRARY_PATH = "${env.LD_LIBRARY_PATH ? "${env.LD_LIBRARY_PATH}:": ''}${GRAALVM_HOME}/lib/server" - echo "LD_LIBRARY_PATH = ${LD_LIBRARY_PATH}" - - env.BUILD_MVN_OPTS_CURRENT = "${env.BUILD_MVN_OPTS_CURRENT ?: ''} ${getBuildMavenOptsCurrent()}" - echo "BUILD_MVN_OPTS_CURRENT = ${BUILD_MVN_OPTS_CURRENT}" - if (mavenDeployArtifacts) { - env.DEPLOY_MVN_OPTS = "${env.DEPLOY_MVN_OPTS ?: ''} -DaltDeploymentRepository=local::default::file://${getMavenDeployLocalDir()}" - echo "DEPLOY_MVN_OPTS = ${DEPLOY_MVN_OPTS}" + stage('Build projects') { + steps { + script { + env.BUILD_MVN_OPTS_CURRENT = "${env.BUILD_MVN_OPTS_CURRENT ?: ''} ${getBuildMavenOptsCurrent()}" + echo "BUILD_MVN_OPTS_CURRENT = ${BUILD_MVN_OPTS_CURRENT}" + if (mavenDeployArtifacts) { + env.DEPLOY_MVN_OPTS = "${env.DEPLOY_MVN_OPTS ?: ''} -DaltDeploymentRepository=local::default::file://${getMavenDeployLocalDir()}" + echo "DEPLOY_MVN_OPTS = ${DEPLOY_MVN_OPTS}" + } + + configFileProvider([configFile(fileId: settingsXmlId, variable: 'MAVEN_SETTINGS_FILE')]) { + withCredentials([string(credentialsId: "${GIT_AUTHOR_TOKEN_CREDENTIALS_ID}", variable: 'GITHUB_TOKEN')]) { + env.BUILD_MVN_OPTS = "${env.BUILD_MVN_OPTS ?: ''} -s ${MAVEN_SETTINGS_FILE} -Dmaven.wagon.http.ssl.insecure=true -Dmaven.test.failure.ignore=true" + echo "BUILD_MVN_OPTS = ${BUILD_MVN_OPTS}" + + sh getBuildChainCommandline(true) + } + } + } } - - configFileProvider([configFile(fileId: settingsXmlId, variable: 'MAVEN_SETTINGS_FILE')]) { - withCredentials([string(credentialsId: "${GIT_AUTHOR_TOKEN_CREDENTIALS_ID}", variable: 'GITHUB_TOKEN')]) { - env.BUILD_MVN_OPTS = "${env.BUILD_MVN_OPTS ?: ''} -s ${MAVEN_SETTINGS_FILE} -Dmaven.wagon.http.ssl.insecure=true -Dmaven.test.failure.ignore=true" - echo "BUILD_MVN_OPTS = ${BUILD_MVN_OPTS}" - - util.runWithPythonVirtualEnv("${getBuildChainCommandline(true)}", 'swf') + post { + always { + script { + // Remove `node_modules` to avoid heap space issues with junit command thereafter + // Related to https://github.com/jenkinsci/junit-plugin/issues/478 and https://github.com/jenkinsci/junit-plugin/issues/467 + sh 'find . -type d -name node_modules -exec rm -rf {} \\; || true' + + junit(testResults: '**/junit.xml, **/target/surefire-reports/**/*.xml, **/target/failsafe-reports/**/*.xml, **/target/invoker-reports/**/*.xml', allowEmptyResults: true) + archiveArtifacts(artifacts: '**/cypress/screenshots/**,**/cypress/videos/**', fingerprint: false, allowEmptyArchive: true) + } + } + unsuccessful { + script { + util.archiveConsoleLog('', 300) + } } } } - } - post { - always { - script { - // Remove `node_modules` to avoid heap space issues with junit command thereafter - // Related to https://github.com/jenkinsci/junit-plugin/issues/478 and https://github.com/jenkinsci/junit-plugin/issues/467 - sh 'find . -type d -name node_modules -exec rm -rf {} \\; || true' - - junit(testResults: '**/junit.xml, **/target/surefire-reports/**/*.xml, **/target/failsafe-reports/**/*.xml, **/target/invoker-reports/**/*.xml', allowEmptyResults: true) - archiveArtifacts(artifacts: '**/cypress/screenshots/**,**/cypress/videos/**', fingerprint: false, allowEmptyArchive: true) + stage('Sonar analysis') { + when { + expression { return enableSonarCloudAnalysis } } - } - unsuccessful { - script { - util.archiveConsoleLog('', 300) + steps { + script { + dir(getProjectFolder()) { + maven.runMavenWithSettingsSonar(settingsXmlId, "-e -nsu validate -Psonarcloud-analysis -Denforcer.skip=true ${env.SONARCLOUD_ANALYSIS_MVN_OPTS ?: ''}", 'SONARCLOUD_TOKEN', 'sonar_analysis.maven.log') + } + } } } - } - } - stage('Sonar analysis') { - when { - expression { return enableSonarCloudAnalysis } - } - tools { - jdk sonarJdkTool - maven mavenTool - } - steps { - script { - dir(getProjectFolder()) { - maven.runMavenWithSettingsSonar(settingsXmlId, "-e -nsu validate -Psonarcloud-analysis -Denforcer.skip=true ${env.SONARCLOUD_ANALYSIS_MVN_OPTS ?: ''}", 'SONARCLOUD_TOKEN', 'sonar_analysis.maven.log') + stage('Upload artifacts to repository') { + when { + expression { return mavenDeployArtifacts } + } + steps { + script { + // Upload to specific repository with credentials + String mavenDeployRepositoryZipUrl = "${mavenDeployRepository.replaceAll('/content/', '/service/local/').replaceFirst('/*$', '')}/content-compressed" + maven.uploadLocalArtifacts(mavenDeployRepositoryCredsId, getMavenDeployLocalDir(), mavenDeployRepositoryZipUrl) + } } - } - } - } - stage('Upload artifacts to repository') { - when { - expression { return mavenDeployArtifacts } - } - tools { - jdk jdkTool - maven mavenTool - } - steps { - script { - // Upload to specific repository with credentials - String mavenDeployRepositoryZipUrl = "${mavenDeployRepository.replaceAll('/content/', '/service/local/').replaceFirst('/*$', '')}/content-compressed" - maven.uploadLocalArtifacts(mavenDeployRepositoryCredsId, getMavenDeployLocalDir(), mavenDeployRepositoryZipUrl) } } } @@ -197,13 +171,6 @@ pipeline { } } } - stage('check space after build') { - steps { - script { - util.spaceLeft() - } - } - } } post { unsuccessful { @@ -217,19 +184,13 @@ pipeline { } } } - cleanup { - script { - // Clean also docker in case of usage of testcontainers lib - util.cleanNode('docker') - sh 'rm -rf /tmp/quarkus' - } - } - } -} - -void setupCypressEnv(String cypressVersion) { - if (env.CYPRESS_BINARY_URL) { - env.CYPRESS_INSTALL_BINARY = "${CYPRESS_BINARY_URL}/cypress-${cypressVersion}.zip" + // cleanup { + // script { + // // Clean also docker in case of usage of testcontainers lib + // util.cleanNode('docker') + // sh 'rm -rf /tmp/quarkus' + // } + // } } } @@ -267,9 +228,9 @@ String getProjectFolder() { // First guessing whether there is a clone defined into the buildchain config // If not, fallback to simple folder structure - String projectFolder = "bc/kiegroup_${project}/${project}" + String projectFolder = "bc/apache_${project}/${project}" if (!fileExists(projectFolder)) { - projectFolder = "bc/kiegroup_${project}" + projectFolder = "bc/apache_${project}" } return projectFolder diff --git a/dsl/seed/jenkinsfiles/Jenkinsfile.environment.integration-branch b/dsl/seed/jenkinsfiles/Jenkinsfile.environment.integration-branch index 4b769bbd3..b766d8beb 100644 --- a/dsl/seed/jenkinsfiles/Jenkinsfile.environment.integration-branch +++ b/dsl/seed/jenkinsfiles/Jenkinsfile.environment.integration-branch @@ -4,7 +4,7 @@ import org.jenkinsci.plugins.workflow.libs.Library pipeline { agent { - label 'kie-rhel8 && !built-in' + label 'ubuntu' } environment { diff --git a/dsl/seed/jenkinsfiles/Jenkinsfile.prod.prepare b/dsl/seed/jenkinsfiles/Jenkinsfile.prod.prepare deleted file mode 100644 index dad5b6f9f..000000000 --- a/dsl/seed/jenkinsfiles/Jenkinsfile.prod.prepare +++ /dev/null @@ -1,664 +0,0 @@ -import org.jenkinsci.plugins.workflow.libs.Library -import java.util.regex.Pattern - -@Library('jenkins-pipeline-shared-libraries')_ - -// parameters -// 1. _RELEASE_BRANCH -// 2. _UPDATE_VERSION (this includes also dependencies) -// 3. QUARKUS_VERSION -// 4. DOWNGRADE_QUARKUS_PR_BRANCH -// 5. ADDITIONAL_BUILD_MAVEN_OPTS -// 6. PROJECTS_TO_REMOVE_FROM_PR_CHECKS - -seedConfig = [:] -mainBranchConfig = [:] -repositoriesToBranch = [] - -pipeline{ - agent{ - label 'rhel8 && !built-in' - } - stages{ - stage('Initialize'){ - steps{ - script { - checkout scm - - String projectListStr = getEcosystemProjects().collect { "${getRepoNameCamelCase(it.name)} ${getBranchFromRepository(it.name)}" }.join(' / ') - currentBuild.displayName = "Productize Branch for ${projectListStr}" - } - } - } - - stage('Create productized branches'){ - steps{ - script { - List repositories = getProdRepositoriesToBranch() - createBranches(repositories) - } - } - } - - // update the main seed config by adding the new prod branch, e.g., - // - name: 8.36.x-prod - // seed: - // branch: seed-drools-8.36.x // this branch MUST already exist - stage('Update main seed config') { - steps { - script { - String seedConfigFile = env.SEED_CONFIG_FILE_PATH - - dir(checkoutMainSeedConfigFileRepo()) { - def seedConfig = readYaml file: seedConfigFile - - if (!seedConfig.git.branches.find { it.name == getEcosystemProductizedBranch() }) { - def branchConfig = [ - name: getEcosystemProductizedBranch(), - seed: [ - branch: getSeedRepoReleaseBranch(), - ] - ] - - seedConfig.git.branches.add(branchConfig) - } - - echo "Write seed config => ${seedConfig}" - sh "rm ${seedConfigFile}" - writeYaml file: seedConfigFile, data: seedConfig, overwrite: true - - pushIfChanged(env.SEED_CONFIG_FILE_GIT_BRANCH, env.SEED_CONFIG_FILE_GIT_AUTHOR_CREDS_ID, "Add productized branch ${getEcosystemProductizedBranch()} after upstream release", "Main config has been updated with release branch ${getEcosystemProductizedBranch()}") - } - } - } - } - - // 1. update seed config repo on productized branch (e.g., 1.37.x-prod) - // - enable productized profile (-Dproductized) on jenkins environments branch.yaml [this MUST be executed only on config file repo, e.g., kogito-pipelines] - // - disable some environments when branching as they make no sense by default [this MUST be executed only on config file repo, e.g., kogito-pipelines] - // - set productized_repositories[].branch to the productized one, e.g., 1.37.x-prod - // - set productized_repositories[].is_branch_config_repo to true for the config file repo - // - enable productized profile (-Dproductized) on build chain config - // - do these changes directly on the prod branch - - - stage("Update branch config on productized branch"){ - steps{ - script { - def prodBranch = getBranchConfigProductizedBranch() - def prodRepositories = getProdRepositoriesToBranch() - - dir(checkoutMainBranchSeedConfigFileRepo()) { - getOrCreateGitBranch(prodBranch, getMainBranchConfigFileGitAuthorCredentialsId()) - - def branchConfigFilePath = getProdBranchConfigFilePath() - def branchConfig = readYaml file: branchConfigFilePath - - branchConfig.productized_repositories = prodRepositories - branchConfig.productized_branch = true - - // keep only those environments that are really needed on productized branch: - // native, quarkus-branch and default if exists - def envsToKeep = ['native', 'quarkus-branch', 'default'] - enabledEnvs = branchConfig.environments.findAll { envName, envData -> envsToKeep.contains(envName) } - - // additional maven opt that enables productized profile - // FIXME do we need -Dproductized here as we are already adding it on buildchain config - def buildMvnOpts = "-Dproductized ${getAdditionalDefaultBuildMavenOpts()} " - - // create default env if not already existing - enabledEnvs.default = enabledEnvs.default ?: [:] - - // apply changes on remaining envs - enabledEnvs.each { envName, envData -> - // ensure env is enabled - envData.enabled = true - - envData.env_vars = envData.env_vars ?: [:] - envData.env_vars.BUILD_MVN_OPTS = buildMvnOpts + (envData.env_vars.BUILD_MVN_OPTS ?: '') - } - - // update environments - branchConfig.environments = enabledEnvs - - // ignore the job creation for jobs in environment which are not defined (apart from the default env) - branchConfig.generation_config = branchConfig.generation_config ?: [:] - branchConfig.generation_config.missing_environment = 'ignore' - - // disable community triggers - branchConfig.disable = branchConfig.disable ?: [:] - branchConfig.disable.triggers = true - - // update repositories branches to the productized one - branchConfig.repositories.each { repo -> - repo.branch = getEcosystemProductizedBranch() - } - - // add cloud.images if operating on kogito project - - branchConfig.cloud = branchConfig.cloud ?: [:] - if (getEcosystemMainProject() == 'kogito') { - branchConfig.cloud.images = 'kogito-data-index-ephemeral,kogito-swf-builder,kogito-swf-devmode' - } - - // override nexus artifacts repository - - def nexusRepo = "${env.BXMS_QE_NEXUS}/content/repositories/osl-prod-nightly/" - branchConfig.maven = branchConfig.maven ?: [:] - branchConfig.maven.artifacts_repository = nexusRepo - - branchConfig.maven.artifacts_upload_repository = branchConfig.maven.artifacts_upload_repository ?: [:] - branchConfig.maven.artifacts_upload_repository.url = nexusRepo - branchConfig.maven.artifacts_upload_repository.creds_id = 'unpacks-zip-on-qa-nexus' - - echo "Write dsl branch_config => ${branchConfig}" - sh "rm -f ${branchConfigFilePath}" - writeYaml file: "${branchConfigFilePath}", data: branchConfig, overwrite: true - - // update build-chain configuration - - def buildChainConfigFilePath = readMainBranchConfig().buildchain_config.git.file_path - def buildChainConfig = readYaml file: buildChainConfigFilePath - - // add -Dproductized in BUILD_MVN_OPTS, then we don't have to add -Dproductized inside DSL configs - - sh "sed -i 's/export BUILD_MVN_OPTS=\"\\\${{ env.BUILD_MVN_OPTS }}/export BUILD_MVN_OPTS=\"\\\${{ env.BUILD_MVN_OPTS }} -Dproductized/' ${buildChainConfigFilePath}" - - // comment out projects we are not interested in from the buildchain-config.yaml - def projectToRemoveFromBCPRChecks = getProjectsToRemoveFromBuildChainConfig() - - if (projectToRemoveFromBCPRChecks.isEmpty()) { - echo "Removing projects ${projectToRemoveFromBCPRChecks} from build chain configuration" - - def builds = buildChainConfig.build.findAll { build -> - !projectToRemoveFromBCPRChecks.contains(build.project) - } - buildChainConfig.build = builds - - echo "Write build-chain configuration => ${buildChainConfig}" - sh "rm -f ${buildChainConfigFilePath}" - writeYaml file: "${buildChainConfigFilePath}", data: buildChainConfig, overwrite: true - - // update build-chain project dependencies - def buildChainConfigDependenciesFilePath = buildChainConfigFilePath.substring(0, buildChainConfigFilePath.lastIndexOf('/') + 1) + "${buildChainConfig.dependencies}" - def buildChainConfigDependencies = readYaml file: buildChainConfigDependenciesFilePath - - def deps = buildChainConfigDependencies.dependencies.findAll { dep -> - !projectToRemoveFromBCPRChecks.contains(dep.project) - } - buildChainConfigDependencies.dependencies = deps - - echo "Write build-chain dependencies configuration => ${buildChainConfigDependencies}" - sh "rm -f ${buildChainConfigDependenciesFilePath}" - writeYaml file: "${buildChainConfigDependenciesFilePath}", data: buildChainConfigDependencies, overwrite: true - } - - pushIfChanged(prodBranch, getMainBranchConfigFileGitAuthorCredentialsId(), "Setup DSL branch config after productized branch creation", "Branch config on ${prodBranch} has been updated with correct configuration") - } - } - } - } - - - // 2. update productized branch on other projects - // - disable PR checks from jobs.groovy - // - comment out projects we are not interested in from the buildchain-config.yaml - // - disable GHA jobs no more needed by commenting out workflows - // - ensure that fullProfile is not enabled on .ci/jenkins/dsl/jobs.groovy, if so remove it - stage("Update prod branch for all projects to be productized"){ - steps{ - script { - def prodRepositories = getProdRepositoriesToBranch() - prodRepositories.each { repoConfig -> - String repository = repoConfig.name - String repoAuthor = repoConfig.author?.name ?: readMainBranchConfig().git.author.name - String repoCredsId = repoConfig.author?.credentials_id ?: readMainBranchConfig().git.author.credentials_id - - String jobsGroovyPath = getJobDefinitionFilePath() - - dir(repository) { - deleteDir() - // here productized branches should have been already created on repos to be productized - checkout(githubscm.resolveRepository(repository, repoAuthor, getProductizedBranchFromRepository(repository), false, repoCredsId)) - - getOrCreateGitBranch(repoConfig.branch, repoCredsId) - - if (fileExists("${jobsGroovyPath}")) { - // def jobsGroovy = readFile file: jobsGroovyPath - // remove fullProfile - sh "sed -i 's/, addFullProfileJobParamsGetter)/)/g' ${jobsGroovyPath}" - // TODO: disable pr checks in getMultijobPRConfig, get them from params - } else { - echo "[WARN] Unable to find jobs.groovy for '${repository}' at '${jobsGroovyPath}'" - } - - - // TODO iterate over PR workflows - // - comment out/remove if not needed (get these from param?) - - // push changes - pushIfChanged(repoConfig.branch, repoCredsId, "Pull request checks updated for productized branch", "Pull request checks updated for productized branch") - } - } - } - } - } - - // 3. update the projects version in order to avoid any kind of conflicts with already existing and deployed artifacts - - stage("Update projects version"){ - when { - expression { return getProjectUpdateVersionKeys().findAll{ it.value != '' } } - } - steps{ - script { - println "Updating projects versions" - - def prodBranch = getEcosystemProductizedBranch() - String jobName = "${prodBranch}/setup-branch/0-setup-branch" - List jobParams = [] - jobParams.add(booleanParam(name: 'DEPLOY', value: false)) - // here we could have a variable number of params - getProjectUpdateVersionKeys().each{ p -> - jobParams.add(stringParam(name: p.key.replace('_UPDATE', ''), value: p.value)) - } - - if (!isDryRun()) { - def job = [:] - try { - // Retry 20 times every 5 min in order to assure seed job ran - retry(20) { - sleep 300 - echo "Build ./${jobName} with parameters ${jobParams}" - job = build(job: "./${jobName}", parameters: jobParams, wait: true, propagate: false) - } - - if (job.result != 'SUCCESS') { - unstable("Update projects version on repositories was not successful") - } - } catch (err) { - unstable("Error executing the ${prodBranch} setup job ...") - String msg = "ERROR: Setup branch job on newly created release branch '${prodBranch}' was unsuccessful." - if(job) { - msg += "\nPlease review job ${job.absoluteUrl}..." - } - sendNotification(msg) - } - } else { - // dry run - just print the job name and params - echo "Build ./${jobName} with parameters ${jobParams}" - } - } - } - } - - // 4. call update quarkus job in order to downgrade quarkus to LTS version (version provided as parameter QUARKUS_VERSION) - // this will: - // - downgrade quarkus, there already exists a job doing this - // - apply patches from .ci/environments/quarkus-lts on the same PR - - stage('Downgrade quarkus to LTS version'){ - when { - expression { return params.QUARKUS_VERSION } - } - steps{ - script { - println "Launch update quarkus all" - - String jobName = "${getEcosystemProductizedBranch()}/tools/update-quarkus-all" - String quarkusDowngradePrBranch = getQuarkusDowngradePrBranch() - List jobParams = [] - jobParams.add(stringParam(name: 'NEW_VERSION', value: getQuarkusVersion())) - jobParams.add(stringParam(name: 'PR_BRANCH', value: quarkusDowngradePrBranch)) - - echo "Build ./${jobName} with parameters ${jobParams}" - if (!isDryRun()) { - def job = [:] - try { - // Retry 20 times every 5 min in order to assure seed job ran - retry(20) { - sleep 300 - echo "Build ./${jobName} with parameters ${jobParams}" - job = build(job: "./${jobName}", parameters: jobParams, wait: true, propagate: false) - } - - if (job.result != 'SUCCESS') { - unstable("Downgrade quarkus version on repositories was not successful") - } else { - def prodRepositories = getProdRepositoriesToBranch() - prodRepositories.each { repoConfig -> - String repoAuthor = repoConfig.author?.name ?: readMainBranchConfig().git.author.name - String repoCredsId = repoConfig.author?.credentials_id ?: readMainBranchConfig().git.author.credentials_id - String repository = repoConfig.name - String baseBranch = repoConfig.branch ?: 'main' - - dir(repository) { - deleteDir() - checkout(githubscm.resolveRepository(repository, repoAuthor, baseBranch, false, repoCredsId)) - - // if quarkusDowngradePrBranch has been created by update-quarkus-all job, checkout it - if (checkoutGitBranchIfExists(quarkusDowngradePrBranch, repoCredsId)) { - // thefollowing script takes care of applying all patches needed when downgrading to Quarkus LTS - // i.e., it will apply all patches found under .ci/environments/quarkus-lts/patches - def updateQuarkusLtsScriptPath = '.ci/environments/update.sh' - if (fileExists(updateQuarkusLtsScriptPath)) { - sh "${updateQuarkusLtsScriptPath} quarkus-lts" - } else { - echo "[WARN] Unable to find '${updateQuarkusLtsScriptPath}' for '${repository}'" - } - - pushIfChanged(quarkusDowngradePrBranch, repoCredsId, "Applied patches after Quarkus downgrade to ${getQuarkusVersion()}", "Quarkus LTS patches on ${quarkusDowngradePrBranch} have been applied") - } - } - } - } - } catch (err) { - unstable("Error executing update quarkus all...") - String msg = "ERROR: Update quarkus all job on newly created prod branch was unsuccessful." - if(job) { - msg += "\nPlease review job ${job.absoluteUrl}..." - } - sendNotification(msg) - } - } - } - } - } - } - post{ - unsuccessful { - script { - sendNotification("${getEcosystemMainProject()}: Branch Productization Pipeline job #${BUILD_NUMBER} was: ${currentBuild.currentResult}\nPlease look here: ${BUILD_URL}") - } - } - success { - script { - sendNotification("${getEcosystemMainProject()}: Branch Productization Pipeline job #${BUILD_NUMBER} was: ${currentBuild.currentResult}") - } - } - } -} - - -void sendNotification(String body) { - echo 'Send Notification' - echo body - if (!isDryRun()) { - emailext body: body, subject: "[${getEcosystemProductizedBranch()}] ${getEcosystemMainProject()}", - to: env.CI_EMAIL_TO - } -} - -// Retrieve job parameters keys or values - -String getBranchParameterKey(String projectName) { - return "${projectName}_RELEASE_BRANCH".toUpperCase() -} - -String getProdBranchSuffixParameterKey(String projectName) { - def key = "${projectName}_PROD_BRANCH_SUFFIX".toUpperCase() - return params."${key}" -} - -def getProjectUpdateVersionKeys() { - return params.findAll{ it.key.contains('_UPDATE_VERSION') } -} - -String getQuarkusVersion() { - return params.QUARKUS_VERSION -} - -String getQuarkusDowngradePrBranch() { - return params.DOWNGRADE_QUARKUS_PR_BRANCH ?: "${getEcosystemProductizedBranch()}-${getQuarkusVersion()}-downgrade" -} - -Boolean isDryRun() { - return params.DRY_RUN -} - - -def getAdditionalDefaultBuildMavenOpts() { - return params.ADDITIONAL_BUILD_MAVEN_OPTS ?: '' -} - -def getProjectsToRemoveFromBuildChainConfig() { - return params.PROJECTS_TO_REMOVE_FROM_PR_CHECKS.split(',') as List ?: [] -} - -// Retrieve seed configuration - -String checkoutMainSeedConfigFileRepo() { - assert env.SEED_CONFIG_FILE_GIT_REPOSITORY : 'Missing main seed config file Git Repository into the environment. Please check the job configuration.' - assert env.SEED_CONFIG_FILE_GIT_AUTHOR_NAME : 'Missing main seed config file Git Author Name into the environment. Please check the job configuration.' - assert env.SEED_CONFIG_FILE_GIT_AUTHOR_CREDS_ID : 'Missing main seed config file Git Author Credentials ID into the environment. Please check the job configuration.' - assert env.SEED_CONFIG_FILE_GIT_BRANCH : 'Missing main seed config file Git Branch into the environment. Please check the job configuration.' - - String dirName = "seed-main-${env.SEED_CONFIG_FILE_GIT_REPOSITORY}" - dir(dirName) { - deleteDir() - checkout(githubscm.resolveRepository(env.SEED_CONFIG_FILE_GIT_REPOSITORY, env.SEED_CONFIG_FILE_GIT_AUTHOR_NAME, env.SEED_CONFIG_FILE_GIT_BRANCH, false, env.SEED_CONFIG_FILE_GIT_AUTHOR_CREDS_ID)) - sh "git checkout ${env.SEED_CONFIG_FILE_GIT_BRANCH}" - } - return dirName -} - -def readSeedConfig() { - assert env.SEED_CONFIG_FILE_PATH : 'Missing main seed config file Path into the environment. Please check the job configuration.' - if (!seedConfig) { - dir(checkoutMainSeedConfigFileRepo()) { - seedConfig = readYaml(file: "${env.SEED_CONFIG_FILE_PATH}") - } - } - return seedConfig -} - -String getProdBranchConfigFilePath() { - return readSeedConfig().seed.config_file.path -} - -String getMainBranchConfigFileGitAuthorCredentialsId() { - return readSeedConfig().seed.config_file.git.author.credentials_id -} - -String checkoutMainBranchSeedConfigFileRepo() { - String dirName = "seed-branch-${getMainBranchConfigFileGitRepository()}" - dir(dirName) { - deleteDir() - checkout(githubscm.resolveRepository(getMainBranchConfigFileGitRepository(), getMainBranchConfigFileGitAuthorName(), getMainBranchConfigFileGitBranch(), false, getMainBranchConfigFileGitAuthorCredentialsId())) - sh "git checkout ${getMainBranchConfigFileGitBranch()}" - } - return dirName -} - -def readMainBranchConfig() { - if (!mainBranchConfig) { - dir(checkoutMainBranchSeedConfigFileRepo()) { - mainBranchConfig = readYaml(file: "${getProdBranchConfigFilePath()}") - } - } - return mainBranchConfig -} - -String getRepoNameCamelCase(String repo) { - List words = repo.split('-') as List - return words.collect { it.isEmpty() ? it : it.substring(0, 1).toUpperCase() + it.substring(1).toLowerCase() }.join(' ') -} - -String getJobDefinitionFilePath() { - return readMainBranchConfig().jenkins?.jobs_definition_file ?: '.ci/jenkins/dsl/jobs.groovy' -} - -// Retrieve impacted projects - -List getEcosystemProjects() { - return readSeedConfig().ecosystem.projects.findAll { ! (it.ignore_productization ? it.ignore_productization.toBoolean() : false) } -} - - -// Retrieve branch to productize starting from current repository/project - -String getProjectFromRepository(String repository) { - return getEcosystemProjects().find { project -> project.regexs?.any { Pattern.compile(it).matcher(repository).matches() } }?.name -} - -String getBranchFromProject(String projectName) { - String key = getBranchParameterKey(projectName) - assert params."${key}" : "No parameter ${key} available for project ${projectName}. Please check job configuration." - return params."${key}" -} - -String getBranchFromRepository(String repository) { - def projectName = getProjectFromRepository(repository) - if (projectName) { - return getBranchFromProject(projectName) - } else { - error "Cannot retrieve repository from repository name: ${repository}. Make sure it matches pattern from ecosystem projects in the main config" - } -} - -// Retrieve repositories to branch - -String getMainBranchConfigFileGitRepository() { - return readSeedConfig().seed.config_file.git.repository -} - -String getMainBranchConfigFileGitAuthorName() { - return readSeedConfig().seed.config_file.git.author.name -} - -String getMainBranchConfigFileGitBranch() { - return readSeedConfig().seed.config_file.git.branch -} - -List getProdRepositoriesToBranch() { - if (!repositoriesToBranch) { - repositoriesToBranch = readMainBranchConfig().productized_repositories.findAll { getProjectFromRepository(it.name) } - def branches = readSeedConfig().git.branches - if (!branches) { - error "No branches are defined in main configuration..." - } - - def releaseBranch = branches.find { it.name == getEcosystemReleaseBranch() }?.name ?: '' - if (!releaseBranch) { - error "No ${getEcosystemReleaseBranch()} branch defined into the main configuration..." - } - repositoriesToBranch.each { repoConfig -> - // set repoConfig.branch to the prod release one (e.g., 1.37.x) - input parameter - def baseBranch = getBranchFromRepository(repoConfig.name) - repoConfig.branch = repoConfig.branch ?: baseBranch ?: releaseBranch - - if (repoConfig.name == getMainBranchConfigFileGitRepository()) { - String gitAuthor = repoConfig.author?.name ?: readMainBranchConfig().git.author.name - if (gitAuthor == getMainBranchConfigFileGitAuthorName() && - repoConfig.branch == getEcosystemReleaseBranch()) { - // we are already going to branch it with the setup of the branch config - echo "Repository ${repoConfig.name} is branch config repository" - repoConfig.is_branch_config_repo = true - } - } - } - } - return repositoriesToBranch -} - -// Create productized branches - -String getProductizedBranchFromProject(String projectName) { - String suffix = getProdBranchSuffixParameterKey(projectName) ?: 'prod' - return "${getBranchFromProject(projectName)}-${suffix}" -} - -String getProductizedBranchFromRepository(String repository) { - def projectName = getProjectFromRepository(repository) - return getProductizedBranchFromProject(projectName) -} - -void createBranches(List repositories) { - repositories.each { repoConfig -> - String repoName = repoConfig.name - String baseBranch = repoConfig.branch - String prodBranch = getProductizedBranchFromRepository(repoConfig.name) - String gitAuthorCredsId = repoConfig.author?.credentials_id ?: readMainBranchConfig().git.author.credentials_id - assert gitAuthorCredsId : "Cannot find any credentials Id for git author ${repoConfig.author.name}. Please check your branch seed config file." - String gitAuthor = repoConfig.author?.name ?: readMainBranchConfig().git.author.name - - echo "Checkout repo ${gitAuthor}/${repoName} on branch ${baseBranch}" - dir(repoName) { - deleteDir() - checkout(githubscm.resolveRepository(repoName, gitAuthor, baseBranch, false, gitAuthorCredsId)) - getOrCreateGitBranch(prodBranch, gitAuthorCredsId) - } - // Update the branch with the new productized branch - repoConfig.branch = prodBranch - } -} - -// Update main seed config - -String getEcosystemMainProject() { - return readSeedConfig().ecosystem.main_project -} - -String getEcosystemProductizedBranch() { - return getProductizedBranchFromProject(getEcosystemMainProject()) -} - -String getEcosystemReleaseBranch() { - return getBranchFromProject(getEcosystemMainProject()) -} - -String getSeedRepoReleaseBranch() { - return readSeedConfig().git.branches.find{ it -> it.name == getEcosystemReleaseBranch() }?.seed.branch ?: "seed-${getEcosystemMainProject()}-${getEcosystemReleaseBranch()}" -} - -String getBranchConfigProductizedBranch() { - // Check if repository already branched - return (getProdRepositoriesToBranch().find { it -> it.is_branch_config_repo }?.branch) ?: "${getEcosystemMainProject()}-${getEcosystemProductizedBranch()}" -} - -// Git utilities - -void getOrCreateGitBranch(String branch, String credentialsId) { - sh 'git fetch origin' - String branchRemoteResult = sh(script: "git ls-remote origin ${branch} | wc -l", returnStdout: true).trim() - if (Integer.parseInt(branchRemoteResult) > 0) { - echo "Branch ${branch} already exist ... will not create it. Checking out !" - sh "git checkout ${branch}" - } else { - echo "Branch ${branch} does not exist ... gonna create it" - githubscm.createBranch(branch) - if (!isDryRun()) { - githubscm.pushObject('origin', branch, credentialsId) - } - } -} - -// If the provided branch exists, checkout it and return true, otherwise return false and skip checkout. -boolean checkoutGitBranchIfExists(String branch, String credentialsId) { - sh 'git fetch origin' - String branchRemoteResult = sh(script: "git ls-remote origin ${branch} | wc -l", returnStdout: true).trim() - if (Integer.parseInt(branchRemoteResult) > 0) { - echo "Branch ${branch} exist. Checking out !" - sh "git checkout ${branch}" - return true - } - - echo "Branch ${branch} does not exist... skipping checkout" - return false -} - -def pushIfChanged(String branch, String credentialsId, String commitMsg, String notificationMsg) { - if (githubscm.isThereAnyChanges()) { - sh 'git diff' - - if (!isDryRun()) { - githubscm.commitChanges(commitMsg) - githubscm.pushObject('origin', branch, credentialsId) - } - - sendNotification(notificationMsg) - } else { - println '[WARN] No changes to commit' - } -} - diff --git a/dsl/seed/jenkinsfiles/Jenkinsfile.release.prepare b/dsl/seed/jenkinsfiles/Jenkinsfile.release.prepare index 0a5514e79..10bcf835b 100644 --- a/dsl/seed/jenkinsfiles/Jenkinsfile.release.prepare +++ b/dsl/seed/jenkinsfiles/Jenkinsfile.release.prepare @@ -9,7 +9,7 @@ repositoriesToBranch = [] pipeline { agent { - label 'rhel8 && !built-in' + label 'ubuntu' } environment { @@ -54,7 +54,6 @@ pipeline { // Update branch config def branchConfig = readYaml file: branchConfigFile branchConfig.repositories = repositories - branchConfig.productized_branch = params.PRODUCTIZED_BRANCH // Disable some environments when branching as they make no sense by default branchConfig.environments.each { envName, envData -> @@ -63,9 +62,6 @@ pipeline { envData.enabled = false // Keep only quarkus-branch } } - if (envData.ids.find { it == 'lts' }) { - envData.enabled = params.PRODUCTIZED_BRANCH // LTS envs depend on param - } } echo "Write dsl branch_config => ${branchConfig}" diff --git a/dsl/seed/jenkinsfiles/Jenkinsfile.remove.branches b/dsl/seed/jenkinsfiles/Jenkinsfile.remove.branches index be29f3e9c..f42833b20 100644 --- a/dsl/seed/jenkinsfiles/Jenkinsfile.remove.branches +++ b/dsl/seed/jenkinsfiles/Jenkinsfile.remove.branches @@ -6,7 +6,7 @@ import org.kie.jenkins.MavenCommand pipeline { agent { - label 'kie-rhel8 && !built-in' + label 'ubuntu' } options{ diff --git a/dsl/seed/jenkinsfiles/Jenkinsfile.seed.branch b/dsl/seed/jenkinsfiles/Jenkinsfile.seed.branch index 5e8891c74..b1e8fa295 100644 --- a/dsl/seed/jenkinsfiles/Jenkinsfile.seed.branch +++ b/dsl/seed/jenkinsfiles/Jenkinsfile.seed.branch @@ -17,7 +17,7 @@ import org.kie.jenkins.jobdsl.utils.SeedJobUtils SeedJobUtils.createSeedJobTrigger( this, - "z-seed-trigger-${REPO_NAME}-job", + "z-seed-trigger-${JOB_DISPLAY_NAME}-job", "${REPO_NAME}", "${GIT_AUTHOR_NAME}", "${GIT_AUTHOR_CREDS_ID}", @@ -32,12 +32,12 @@ pipeline { } tools { - jdk 'kie-jdk11' + jdk 'jdk_11_latest' } options { timestamps() - timeout(time: 60, unit: 'MINUTES') + timeout(time: 180, unit: 'MINUTES') } // parameters { @@ -130,6 +130,7 @@ pipeline { // Add other repos `jenkins_config_path` var (useful if multijob PR checks for example) envProps += all_repos.collectEntries { [ + (generateEnvKey(it, 'job_display_name')): getRepoConfig(it).job_display_name, (generateEnvKey(it, 'jenkins_config_path')): getRepoConfig(it).git.jenkins_config_path, (generateEnvKey(it, 'git_branch')): getRepoConfig(it).git.branch, (generateEnvKey(it, 'git_author')): getRepoConfig(it).git.author.name, @@ -170,6 +171,7 @@ pipeline { GIT_AUTHOR_CREDS_ID: repoConfig.git.author.credentials_id, GIT_JENKINS_CONFIG_PATH: repoConfig.git.jenkins_config_path, JOB_NAME: "${JOB_NAME}", + JOB_DISPLAY_NAME: repoConfig.job_display_name, SEED_CONFIG_FILE_GIT_REPOSITORY: "${SEED_CONFIG_FILE_GIT_REPOSITORY}", SEED_CONFIG_FILE_GIT_AUTHOR_NAME: "${SEED_CONFIG_FILE_GIT_AUTHOR_NAME}", SEED_CONFIG_FILE_GIT_AUTHOR_CREDS_ID: "${SEED_CONFIG_FILE_GIT_AUTHOR_CREDS_ID}", @@ -256,6 +258,8 @@ def getRepoConfig(String repository, String generationBranch = "${GENERATION_BRA def cfg = scriptUtils.deepCopyObject(branchConfig) cfg.remove('repositories') + cfg.job_display_name = repoConfig.job_display_name + // In case repository is disabled cfg.disabled = repoConfig.disabled ?: false @@ -309,7 +313,7 @@ void fillEnvProperties(Map envProperties, String envKeyPrefix, Map propsMap) { } if (value instanceof Map) { fillEnvProperties(envProperties, newKey, value as Map) - } else if ([List, net.sf.json.JSONArray].any { it.isAssignableFrom(value.getClass()) }) { + } else if (value instanceof List || value instanceof net.sf.json.JSONArray) { envProperties.put(newKey, (value as List).collect { it }.join(',')) } else { envProperties.put(newKey, value) diff --git a/dsl/seed/jenkinsfiles/Jenkinsfile.seed.main b/dsl/seed/jenkinsfiles/Jenkinsfile.seed.main index 4a56f485c..98b7ead05 100644 --- a/dsl/seed/jenkinsfiles/Jenkinsfile.seed.main +++ b/dsl/seed/jenkinsfiles/Jenkinsfile.seed.main @@ -16,11 +16,11 @@ pipeline { options { timestamps() - timeout(time: 20, unit: 'MINUTES') + timeout(time: 180, unit: 'MINUTES') } tools { - jdk 'kie-jdk11' + jdk 'jdk_11_latest' } environment { @@ -43,11 +43,9 @@ pipeline { steps { script { def releaseProjects = readSeedConfig().ecosystem.projects.findAll { ! (it.ignore_release ? it.ignore_release.toBoolean() : false) } - def productizeProjects = readSeedConfig().ecosystem.projects.findAll { ! (it.ignore_productization ? it.ignore_productization.toBoolean() : false) } def envProperties = [ JENKINS_EMAIL_CREDS_ID: readSeedConfig().jenkins.email_creds_id, RELEASE_PROJECTS: releaseProjects.collect { it.name }.join(','), - PRODUCTIZED_PROJECTS: productizeProjects.collect { it.name }.join(','), DEPENDENCY_PROJECTS: readSeedConfig().ecosystem.dependencies.collect { it.name } .join(','), SEED_REPO: "${SEED_REPO}", SEED_AUTHOR: "${SEED_AUTHOR}", diff --git a/dsl/seed/jenkinsfiles/Jenkinsfile.seed.trigger b/dsl/seed/jenkinsfiles/Jenkinsfile.seed.trigger index b7f556677..4fe35ac79 100644 --- a/dsl/seed/jenkinsfiles/Jenkinsfile.seed.trigger +++ b/dsl/seed/jenkinsfiles/Jenkinsfile.seed.trigger @@ -9,7 +9,7 @@ pipeline { options { timestamps() - timeout(time: 20, unit: 'MINUTES') + timeout(time: 180, unit: 'MINUTES') } stages { diff --git a/dsl/seed/jenkinsfiles/Jenkinsfile.tools.toggle-triggers b/dsl/seed/jenkinsfiles/Jenkinsfile.tools.toggle-triggers index e0f3e68f8..d63cd573d 100644 --- a/dsl/seed/jenkinsfiles/Jenkinsfile.tools.toggle-triggers +++ b/dsl/seed/jenkinsfiles/Jenkinsfile.tools.toggle-triggers @@ -5,7 +5,7 @@ import org.kie.jenkins.MavenCommand pipeline { agent { - label 'kie-rhel8 && !built-in' + label 'ubuntu' } options { diff --git a/dsl/seed/jenkinsfiles/Jenkinsfile.update-dependency-version b/dsl/seed/jenkinsfiles/Jenkinsfile.update-dependency-version index 5d5480619..969fd622a 100644 --- a/dsl/seed/jenkinsfiles/Jenkinsfile.update-dependency-version +++ b/dsl/seed/jenkinsfiles/Jenkinsfile.update-dependency-version @@ -7,25 +7,17 @@ pipelineProperties = [:] pipeline { agent { - label 'rhel8 && !built-in' - } - - tools { - maven env.BUILD_MAVEN_TOOL - jdk env.BUILD_JDK_TOOL + docker { + image env.AGENT_DOCKER_BUILDER_IMAGE + args env.AGENT_DOCKER_BUILDER_ARGS + } } options { timeout(time: 30, unit: 'MINUTES') } - // parameters { - // For parameters, check into .jenkins/dsl/jobs.groovy file - // } - environment { - // Static env is defined into .jenkins/dsl/jobs.groovy file - KOGITO_CI_EMAIL_TO = credentials("${JENKINS_EMAIL_CREDS_ID}") } diff --git a/dsl/seed/jenkinsfiles/Jenkinsfile.update-quarkus-platform b/dsl/seed/jenkinsfiles/Jenkinsfile.update-quarkus-platform index d318e9883..8843ea7a2 100644 --- a/dsl/seed/jenkinsfiles/Jenkinsfile.update-quarkus-platform +++ b/dsl/seed/jenkinsfiles/Jenkinsfile.update-quarkus-platform @@ -3,14 +3,15 @@ import org.jenkinsci.plugins.workflow.libs.Library import org.kie.jenkins.MavenCommand -AGENT_LABEL="kie-rhel8 && docker && kie-mem16g && !built-in" quarkusPlatformRepo="quarkus-platform" branchCreated = false pipeline { - agent { - label "${AGENT_LABEL}" + docker { + image env.AGENT_DOCKER_BUILDER_IMAGE + args env.AGENT_DOCKER_BUILDER_ARGS + } } options{ @@ -18,14 +19,7 @@ pipeline { timeout(time: 60, unit: 'MINUTES') } - tools { - maven "${BUILD_MAVEN_TOOL}" - jdk "${BUILD_JDK_TOOL}" - } - environment { - // Static env is defined into .jenkins/dsl/jobs.groovy file - KOGITO_CI_EMAIL_TO = credentials("${JENKINS_EMAIL_CREDS_ID}") } diff --git a/dsl/seed/jenkinsfiles/Jenkinsfile.update-quarkus-version b/dsl/seed/jenkinsfiles/Jenkinsfile.update-quarkus-version index 000b75db9..d1c28fcf8 100644 --- a/dsl/seed/jenkinsfiles/Jenkinsfile.update-quarkus-version +++ b/dsl/seed/jenkinsfiles/Jenkinsfile.update-quarkus-version @@ -7,16 +7,10 @@ failingRepos = [] pipeline { agent { - label 'rhel8 && !built-in' + label 'ubuntu' } - // parameters { - // For parameters, check into ./dsl/jobs.groovy file - // } - environment { - // Static env is defined into .jenkins/dsl/jobs.groovy file - KOGITO_CI_EMAIL_TO = credentials("${JENKINS_EMAIL_CREDS_ID}") } diff --git a/dsl/seed/jobs/root_jobs.groovy b/dsl/seed/jobs/root_jobs.groovy index 95a21e80d..f3e3dd006 100644 --- a/dsl/seed/jobs/root_jobs.groovy +++ b/dsl/seed/jobs/root_jobs.groovy @@ -30,8 +30,6 @@ KogitoJobTemplate.createPipelineJob(this, jobParams)?.with { stringParam("${projectName}_VERSION".toUpperCase(), '', "${Utils.getRepoNameCamelCase(projectName)} dependency version which this will depend on") } } - - booleanParam('PRODUCTIZED_BRANCH', false, 'Is the created branch a productized one ?') } environmentVariables { @@ -85,61 +83,3 @@ if (nonMainBranches) { } else { println 'No branches to remove ...' } - -def jobParamsProd = [ - job: [ - name: '0-prepare-productized-branch', - description: 'Prepare productized branch', - ], - git: [ - repository: Utils.getSeedRepo(this), - author: Utils.getSeedAuthor(this), - credentials: Utils.getSeedAuthorCredsId(this), - branch: Utils.getSeedBranch(this), - ], - env: [:], - jenkinsfile: 'dsl/seed/jenkinsfiles/Jenkinsfile.prod.prepare', -] - -List communityReleaseBranches = ALL_BRANCHES.split(',').findAll { it != MAIN_BRANCH_NAME && !it.endsWith('-prod') } -if (communityReleaseBranches) { - KogitoJobTemplate.createPipelineJob(this, jobParamsProd)?.with { - parameters { - PRODUCTIZED_PROJECTS.split(',').each { projectName -> - choiceParam("${projectName}_RELEASE_BRANCH".toUpperCase(), communityReleaseBranches, "${Utils.getRepoNameCamelCase(projectName)} community branch to which to create the productized branch from") - stringParam("${projectName}_PROD_BRANCH_SUFFIX".toUpperCase(), 'prod', "${Utils.getRepoNameCamelCase(projectName)} productized branch suffix") - stringParam("${projectName}_UPDATE_VERSION".toUpperCase(), '', "${Utils.getRepoNameCamelCase(projectName)} dependency version which this will depend on") - } - - if (DEPENDENCY_PROJECTS) { - DEPENDENCY_PROJECTS.split(',').each { projectName -> - stringParam("${projectName}_UPDATE_VERSION".toUpperCase(), '', "${Utils.getRepoNameCamelCase(projectName)} dependency version which this will depend on") - } - } - - stringParam('QUARKUS_VERSION', '', 'Quarkus version to which to update all productized branches, usually latest LTS version') - stringParam('DOWNGRADE_QUARKUS_PR_BRANCH', '', 'Which PR branch name to use for Quarkus downgrade? If none given, a name will be generated automatically.') - stringParam('ADDITIONAL_BUILD_MAVEN_OPTS', '', 'Additional default maven opts for jenkins jobs, e.g., -Ddata-index-ephemeral.image=quay.io/kiegroup/kogito-data-index-ephemeral') - stringParam('PROJECTS_TO_REMOVE_FROM_PR_CHECKS', '', 'Comma-separated list of projects (/) to be removed/disabled from build chain pull request config') - - booleanParam('DRY_RUN', false, 'If enabled no changes will be applied to remote branches') - } - - environmentVariables { - env('JENKINS_EMAIL_CREDS_ID', Utils.getJenkinsEmailCredsId(this)) - - env('SEED_CONFIG_FILE_GIT_REPOSITORY', "${SEED_CONFIG_FILE_GIT_REPOSITORY}") - env('SEED_CONFIG_FILE_GIT_AUTHOR_NAME', "${SEED_CONFIG_FILE_GIT_AUTHOR_NAME}") - env('SEED_CONFIG_FILE_GIT_AUTHOR_CREDS_ID', "${SEED_CONFIG_FILE_GIT_AUTHOR_CREDS_ID}") - env('SEED_CONFIG_FILE_GIT_BRANCH', "${SEED_CONFIG_FILE_GIT_BRANCH}") - env('SEED_CONFIG_FILE_PATH', "${SEED_CONFIG_FILE_PATH}") - - env('SEED_REPO', Utils.getSeedRepo(this)) - env('SEED_AUTHOR', Utils.getSeedAuthor(this)) - env('SEED_BRANCH', Utils.getSeedBranch(this)) - env('SEED_CREDENTIALS_ID', Utils.getSeedAuthorCredsId(this)) - } - } -} else { - println 'No branches to productize ...' -} \ No newline at end of file diff --git a/dsl/seed/jobs/seed_job_branch.groovy b/dsl/seed/jobs/seed_job_branch.groovy index c99c17743..fd30e8505 100644 --- a/dsl/seed/jobs/seed_job_branch.groovy +++ b/dsl/seed/jobs/seed_job_branch.groovy @@ -45,7 +45,7 @@ SeedJobUtils.createSeedJobTrigger( // Configuration of the seed and generated jobs is done via `dsl/seed/config.yaml` pipelineJob("${GENERATION_BRANCH}/${JOB_NAME}") { - description("This job creates all needed Jenkins jobs on branch ${GENERATION_BRANCH}. DO NOT USE FOR TESTING !!!! See https://github.com/kiegroup/kogito-pipelines/blob/main/docs/jenkins.md#test-specific-jobs") + description("This job creates all needed Jenkins jobs on branch ${GENERATION_BRANCH}. DO NOT USE FOR TESTING !!!! See https://github.com/apache/incubator-kie-kogito-pipelines/blob/main/docs/jenkins.md#test-specific-jobs") logRotator { numToKeep(5) @@ -76,7 +76,7 @@ pipelineJob("${GENERATION_BRANCH}/${JOB_NAME}") { env('SEED_AUTHOR_CREDS_ID', Utils.getSeedAuthorCredsId(this)) env('SEED_BRANCH', Utils.getSeedBranch(this)) - env('AGENT_LABEL', Utils.isProdEnvironment(this) ? 'kie-rhel8 && !built-in' : 'kie-rhel8-priority') + env('AGENT_LABEL', 'ubuntu') env('JENKINS_EMAIL_CREDS_ID', Utils.getJenkinsEmailCredsId(this)) } diff --git a/dsl/seed/jobs/seed_job_main.groovy b/dsl/seed/jobs/seed_job_main.groovy index c43c2d3be..3d4a10b73 100644 --- a/dsl/seed/jobs/seed_job_main.groovy +++ b/dsl/seed/jobs/seed_job_main.groovy @@ -3,7 +3,7 @@ import org.kie.jenkins.jobdsl.utils.SeedJobUtils import org.kie.jenkins.jobdsl.Utils -KOGITO_PIPELINES_REPOSITORY = 'kogito-pipelines' +KOGITO_PIPELINES_REPOSITORY = 'incubator-kie-kogito-pipelines' DEFAULT_CREDENTIALS_ID = 'kie-ci' String getSeedRepo() { @@ -11,7 +11,7 @@ String getSeedRepo() { } String getSeedAuthor() { - return Utils.getSeedAuthor(this) ?: 'kiegroup' + return Utils.getSeedAuthor(this) ?: 'apache' } String getSeedAuthorCredsId() { @@ -27,7 +27,7 @@ String getSeedConfigFileGitRepository() { } String getSeedConfigFileGitAuthorName() { - return SEED_CONFIG_FILE_GIT_AUTHOR_NAME ?: 'kiegroup' + return SEED_CONFIG_FILE_GIT_AUTHOR_NAME ?: 'apache' } String getSeedConfigFileGitAuthorCredsId() { @@ -72,7 +72,7 @@ SeedJobUtils.createSeedJobTrigger( // Configuration of the seed and generated jobs is done via `dsl/seed/config.yaml` pipelineJob('0-seed-job') { - description('This job creates all needed Jenkins jobs. DO NOT USE FOR TESTING !!!! See https://github.com/kiegroup/kogito-pipelines/blob/main/docs/jenkins.md#test-specific-jobs') + description('This job creates all needed Jenkins jobs. DO NOT USE FOR TESTING !!!! See https://github.com/apache/incubator-kie-kogito-pipelines/blob/main/docs/jenkins.md#test-specific-jobs') logRotator { numToKeep(5) @@ -83,7 +83,7 @@ pipelineJob('0-seed-job') { } environmentVariables { - env('AGENT_LABEL', Utils.isProdEnvironment(this) ? 'kie-rhel8 && !built-in' : 'kie-rhel8-priority') + env('AGENT_LABEL', 'ubuntu') env('JENKINS_EMAIL_CREDS_ID', Utils.getJenkinsEmailCredsId(this)) } diff --git a/dsl/seed/scripts/seed_test.sh b/dsl/seed/scripts/seed_test.sh index e0e36809f..d5d01343b 100755 --- a/dsl/seed/scripts/seed_test.sh +++ b/dsl/seed/scripts/seed_test.sh @@ -6,7 +6,7 @@ # #!/bin/bash -e # file=$(mktemp) # # For more usage of the script, use ./test.sh -h -# curl -o ${file} https://raw.githubusercontent.com/kiegroup/kogito-pipelines/main/dsl/seed/scripts/seed_test.sh +# curl -o ${file} https://raw.githubusercontent.com/apache/incubator-kie-kogito-pipelines/main/dsl/seed/scripts/seed_test.sh # chmod u+x ${file} # ${file} $@ @@ -24,7 +24,7 @@ usage() { echo ' DSL_DEFAULT_MAIN_CONFIG_FILE_REPO Main config file repository (owner/repo)' echo ' DSL_DEFAULT_MAIN_CONFIG_FILE_REF Main config file reference' echo ' DSL_DEFAULT_MAIN_CONFIG_FILE_PATH Main config file path. Default is `.ci/jenkins/config/main.yaml`' - echo ' DSL_DEFAULT_FALLBACK_MAIN_CONFIG_FILE_REPO Fallback main config repository (owner/repo). Default is `kiegroup/kogito-pipelines`' + echo ' DSL_DEFAULT_FALLBACK_MAIN_CONFIG_FILE_REPO Fallback main config repository (owner/repo). Default is `apache/incubator-kie-kogito-pipelines`' echo ' DSL_DEFAULT_FALLBACK_MAIN_CONFIG_FILE_REF Fallback main config reference. Default is `main`' echo ' DSL_DEFAULT_MAIN_CONFIG_FILE_LOCAL_PATH Main config file local path. If set, the other `DSL_DEFAULT_MAIN_CONFIG_FILE_*` envs will be ignored' echo @@ -39,7 +39,7 @@ usage() { echo ' DSL_DEFAULT_SEED_REPO DSL seed repository (owner/repo). Else it will be calculated from the test repository.' echo ' DSL_DEFAULT_SEED_REF DSL seed reference. Else it will be calculated from the branch config' echo ' DSL_DEFAULT_SEED_REPO_LOCAL_PATH DSL seed repository local path. If set, the other `DSL_DEFAULT_SEED_*` envs will be ignored' - echo ' DSL_DEFAULT_FALLBACK_SEED_REPO Fallback seed repository (owner/repo). Default is `kiegroup/kogito-pipelines`' + echo ' DSL_DEFAULT_FALLBACK_SEED_REPO Fallback seed repository (owner/repo). Default is `apache/incubator-kie-kogito-pipelines`' echo ' DSL_DEFAULT_FALLBACK_SEED_REF Fallback seed reference. Default is `main`' echo echo ' Test repository configuration:' @@ -179,9 +179,9 @@ branch_config_file_ref=${DSL_DEFAULT_BRANCH_CONFIG_FILE_REF} branch_config_file_path=${DSL_DEFAULT_BRANCH_CONFIG_FILE_PATH} branch_config_file_local_path=${DSL_DEFAULT_BRANCH_CONFIG_FILE_LOCAL_PATH:-''} -fallback_main_config_file_repo=${DSL_DEFAULT_FALLBACK_MAIN_CONFIG_FILE_REPO:-'kiegroup/kogito-pipelines'} +fallback_main_config_file_repo=${DSL_DEFAULT_FALLBACK_MAIN_CONFIG_FILE_REPO:-'apache/incubator-kie-kogito-pipelines'} fallback_main_config_file_ref=${DSL_DEFAULT_FALLBACK_MAIN_CONFIG_FILE_REF:-'main'} -fallback_branch_config_file_repo=${DSL_DEFAULT_FALLBACK_BRANCH_CONFIG_FILE_REPO:-'kiegroup/kogito-pipelines'} +fallback_branch_config_file_repo=${DSL_DEFAULT_FALLBACK_BRANCH_CONFIG_FILE_REPO:-'apache/incubator-kie-kogito-pipelines'} fallback_branch_config_file_ref=${DSL_DEFAULT_FALLBACK_BRANCH_CONFIG_FILE_REF:-'main'} fallback_branch_config_name="${fallback_main_config_file_ref}" @@ -315,7 +315,7 @@ seed_repo=${DSL_DEFAULT_SEED_REPO} seed_ref=${DSL_DEFAULT_SEED_REF} seed_local_path=${DSL_DEFAULT_SEED_REPO_LOCAL_PATH} -fallback_seed_repo=${DSL_DEFAULT_FALLBACK_SEED_REPO:-'kiegroup/kogito-pipelines'} +fallback_seed_repo=${DSL_DEFAULT_FALLBACK_SEED_REPO:-'apache/incubator-kie-kogito-pipelines'} fallback_seed_ref=${DSL_DEFAULT_FALLBACK_SEED_REF} if [ -z ${fallback_seed_ref} ]; then fallback_seed_ref="$(yq -e ".git.branches[] | select(.name == \"${branch_config_name}\") | .seed.branch" ${main_config_file_path} 2> /dev/null)" diff --git a/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/KogitoJobTemplate.groovy b/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/KogitoJobTemplate.groovy index e4b84a259..79dfbf5b8 100644 --- a/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/KogitoJobTemplate.groovy +++ b/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/KogitoJobTemplate.groovy @@ -44,7 +44,7 @@ class KogitoJobTemplate { String jobFolderName = '' def jobFolder = jobParams.job.folder if (jobFolder) { - if (![JenkinsFolder].any { it.isAssignableFrom(jobFolder.getClass()) }) { + if (!jobFolder instanceof JenkinsFolder) { throw new RuntimeException('Folder is not of type org.kie.jenkins.jobdsl.model.JenkinsFolder') } @@ -182,14 +182,14 @@ class KogitoJobTemplate { jobParams.pr = jobParams.pr ?: [:] jobParams.pr.putAll ([ run_only_for_labels: (jobParams.pr.run_only_for_labels ?: []) + [KogitoConstants.LABEL_DSL_TEST], - run_only_for_branches: [ jobParams.git.repository == 'optaplanner-quickstarts' ? 'development' : 'main' ], - authorized_users: [ 'kiegroup' ], - authorized_groups: [ 'kiegroup' ], + run_only_for_branches: [ VersionUtils.isOptaplannerQuickstartsProject(jobParams.git.repository) ? 'development' : 'main' ], + authorized_users: [ 'apache' ], + authorized_groups: [ 'apache' ], ]) // Enable PR test only if main branch if (Utils.isMainBranch(script)) { - jobParams.git.project_url = "https://github.com/kiegroup/${jobParams.pr.target_repository ?: jobParams.git.repository}/" + jobParams.git.project_url = "https://github.com/apache/${jobParams.pr.target_repository ?: jobParams.git.repository}/" } } @@ -360,7 +360,7 @@ class KogitoJobTemplate { jobParams.job.folder = prFolder jobParams.env = jobParams.env ?: [:] jobParams.pr = jobParams.pr ?: [:] - JobParamsUtils.setupJobParamsDefaultMavenConfiguration(script, jobParams) + JobParamsUtils.setupJobParamsAgentDockerBuilderImageConfiguration(script, jobParams) // Kept for backward compatibility jobParams.job.name += testTypeId ? ".${testTypeId}" : '' @@ -466,4 +466,94 @@ class KogitoJobTemplate { return "(.*${RegexUtils.getRegexFirstLetterCase('jenkins')},?.*(rerun|run) ${idStr}${testType}.*)" } + static def createPullRequestMultibranchPipelineJob(def script, String jenkinsFilePath = '.ci/jenkins/Jenkinsfile') { + String jobName = "${Utils.getJobDisplayName(script)}-pr" + PrintUtils.debug(script, "Create pull request multibranch job ${jobName}") + script.folder('pullrequest_jobs') + return script.multibranchPipelineJob("pullrequest_jobs/${Utils.getJobDisplayName(script)}-pr")?.with { + triggers { + periodicFolderTrigger { + // The maximum amount of time since the last indexing that is allowed to elapse before an indexing is triggered. + interval('5m') + } + } + factory { + workflowBranchProjectFactory { + scriptPath(jenkinsFilePath) + } + } + branchSources { + github { + id(Utils.getRepoName(script)) // IMPORTANT: use a constant and unique identifier + repoOwner(Utils.getGitAuthor(script)) + repository(Utils.getRepoName(script)) + checkoutCredentialsId(Utils.getGitAuthorCredsId(script)) + scanCredentialsId(Utils.getGitAuthorCredsId(script)) + // Build fork PRs (unmerged head). + buildForkPRHead(false) + // Build fork PRs (merged with base branch). + buildForkPRMerge(true) + // Build origin branches. + buildOriginBranch(false) + // Build origin branches also filed as PRs. + buildOriginBranchWithPR(false) + // Build origin PRs (unmerged head). + buildOriginPRHead(false) + // Build origin PRs (merged with base branch). + buildOriginPRMerge(true) + } + } + orphanedItemStrategy { + discardOldItems { + daysToKeep(10) + } + } + } + } + + static def createBranchMultibranchPipelineJob(def script, String jobName = '', String jenkinsFilePath = '.ci/jenkins/Jenkinsfile') { + jobName = jobName ?: Utils.getJobDisplayName(script) + PrintUtils.debug(script, "Create branch multibranch job ${jobName}") + script.folder('branch_jobs') + return script.multibranchPipelineJob("branch_jobs/${jobName}")?.with { + triggers { + periodicFolderTrigger { + // The maximum amount of time since the last indexing that is allowed to elapse before an indexing is triggered. + interval('5m') + } + } + factory { + workflowBranchProjectFactory { + scriptPath(jenkinsFilePath) + } + } + branchSources { + github { + id(Utils.getRepoName(script)) // IMPORTANT: use a constant and unique identifier + repoOwner(Utils.getGitAuthor(script)) + repository(Utils.getRepoName(script)) + checkoutCredentialsId(Utils.getGitAuthorCredsId(script)) + scanCredentialsId(Utils.getGitAuthorCredsId(script)) + // Build fork PRs (unmerged head). + buildForkPRHead(false) + // Build fork PRs (merged with base branch). + buildForkPRMerge(false) + // Build origin branches. + buildOriginBranch(true) + // Build origin branches also filed as PRs. + buildOriginBranchWithPR(false) + // Build origin PRs (unmerged head). + buildOriginPRHead(false) + // Build origin PRs (merged with base branch). + buildOriginPRMerge(false) + } + } + orphanedItemStrategy { + discardOldItems { + daysToKeep(10) + } + } + } + } + } diff --git a/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/KogitoJobUtils.groovy b/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/KogitoJobUtils.groovy index 929adcdc9..72ba7b3fb 100644 --- a/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/KogitoJobUtils.groovy +++ b/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/KogitoJobUtils.groovy @@ -42,8 +42,8 @@ class KogitoJobUtils { * */ static def createVersionUpdateToolsJob(def script, String repository, String dependencyName, def mavenUpdate = [:], def gradleUpdate = [:], def filepathReplaceRegex = [], def scriptCalls = []) { - def jobParams = JobParamsUtils.getSeedJobParams(script, "update-${dependencyName.toLowerCase()}-${repository}", JobType.TOOLS, 'Jenkinsfile.update-dependency-version', "Update ${dependencyName} version for ${repository}") - JobParamsUtils.setupJobParamsDefaultMavenConfiguration(script, jobParams) + def jobParams = JobParamsUtils.getSeedJobParams(script, "update-${dependencyName.toLowerCase()}-${Utils.getRepositoryJobDisplayName(script, repository)}", JobType.TOOLS, 'Jenkinsfile.update-dependency-version', "Update ${dependencyName} version for ${repository}") + JobParamsUtils.setupJobParamsAgentDockerBuilderImageConfiguration(script, jobParams) // Setup correct checkout branch for pipelines jobParams.env.putAll([ JENKINS_EMAIL_CREDS_ID: Utils.getJenkinsEmailCredsId(script), @@ -140,7 +140,7 @@ class KogitoJobUtils { */ static def createQuarkusPlatformUpdateToolsJob(def script, String project) { def jobParams = JobParamsUtils.getSeedJobParams(script, "update-quarkus-platform-${project}", JobType.TOOLS, 'Jenkinsfile.update-quarkus-platform', "Update Quarkus platform with new version of ${project}") - JobParamsUtils.setupJobParamsDefaultMavenConfiguration(script, jobParams) + JobParamsUtils.setupJobParamsAgentDockerBuilderImageConfiguration(script, jobParams) jobParams.env.putAll([ JENKINS_EMAIL_CREDS_ID: Utils.getJenkinsEmailCredsId(script), BUILD_BRANCH_NAME: Utils.getGitBranch(script), @@ -178,7 +178,7 @@ class KogitoJobUtils { * */ static def createEnvironmentIntegrationBranchNightlyJob(def script, String envName, def scriptCalls = [], Closure defaultJobParamsGetter = JobParamsUtils.DEFAULT_PARAMS_GETTER) { - def jobParams = JobParamsUtils.getSeedJobParamsWithEnv(script, "${Utils.getRepoName(script)}.integration", JobType.NIGHTLY, envName, 'Jenkinsfile.environment.integration-branch', "Generic Integration branch job for specific environment", defaultJobParamsGetter) + def jobParams = JobParamsUtils.getSeedJobParamsWithEnv(script, "${Utils.getJobDisplayName(script)}.integration", JobType.NIGHTLY, envName, 'Jenkinsfile.environment.integration-branch', "Generic Integration branch job for specific environment", defaultJobParamsGetter) jobParams.triggers = jobParams.triggers ?: [ cron : '@midnight' ] // To remove once environment nightlies are managed by main nightly pipeline if (!envName) { throw new RuntimeException('Please provide a non-empty environment to generate an integration branch job...') @@ -237,7 +237,7 @@ class KogitoJobUtils { * See also `createBranchBuildChainJob` method */ static def createNightlyBuildChainBuildAndTestJob(def script, String envName = '', String repository, Map extraEnv = [:], boolean enableNotification = false, Closure defaultJobParamsGetter = JobParamsUtils.DEFAULT_PARAMS_GETTER) { - def jobParams = JobParamsUtils.getSeedJobParamsWithEnv(script, "${repository}.build-and-test", JobType.NIGHTLY, envName, KogitoConstants.BUILD_CHAIN_JENKINSFILE, "Build & Test for ${repository} using the build-chain", defaultJobParamsGetter) + def jobParams = JobParamsUtils.getSeedJobParamsWithEnv(script, "${Utils.getRepositoryJobDisplayName(script, repository)}.build-and-test", JobType.NIGHTLY, envName, KogitoConstants.BUILD_CHAIN_JENKINSFILE, "Build & Test for ${repository} using the build-chain", defaultJobParamsGetter) jobParams.env.putAll(extraEnv) jobParams.triggers = jobParams.triggers ?: [ cron : '@midnight' ] // To remove once environment nightlies are managed by main nightly pipeline return createBranchBuildChainJob(script, jobParams, repository, enableNotification, envName) @@ -258,7 +258,7 @@ class KogitoJobUtils { * See also `createBranchBuildChainJob` method */ static def createNightlyBuildChainBuildAndDeployJob(def script, String envName = '', String repository, Map extraEnv = [:], boolean enableNotification = false, Closure defaultJobParamsGetter = JobParamsUtils.DEFAULT_PARAMS_GETTER) { - def jobParams = JobParamsUtils.getSeedJobParamsWithEnv(script, "${repository}.build-and-deploy", JobType.NIGHTLY, envName, KogitoConstants.BUILD_CHAIN_JENKINSFILE, "Build & Test for ${repository} using the build-chain", defaultJobParamsGetter) + def jobParams = JobParamsUtils.getSeedJobParamsWithEnv(script, "${Utils.getRepositoryJobDisplayName(script, repository)}.build-and-deploy", JobType.NIGHTLY, envName, KogitoConstants.BUILD_CHAIN_JENKINSFILE, "Build & Test for ${repository} using the build-chain", defaultJobParamsGetter) JobParamsUtils.setupJobParamsDeployConfiguration(script, jobParams) jobParams.env.putAll(extraEnv) return createBranchBuildChainJob(script, jobParams, repository, enableNotification, envName) @@ -285,7 +285,7 @@ class KogitoJobUtils { * */ static def createBuildChainIntegrationJob(def script, String envName, String repository, boolean enableNotification = false, Closure defaultJobParamsGetter = JobParamsUtils.DEFAULT_PARAMS_GETTER) { - def jobParams = JobParamsUtils.getSeedJobParamsWithEnv(script, "${repository}.integration", JobType.NIGHTLY, envName, KogitoConstants.BUILD_CHAIN_JENKINSFILE, "Integration with Quarkus for ${repository} using the build-chain", defaultJobParamsGetter) + def jobParams = JobParamsUtils.getSeedJobParamsWithEnv(script, "${Utils.getRepositoryJobDisplayName(script, repository)}.integration", JobType.NIGHTLY, envName, KogitoConstants.BUILD_CHAIN_JENKINSFILE, "Integration with Quarkus for ${repository} using the build-chain", defaultJobParamsGetter) if (!envName) { throw new RuntimeException('Please provide a non-empty environment to generate an integration branch job...') } @@ -303,7 +303,6 @@ class KogitoJobUtils { * - notificationJobName: Identifier for the notification stream */ static def createBranchBuildChainJob(def script, def jobParams, String repository, boolean enableNotification = false, String notificationJobName = '') { - JobParamsUtils.setupJobParamsDefaultMavenConfiguration(script, jobParams) JobParamsUtils.setupJobParamsBuildChainConfiguration(script, jobParams, repository, 'branch', notificationJobName) jobParams.parametersClosures.add({ diff --git a/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/Utils.groovy b/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/Utils.groovy index 9514b2bd9..d04c3a47f 100644 --- a/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/Utils.groovy +++ b/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/Utils.groovy @@ -48,10 +48,6 @@ class Utils { static boolean hasBindingValuesStartingWith(def script, String keyPrefix) { return getBindingValuesStartingWith(script, keyPrefix).size() > 0 } - - static boolean isProductizedBranch(def script) { - return getBindingValue(script, 'PRODUCTIZED_BRANCH').toBoolean() - } static String getGenerationBranch(def script) { return getBindingValue(script, 'GENERATION_BRANCH') @@ -67,6 +63,14 @@ class Utils { return getBindingValue(script, 'REPO_NAME') } + static String getRepositoryJobDisplayName(def script, String repository) { + return getBindingValue(script, "${repository.toUpperCase()}_JOB_DISPLAY_NAME") ?: repository + } + + static String getJobDisplayName(def script) { + return getRepositoryJobDisplayName(script, getRepoName(script)) + } + static String getGitBranch(def script) { return getBindingValue(script, 'GIT_BRANCH') } @@ -102,9 +106,9 @@ class Utils { static boolean isProdEnvironment(def script) { // Check for all possible `GIT_AUTHOR_NAME` variables - return getSeedAuthor(script) == 'kiegroup' && - (hasGitAuthor(script) ? getGitAuthor(script) == 'kiegroup' : true) && - (hasSeedConfigFileGitAuthor(script) ? getSeedConfigFileGitAuthor(script) == 'kiegroup' : true) + return getSeedAuthor(script) == 'kieapachegroup' && + (hasGitAuthor(script) ? getGitAuthor(script) == 'apache' : true) && + (hasSeedConfigFileGitAuthor(script) ? getSeedConfigFileGitAuthor(script) == 'apache' : true) } static boolean isTestEnvironment(def script) { @@ -223,6 +227,14 @@ class Utils { return getBindingValue(script, 'MAVEN_QUARKUS_PLATFORM_REPOSITORY_CREDS_ID') } + static String getJenkinsAgentDockerImage(def script, String imageId) { + return getBindingValue(script, "JENKINS_AGENT_DOCKER_${imageId.toUpperCase()}_IMAGE") + } + + static String getJenkinsAgentDockerArgs(def script, String imageId) { + return getBindingValue(script, "JENKINS_AGENT_DOCKER_${imageId.toUpperCase()}_ARGS") + } + static String getSeedJenkinsfilePath(def script, String jenkinsfileName) { return "${KogitoConstants.SEED_JENKINSFILES_PATH}/${jenkinsfileName}" } @@ -263,4 +275,8 @@ class Utils { return words.collect { it.isEmpty() ? it : it.substring(0, 1).toUpperCase() + it.substring(1).toLowerCase() }.join(' ') } + static boolean isDeployDisabled(def script) { + return getBindingValue(script, 'DISABLE_DEPLOY').toBoolean() || isTestEnvironment(script) + } + } diff --git a/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/utils/JobParamsUtils.groovy b/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/utils/JobParamsUtils.groovy index 7c946f034..d1da98aa2 100644 --- a/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/utils/JobParamsUtils.groovy +++ b/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/utils/JobParamsUtils.groovy @@ -16,7 +16,7 @@ class JobParamsUtils { repository = repository ?: Utils.getRepoName(script) def jobParams = [ job: [ - name: repository, + name: Utils.getRepositoryJobDisplayName(script, repository), ], git: [ author: Utils.getGitAuthor(script), @@ -135,9 +135,10 @@ class JobParamsUtils { static def setupJobParamsBuildChainConfiguration(def script, def jobParams, String repository, String buildchainType, String notificationJobName) { setupJobParamsSeedPipelineConfiguration(script, jobParams, KogitoConstants.BUILD_CHAIN_JENKINSFILE) + setupJobParamsAgentDockerBuilderImageConfiguration(script, jobParams) jobParams.env = jobParams.env ?: [:] jobParams.env.putAll([ - BUILDCHAIN_PROJECT: "kiegroup/${repository}", + BUILDCHAIN_PROJECT: "apache/${repository}", BUILDCHAIN_TYPE: buildchainType, BUILDCHAIN_CONFIG_REPO: Utils.getBuildChainConfigRepo(script) ?: Utils.getSeedRepo(script), BUILDCHAIN_CONFIG_AUTHOR: Utils.getBuildChainConfigAuthor(script) ?: Utils.getSeedAuthor(script), @@ -168,8 +169,14 @@ class JobParamsUtils { static def setupJobParamsDeployConfiguration(def script, def jobParams) { jobParams.env = jobParams.env ?: [:] - jobParams.env.put('ENABLE_DEPLOY', 'true') + jobParams.env.put('ENABLE_DEPLOY', String.valueOf(!Utils.isDeployDisabled(script))) addJobParamsEnvIfNotExisting(script, jobParams, 'MAVEN_DEPLOY_REPOSITORY', Utils.getMavenArtifactsUploadRepositoryUrl(script)) addJobParamsEnvIfNotExisting(script, jobParams, 'MAVEN_DEPLOY_REPOSITORY_CREDS_ID', Utils.getMavenArtifactsUploadRepositoryCredentialsId(script)) } + + static def setupJobParamsAgentDockerBuilderImageConfiguration(def script, def jobParams) { + jobParams.env = jobParams.env ?: [:] + addJobParamsEnvIfNotExisting(script, jobParams, 'AGENT_DOCKER_BUILDER_IMAGE', Utils.getJenkinsAgentDockerImage(script, 'builder')) + addJobParamsEnvIfNotExisting(script, jobParams, 'AGENT_DOCKER_BUILDER_ARGS', Utils.getJenkinsAgentDockerArgs(script, 'builder')) + } } diff --git a/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/utils/SeedJobUtils.groovy b/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/utils/SeedJobUtils.groovy index 8abdbb183..0d9b9a1bf 100644 --- a/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/utils/SeedJobUtils.groovy +++ b/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/utils/SeedJobUtils.groovy @@ -15,7 +15,7 @@ class SeedJobUtils { throw new RuntimeException('pathsToListen cannot be empty, else it would end up in an infinite loop ...'); } def job = jenkinsScript.pipelineJob(jobName) { - description('This job listens to pipelines repo and launch the seed job if needed. DO NOT USE FOR TESTING !!!! See https://github.com/kiegroup/kogito-pipelines/blob/main/docs/jenkins.md#test-specific-jobs') + description('This job listens to pipelines repo and launch the seed job if needed. DO NOT USE FOR TESTING !!!! See https://github.com/apache/incubator-kie-kogito-pipelines/blob/main/docs/jenkins.md#test-specific-jobs') logRotator { numToKeep(5) @@ -38,12 +38,13 @@ class SeedJobUtils { env('JOB_RELATIVE_PATH_TO_TRIGGER', jobRelativePathToTrigger) env('LISTEN_TO_MODIFIED_PATHS', new groovy.json.JsonBuilder(pathsToListen).toString()) - env('AGENT_LABEL', Utils.isProdEnvironment(jenkinsScript) ? 'rhel8 && !built-in' : 'kie-rhel8-priority') + env('AGENT_LABEL', 'ubuntu') } definition { cps { script(jenkinsScript.readFileFromWorkspace('jenkinsfiles/Jenkinsfile.seed.trigger')) + sandbox() } } diff --git a/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/utils/VersionUtils.groovy b/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/utils/VersionUtils.groovy index 7a8adeda0..ac9472b3f 100644 --- a/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/utils/VersionUtils.groovy +++ b/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/utils/VersionUtils.groovy @@ -28,15 +28,15 @@ class VersionUtils { } static boolean isKogitoProject(String project) { - return project.startsWith('kogito') + return project.startsWith('kogito') || project.startsWith('incubator-kie-kogito') } static boolean isOptaplannerProject(String project) { - return project.startsWith('opta') + return project.startsWith('opta') || project.startsWith('incubator-kie-opta') } static boolean isDroolsProject(String project) { - return project.startsWith('drools') + return project.startsWith('drools') || project.startsWith('incubator-kie-drools') } static boolean isDroolsOrOptaPlannerProject(String project) { @@ -44,7 +44,7 @@ class VersionUtils { } static boolean isOptaplannerQuickstartsProject(String project) { - return project == 'optaplanner-quickstarts' + return project == 'optaplanner-quickstarts' || project == 'incubator-kie-optaplanner-quickstarts' } } diff --git a/dsl/seed/src/test/groovy/org/kie/jenkins/jobdsl/JobScriptsSpec.groovy b/dsl/seed/src/test/groovy/org/kie/jenkins/jobdsl/JobScriptsSpec.groovy index 555ab0e31..0e7d74138 100644 --- a/dsl/seed/src/test/groovy/org/kie/jenkins/jobdsl/JobScriptsSpec.groovy +++ b/dsl/seed/src/test/groovy/org/kie/jenkins/jobdsl/JobScriptsSpec.groovy @@ -44,7 +44,6 @@ class JobScriptsSpec extends Specification { envVars.put('GENERATION_BRANCH', 'GENERATION_BRANCH') envVars.put('RELEASE_PROJECTS', 'RELEASE_PROJECTS') - envVars.put('PRODUCTIZED_PROJECTS', 'PRODUCTIZED_PROJECTS') envVars.put('DEPENDENCY_PROJECTS', 'DEPENDENCY_PROJECTS') envVars.put('REPO_NAME', 'REPO_NAME') envVars.put('GIT_MAIN_BRANCH', 'GIT_MAIN_BRANCH') diff --git a/tools/update-kie7-version.sh b/tools/update-kie7-version.sh index a1582c1c3..6e5e2ec70 100755 --- a/tools/update-kie7-version.sh +++ b/tools/update-kie7-version.sh @@ -85,7 +85,7 @@ if [ "$FORK" = "" ]; then fi REPO=kogito-runtimes -ORIGIN=kiegroup/$REPO +ORIGIN=apache/$REPO PR_FORK=$FORK/$REPO PR_BRANCH=${BRANCH}-bump-${PREFIX}kie-$KIE_VERSION @@ -120,7 +120,7 @@ if [ "$DRY_RUN" = "false" ]; then git push -u ${GITHUB_URL}${PR_FORK} ${PR_BRANCH} # Open a PR to kogito-runtimes using the commit as a title - # e.g. see https://github.com/kiegroup/kogito-runtimes/pull/1200 + # e.g. see https://github.com/apache/incubator-kie-kogito-runtimes/pull/1200 gh pr create --fill --base $BRANCH -R $ORIGIN fi diff --git a/tools/update-quarkus-versions.sh b/tools/update-quarkus-versions.sh index ad4cf80a0..19b72ee8f 100755 --- a/tools/update-quarkus-versions.sh +++ b/tools/update-quarkus-versions.sh @@ -113,7 +113,7 @@ if [ "$FORK" = "" ]; then exit 2 fi -ORIGIN=kiegroup/$REPO +ORIGIN=apache/$REPO PR_FORK=$FORK/$REPO PR_BRANCH=${BRANCH}-bump-quarkus-$QUARKUS_VERSION @@ -183,6 +183,6 @@ if [ "$DRY_RUN" = "false" ]; then git push -u ${GITHUB_URL}${PR_FORK} ${PR_BRANCH} # Open a PR to kogito-runtimes using the commit as a title - # e.g. see https://github.com/kiegroup/kogito-runtimes/pull/1200 + # e.g. see https://github.com/apache/incubator-kie-kogito-runtimes/pull/1200 gh pr create --fill --base $BRANCH -R $ORIGIN fi \ No newline at end of file From aa8afba484e07cadf0003853abdf7e4c2db7b456 Mon Sep 17 00:00:00 2001 From: Jan Stastny Date: Wed, 20 Sep 2023 09:19:55 +0200 Subject: [PATCH 09/25] kie-issues#578 install freetype package using dnf (#1091) * kie-issues#578 install freetype package using dnf * adding further deps --------- Co-authored-by: jstastny-cz --- apache-nodes/Dockerfile.kogito-ci-build | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/apache-nodes/Dockerfile.kogito-ci-build b/apache-nodes/Dockerfile.kogito-ci-build index d5cafc03b..2f130382a 100644 --- a/apache-nodes/Dockerfile.kogito-ci-build +++ b/apache-nodes/Dockerfile.kogito-ci-build @@ -33,6 +33,12 @@ sudo \ wget \ which \ # system (END) +# drools (BEGIN) +fontconfig \ +freetype \ +# couldn't get it for pre-defined repositories +https://mirror.stream.centos.org/9-stream/AppStream/aarch64/os/Packages/overpass-fonts-3.0.4-8.el9.noarch.rpm \ +# drools (END) && dnf clean all RUN sudo alternatives --install /usr/local/bin/python python $(which python${PYTHON_MAJOR_MINOR_VERSION}) 1 && \ From fe9b050d5851c23869b22a364f1115d6f34da0f0 Mon Sep 17 00:00:00 2001 From: Jan Stastny Date: Wed, 20 Sep 2023 14:50:01 +0200 Subject: [PATCH 10/25] kie-issues#579: remove wrong argument being passed in KogitoJobUtils (#1092) Co-authored-by: jstastny-cz --- .../main/groovy/org/kie/jenkins/jobdsl/KogitoJobUtils.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/KogitoJobUtils.groovy b/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/KogitoJobUtils.groovy index 72ba7b3fb..105d1ce3b 100644 --- a/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/KogitoJobUtils.groovy +++ b/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/KogitoJobUtils.groovy @@ -86,7 +86,7 @@ class KogitoJobUtils { * Create a Quarkus update job which allow to update the quarkus version into current repository, via Maven or Gradle */ static def createQuarkusVersionUpdateToolsJobForCurrentRepo(def script, def mavenUpdate = [:], def gradleUpdate = [:], def filepathReplaceRegex = [:]) { - return createQuarkusUpdateToolsJob(script, Utils.getRepoName(script), 'Quarkus', mavenUpdate, gradleUpdate, filepathReplaceRegex) + return createQuarkusUpdateToolsJob(script, Utils.getRepoName(script), mavenUpdate, gradleUpdate, filepathReplaceRegex) } /** From 93c6bffbe0438c2dc051a739960c12472eddf511 Mon Sep 17 00:00:00 2001 From: Jan Stastny Date: Wed, 20 Sep 2023 16:30:53 +0200 Subject: [PATCH 11/25] kie-issues#581: start kogito-ci-build container with host network for PRs (#1093) Co-authored-by: jstastny-cz --- dsl/scripts/pr_check.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dsl/scripts/pr_check.groovy b/dsl/scripts/pr_check.groovy index ea74d7ab9..6be210507 100644 --- a/dsl/scripts/pr_check.groovy +++ b/dsl/scripts/pr_check.groovy @@ -6,10 +6,11 @@ dockerGroups = [ ] dockerArgs = [ '-v /var/run/docker.sock:/var/run/docker.sock', + '--network host', ] + dockerGroups.collect { group -> "--group-add ${group}" } void launch() { - String builderImage = 'quay.io/kiegroup/kogito-ci-build:latest' + String builderImage = 'quay.io/kiegroup/kogito-ci-build:main-latest' sh "docker rmi -f ${builderImage} || true" // Remove before launching try { From 84acfcc170f08d5aa125bc9a259a67b4db0f7245 Mon Sep 17 00:00:00 2001 From: Jan Stastny Date: Thu, 21 Sep 2023 10:34:22 +0200 Subject: [PATCH 12/25] kie-issues#584: add docker-compose on PATH in kogito-ci-build image (#1094) Co-authored-by: jstastny-cz --- apache-nodes/Dockerfile.kogito-ci-build | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apache-nodes/Dockerfile.kogito-ci-build b/apache-nodes/Dockerfile.kogito-ci-build index 2f130382a..7208c224d 100644 --- a/apache-nodes/Dockerfile.kogito-ci-build +++ b/apache-nodes/Dockerfile.kogito-ci-build @@ -52,7 +52,8 @@ RUN dnf -y update && dnf install -y yum-utils device-mapper-persistent-data lvm dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo && \ dnf remove podman buildah && \ dnf install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin --nobest && \ - dnf clean all + dnf clean all && \ + alternatives --install /usr/local/bin/docker-compose docker-compose /usr/libexec/docker/cli-plugins/docker-compose 1 USER nonrootuser From 8718cd516044cbcb39d94e8d6da8f34c97fb4ab8 Mon Sep 17 00:00:00 2001 From: Jan Stastny Date: Fri, 22 Sep 2023 14:30:52 +0200 Subject: [PATCH 13/25] kie-issues#586: handle ENABLE_DEPLOY env as boolean (#1096) Co-authored-by: jstastny-cz --- dsl/seed/jenkinsfiles/Jenkinsfile.buildchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dsl/seed/jenkinsfiles/Jenkinsfile.buildchain b/dsl/seed/jenkinsfiles/Jenkinsfile.buildchain index dc658eac9..6725a9197 100644 --- a/dsl/seed/jenkinsfiles/Jenkinsfile.buildchain +++ b/dsl/seed/jenkinsfiles/Jenkinsfile.buildchain @@ -9,7 +9,7 @@ jdkTool = env.BUILD_JDK_TOOL sonarJdkTool = env.SONAR_JDK_TOOL mavenTool = env.BUILD_MAVEN_TOOL -mavenDeployArtifacts = env.ENABLE_DEPLOY +mavenDeployArtifacts = Boolean.valueof(env.ENABLE_DEPLOY) mavenDeployRepository = env.MAVEN_DEPLOY_REPOSITORY ?: '' mavenDeployRepositoryCredsId = env.MAVEN_DEPLOY_REPOSITORY_CREDS_ID ?: '' mavenDeployLocalDir = env.MAVEN_DEPLOY_LOCAL_DIR ?: '' From 55f13d6c23fc598b6650e8daf97555f2a687876c Mon Sep 17 00:00:00 2001 From: Jan Stastny Date: Sun, 24 Sep 2023 12:50:49 +0200 Subject: [PATCH 14/25] Update typo in Jenkinsfile.buildchain --- dsl/seed/jenkinsfiles/Jenkinsfile.buildchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dsl/seed/jenkinsfiles/Jenkinsfile.buildchain b/dsl/seed/jenkinsfiles/Jenkinsfile.buildchain index 6725a9197..ce3adbe43 100644 --- a/dsl/seed/jenkinsfiles/Jenkinsfile.buildchain +++ b/dsl/seed/jenkinsfiles/Jenkinsfile.buildchain @@ -9,7 +9,7 @@ jdkTool = env.BUILD_JDK_TOOL sonarJdkTool = env.SONAR_JDK_TOOL mavenTool = env.BUILD_MAVEN_TOOL -mavenDeployArtifacts = Boolean.valueof(env.ENABLE_DEPLOY) +mavenDeployArtifacts = Boolean.valueOf(env.ENABLE_DEPLOY) mavenDeployRepository = env.MAVEN_DEPLOY_REPOSITORY ?: '' mavenDeployRepositoryCredsId = env.MAVEN_DEPLOY_REPOSITORY_CREDS_ID ?: '' mavenDeployLocalDir = env.MAVEN_DEPLOY_LOCAL_DIR ?: '' From 8a5abd4f1ef40823a7f3c534ad297af3d78da89c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tibor=20Zim=C3=A1nyi?= Date: Mon, 2 Oct 2023 12:31:06 +0200 Subject: [PATCH 15/25] Don't rebuild PR checks when origin changes (#1097) --- .../groovy/org/kie/jenkins/jobdsl/KogitoJobTemplate.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/KogitoJobTemplate.groovy b/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/KogitoJobTemplate.groovy index 79dfbf5b8..7857df9ca 100644 --- a/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/KogitoJobTemplate.groovy +++ b/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/KogitoJobTemplate.groovy @@ -490,9 +490,9 @@ class KogitoJobTemplate { checkoutCredentialsId(Utils.getGitAuthorCredsId(script)) scanCredentialsId(Utils.getGitAuthorCredsId(script)) // Build fork PRs (unmerged head). - buildForkPRHead(false) + buildForkPRHead(true) // Build fork PRs (merged with base branch). - buildForkPRMerge(true) + buildForkPRMerge(false) // Build origin branches. buildOriginBranch(false) // Build origin branches also filed as PRs. @@ -500,7 +500,7 @@ class KogitoJobTemplate { // Build origin PRs (unmerged head). buildOriginPRHead(false) // Build origin PRs (merged with base branch). - buildOriginPRMerge(true) + buildOriginPRMerge(false) } } orphanedItemStrategy { From 4b0c19f88738d3302a2d1cfb983cafaba6f1e88b Mon Sep 17 00:00:00 2001 From: Jan Stastny Date: Mon, 2 Oct 2023 12:39:56 +0200 Subject: [PATCH 16/25] kie-issues#591: Install gcc in kogito-ci-build image (#1098) Co-authored-by: jstastny-cz --- apache-nodes/Dockerfile.kogito-ci-build | 1 + 1 file changed, 1 insertion(+) diff --git a/apache-nodes/Dockerfile.kogito-ci-build b/apache-nodes/Dockerfile.kogito-ci-build index 7208c224d..f9597587c 100644 --- a/apache-nodes/Dockerfile.kogito-ci-build +++ b/apache-nodes/Dockerfile.kogito-ci-build @@ -28,6 +28,7 @@ krb5-devel \ gcc \ # python3 (END) # system (BEGIN) +gcc-c++ \ shadow-utils \ sudo \ wget \ From fea26e17fc0566ef5858365360df43c4fbd2b0cd Mon Sep 17 00:00:00 2001 From: Jan Stastny Date: Mon, 2 Oct 2023 12:41:38 +0200 Subject: [PATCH 17/25] kie-issues#592: fix postComment for current plugin (#1099) Co-authored-by: jstastny-cz --- dsl/scripts/pr_check.groovy | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/dsl/scripts/pr_check.groovy b/dsl/scripts/pr_check.groovy index 6be210507..b5bee9692 100644 --- a/dsl/scripts/pr_check.groovy +++ b/dsl/scripts/pr_check.groovy @@ -31,7 +31,10 @@ void launchInDocker(String builderImage) { echo "Got build result ${currentBuild.currentResult}" if (currentBuild.currentResult != 'SUCCESS') { // TODO ci token as env ? - pullrequest.postComment(util.getMarkdownTestSummary('PR', getReproducer(true), "${BUILD_URL}", 'GITHUB'), "kie-ci3-token") + postComment( + util.getMarkdownTestSummary('PR', getReproducer(true), "${BUILD_URL}", 'GITHUB'), + 'kie-ci3-token' + ) } } } @@ -133,4 +136,23 @@ ${reproducer} } } +/** +* This method add a comment to current PR (for Github Branch Source plugin) +*/ +void postComment(String commentText, String githubTokenCredsId = "kie-ci1-token") { + if (!CHANGE_ID) { + error "Pull Request Id variable (CHANGE_ID) is not set. Are you sure you are running with Github Branch Source plugin ?" + } + String filename = "${util.generateHash(10)}.build.summary" + def jsonComment = [ + body : commentText + ] + writeJSON(json: jsonComment, file: filename) + sh "cat ${filename}" + withCredentials([string(credentialsId: githubTokenCredsId, variable: 'GITHUB_TOKEN')]) { + sh "curl -s -H \"Authorization: token ${GITHUB_TOKEN}\" -X POST -d '@${filename}' \"https://api.github.com/repos/${BUILDCHAIN_PROJECT}/issues/${CHANGE_ID}/comments\"" + } + sh "rm ${filename}" +} + return this From 9df49c160ff78c94b6c3e823d6281f3c3e016744 Mon Sep 17 00:00:00 2001 From: Jan Stastny Date: Mon, 2 Oct 2023 14:31:49 +0200 Subject: [PATCH 18/25] kie-issues#593: switch to compile downstream in Jenkins (#1100) Co-authored-by: jstastny-cz --- .ci/buildchain-config-pr-cdb.yaml | 82 +++++++++++++++++++++++++++++++ dsl/scripts/pr_check.groovy | 2 +- 2 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 .ci/buildchain-config-pr-cdb.yaml diff --git a/.ci/buildchain-config-pr-cdb.yaml b/.ci/buildchain-config-pr-cdb.yaml new file mode 100644 index 000000000..a7c1c09fe --- /dev/null +++ b/.ci/buildchain-config-pr-cdb.yaml @@ -0,0 +1,82 @@ +version: "2.1" + +dependencies: ./project-dependencies.yaml + +pre: | + export BUILD_MVN_OPTS="${{ env.BUILD_MVN_OPTS }} -nsu -ntp -fae -e -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 -Dmaven.wagon.http.retryHandler.count=3" + echo "BUILD_MVN_OPTS=${{ env.BUILD_MVN_OPTS }}" + export BUILD_MVN_OPTS_CURRENT="${{ env.BUILD_MVN_OPTS_CURRENT }}" + echo "BUILD_MVN_OPTS_CURRENT=${{ env.BUILD_MVN_OPTS_CURRENT }}" + echo "QUARKUS_VERSION=${{ env.QUARKUS_VERSION }}" + echo "ENABLE_DEPLOY=${{ env.ENABLE_DEPLOY }}" + +default: + build-command: + before: + current: | + export INTEGRATION_BRANCH=${{ env.INTEGRATION_BRANCH_CURRENT }} + bash -c "if [ ! -z '${{ env.BUILD_ENVIRONMENT }}' ] && [ -f .ci/environments/update.sh ]; then .ci/environments/update.sh ${{ env.BUILD_ENVIRONMENT }} ${{ env.BUILD_ENVIRONMENT_OPTIONS_CURRENT }}; fi" + upstream: | + export INTEGRATION_BRANCH=${{ env.INTEGRATION_BRANCH_UPSTREAM }} + bash -c "if [ ! -z '${{ env.BUILD_ENVIRONMENT }}' ] && [ -f .ci/environments/update.sh ]; then .ci/environments/update.sh ${{ env.BUILD_ENVIRONMENT }} ${{ env.BUILD_ENVIRONMENT_OPTIONS_UPSTREAM }}; fi" + current: | + mvn dependency:tree clean install ${{ env.BUILD_MVN_OPTS }} ${{ env.BUILD_MVN_OPTS_CURRENT }} + upstream: | + mvn dependency:tree clean install -Dquickly ${{ env.BUILD_MVN_OPTS }} ${{ env.BUILD_MVN_OPTS_UPSTREAM }} + +build: + - project: apache/incubator-kie-drools + build-command: + current: | + export MVN_CMD=`bash -c "if [ '${{ env.ENABLE_DEPLOY }}' = 'true' ]; then printf 'deploy ${{ env.DEPLOY_MVN_OPTS }} ${{ env.DROOLS_DEPLOY_MVN_OPTS }}'; else printf 'install'; fi"` + mvn dependency:tree clean ${{ env.MVN_CMD }} ${{ env.BUILD_MVN_OPTS }} ${{ env.BUILD_MVN_OPTS_CURRENT }} ${{ env.DROOLS_BUILD_MVN_OPTS }} + upstream: | + mvn dependency:tree clean install -Dquickly ${{ env.BUILD_MVN_OPTS }} ${{ env.BUILD_MVN_OPTS_UPSTREAM }} ${{ env.DROOLS_BUILD_MVN_OPTS_UPSTREAM }} + downstream: | + mvn dependency:tree clean compile ${{ env.BUILD_MVN_OPTS }} ${{ env.BUILD_MVN_OPTS_DOWNSTREAM }} ${{ env.DROOLS_BUILD_MVN_OPTS_DOWNSTREAM }} + + - project: apache/incubator-kie-kogito-runtimes + build-command: + current: | + export MVN_CMD=`bash -c "if [ '${{ env.ENABLE_DEPLOY }}' = 'true' ]; then printf 'deploy ${{ env.DEPLOY_MVN_OPTS }} ${{ env.KOGITO_RUNTIMES_DEPLOY_MVN_OPTS }}'; else printf 'install'; fi"` + mvn dependency:tree clean -Dfull ${{ env.MVN_CMD }} ${{ env.BUILD_MVN_OPTS }} ${{ env.BUILD_MVN_OPTS_CURRENT }} ${{ env.KOGITO_RUNTIMES_BUILD_MVN_OPTS }} + upstream: | + mvn dependency:tree clean install -Dquickly -Dfull ${{ env.BUILD_MVN_OPTS }} ${{ env.BUILD_MVN_OPTS_UPSTREAM }} ${{ env.KOGITO_RUNTIMES_BUILD_MVN_OPTS_UPSTREAM }} + downstream: | + mvn dependency:tree clean compile -Dfull ${{ env.BUILD_MVN_OPTS }} ${{ env.BUILD_MVN_OPTS_DOWNSTREAM }} ${{ env.KOGITO_RUNTIMES_BUILD_MVN_OPTS_DOWNSTREAM }} + + - project: apache/incubator-kie-kogito-apps + build-command: + current: | + export MVN_CMD=`bash -c "if [ '${{ env.ENABLE_DEPLOY }}' = 'true' ]; then printf 'deploy ${{ env.DEPLOY_MVN_OPTS }} ${{ env.KOGITO_APPS_DEPLOY_MVN_OPTS }}'; else printf 'install'; fi"` + mvn dependency:tree clean ${{ env.MVN_CMD }} ${{ env.BUILD_MVN_OPTS }} ${{ env.BUILD_MVN_OPTS_CURRENT }} ${{ env.KOGITO_APPS_BUILD_MVN_OPTS }} + upstream: | + mvn dependency:tree clean install -Dquickly ${{ env.BUILD_MVN_OPTS }} ${{ env.BUILD_MVN_OPTS_UPSTREAM }} ${{ env.KOGITO_APPS_BUILD_MVN_OPTS_UPSTREAM }} + downstream: | + mvn dependency:tree clean compile ${{ env.BUILD_MVN_OPTS }} ${{ env.BUILD_MVN_OPTS_DOWNSTREAM }} ${{ env.KOGITO_APPS_BUILD_MVN_OPTS_DOWNSTREAM }} + archive-artifacts: + path: | + **/*.log + **/cypress/screenshots/** + **/cypress/videos/** + + - project: apache/incubator-kie-kogito-examples + build-command: + # First install the main pom + # Then build the required submodule pom + current: | + export MVN_CMD=`bash -c "if [ '${{ env.ENABLE_DEPLOY }}' = 'true' ]; then printf 'deploy ${{ env.DEPLOY_MVN_OPTS }} ${{ env.KOGITO_EXAMPLES_DEPLOY_MVN_OPTS }}'; else printf 'install'; fi"` + mvn dependency:tree -pl :kogito-examples clean ${{ env.MVN_CMD }} ${{ env.BUILD_MVN_OPTS }} ${{ env.BUILD_MVN_OPTS_CURRENT }} ${{ env.KOGITO_EXAMPLES_BUILD_MVN_OPTS }} + mvn dependency:tree -f ${{ env.KOGITO_EXAMPLES_SUBFOLDER_POM }}pom.xml clean ${{ env.MVN_CMD }} ${{ env.BUILD_MVN_OPTS }} ${{ env.BUILD_MVN_OPTS_CURRENT }} ${{ env.KOGITO_EXAMPLES_BUILD_MVN_OPTS }} + upstream: | + mvn dependency:tree clean install -DskipTests -DskipITs ${{ env.BUILD_MVN_OPTS }} ${{ env.BUILD_MVN_OPTS_UPSTREAM }} ${{ env.KOGITO_EXAMPLES_BUILD_MVN_OPTS_UPSTREAM }} + downstream: | + mvn dependency:tree clean compile ${{ env.BUILD_MVN_OPTS }} ${{ env.BUILD_MVN_OPTS_DOWNSTREAM }} ${{ env.KOGITO_EXAMPLES_BUILD_MVN_OPTS_DOWNSTREAM }} + + # - project: kiegroup/kie-jpmml-integration + # build-command: + # current: | + # export MVN_CMD=`bash -c "if [ '${{ env.ENABLE_DEPLOY }}' = 'true' ]; then printf 'deploy ${{ env.DEPLOY_MVN_OPTS }} ${{ env.KIE_JPMML_INTEGRATION_DEPLOY_MVN_OPTS }}'; else printf 'install'; fi"` + # mvn clean ${{ env.MVN_CMD }} ${{ env.BUILD_MVN_OPTS }} ${{ env.BUILD_MVN_OPTS_CURRENT }} ${{ env.KIE_JPMML_INTEGRATION_BUILD_MVN_OPTS }} + # upstream: | + # mvn clean install -DskipTests ${ env.BUILD_MVN_OPTS }} ${{ env.BUILD_MVN_OPTS_UPSTREAM }} ${{ env.KIE_JPMML_INTEGRATION_BUILD_MVN_OPTS_UPSTREAM }} diff --git a/dsl/scripts/pr_check.groovy b/dsl/scripts/pr_check.groovy index b5bee9692..8cbb7b55b 100644 --- a/dsl/scripts/pr_check.groovy +++ b/dsl/scripts/pr_check.groovy @@ -97,7 +97,7 @@ String getBuildChainCommandline() { String buildChainConfigRepo = env.BUILDCHAIN_CONFIG_REPO ?: 'incubator-kie-kogito-pipelines' String buildChainConfigBranch = env.BUILDCHAIN_CONFIG_BRANCH ?: '\${BRANCH:main}' String buildChainConfigGitAuthor = env.BUILDCHAIN_CONFIG_AUTHOR ?: '\${AUTHOR:apache}' - String buildChainConfigDefinitionFilePath = env.BUILDCHAIN_CONFIG_FILE_PATH ?: '.ci/pull-request-config.yaml' + String buildChainConfigDefinitionFilePath = env.BUILDCHAIN_CONFIG_FILE_PATH ?: '.ci/buildchain-config-pr-cdb.yaml' List buildChainAdditionalArguments = [ "-p ${buildChainProject}", From 576168d3e46ad466819d71008441a73eeb198092 Mon Sep 17 00:00:00 2001 From: Jan Stastny Date: Tue, 3 Oct 2023 09:07:31 +0200 Subject: [PATCH 19/25] kie-issues#593: use install goal for downstream projects (#1102) Co-authored-by: jstastny-cz --- .ci/buildchain-config-pr-cdb.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.ci/buildchain-config-pr-cdb.yaml b/.ci/buildchain-config-pr-cdb.yaml index a7c1c09fe..c889e6d7c 100644 --- a/.ci/buildchain-config-pr-cdb.yaml +++ b/.ci/buildchain-config-pr-cdb.yaml @@ -33,7 +33,7 @@ build: upstream: | mvn dependency:tree clean install -Dquickly ${{ env.BUILD_MVN_OPTS }} ${{ env.BUILD_MVN_OPTS_UPSTREAM }} ${{ env.DROOLS_BUILD_MVN_OPTS_UPSTREAM }} downstream: | - mvn dependency:tree clean compile ${{ env.BUILD_MVN_OPTS }} ${{ env.BUILD_MVN_OPTS_DOWNSTREAM }} ${{ env.DROOLS_BUILD_MVN_OPTS_DOWNSTREAM }} + mvn dependency:tree clean install -Dquickly ${{ env.BUILD_MVN_OPTS }} ${{ env.BUILD_MVN_OPTS_DOWNSTREAM }} ${{ env.DROOLS_BUILD_MVN_OPTS_DOWNSTREAM }} - project: apache/incubator-kie-kogito-runtimes build-command: @@ -43,7 +43,7 @@ build: upstream: | mvn dependency:tree clean install -Dquickly -Dfull ${{ env.BUILD_MVN_OPTS }} ${{ env.BUILD_MVN_OPTS_UPSTREAM }} ${{ env.KOGITO_RUNTIMES_BUILD_MVN_OPTS_UPSTREAM }} downstream: | - mvn dependency:tree clean compile -Dfull ${{ env.BUILD_MVN_OPTS }} ${{ env.BUILD_MVN_OPTS_DOWNSTREAM }} ${{ env.KOGITO_RUNTIMES_BUILD_MVN_OPTS_DOWNSTREAM }} + mvn dependency:tree clean install -Dquickly -Dfull ${{ env.BUILD_MVN_OPTS }} ${{ env.BUILD_MVN_OPTS_DOWNSTREAM }} ${{ env.KOGITO_RUNTIMES_BUILD_MVN_OPTS_DOWNSTREAM }} - project: apache/incubator-kie-kogito-apps build-command: @@ -53,7 +53,7 @@ build: upstream: | mvn dependency:tree clean install -Dquickly ${{ env.BUILD_MVN_OPTS }} ${{ env.BUILD_MVN_OPTS_UPSTREAM }} ${{ env.KOGITO_APPS_BUILD_MVN_OPTS_UPSTREAM }} downstream: | - mvn dependency:tree clean compile ${{ env.BUILD_MVN_OPTS }} ${{ env.BUILD_MVN_OPTS_DOWNSTREAM }} ${{ env.KOGITO_APPS_BUILD_MVN_OPTS_DOWNSTREAM }} + mvn dependency:tree clean install -Dquickly ${{ env.BUILD_MVN_OPTS }} ${{ env.BUILD_MVN_OPTS_DOWNSTREAM }} ${{ env.KOGITO_APPS_BUILD_MVN_OPTS_DOWNSTREAM }} archive-artifacts: path: | **/*.log @@ -71,7 +71,7 @@ build: upstream: | mvn dependency:tree clean install -DskipTests -DskipITs ${{ env.BUILD_MVN_OPTS }} ${{ env.BUILD_MVN_OPTS_UPSTREAM }} ${{ env.KOGITO_EXAMPLES_BUILD_MVN_OPTS_UPSTREAM }} downstream: | - mvn dependency:tree clean compile ${{ env.BUILD_MVN_OPTS }} ${{ env.BUILD_MVN_OPTS_DOWNSTREAM }} ${{ env.KOGITO_EXAMPLES_BUILD_MVN_OPTS_DOWNSTREAM }} + mvn dependency:tree clean install -DskipTests -DskipITs ${{ env.BUILD_MVN_OPTS }} ${{ env.BUILD_MVN_OPTS_DOWNSTREAM }} ${{ env.KOGITO_EXAMPLES_BUILD_MVN_OPTS_DOWNSTREAM }} # - project: kiegroup/kie-jpmml-integration # build-command: From e5d0c77daedde41e7d35da9b92d34b7e6b2f60fd Mon Sep 17 00:00:00 2001 From: Jan Stastny Date: Wed, 4 Oct 2023 09:04:47 +0200 Subject: [PATCH 20/25] kie-issues#594: fix seed author literal (#1101) Co-authored-by: jstastny-cz --- dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/Utils.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/Utils.groovy b/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/Utils.groovy index d04c3a47f..91eb26124 100644 --- a/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/Utils.groovy +++ b/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/Utils.groovy @@ -106,7 +106,7 @@ class Utils { static boolean isProdEnvironment(def script) { // Check for all possible `GIT_AUTHOR_NAME` variables - return getSeedAuthor(script) == 'kieapachegroup' && + return getSeedAuthor(script) == 'apache' && (hasGitAuthor(script) ? getGitAuthor(script) == 'apache' : true) && (hasSeedConfigFileGitAuthor(script) ? getSeedConfigFileGitAuthor(script) == 'apache' : true) } From da32d41e00d2914ab34c8e51027e4f83c6ece17e Mon Sep 17 00:00:00 2001 From: Jan Stastny Date: Wed, 4 Oct 2023 15:34:36 +0200 Subject: [PATCH 21/25] kie-issues#600: set credentials for buildchain (#1103) Co-authored-by: jstastny-cz --- .ci/jenkins/config/branch.yaml | 1 + dsl/seed/jenkinsfiles/Jenkinsfile.buildchain | 7 +++++-- .../src/main/groovy/org/kie/jenkins/jobdsl/Utils.groovy | 4 ++++ .../org/kie/jenkins/jobdsl/utils/JobParamsUtils.groovy | 2 +- 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.ci/jenkins/config/branch.yaml b/.ci/jenkins/config/branch.yaml index d1509a9fa..248f13250 100644 --- a/.ci/jenkins/config/branch.yaml +++ b/.ci/jenkins/config/branch.yaml @@ -82,6 +82,7 @@ buildchain_config: git: repository: incubator-kie-kogito-pipelines file_path: .ci/pull-request-config.yaml + token_credentials_id: kie-ci3-token maven: settings_file_id: kie-release-settings nexus: diff --git a/dsl/seed/jenkinsfiles/Jenkinsfile.buildchain b/dsl/seed/jenkinsfiles/Jenkinsfile.buildchain index ce3adbe43..9318bcc14 100644 --- a/dsl/seed/jenkinsfiles/Jenkinsfile.buildchain +++ b/dsl/seed/jenkinsfiles/Jenkinsfile.buildchain @@ -105,7 +105,7 @@ pipeline { } configFileProvider([configFile(fileId: settingsXmlId, variable: 'MAVEN_SETTINGS_FILE')]) { - withCredentials([string(credentialsId: "${GIT_AUTHOR_TOKEN_CREDENTIALS_ID}", variable: 'GITHUB_TOKEN')]) { + withCredentials([string(credentialsId: "${BUILDCHAIN_CONFIG_GIT_TOKEN_CREDENTIALS_ID}", variable: 'GITHUB_TOKEN')]) { env.BUILD_MVN_OPTS = "${env.BUILD_MVN_OPTS ?: ''} -s ${MAVEN_SETTINGS_FILE} -Dmaven.wagon.http.ssl.insecure=true -Dmaven.test.failure.ignore=true" echo "BUILD_MVN_OPTS = ${BUILD_MVN_OPTS}" @@ -176,7 +176,7 @@ pipeline { unsuccessful { script { if (isPRBuildChainType()) { - pullrequest.postComment(util.getMarkdownTestSummary(notificationJobName, getReproducer(true), "${BUILD_URL}", 'GITHUB'), "${GIT_AUTHOR_TOKEN_CREDENTIALS_ID}") + pullrequest.postComment(util.getMarkdownTestSummary(notificationJobName, getReproducer(true), "${BUILD_URL}", 'GITHUB'), "${BUILDCHAIN_CONFIG_GIT_TOKEN_CREDENTIALS_ID}") } else if (shouldNotify()) { withCredentials([string(credentialsId: "${JENKINS_EMAIL_CREDS_ID}", variable: 'KOGITO_CI_EMAIL_TO')]) { mailer.sendMarkdownTestSummaryNotification(env.NOTIFICATION_JOB_NAME ?: '', "[${env.GIT_BRANCH_NAME}] ${getRepoNameCamelCase(env.REPO_NAME)}", [env.KOGITO_CI_EMAIL_TO], getReproducer()) @@ -184,6 +184,9 @@ pipeline { } } } + cleanup { + cleanWs() + } // cleanup { // script { // // Clean also docker in case of usage of testcontainers lib diff --git a/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/Utils.groovy b/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/Utils.groovy index 91eb26124..c11b65f35 100644 --- a/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/Utils.groovy +++ b/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/Utils.groovy @@ -211,6 +211,10 @@ class Utils { return getBindingValue(script, 'BUILDCHAIN_CONFIG_GIT_FILE_PATH') } + static String getBuildChainConfigTokenCredentialsId(def script) { + return getBindingValue(script, 'BUILDCHAIN_CONFIG_GIT_TOKEN_CREDENTIALS_ID') + } + static String getMavenArtifactsUploadRepositoryUrl(def script) { return getBindingValue(script, 'MAVEN_ARTIFACTS_UPLOAD_REPOSITORY_URL') } diff --git a/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/utils/JobParamsUtils.groovy b/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/utils/JobParamsUtils.groovy index d1da98aa2..c60317b41 100644 --- a/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/utils/JobParamsUtils.groovy +++ b/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/utils/JobParamsUtils.groovy @@ -144,8 +144,8 @@ class JobParamsUtils { BUILDCHAIN_CONFIG_AUTHOR: Utils.getBuildChainConfigAuthor(script) ?: Utils.getSeedAuthor(script), BUILDCHAIN_CONFIG_BRANCH: Utils.getBuildChainConfigBranch(script) ?: Utils.getSeedBranch(script), BUILDCHAIN_CONFIG_FILE_PATH: Utils.getBuildChainConfigFilePath(script), + BUILDCHAIN_CONFIG_GIT_TOKEN_CREDENTIALS_ID: Utils.getBuildChainConfigTokenCredentialsId(script) ?: Utils.getGitAuthorTokenCredsId(script), NOTIFICATION_JOB_NAME: notificationJobName, - GIT_AUTHOR_TOKEN_CREDENTIALS_ID: Utils.getGitAuthorTokenCredsId(script), ]) addJobParamsEnvIfNotExisting(script, jobParams, 'BUILD_ENVIRONMENT', jobParams.job.folder.getEnvironmentName()) } From 664ee54f4f86d68aab8ad2e35c2b2e2971d5b5b2 Mon Sep 17 00:00:00 2001 From: Jan Stastny Date: Thu, 5 Oct 2023 09:22:52 +0200 Subject: [PATCH 22/25] kie-issues#601: enable kogito nightly (#1105) Co-authored-by: jstastny-cz --- .ci/jenkins/config/branch.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.ci/jenkins/config/branch.yaml b/.ci/jenkins/config/branch.yaml index 248f13250..8dc37df17 100644 --- a/.ci/jenkins/config/branch.yaml +++ b/.ci/jenkins/config/branch.yaml @@ -38,7 +38,6 @@ environments: ids: - ecosystem disable: - triggers: true # TODO to set back deploy: true repositories: - name: incubator-kie-kogito-pipelines @@ -113,7 +112,7 @@ jenkins: # but we need to make sure the image exists first ... simple tag before setting up the branch ? # See https://github.com/kiegroup/kie-issues/issues/551 image: quay.io/kiegroup/kogito-ci-build:main-latest - args: -v /var/run/docker.sock:/var/run/docker.sock --group-add docker --group-add input --group-add render + args: -v /var/run/docker.sock:/var/run/docker.sock --network host --group-add docker --group-add input --group-add render default_tools: jdk: jdk_11_latest maven: maven_3.8.6 From 97cfc40d654192bfe4c6442f3e4121780e3566ce Mon Sep 17 00:00:00 2001 From: Jan Stastny Date: Fri, 6 Oct 2023 08:56:08 +0200 Subject: [PATCH 23/25] kie-issues#604: install libs for kogito-examples (#1106) Co-authored-by: jstastny-cz --- apache-nodes/Dockerfile.kogito-ci-build | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/apache-nodes/Dockerfile.kogito-ci-build b/apache-nodes/Dockerfile.kogito-ci-build index f9597587c..df28cfbf4 100644 --- a/apache-nodes/Dockerfile.kogito-ci-build +++ b/apache-nodes/Dockerfile.kogito-ci-build @@ -28,7 +28,7 @@ krb5-devel \ gcc \ # python3 (END) # system (BEGIN) -gcc-c++ \ +nc \ shadow-utils \ sudo \ wget \ @@ -40,6 +40,10 @@ freetype \ # couldn't get it for pre-defined repositories https://mirror.stream.centos.org/9-stream/AppStream/aarch64/os/Packages/overpass-fonts-3.0.4-8.el9.noarch.rpm \ # drools (END) +# kogito python integration (BEGIN) +gcc-c++ \ +libglvnd-glx \ +# kogito python integration (END) && dnf clean all RUN sudo alternatives --install /usr/local/bin/python python $(which python${PYTHON_MAJOR_MINOR_VERSION}) 1 && \ From 15aded13e1677d935d6a634019f29b7765917127 Mon Sep 17 00:00:00 2001 From: Ricardo Zanini <1538000+ricardozanini@users.noreply.github.com> Date: Fri, 6 Oct 2023 11:33:17 -0300 Subject: [PATCH 24/25] Remove maven actions to follow Apache policies (#1104) * Remove maven actions to follow Apache policies Signed-off-by: Ricardo Zanini * Include Maven version Signed-off-by: Ricardo Zanini * Adding testing repo Signed-off-by: Ricardo Zanini * Fix tar command Signed-off-by: Ricardo Zanini * Fix Maven Download URL Signed-off-by: Ricardo Zanini * Reverting from test Signed-off-by: Ricardo Zanini * Remove JBoss URLs Signed-off-by: Ricardo Zanini --------- Signed-off-by: Ricardo Zanini --- .ci/actions/maven/action.yml | 91 +++++++------------------ .ci/actions/maven/install-maven.sh | 35 ++++++++++ .ci/actions/maven/settings-template.xml | 25 +++++++ 3 files changed, 84 insertions(+), 67 deletions(-) create mode 100755 .ci/actions/maven/install-maven.sh create mode 100644 .ci/actions/maven/settings-template.xml diff --git a/.ci/actions/maven/action.yml b/.ci/actions/maven/action.yml index 58f410ec2..460f44d64 100644 --- a/.ci/actions/maven/action.yml +++ b/.ci/actions/maven/action.yml @@ -30,10 +30,27 @@ runs: distribution: 'temurin' java-version: ${{ inputs.java-version }} check-latest: true - - name: Set up Maven - uses: stCarolas/setup-maven@v4.2 + # We have to download the required scripts to our local workspace + - name: Checkout Scripts + uses: actions/checkout@v4 + with: + repository: apache/incubator-kie-kogito-pipelines + sparse-checkout: | + .ci/actions/maven + path: maven-tools + - name: Cache Maven Download + id: cache-maven-download + uses: actions/cache@v3 with: - maven-version: ${{ inputs.maven-version }} + key: ${{ runner.os }}-maven-${{ inputs.maven-version }} + path: | + ~/.maven/download/ + - name: Set up Maven + shell: sh + run: | + ./maven-tools/.ci/actions/maven/install-maven.sh "$HOME/.local/mvn" ${{ inputs.maven-version }} + echo "$HOME/.local/mvn/bin" >> $GITHUB_PATH + - name: Cache Maven packages uses: actions/cache@v3 with: @@ -48,70 +65,10 @@ runs: key: ${{ inputs.cache-key-prefix }}-m2-${{ hashFiles('**/pom.xml') }} restore-keys: ${{ inputs.cache-key-prefix }}-m2 - name: Setup Maven Settings - uses: whelk-io/maven-settings-xml-action@v21 - with: - repositories: > - [ - { - "id": "jboss-public-repository-group", - "name": "JBoss Public Repository Group", - "url": "https://repository.jboss.org/nexus/content/groups/public", - "releases": { - "enabled": "true", - "updatePolicy": "never" - }, - "snapshots": { - "enabled": "${{ inputs.allow-snapshots }}", - "updatePolicy": "never" - } - }, - { - "id": "kogito-staging-repository-group", - "name": "Kogito Staging Repositories", - "url": "https://repository.jboss.org/nexus/content/groups/kogito-public", - "releases": { - "enabled": "true", - "updatePolicy": "never" - }, - "snapshots": { - "enabled": "${{ inputs.allow-snapshots }}", - "updatePolicy": "never" - } - } - ] - plugin_repositories: > - [ - { - "id": "jboss-public-repository-group", - "name": "JBoss Public Repository Group", - "url": "https://repository.jboss.org/nexus/content/groups/public", - "releases": { - "enabled": "true", - "updatePolicy": "never" - }, - "snapshots": { - "enabled": "${{ inputs.allow-snapshots }}", - "updatePolicy": "never" - } - }, - { - "id": "kogito-staging-repository-group", - "name": "Kogito Staging Repositories", - "url": "https://repository.jboss.org/nexus/content/groups/kogito-public", - "releases": { - "enabled": "true", - "updatePolicy": "never" - }, - "snapshots": { - "enabled": "${{ inputs.allow-snapshots }}", - "updatePolicy": "never" - } - } - ] - plugin_groups: > - [ - "org.zanata" - ] + shell: bash + run: | + cp maven-tools/.ci/actions/maven/settings-template.xml ${HOME}/.m2/settings.xml + - name: Debug settings.xml if: ${{ inputs.debug }} shell: bash diff --git a/.ci/actions/maven/install-maven.sh b/.ci/actions/maven/install-maven.sh new file mode 100755 index 000000000..67957fa49 --- /dev/null +++ b/.ci/actions/maven/install-maven.sh @@ -0,0 +1,35 @@ +#!/bin/sh +set -e + +install_path=$1 +maven_version=$2 + +if [[ -z ${maven_version} ]]; then + maven_version=$(curl https://maven.apache.org/download.cgi --silent | grep "Downloading Apache Maven " | grep -oE '[0-9].[0-9]+.[0-9]+') +fi +maven_file="apache-maven-${maven_version}-bin.tar.gz" +download_url="https://archive.apache.org/dist/maven/maven-3/${maven_version}/binaries/${maven_file}" +download_path="${HOME}/.maven/download/" + +echo "---> Maven version to install is ${maven_version}" + +if [ -e "${download_path}/${maven_file}" ]; then + echo "---> Maven ${maven_version} already exists in '${download_path}', skipping downloading" +else + mkdir -p "${download_path}" + cd "${download_path}" + echo "---> Downloading maven ${maven_version} to ${download_path}" + curl -LO "${download_url}" + cd - +fi + +if [ -z "${install_path}" ]; then + install_path="${HOME}/runner/mvn" +fi + +echo "---> Ensuring maven installation at ${install_path}" + +mkdir -p "${install_path}" +tar -xf "${download_path}/${maven_file}" --strip-components=1 -C ${install_path} + +sh "${install_path}/bin/mvn" --version \ No newline at end of file diff --git a/.ci/actions/maven/settings-template.xml b/.ci/actions/maven/settings-template.xml new file mode 100644 index 000000000..0d66e68a0 --- /dev/null +++ b/.ci/actions/maven/settings-template.xml @@ -0,0 +1,25 @@ + + + + + + + + + + + + org.zanata + + + + + + + + + + \ No newline at end of file From 5ec21a99beeb6e35222eb76244d60672b90cd9a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tibor=20Zim=C3=A1nyi?= Date: Wed, 11 Oct 2023 14:49:42 +0200 Subject: [PATCH 25/25] Build PRs from origin branches (#1108) --- .../main/groovy/org/kie/jenkins/jobdsl/KogitoJobTemplate.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/KogitoJobTemplate.groovy b/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/KogitoJobTemplate.groovy index 7857df9ca..438a10e5c 100644 --- a/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/KogitoJobTemplate.groovy +++ b/dsl/seed/src/main/groovy/org/kie/jenkins/jobdsl/KogitoJobTemplate.groovy @@ -498,7 +498,7 @@ class KogitoJobTemplate { // Build origin branches also filed as PRs. buildOriginBranchWithPR(false) // Build origin PRs (unmerged head). - buildOriginPRHead(false) + buildOriginPRHead(true) // Build origin PRs (merged with base branch). buildOriginPRMerge(false) }