diff --git a/.ci/jenkins/Jenkinsfile.setup-branch b/.ci/jenkins/Jenkinsfile.setup-branch index 5988c9728..3d718abac 100644 --- a/.ci/jenkins/Jenkinsfile.setup-branch +++ b/.ci/jenkins/Jenkinsfile.setup-branch @@ -181,6 +181,9 @@ pipeline { // Launch the nightly to deploy all artifacts from the branch stage('Launch the nightly') { + when { + return params.DEPLOY_ARTIFACTS + } steps { script { def buildParams = getDefaultBuildParams() diff --git a/.ci/jenkins/dsl/jobs.groovy b/.ci/jenkins/dsl/jobs.groovy index 37c9071f3..889bd7e11 100644 --- a/.ci/jenkins/dsl/jobs.groovy +++ b/.ci/jenkins/dsl/jobs.groovy @@ -121,6 +121,7 @@ void createSetupBranchJob() { parameters { stringParam('KOGITO_VERSION', '', 'Kogito version') stringParam('DROOLS_VERSION', '', 'Drools version') + booleanParam('DEPLOY_ARTIFACTS', true, 'Deploy artifacts') } } } diff --git a/dsl/config/branch.yaml b/dsl/config/branch.yaml index dc206e17f..266a19feb 100644 --- a/dsl/config/branch.yaml +++ b/dsl/config/branch.yaml @@ -109,6 +109,7 @@ git: buildchain_config: git: repository: kogito-pipelines + file_path: .ci/pull-request-config.yaml maven: settings_file_id: kogito_release_settings nexus: @@ -140,3 +141,4 @@ jenkins: default_tools: jdk: kie-jdk11 maven: kie-maven-3.8.7 + jobs_definition_file: .ci/jenkins/dsl/jobs.groovy diff --git a/dsl/seed/jenkinsfiles/Jenkinsfile.prod.prepare b/dsl/seed/jenkinsfiles/Jenkinsfile.prod.prepare index b6fc8f7f3..7ef1c6f43 100644 --- a/dsl/seed/jenkinsfiles/Jenkinsfile.prod.prepare +++ b/dsl/seed/jenkinsfiles/Jenkinsfile.prod.prepare @@ -5,8 +5,11 @@ import java.util.regex.Pattern // parameters // 1. _RELEASE_BRANCH -// 2. QUARKUS_VERSION -// 3. ADDITIONAL_BUILD_MAVEN_OPTS +// 2. _UPDATE_VERSION (this includes also dependencies) +// 3. QUARKUS_VERSION +// 4. DOWNGRADE_QUARKUS_PR_BRANCH +// 5. ADDITIONAL_BUILD_MAVEN_OPTS +// 6. PROJECTS_TO_REMOVE_FROM_PR_CHECKS seedConfig = [:] mainBranchConfig = [:] @@ -133,6 +136,38 @@ pipeline{ sh "rm -f ${branchConfigFilePath}" writeYaml file: "${branchConfigFilePath}", data: branchConfig, overwrite: true + // update build-chain configuration + // comment out projects we are not interested in from the buildchain-config.yaml + def projectToRemoveFromBCPRChecks = getProjectsToRemoveFromBuildChainConfig() + + if (projectToRemoveFromBCPRChecks) { + // update build-chain config + def buildChainConfigFilePath = readMainBranchConfig().buildchain_config.git.file_path + def buildChainConfig = readYaml file: buildChainConfigFilePath + + def builds = buildChainConfig.build.findAll { build -> + !projectToRemoveFromBCPRChecks.contains(build.project) + } + buildChainConfig.build = builds + + echo "Write build-chain configuration => ${buildChainConfig}" + sh "rm -f ${buildChainConfigFilePath}" + writeYaml file: "${buildChainConfigFilePath}", data: buildChainConfig, overwrite: true + + // update build-chain project dependencies + def buildChainConfigDependenciesFilePath = buildChainConfigFilePath.substring(0, buildChainConfigFilePath.lastIndexOf('/') + 1) + "${buildChainConfig.dependencies}" + def buildChainConfigDependencies = readYaml file: buildChainConfigDependenciesFilePath + + def deps = buildChainConfigDependencies.dependencies.findAll { dep -> + !projectToRemoveFromBCPRChecks.contains(dep.project) + } + buildChainConfigDependencies.dependencies = deps + + echo "Write build-chain dependencies configuration => ${buildChainConfigDependencies}" + sh "rm -f ${buildChainConfigDependenciesFilePath}" + writeYaml file: "${buildChainConfigDependenciesFilePath}", data: buildChainConfigDependencies, overwrite: true + } + pushIfChanged(prodBranch, getMainBranchConfigFileGitAuthorCredentialsId(), "Setup DSL branch config after productized branch creation", "Branch config on ${prodBranch} has been updated with correct configuration") } } @@ -146,27 +181,124 @@ pipeline{ // - disable GHA jobs no more needed by commenting out workflows // - enable productized profile (-Dproductized) on GHA PR checks workflows // - ensure that fullProfile is not enabled on .ci/jenkins/dsl/jobs.groovy, if so remove it + stage("Update prod branch for all projects to be productized"){ + steps{ + script { + def prodRepositories = getProdRepositoriesToBranch() + prodRepositories.each { repoConfig -> + String repository = repoConfig.name + String repoAuthor = repoConfig.author?.name ?: readMainBranchConfig().git.author.name + String repoCredsId = repoConfig.author?.credentials_id ?: readMainBranchConfig().git.author.credentials_id + + String jobsGroovyPath = getJobDefinitionFilePath() + + dir(repository) { + deleteDir() + // here productized branches should have been already created on repos to be productized + checkout(githubscm.resolveRepository(repository, repoAuthor, getProductizedBranchFromRepository(repository), false, repoCredsId)) + + getOrCreateGitBranch(repoConfig.branch, getRepositoryAuthorCredsId()) + + if (fileExists("${jobsGroovyPath}")) { + // def jobsGroovy = readFile file: jobsGroovyPath + // remove fullProfile + sh "sed -i 's/, addFullProfileJobParamsGetter)/)/g' ${jobsGroovyPath}" + // TODO: disable pr checks in getMultijobPRConfig, get them from params + } else { + echo "[WARN] Unable to find jobs.groovy for '${repository}' at '${jobsGroovyPath}'" + } + + + // TODO iterate over PR workflows + // - comment out if not needed (get these from param?) + // - if PR check is needed, keep it and add -Dproductized + } + } + } + } + } + + // 3. update the projects version in order to avoid any kind of conflicts with already existing and deployed artifacts + + stage("Update projects version"){ + when { + expression { return getProjectUpdateVersionKeys().findAll{ it.value != '' } } + } + steps{ + script { + println "Updating projects versions" + + String jobName = "${getEcosystemProductizedBranch()}/setup-branch/0-setup-branch" + List jobParams = [] + jobParams.add(booleanParam(name: 'DEPLOY_ARTIFACTS', value: false)) + // here we could have a variable number of params + getProjectUpdateVersionKeys().each{ p -> + jobParams.add(stringParam(name: p.key.replace('_UPDATE', ''), value: p.value)) + } - // stage("Update prod branch for all projects to be productized"){ - // steps{ - // script { - // // TODO: implement - // } - // } - // } + echo "Build ./${jobName} with parameters ${jobParams}" + if (!isDryRun()) { + def job = build(job: "./${jobName}", parameters: jobParams, wait: true, propagate: false) + if (job.result != 'SUCCESS') { + unstable("Update projects version on repositories was not successful") + } + } + } + } + } - // 3. call update quarkus job in order to downgrade quarkus to LTS version (version provided as parameter QUARKUS_VERSION) + // 4. call update quarkus job in order to downgrade quarkus to LTS version (version provided as parameter QUARKUS_VERSION) // this will: // - downgrade quarkus, there already exists a job doing this // - apply patches from .ci/environments/quarkus-lts on the same PR - // stage('Downgrade quarkus to LTS version'){ - // steps{ - // script { - // // TODO: implement - // } - // } - // } + stage('Downgrade quarkus to LTS version'){ + when { + expression { return params.QUARKUS_VERSION } + } + steps{ + script { + println "Launch update quarkus all" + + String jobName = "${getEcosystemProductizedBranch()}/tools/update-quarkus-all" + String quarkusDowngradePrBranch = getQuarkusDowngradePrBranch() + List jobParams = [] + jobParams.add(stringParam(name: 'NEW_VERSION', value: getQuarkusVersion())) + jobParams.add(stringParam(name: 'PR_BRANCH', value: quarkusDowngradePrBranch)) + + echo "Build ./${jobName} with parameters ${jobParams}" + if (!isDryRun()) { + def job = build(job: "./${jobName}", parameters: jobParams, wait: true, propagate: false) + + if (job.result == 'SUCCESS') { + def prodRepositories = getProdRepositoriesToBranch() + prodRepositories.each { repoConfig -> + String repoAuthor = repoConfig.author?.name ?: readMainBranchConfig().git.author.name + String repoCredsId = repoConfig.author?.credentials_id ?: readMainBranchConfig().git.author.credentials_id + String repository = repoConfig.name + String baseBranch = repoConfig.branch ?: 'main' + + dir(repository) { + deleteDir() + checkout(githubscm.resolveRepository(repository, repoAuthor, baseBranch, false, repoCredsId)) + + // if quarkusDowngradePrBranch has been created by update-quarkus-all job, checkout it + if (checkoutGitBranchIfExists(quarkusDowngradePrBranch, repoCredsId)) { + // thefollowing script takes care of applying all patches needed when downgrading to Quarkus LTS + // i.e., it will apply all patches found under .ci/environments/quarkus-lts/patches + sh '.ci/environments/update.sh quarkus-lts' + + pushIfChanged(quarkusDowngradePrBranch, repoCredsId, "Applied patches after Quarkus downgrade to ${getQuarkusVersion()}", "Quarkus LTS patches on ${quarkusDowngradePrBranch} have been applied") + } + } + } + } else { + unstable("Update Quarkus on repositories was not successful") + } + } + } + } + } } post{ unsuccessful { @@ -203,10 +335,18 @@ String getProdBranchSuffixParameterKey(String projectName) { return params."${key}" } +def getProjectUpdateVersionKeys() { + return params.findAll{ it.key.contains('_UPDATE_VERSION') } +} + String getQuarkusVersion() { return params.QUARKUS_VERSION } +String getQuarkusDowngradePrBranch() { + return params.DOWNGRADE_QUARKUS_PR_BRANCH ?: "${getEcosystemProductizedBranch()}-${getQuarkusVersion()}-downgrade" +} + Boolean isDryRun() { return params.DRY_RUN } @@ -216,6 +356,10 @@ def getAdditionalDefaultBuildMavenOpts() { return params.ADDITIONAL_BUILD_MAVEN_OPTS ?: '' } +def getProjectsToRemoveFromBuildChainConfig() { + return params.PROJECTS_TO_REMOVE_FROM_PR_CHECKS.split(',') as List ?: [] +} + // Retrieve seed configuration String checkoutMainSeedConfigFileRepo() { @@ -275,6 +419,9 @@ String getRepoNameCamelCase(String repo) { return words.collect { it.isEmpty() ? it : it.substring(0, 1).toUpperCase() + it.substring(1).toLowerCase() }.join(' ') } +String getJobDefinitionFilePath() { + return readMainBranchConfig().jenkins?.jobs_definition_file ?: '.ci/jenkins/dsl/jobs.groovy' +} // Retrieve impacted projects @@ -421,6 +568,20 @@ void getOrCreateGitBranch(String branch, String credentialsId) { } } +// If the provided branch exists, checkout it and return true, otherwise return false and skip checkout. +boolean checkoutGitBranchIfExists(String branch, String credentialsId) { + sh 'git fetch origin' + String branchRemoteResult = sh(script: "git ls-remote origin ${branch} | wc -l", returnStdout: true).trim() + if (Integer.parseInt(branchRemoteResult) > 0) { + echo "Branch ${branch} exist. Checking out !" + sh "git checkout ${branch}" + return true + } + + echo "Branch ${branch} does not exist... skipping checkout" + return false +} + def pushIfChanged(String branch, String credentialsId, String commitMsg, String notificationMsg) { if (githubscm.isThereAnyChanges()) { sh 'git diff' diff --git a/dsl/seed/jobs/root_jobs.groovy b/dsl/seed/jobs/root_jobs.groovy index b3d88f96d..d170f6032 100644 --- a/dsl/seed/jobs/root_jobs.groovy +++ b/dsl/seed/jobs/root_jobs.groovy @@ -107,10 +107,19 @@ if (communityReleaseBranches) { PRODUCTIZED_PROJECTS.split(',').each { projectName -> choiceParam("${projectName}_RELEASE_BRANCH".toUpperCase(), communityReleaseBranches, "${Utils.getRepoNameCamelCase(projectName)} community branch to which to create the productized branch from") stringParam("${projectName}_PROD_BRANCH_SUFFIX".toUpperCase(), 'prod', "${Utils.getRepoNameCamelCase(projectName)} productized branch suffix") + stringParam("${projectName}_UPDATE_VERSION".toUpperCase(), '', "${Utils.getRepoNameCamelCase(projectName)} dependency version which this will depend on") + } + + if (DEPENDENCY_PROJECTS) { + DEPENDENCY_PROJECTS.split(',').each { projectName -> + stringParam("${projectName}_UPDATE_VERSION".toUpperCase(), '', "${Utils.getRepoNameCamelCase(projectName)} dependency version which this will depend on") + } } stringParam('QUARKUS_VERSION', '', 'Quarkus version to which to update all productized branches, usually latest LTS version') + stringParam('DOWNGRADE_QUARKUS_PR_BRANCH', '', 'Which PR branch name to use for Quarkus downgrade? If none given, a name will be generated automatically.') stringParam('ADDITIONAL_BUILD_MAVEN_OPTS', '', 'Additional default maven opts for jenkins jobs, e.g., -Ddata-index-ephemeral.image=quay.io/kiegroup/kogito-data-index-ephemeral') + stringParam('PROJECTS_TO_REMOVE_FROM_PR_CHECKS', '', 'Comma-separated list of projects (/) to be removed/disabled from build chain pull request config') booleanParam('DRY_RUN', false, 'If enabled no changes will be applied to remote branches') }