Skip to content

Commit

Permalink
[BXMSPROD-2019] Productized branch creation (#967)
Browse files Browse the repository at this point in the history 
* [BXMSPROD-2019] update build-chain config

* [BXMSPROD-2019] Update prod branch and downgrade quarkus

* fix path to file_path on branch.yaml

* Trigger setup-branch job to update versions

* Apply suggestions from code review

Co-authored-by: Tristan Radisson <[email protected]>

* Applied suggestions after code review

* applying patches running .ci/environments/update.sh script

---------

Co-authored-by: Tristan Radisson <[email protected]>
  • Loading branch information
@lampajr @radtriste
lampajr and radtriste authored Jun 26, 2023
1 parent 9f72b30 commit 9f2ae9e
Show file tree
Hide file tree
Showing 5 changed files with 193 additions and 17 deletions.
3 changes: 3 additions & 0 deletions .ci/jenkins/Jenkinsfile.setup-branch
View file
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,9 @@ pipeline {

// Launch the nightly to deploy all artifacts from the branch
stage('Launch the nightly') {
when {
return params.DEPLOY_ARTIFACTS
}
steps {
script {
def buildParams = getDefaultBuildParams()
Expand Down
1 change: 1 addition & 0 deletions .ci/jenkins/dsl/jobs.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,7 @@ void createSetupBranchJob() {
parameters {
stringParam('KOGITO_VERSION', '', 'Kogito version')
stringParam('DROOLS_VERSION', '', 'Drools version')
booleanParam('DEPLOY_ARTIFACTS', true, 'Deploy artifacts')
}
}
}
Expand Down
2 changes: 2 additions & 0 deletions dsl/config/branch.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,7 @@ git:
buildchain_config:
git:
repository: kogito-pipelines
file_path: .ci/pull-request-config.yaml
maven:
settings_file_id: kogito_release_settings
nexus:
Expand Down Expand Up @@ -140,3 +141,4 @@ jenkins:
default_tools:
jdk: kie-jdk11
maven: kie-maven-3.8.7
jobs_definition_file: .ci/jenkins/dsl/jobs.groovy
195 changes: 178 additions & 17 deletions dsl/seed/jenkinsfiles/Jenkinsfile.prod.prepare
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,11 @@ import java.util.regex.Pattern

// parameters
// 1. <prj>_RELEASE_BRANCH
// 2. QUARKUS_VERSION
// 3. ADDITIONAL_BUILD_MAVEN_OPTS
// 2. <prj>_UPDATE_VERSION (this includes also dependencies)
// 3. QUARKUS_VERSION
// 4. DOWNGRADE_QUARKUS_PR_BRANCH
// 5. ADDITIONAL_BUILD_MAVEN_OPTS
// 6. PROJECTS_TO_REMOVE_FROM_PR_CHECKS

seedConfig = [:]
mainBranchConfig = [:]
Expand Down Expand Up @@ -133,6 +136,38 @@ pipeline{
sh "rm -f ${branchConfigFilePath}"
writeYaml file: "${branchConfigFilePath}", data: branchConfig, overwrite: true

// update build-chain configuration
// comment out projects we are not interested in from the buildchain-config.yaml
def projectToRemoveFromBCPRChecks = getProjectsToRemoveFromBuildChainConfig()

if (projectToRemoveFromBCPRChecks) {
// update build-chain config
def buildChainConfigFilePath = readMainBranchConfig().buildchain_config.git.file_path
def buildChainConfig = readYaml file: buildChainConfigFilePath

def builds = buildChainConfig.build.findAll { build ->
!projectToRemoveFromBCPRChecks.contains(build.project)
}
buildChainConfig.build = builds

echo "Write build-chain configuration => ${buildChainConfig}"
sh "rm -f ${buildChainConfigFilePath}"
writeYaml file: "${buildChainConfigFilePath}", data: buildChainConfig, overwrite: true

// update build-chain project dependencies
def buildChainConfigDependenciesFilePath = buildChainConfigFilePath.substring(0, buildChainConfigFilePath.lastIndexOf('/') + 1) + "${buildChainConfig.dependencies}"
def buildChainConfigDependencies = readYaml file: buildChainConfigDependenciesFilePath

def deps = buildChainConfigDependencies.dependencies.findAll { dep ->
!projectToRemoveFromBCPRChecks.contains(dep.project)
}
buildChainConfigDependencies.dependencies = deps

echo "Write build-chain dependencies configuration => ${buildChainConfigDependencies}"
sh "rm -f ${buildChainConfigDependenciesFilePath}"
writeYaml file: "${buildChainConfigDependenciesFilePath}", data: buildChainConfigDependencies, overwrite: true
}

pushIfChanged(prodBranch, getMainBranchConfigFileGitAuthorCredentialsId(), "Setup DSL branch config after productized branch creation", "Branch config on ${prodBranch} has been updated with correct configuration")
}
}
Expand All @@ -146,27 +181,124 @@ pipeline{
// - disable GHA jobs no more needed by commenting out workflows
// - enable productized profile (-Dproductized) on GHA PR checks workflows
// - ensure that fullProfile is not enabled on .ci/jenkins/dsl/jobs.groovy, if so remove it
stage("Update prod branch for all projects to be productized"){
steps{
script {
def prodRepositories = getProdRepositoriesToBranch()
prodRepositories.each { repoConfig ->
String repository = repoConfig.name
String repoAuthor = repoConfig.author?.name ?: readMainBranchConfig().git.author.name
String repoCredsId = repoConfig.author?.credentials_id ?: readMainBranchConfig().git.author.credentials_id

String jobsGroovyPath = getJobDefinitionFilePath()

dir(repository) {
deleteDir()
// here productized branches should have been already created on repos to be productized
checkout(githubscm.resolveRepository(repository, repoAuthor, getProductizedBranchFromRepository(repository), false, repoCredsId))

getOrCreateGitBranch(repoConfig.branch, getRepositoryAuthorCredsId())

if (fileExists("${jobsGroovyPath}")) {
// def jobsGroovy = readFile file: jobsGroovyPath
// remove fullProfile
sh "sed -i 's/, addFullProfileJobParamsGetter)/)/g' ${jobsGroovyPath}"
// TODO: disable pr checks in getMultijobPRConfig, get them from params
} else {
echo "[WARN] Unable to find jobs.groovy for '${repository}' at '${jobsGroovyPath}'"
}


// TODO iterate over PR workflows
// - comment out if not needed (get these from param?)
// - if PR check is needed, keep it and add -Dproductized
}
}
}
}
}

// 3. update the projects version in order to avoid any kind of conflicts with already existing and deployed artifacts

stage("Update projects version"){
when {
expression { return getProjectUpdateVersionKeys().findAll{ it.value != '' } }
}
steps{
script {
println "Updating projects versions"

String jobName = "${getEcosystemProductizedBranch()}/setup-branch/0-setup-branch"
List jobParams = []
jobParams.add(booleanParam(name: 'DEPLOY_ARTIFACTS', value: false))
// here we could have a variable number of params
getProjectUpdateVersionKeys().each{ p ->
jobParams.add(stringParam(name: p.key.replace('_UPDATE', ''), value: p.value))
}

// stage("Update prod branch for all projects to be productized"){
// steps{
// script {
// // TODO: implement
// }
// }
// }
echo "Build ./${jobName} with parameters ${jobParams}"
if (!isDryRun()) {
def job = build(job: "./${jobName}", parameters: jobParams, wait: true, propagate: false)
if (job.result != 'SUCCESS') {
unstable("Update projects version on repositories was not successful")
}
}
}
}
}

// 3. call update quarkus job in order to downgrade quarkus to LTS version (version provided as parameter QUARKUS_VERSION)
// 4. call update quarkus job in order to downgrade quarkus to LTS version (version provided as parameter QUARKUS_VERSION)
// this will:
// - downgrade quarkus, there already exists a job doing this
// - apply patches from .ci/environments/quarkus-lts on the same PR

// stage('Downgrade quarkus to LTS version'){
// steps{
// script {
// // TODO: implement
// }
// }
// }
stage('Downgrade quarkus to LTS version'){
when {
expression { return params.QUARKUS_VERSION }
}
steps{
script {
println "Launch update quarkus all"

String jobName = "${getEcosystemProductizedBranch()}/tools/update-quarkus-all"
String quarkusDowngradePrBranch = getQuarkusDowngradePrBranch()
List jobParams = []
jobParams.add(stringParam(name: 'NEW_VERSION', value: getQuarkusVersion()))
jobParams.add(stringParam(name: 'PR_BRANCH', value: quarkusDowngradePrBranch))

echo "Build ./${jobName} with parameters ${jobParams}"
if (!isDryRun()) {
def job = build(job: "./${jobName}", parameters: jobParams, wait: true, propagate: false)

if (job.result == 'SUCCESS') {
def prodRepositories = getProdRepositoriesToBranch()
prodRepositories.each { repoConfig ->
String repoAuthor = repoConfig.author?.name ?: readMainBranchConfig().git.author.name
String repoCredsId = repoConfig.author?.credentials_id ?: readMainBranchConfig().git.author.credentials_id
String repository = repoConfig.name
String baseBranch = repoConfig.branch ?: 'main'

dir(repository) {
deleteDir()
checkout(githubscm.resolveRepository(repository, repoAuthor, baseBranch, false, repoCredsId))

// if quarkusDowngradePrBranch has been created by update-quarkus-all job, checkout it
if (checkoutGitBranchIfExists(quarkusDowngradePrBranch, repoCredsId)) {
// thefollowing script takes care of applying all patches needed when downgrading to Quarkus LTS
// i.e., it will apply all patches found under .ci/environments/quarkus-lts/patches
sh '.ci/environments/update.sh quarkus-lts'

pushIfChanged(quarkusDowngradePrBranch, repoCredsId, "Applied patches after Quarkus downgrade to ${getQuarkusVersion()}", "Quarkus LTS patches on ${quarkusDowngradePrBranch} have been applied")
}
}
}
} else {
unstable("Update Quarkus on repositories was not successful")
}
}
}
}
}
}
post{
unsuccessful {
Expand Down Expand Up @@ -203,10 +335,18 @@ String getProdBranchSuffixParameterKey(String projectName) {
return params."${key}"
}

def getProjectUpdateVersionKeys() {
return params.findAll{ it.key.contains('_UPDATE_VERSION') }
}

String getQuarkusVersion() {
return params.QUARKUS_VERSION
}

String getQuarkusDowngradePrBranch() {
return params.DOWNGRADE_QUARKUS_PR_BRANCH ?: "${getEcosystemProductizedBranch()}-${getQuarkusVersion()}-downgrade"
}

Boolean isDryRun() {
return params.DRY_RUN
}
Expand All @@ -216,6 +356,10 @@ def getAdditionalDefaultBuildMavenOpts() {
return params.ADDITIONAL_BUILD_MAVEN_OPTS ?: ''
}

def getProjectsToRemoveFromBuildChainConfig() {
return params.PROJECTS_TO_REMOVE_FROM_PR_CHECKS.split(',') as List ?: []
}

// Retrieve seed configuration

String checkoutMainSeedConfigFileRepo() {
Expand Down Expand Up @@ -275,6 +419,9 @@ String getRepoNameCamelCase(String repo) {
return words.collect { it.isEmpty() ? it : it.substring(0, 1).toUpperCase() + it.substring(1).toLowerCase() }.join(' ')
}

String getJobDefinitionFilePath() {
return readMainBranchConfig().jenkins?.jobs_definition_file ?: '.ci/jenkins/dsl/jobs.groovy'
}

// Retrieve impacted projects

Expand Down Expand Up @@ -421,6 +568,20 @@ void getOrCreateGitBranch(String branch, String credentialsId) {
}
}

// If the provided branch exists, checkout it and return true, otherwise return false and skip checkout.
boolean checkoutGitBranchIfExists(String branch, String credentialsId) {
sh 'git fetch origin'
String branchRemoteResult = sh(script: "git ls-remote origin ${branch} | wc -l", returnStdout: true).trim()
if (Integer.parseInt(branchRemoteResult) > 0) {
echo "Branch ${branch} exist. Checking out !"
sh "git checkout ${branch}"
return true
}

echo "Branch ${branch} does not exist... skipping checkout"
return false
}

def pushIfChanged(String branch, String credentialsId, String commitMsg, String notificationMsg) {
if (githubscm.isThereAnyChanges()) {
sh 'git diff'
Expand Down
9 changes: 9 additions & 0 deletions dsl/seed/jobs/root_jobs.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,10 +107,19 @@ if (communityReleaseBranches) {
PRODUCTIZED_PROJECTS.split(',').each { projectName ->
choiceParam("${projectName}_RELEASE_BRANCH".toUpperCase(), communityReleaseBranches, "${Utils.getRepoNameCamelCase(projectName)} community branch to which to create the productized branch from")
stringParam("${projectName}_PROD_BRANCH_SUFFIX".toUpperCase(), 'prod', "${Utils.getRepoNameCamelCase(projectName)} productized branch suffix")
stringParam("${projectName}_UPDATE_VERSION".toUpperCase(), '', "${Utils.getRepoNameCamelCase(projectName)} dependency version which this will depend on")
}

if (DEPENDENCY_PROJECTS) {
DEPENDENCY_PROJECTS.split(',').each { projectName ->
stringParam("${projectName}_UPDATE_VERSION".toUpperCase(), '', "${Utils.getRepoNameCamelCase(projectName)} dependency version which this will depend on")
}
}

stringParam('QUARKUS_VERSION', '', 'Quarkus version to which to update all productized branches, usually latest LTS version')
stringParam('DOWNGRADE_QUARKUS_PR_BRANCH', '', 'Which PR branch name to use for Quarkus downgrade? If none given, a name will be generated automatically.')
stringParam('ADDITIONAL_BUILD_MAVEN_OPTS', '', 'Additional default maven opts for jenkins jobs, e.g., -Ddata-index-ephemeral.image=quay.io/kiegroup/kogito-data-index-ephemeral')
stringParam('PROJECTS_TO_REMOVE_FROM_PR_CHECKS', '', 'Comma-separated list of projects (<owner>/<repo>) to be removed/disabled from build chain pull request config')

booleanParam('DRY_RUN', false, 'If enabled no changes will be applied to remote branches')
}
Expand Down

0 comments on commit 9f2ae9e

Please sign in to comment.