From eae00a2c212875f372f43f385ca62e34a689cafd Mon Sep 17 00:00:00 2001 From: Rodrigo Antunes Date: Tue, 9 Jan 2024 09:13:29 -0300 Subject: [PATCH] kie-issues#755: Fix kogito operator nightly deploy pipelines (#323) * kie-issues#755: initial adjustment for ASF jenkins * Remove directory deletion before git checkout * Add default container engine * replace local var with env * Add CONTAINER_ENGINE env var * Fix minikube installation * Force bash shell usage on sh step * Force bash shell usage on sh step * set user config for git * Reuse existing script to install minikube * Reuse existing script to install minikube * fix minikube installation * fix minikube installation * fix minikube installation * fix minikube permission --------- Co-authored-by: jstastny-cz --- .ci/jenkins/Jenkinsfile.deploy | 22 ++++++++++------------ .ci/jenkins/Jenkinsfile.promote | 20 +++++++++----------- .ci/jenkins/Jenkinsfile.setup-branch | 3 ++- .ci/jenkins/dsl/jobs.groovy | 14 ++++++++------ .ci/jenkins/scripts/helper.groovy | 20 +++++++++++++------- .ci/jenkins/scripts/minikube.groovy | 4 ++-- 6 files changed, 44 insertions(+), 39 deletions(-) diff --git a/.ci/jenkins/Jenkinsfile.deploy b/.ci/jenkins/Jenkinsfile.deploy index a75e6a6c9..97378f1fa 100644 --- a/.ci/jenkins/Jenkinsfile.deploy +++ b/.ci/jenkins/Jenkinsfile.deploy @@ -4,8 +4,6 @@ helper = null commitDone = false -def localRegistryUrl = ''; - pipeline { agent { docker { @@ -25,6 +23,8 @@ pipeline { PR_BRANCH_HASH = "${util.generateHash(10)}" IMAGE_BUILD_PLATFORMS = 'linux/amd64,linux/arm64' + + CONTAINER_ENGINE = 'docker' } stages { @@ -55,11 +55,10 @@ pipeline { // Prepare for multiplatform build int freePort = cloud.findFreePort() - localRegistryUrl = cloud.startLocalRegistry(freePort) + env.localRegistryUrl = cloud.startLocalRegistry(freePort) // TODO docker buildx could be preinstalled onto the docker image - cloud.prepareForDockerMultiplatformBuild([localRegistryUrl],[cloud.getDockerIOMirrorRegistryConfig()], false) - cloud.installSkopeo() + cloud.prepareForDockerMultiplatformBuild([env.localRegistryUrl],[cloud.getDockerIOMirrorRegistryConfig()], false) } } post { @@ -81,7 +80,7 @@ pipeline { steps { script { if (githubscm.isBranchExist('origin', helper.getPRBranch())) { - githubscm.removeRemoteBranch('origin', helper.getPRBranch()) + githubscm.removeRemoteBranch('origin', helper.getPRBranch(), helper.getGitAuthorPushCredsId()) } githubscm.createBranch(helper.getPRBranch()) } @@ -96,6 +95,7 @@ pipeline { script { runPythonCommand("make bump-version new_version=${helper.getProjectVersion()}") + githubscm.setUserConfigFromCreds(getGitAuthorPushCredsId()) // Commit changes githubscm.commitChanges(getCommitMessage(), { sh ''' @@ -206,14 +206,14 @@ pipeline { } steps { script { - githubscm.pushObject('origin', helper.getPRBranch(), helper.getGitAuthorCredsID()) + githubscm.pushObject('origin', helper.getPRBranch(), helper.getGitAuthorPushCredsId()) def prMsg = getCommitMessage() def prBody = "Generated by build ${BUILD_TAG}: ${BUILD_URL}.\nPlease do not merge, it shoud be merged automatically." - String prLink = githubscm.createPR(prMsg, prBody, helper.getBuildBranch(), helper.getGitAuthorCredsID()) + String prLink = githubscm.createPR(prMsg, prBody, helper.getBuildBranch(), helper.getGitAuthorPushCredsId()) properties.add("${helper.getRepoName()}.pr.link", prLink) - properties.add("${helper.getRepoName()}.pr.source.uri", "https://github.com/${helper.getGitAuthorCredsID()}/${helper.getRepoName()}") + properties.add("${helper.getRepoName()}.pr.source.uri", "https://github.com/${helper.getGitAuthor()}/${helper.getRepoName()}") properties.add("${helper.getRepoName()}.pr.source.ref", helper.getPRBranch()) properties.add("${helper.getRepoName()}.pr.target.uri", "https://github.com/${helper.getGitAuthor()}/${helper.getRepoName()}") properties.add("${helper.getRepoName()}.pr.target.ref", helper.getBuildBranch()) @@ -236,8 +236,6 @@ pipeline { helper.cleanGoPath() util.cleanNode(env.CONTAINER_ENGINE) cloud.cleanDockerMultiplatformBuild() - cloud.cleanLocalRegistry() - cloud.cleanSkopeo() } } } @@ -264,7 +262,7 @@ String getBuiltImage() { } String getTempBuiltImageTag() { - return "${localRegistryUrl}/kogito-serverless-operator:${getOperatorVersion()}" + return "${env.localRegistryUrl}/kogito-serverless-operator:${getOperatorVersion()}" } void runPythonCommand(String cmd, boolean stdout = false) { diff --git a/.ci/jenkins/Jenkinsfile.promote b/.ci/jenkins/Jenkinsfile.promote index 5e240a458..02eb68db0 100644 --- a/.ci/jenkins/Jenkinsfile.promote +++ b/.ci/jenkins/Jenkinsfile.promote @@ -48,8 +48,6 @@ pipeline { // Login old and new registries helper.loginRegistry(baseImageParamsPrefix) helper.loginRegistry(promoteImageParamsPrefix) - - cloud.installSkopeo() } } } @@ -86,8 +84,8 @@ pipeline { // Merge PR String prLink = properties.retrieve("${helper.getRepoName()}.pr.link") if (prLink) { - githubscm.mergePR(prLink, helper.getGitAuthorCredsID()) - githubscm.pushObject('origin', helper.getBuildBranch(), helper.getGitAuthorCredsID()) + githubscm.mergePR(prLink, helper.getGitAuthorPushCredsId()) + githubscm.pushObject('origin', helper.getBuildBranch(), helper.getGitAuthorPushCredsId()) } // Tag api / container-builder @@ -117,13 +115,13 @@ pipeline { steps { script { dir(helper.getRepoName()) { - if(githubscm.isReleaseExist(helper.getGitTag(), helper.getGitAuthorCredsID())) { - githubscm.deleteReleaseAndTag(helper.getGitTag(), helper.getGitAuthorCredsID()) + if(githubscm.isReleaseExist(helper.getGitTag(), helper.getGitAuthorCredsId())) { + githubscm.deleteReleaseAndTag(helper.getGitTag(), helper.getGitAuthorPushCredsId()) } - githubscm.createReleaseWithGeneratedReleaseNotes(helper.getGitTag(), helper.getBuildBranch(), githubscm.getPreviousTagFromVersion(helper.getProjectVersion(), 'v', '', ['/']), helper.getGitAuthorCredsID()) - githubscm.updateReleaseBody(helper.getGitTag(), helper.getGitAuthorCredsID()) + githubscm.createReleaseWithGeneratedReleaseNotes(helper.getGitTag(), helper.getBuildBranch(), githubscm.getPreviousTagFromVersion(helper.getProjectVersion(), 'v', '', ['/']), helper.getGitAuthorPushCredsId()) + githubscm.updateReleaseBody(helper.getGitTag(), helper.getGitAuthorPushCredsId()) - withCredentials([usernamePassword(credentialsId: helper.getGitAuthorCredsID(), usernameVariable: 'GH_USER', passwordVariable: 'GH_TOKEN')]) { + withCredentials([usernamePassword(credentialsId: helper.getGitAuthorPushCredsId(), usernameVariable: 'GH_USER', passwordVariable: 'GH_TOKEN')]) { sh """ gh release upload ${helper.getGitTag()} "operator.yaml" """ @@ -141,7 +139,6 @@ pipeline { script { helper.cleanGoPath() util.cleanNode(containerEngine) - cloud.cleanSkopeo() } } } @@ -168,8 +165,9 @@ void updateInternalGoModuleWithTag(String moduleName, String tagName) { void commitAndPush(String remote, String commitMsg, Closure preCommit) { if (githubscm.isThereAnyChanges()) { + githubscm.setUserConfigFromCreds(getGitAuthorPushCredsId()) githubscm.commitChanges(commitMsg, preCommit) - githubscm.pushObject(remote, helper.getBuildBranch(), helper.getGitAuthorCredsID()) + githubscm.pushObject(remote, helper.getBuildBranch(), helper.getGitAuthorPushCredsId()) } else { echo "No change... Skipping commit and push" } diff --git a/.ci/jenkins/Jenkinsfile.setup-branch b/.ci/jenkins/Jenkinsfile.setup-branch index 63f2cf938..68cb5822c 100644 --- a/.ci/jenkins/Jenkinsfile.setup-branch +++ b/.ci/jenkins/Jenkinsfile.setup-branch @@ -69,12 +69,13 @@ pipeline { git checkout -- go.sum ''' if (githubscm.isThereAnyChanges()) { + githubscm.setUserConfigFromCreds(getGitAuthorPushCredsId()) githubscm.commitChanges("[${helper.getBuildBranch()}] Update version to ${helper.getProjectVersion()}", { sh ''' git add . ''' }) - githubscm.pushObject('origin', helper.getBuildBranch(), helper.getGitAuthorCredsID()) + githubscm.pushObject('origin', helper.getBuildBranch(), helper.getGitAuthorPushCredsId()) } else { println '[WARN] no changes to commit' } diff --git a/.ci/jenkins/dsl/jobs.groovy b/.ci/jenkins/dsl/jobs.groovy index a38d710a5..c323f3ad5 100644 --- a/.ci/jenkins/dsl/jobs.groovy +++ b/.ci/jenkins/dsl/jobs.groovy @@ -61,8 +61,8 @@ void createSetupBranchJob() { JENKINS_EMAIL_CREDS_ID: "${JENKINS_EMAIL_CREDS_ID}", GIT_AUTHOR: "${GIT_AUTHOR_NAME}", - AUTHOR_CREDS_ID: "${GIT_AUTHOR_CREDENTIALS_ID}", - GITHUB_TOKEN_CREDS_ID: "${GIT_AUTHOR_TOKEN_CREDENTIALS_ID}", + GIT_AUTHOR_CREDS_ID: "${GIT_AUTHOR_CREDENTIALS_ID}", + GIT_AUTHOR_PUSH_CREDS_ID: "${GIT_AUTHOR_PUSH_CREDENTIALS_ID}", IS_MAIN_BRANCH: "${Utils.isMainBranch(this)}" ]) @@ -86,8 +86,8 @@ void setupDeployJob(JobType jobType) { JENKINS_EMAIL_CREDS_ID: "${JENKINS_EMAIL_CREDS_ID}", GIT_AUTHOR: "${GIT_AUTHOR_NAME}", - AUTHOR_CREDS_ID: "${GIT_AUTHOR_CREDENTIALS_ID}", - GITHUB_TOKEN_CREDS_ID: "${GIT_AUTHOR_TOKEN_CREDENTIALS_ID}", + GIT_AUTHOR_CREDS_ID: "${GIT_AUTHOR_CREDENTIALS_ID}", + GIT_AUTHOR_PUSH_CREDS_ID: "${GIT_AUTHOR_PUSH_CREDENTIALS_ID}", OPERATOR_IMAGE_NAME: 'kogito-serverless-operator', MAX_REGISTRY_RETRIES: 3, @@ -137,8 +137,8 @@ void setupPromoteJob(JobType jobType) { GIT_AUTHOR: "${GIT_AUTHOR_NAME}", - AUTHOR_CREDS_ID: "${GIT_AUTHOR_CREDENTIALS_ID}", - GITHUB_TOKEN_CREDS_ID: "${GIT_AUTHOR_TOKEN_CREDENTIALS_ID}", + GIT_AUTHOR_CREDS_ID: "${GIT_AUTHOR_CREDENTIALS_ID}", + GIT_AUTHOR_PUSH_CREDS_ID: "${GIT_AUTHOR_PUSH_CREDENTIALS_ID}", ]) KogitoJobTemplate.createPipelineJob(this, jobParams)?.with { parameters { @@ -182,6 +182,8 @@ void setupE2EJob(JobType jobType, String clusterName, Map extraEnv = [:]) { GIT_AUTHOR: "${GIT_AUTHOR_NAME}", + GIT_AUTHOR_CREDS_ID: "${GIT_AUTHOR_CREDENTIALS_ID}", + OPERATOR_IMAGE_NAME: 'kogito-serverless-operator', MAX_REGISTRY_RETRIES: 3, PROPERTIES_FILE_NAME: 'deployment.properties', diff --git a/.ci/jenkins/scripts/helper.groovy b/.ci/jenkins/scripts/helper.groovy index 616e8703b..38542b708 100644 --- a/.ci/jenkins/scripts/helper.groovy +++ b/.ci/jenkins/scripts/helper.groovy @@ -15,7 +15,7 @@ void initPipeline() { openshift.openshiftApiCredsKey = env.OPENSHIFT_CREDS_KEY container = load '.ci/jenkins/scripts/container.groovy' - container.containerEngine = env.CONTAINER_ENGINE + container.containerEngine = env.CONTAINER_ENGINE ?: 'docker' container.containerEngineTlsOptions = env.CONTAINER_ENGINE_TLS_OPTIONS ?: '' container.containerOpenshift = openshift @@ -48,14 +48,16 @@ String getTempTag() { void checkoutRepo(String repoName = '', String directory = '') { repoName = repoName ?: getRepoName() closure = { - deleteDir() - checkout(githubscm.resolveRepository(repoName, getGitAuthor(), getBuildBranch(), false)) + checkout(githubscm.resolveRepository(repoName, getGitAuthor(), getBuildBranch(), false, getGitAuthorCredsId())) // need to manually checkout branch since on a detached branch after checkout command //sh "git checkout ${getBuildBranch()}" } if (directory) { - dir(directory, closure) + dir(directory) { + deleteDir() + closure() + } } else { closure() } @@ -70,7 +72,7 @@ void loginRegistry(String paramsPrefix = defaultImageParamsPrefix) { } void createTag(String tagName = getGitTag()) { - githubscm.tagLocalAndRemoteRepository('origin', tagName, getGitAuthorCredsID(), '', true) + githubscm.tagLocalAndRemoteRepository('origin', tagName, getGitAuthorPushCredsId(), '', true) } // Set images public on quay. Useful when new images are introduced. @@ -237,8 +239,12 @@ String getGitAuthor() { return "${GIT_AUTHOR}" } -String getGitAuthorCredsID() { - return env.AUTHOR_CREDS_ID +String getGitAuthorCredsId() { + return env.GIT_AUTHOR_CREDS_ID +} + +String getGitAuthorPushCredsId() { + return env.GIT_AUTHOR_PUSH_CREDS_ID } String getPRBranch() { diff --git a/.ci/jenkins/scripts/minikube.groovy b/.ci/jenkins/scripts/minikube.groovy index 31e254c08..1b921db4e 100644 --- a/.ci/jenkins/scripts/minikube.groovy +++ b/.ci/jenkins/scripts/minikube.groovy @@ -100,8 +100,8 @@ void preChecks() { ${minikubeContainerEngine} info if [[ ! \$(command -v minikube) ]]; then - curl -LO https://github.com/kubernetes/minikube/releases/download/v${minikubeVersion}/minikube-${minikubeVersion}-0.x86_64.rpm - sudo yum localinstall -y minikube-${minikubeVersion}-0.x86_64.rpm + sudo ./hack/ci/install-minikube.sh /usr/local/bin + sudo chmod +x /usr/local/bin/minikube fi """ }