From 8e212f5393d11aa57882a5b920acd04bef2812f3 Mon Sep 17 00:00:00 2001 From: Rodrigo Antunes Date: Wed, 7 Feb 2024 08:05:54 -0300 Subject: [PATCH] kie-issues#821: Kogito Serverless Operator weekly job (#374) * Serverless Operator weekly job --- .ci/jenkins/Jenkinsfile.weekly.deploy | 257 ++++++++++++++++++ .ci/jenkins/dsl/jobs.groovy | 46 ++++ Makefile | 2 +- ...taflow-operator.clusterserviceversion.yaml | 4 +- image.yaml | 2 +- version/version.go | 4 +- 6 files changed, 309 insertions(+), 6 deletions(-) create mode 100644 .ci/jenkins/Jenkinsfile.weekly.deploy diff --git a/.ci/jenkins/Jenkinsfile.weekly.deploy b/.ci/jenkins/Jenkinsfile.weekly.deploy new file mode 100644 index 000000000..c29a96428 --- /dev/null +++ b/.ci/jenkins/Jenkinsfile.weekly.deploy @@ -0,0 +1,257 @@ +@Library('jenkins-pipeline-shared-libraries')_ + +helper = null + +pipeline { + agent { + docker { + image env.AGENT_DOCKER_BUILDER_IMAGE + args env.AGENT_DOCKER_BUILDER_ARGS + } + } + + options { + timeout(time: 10, unit: 'HOURS') + timestamps() + } + + environment { + KOGITO_CI_EMAIL_TO = credentials("${JENKINS_EMAIL_CREDS_ID}") + + PR_BRANCH_HASH = "${util.generateHash(10)}" + + IMAGE_BUILD_PLATFORMS = 'linux/amd64,linux/arm64' + + CONTAINER_ENGINE = 'docker' + } + + stages { + stage('Setup pipeline') { + steps { + script { + helper = load '.ci/jenkins/scripts/helper.groovy' + helper.initPipeline() + } + } + } + stage('Initialize') { + steps { + script { + helper.cleanGoPath() + + helper.updateDisplayName() + + checkoutRepo() + + // Login to final registry + helper.loginRegistry() + + // Prepare for multiplatform build + int freePort = cloud.findFreePort() + env.localRegistryUrl = cloud.startLocalRegistry(freePort) + + // TODO docker buildx could be preinstalled onto the docker image + cloud.prepareForDockerMultiplatformBuild([env.localRegistryUrl],[cloud.getDockerIOMirrorRegistryConfig()], false) + + env.PROJECT_VERSION = getOperatorVersion() + } + } + post { + success { + script { + properties.add('git.branch', helper.getBuildBranch()) + properties.add('git.author', helper.getGitAuthor()) + properties.add('project.version', getProjectVersion()) + } + } + } + } + + stage('Update version') { + steps { + script { + runPythonCommand("make bump-version new_version=${getProjectVersion()}") + } + } + } + + stage('Test Operator') { + when { + expression { + return helper.shouldLaunchTests() + } + } + steps { + runPythonCommand('make test') + } + post { + unsuccessful { + script { + util.archiveConsoleLog() + } + } + } + } + + stage('Build Operator') { + steps { + script { + String tempBuiltImageTag = getTempBuiltImageTag() + + // Generate the Dockerfile + runPythonCommand("make container-build BUILDER=${env.CONTAINER_ENGINE} IMG=${tempBuiltImageTag} ignore_tag=true build_options='--dry-run'") + + // Build multiplatform from generated Dockerfile + dir('target/image') { + cloud.dockerBuildMultiPlatformImages(tempBuiltImageTag, getImageBuildPlatforms(), true, 'Kogito Serverless Operator squashed image') + } + } + } + post { + unsuccessful { + script { + util.archiveConsoleLog() + } + } + } + } + + stage('Push to registry') { + steps { + script { + // Push the snapshot image to registry + pushFinalImage(getTempBuiltImageTag(), getBuiltImage()) + + // Tag with `latest` tag if asked for as parameter + if (helper.isDeployLatestTag()) { + pushFinalImage(getTempBuiltImageTag(), "${getOperatorImageName()}:weekly-latest")) + } + + // Store image deployment information + properties.add(helper.getImageRegistryProperty(), helper.getImageRegistry()) + properties.add(helper.getImageNamespaceProperty(), helper.getImageNamespace()) + properties.add(helper.getImageTagProperty(), getTempBuiltImageTag()) + } + } + } + + stage('Create and push a new tag') { + steps { + script { + projectVersion = getProjectVersion(false) + githubscm.setUserConfigFromCreds(helper.getGitAuthorPushCredsId()) + githubscm.tagRepository(projectVersion) + githubscm.pushRemoteTag('origin', projectVersion, helper.getGitAuthorPushCredsId()) + } + } + } + + stage('Run e2e tests on Minikube') { + when { + expression { + return helper.shouldLaunchTests() + } + } + steps { + script { + launchE2ETestsJob('minikube') + } + } + } + } + post { + always { + script { + properties.writeToFile(env.PROPERTIES_FILE_NAME) + archiveArtifacts(artifacts: env.PROPERTIES_FILE_NAME) + } + } + unsuccessful { + sendNotification() + } + cleanup { + script { + helper.cleanGoPath() + util.cleanNode(env.CONTAINER_ENGINE) + cloud.cleanDockerMultiplatformBuild() + } + } + } +} + +void sendNotification() { + if (params.SEND_NOTIFICATION) { + mailer.sendMarkdownTestSummaryNotification('Deploy', "[${helper.getBuildBranch()}] Kogito Serverless Operator", [env.KOGITO_CI_EMAIL_TO]) + } else { + echo 'No notification sent per configuration' + } +} + +void checkoutRepo() { + checkout(githubscm.resolveRepository(helper.getRepoName(), helper.getGitAuthor(), helper.getBuildBranch(), false, helper.getGitAuthorCredsId())) + // need to manually checkout branch since on a detached branch after checkout command + sh "git checkout ${helper.getBuildBranch()}" + checkoutDatetime = getCheckoutDatetime() + if (checkoutDatetime) { + sh "git checkout `git rev-list -n 1 --before=\"${checkoutDatetime}\" ${helper.getBuildBranch()}`" + } +} + +String getOperatorVersion() { + return sh(script: 'source ./hack/env.sh > /dev/null && echo $(getOperatorVersion)', returnStdout: true).trim() +} + +String getOperatorImageName() { + return sh(script: 'source ./hack/env.sh > /dev/null && echo $(getOperatorImageName)', returnStdout: true).trim() +} + +String getBuiltImage() { + return "${getOperatorImageName()}:${getProjectVersion(false)}" +} + +String getTempBuiltImageTag() { + return "${env.localRegistryUrl}/kogito-serverless-operator:${getProjectVersion(false)}" +} + +void pushFinalImage(String oldImageName, String newImageName) { + cloud.skopeoCopyRegistryImages(oldImageName, newImageName, Integer.parseInt(env.MAX_REGISTRY_RETRIES)) +} + +void runPythonCommand(String cmd, boolean stdout = false) { + return sh(returnStdout: stdout, script: cmd) +} + +void launchE2ETestsJob(String clusterName) { + String jobName = "kogito-serverless-operator.e2e.${clusterName}" + def buildParams = [ + string(name: 'DISPLAY_NAME', value: params.DISPLAY_NAME), + string(name: 'BUILD_BRANCH_NAME', value: params.BUILD_BRANCH_NAME), + string(name: 'TEST_IMAGE_FULL_TAG', value: getBuiltImage()) + ] + echo "Build ${jobName} with params ${buildParams}" + def job = build(job: "${jobName}", wait: true, parameters: buildParams, propagate: false) + if (job.result != 'SUCCESS') { + unstable("Tests on cluster ${clusterName} finished with result ${job.result}") + } +} + +List getImageBuildPlatforms() { + return "${IMAGE_BUILD_PLATFORMS}".split(',') as List +} + +String getCheckoutDatetime() { + return params.GIT_CHECKOUT_DATETIME +} + +String getProjectVersionDate() { + def projectVersionDate = (getCheckoutDatetime() =~ /(\d{4}-\d{2}-\d{2})/)[0][0] + return projectVersionDate.replace('-', '') +} + +String getProjectVersion(boolean keepSnapshotSuffix = true) { + def projectVersion = env.PROJECT_VERSION + if (keepSnapshotSuffix) { + return projectVersion.replace("-snapshot", "-${getProjectVersionDate()}-snapshot") + } + return projectVersion.replace("-snapshot", "-${getProjectVersionDate()}") +} diff --git a/.ci/jenkins/dsl/jobs.groovy b/.ci/jenkins/dsl/jobs.groovy index c323f3ad5..f19e84926 100644 --- a/.ci/jenkins/dsl/jobs.groovy +++ b/.ci/jenkins/dsl/jobs.groovy @@ -46,6 +46,9 @@ createSetupBranchJob() // Nightly setupDeployJob(JobType.NIGHTLY) +// Weekly +setupWeeklyDeployJob(JobType.OTHER) + // Release setupDeployJob(JobType.RELEASE) setupPromoteJob(JobType.RELEASE) @@ -197,3 +200,46 @@ void setupE2EJob(JobType jobType, String clusterName, Map extraEnv = [:]) { } } } + +void setupWeeklyDeployJob(JobType jobType) { + def jobParams = JobParamsUtils.getBasicJobParams(this, 'kogito-serverless-operator.weekly-deploy', jobType, "${jenkins_path}/Jenkinsfile.weekly.deploy", 'Kogito Serverless Cloud Operator Weekly Deploy') + JobParamsUtils.setupJobParamsAgentDockerBuilderImageConfiguration(this, jobParams) + jobParams.env.putAll([ + JENKINS_EMAIL_CREDS_ID: "${JENKINS_EMAIL_CREDS_ID}", + + GIT_AUTHOR: "${GIT_AUTHOR_NAME}", + GIT_AUTHOR_CREDS_ID: "${GIT_AUTHOR_CREDENTIALS_ID}", + GIT_AUTHOR_PUSH_CREDS_ID: "${GIT_AUTHOR_PUSH_CREDENTIALS_ID}", + + OPERATOR_IMAGE_NAME: 'kogito-serverless-operator', + MAX_REGISTRY_RETRIES: 3, + PROPERTIES_FILE_NAME: 'deployment.properties', + + TEST_CLUSTER_NAMES: clustersConfig.keySet().join(','), + ]) + KogitoJobTemplate.createPipelineJob(this, jobParams)?.with { + parameters { + stringParam('DISPLAY_NAME', '', 'Setup a specific build display name') + + stringParam('BUILD_BRANCH_NAME', "${GIT_BRANCH}", 'Set the Git branch to checkout') + + // Build&Test information + booleanParam('SKIP_TESTS', false, 'Skip tests') + + // Deploy information + stringParam('IMAGE_REGISTRY_CREDENTIALS', "${CLOUD_IMAGE_REGISTRY_CREDENTIALS}", 'Image registry credentials to use to deploy images. Will be ignored if no IMAGE_REGISTRY is given') + stringParam('IMAGE_REGISTRY', "${CLOUD_IMAGE_REGISTRY}", 'Image registry to use to deploy images') + stringParam('IMAGE_NAMESPACE', "${CLOUD_IMAGE_NAMESPACE}", 'Image namespace to use to deploy images') + booleanParam('DEPLOY_WITH_LATEST_TAG', false, 'Set to true if you want the deployed images to also be with the `weekly-latest` tag') + + stringParam('GIT_CHECKOUT_DATETIME', '', 'Git checkout date and time - (Y-m-d H:i)') + + booleanParam('SEND_NOTIFICATION', false, 'In case you want the pipeline to send a notification on CI channel for this run.') + } + } + + // Create E2E jobs + clustersConfig.each { clusterName, clusterEnv -> + setupE2EJob(jobType, clusterName, clusterEnv) + } +} diff --git a/Makefile b/Makefile index f8f4e43d7..2d178f614 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ # To re-generate a bundle for another specific version without changing the standard setup, you can: # - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2) # - use environment variables to overwrite this value (e.g export VERSION=0.0.2) -VERSION ?= 2.0.0-snapshot +VERSION ?= 999.0.0-snapshot REDUCED_VERSION ?= latest # CHANNELS define the bundle channels used in the bundle. diff --git a/bundle/manifests/sonataflow-operator.clusterserviceversion.yaml b/bundle/manifests/sonataflow-operator.clusterserviceversion.yaml index 65359c86e..577f07565 100644 --- a/bundle/manifests/sonataflow-operator.clusterserviceversion.yaml +++ b/bundle/manifests/sonataflow-operator.clusterserviceversion.yaml @@ -127,7 +127,7 @@ metadata: operators.operatorframework.io/project_layout: go.kubebuilder.io/v3 repository: https://github.com/apache/incubator-kie-kogito-serverless-operator support: Red Hat - name: sonataflow-operator.v2.0.0-snapshot + name: sonataflow-operator.v999.0.0-snapshot namespace: placeholder spec: apiservicedefinitions: {} @@ -793,4 +793,4 @@ spec: minKubeVersion: 1.23.0 provider: name: Red Hat - version: 2.0.0-snapshot + version: 999.0.0-snapshot diff --git a/image.yaml b/image.yaml index afb4e908b..cc53f31ff 100644 --- a/image.yaml +++ b/image.yaml @@ -11,7 +11,7 @@ - name: org.kie.kogito.app.builder - name: kogito-serverless-operator - version: 2.0.0-snapshot + version: 999.0.0-snapshot from: registry.access.redhat.com/ubi9/ubi-micro:latest description: Runtime Image for the Operator diff --git a/version/version.go b/version/version.go index c3f8102ae..942dc66c4 100644 --- a/version/version.go +++ b/version/version.go @@ -25,11 +25,11 @@ import ( const ( // Current version - OperatorVersion = "2.0.0-snapshot" + OperatorVersion = "999.0.0-snapshot" // Should not be changed snapshotSuffix = "snapshot" - latestVersion = "2.0.0-snapshot" + latestVersion = "999.0.0-snapshot" ) func IsSnapshot() bool {