From 418b01b3b6fd04174ef3620030c90d0e7a83dc6f Mon Sep 17 00:00:00 2001 From: Dominik Hanak Date: Mon, 5 Feb 2024 11:35:39 +0100 Subject: [PATCH] kso-issue-376: Adjust nightly E2E job to use kind cluster --- .ci/jenkins/Jenkinsfile.e2e.cluster | 67 ++++++++++++++++++++++------- .ci/jenkins/scripts/helper.groovy | 4 +- .ci/jenkins/scripts/kind.groovy | 32 ++++++++++++++ Makefile | 2 +- test/e2e/helpers.go | 12 ++++-- test/e2e/platform_test.go | 4 +- test/e2e/workflow_test.go | 18 ++++---- 7 files changed, 107 insertions(+), 32 deletions(-) create mode 100644 .ci/jenkins/scripts/kind.groovy diff --git a/.ci/jenkins/Jenkinsfile.e2e.cluster b/.ci/jenkins/Jenkinsfile.e2e.cluster index 3b90cab28..5568f1717 100644 --- a/.ci/jenkins/Jenkinsfile.e2e.cluster +++ b/.ci/jenkins/Jenkinsfile.e2e.cluster @@ -2,9 +2,12 @@ helper = null -minikubeClusterPlatform = 'minikube' +KIND_VERSION = "v0.20.0" +kindClusterPlatform = 'kind' openshiftClusterPlatform = 'openshift' +kindLogsFolder = "/tmp/kind-logs" + pipeline { agent { docker { @@ -47,6 +50,31 @@ pipeline { } } } + stage('Load image into Kind') { + when { + expression { + return getClusterName() == kindClusterPlatform + } + } + steps { + script { + kind.loadImage(getTestImage()) + } + } + } + stage('Deploy the operator') { + when { + expression { + return getClusterName() == kindClusterPlatform + } + } + steps { + script { + sh "make deploy IMG=${getTestImage()}" + sh "kubectl wait pod -A -l control-plane=sonataflow-operator --for condition=Ready" + } + } + } stage('Prepare for e2e tests') { when { expression { @@ -83,16 +111,24 @@ pipeline { make test-e2e """ } catch (err) { + kind.exportLogs(kindLogsFolder) sh 'make undeploy' + deleteKindCluster() throw err } sh 'kubectl get pods -A' + deleteKindCluster() } } } } } post { + always { + script { + archiveArtifacts(artifacts: "**${kindLogsFolder}/**/*.*,**/e2e-test-report.json") + } + } cleanup { script { clean() @@ -104,7 +140,6 @@ pipeline { void clean() { helper.cleanGoPath() util.cleanNode(containerEngine) - cleanupCluster() } String getTestImage() { @@ -121,10 +156,12 @@ String getOperatorVersion() { void setupCluster() { switch (getClusterName()) { - case minikubeClusterPlatform: - setupMinikube() + case kindClusterPlatform: + echo 'Creating kind cluster' + createKindCluster() break case openshiftClusterPlatform: + echo 'Setting up Openshift' setupOpenshift() break default: @@ -132,13 +169,12 @@ void setupCluster() { } } -void setupMinikube() { - // Start minikube - minikube.minikubeMemory = '12g' - minikube.start() +void createKindCluster() { + sh(script: "make KIND_VERSION=${KIND_VERSION} create-cluster", returnStdout: true) +} - minikube.waitForMinikubeStarted() - minikube.waitForMinikubeRegistry() +void deleteKindCluster() { + sh(script: "make delete-cluster", returnStdout: true) } void setupOpenshift() { @@ -148,8 +184,9 @@ void setupOpenshift() { void cleanupCluster() { switch (getClusterName()) { - case minikubeClusterPlatform: - minikube.stop() + case kindClusterPlatform: + echo 'Deleting kind cluster' + deleteKindCluster() break case openshiftClusterPlatform: echo 'Nothing to cleanup on openshift. All good !' @@ -161,8 +198,8 @@ void cleanupCluster() { void executeInCluster(Closure executeClosure) { switch (getClusterName()) { - case minikubeClusterPlatform: - echo "Execute in minikube" + case kindClusterPlatform: + echo "Execute in kind" executeClosure() break case openshiftClusterPlatform: @@ -178,7 +215,7 @@ void executeInCluster(Closure executeClosure) { void getPlatformCRFilePath() { switch (getClusterName()) { - case minikubeClusterPlatform: + case kindClusterPlatform: return 'test/testdata/sonataflow.org_v1alpha08_sonataflowplatform_withCache_minikube.yaml' case openshiftClusterPlatform: return 'test/testdata/sonataflow.org_v1alpha08_sonataflowplatform_openshift.yaml' diff --git a/.ci/jenkins/scripts/helper.groovy b/.ci/jenkins/scripts/helper.groovy index 38542b708..85326e70e 100644 --- a/.ci/jenkins/scripts/helper.groovy +++ b/.ci/jenkins/scripts/helper.groovy @@ -1,7 +1,7 @@ openshift = null container = null properties = null -minikube = null +kind = null defaultImageParamsPrefix = 'IMAGE' baseImageParamsPrefix = 'BASE_IMAGE' @@ -19,7 +19,7 @@ void initPipeline() { container.containerEngineTlsOptions = env.CONTAINER_ENGINE_TLS_OPTIONS ?: '' container.containerOpenshift = openshift - minikube = load '.ci/jenkins/scripts/minikube.groovy' + kind = load '.ci/jenkins/scripts/kind.groovy' } void updateDisplayName() { diff --git a/.ci/jenkins/scripts/kind.groovy b/.ci/jenkins/scripts/kind.groovy new file mode 100644 index 000000000..bd28dfbd3 --- /dev/null +++ b/.ci/jenkins/scripts/kind.groovy @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +void exportLogs(String destination) { + println "Exporting kind logs to ${destination}" + def exportKindLogs = sh(returnStatus: true, script: """ + mkdir -p .${destination} + kind export logs --loglevel=debug .${destination} + """) +} + +void loadImage(String imageName) { + sh "kind load docker-image ${imageName}" +} + +return this diff --git a/Makefile b/Makefile index f8f4e43d7..edac40a43 100644 --- a/Makefile +++ b/Makefile @@ -347,7 +347,7 @@ generate-all: generate generate-deploy bundle addheaders vet fmt .PHONY: test-e2e # You will need to have a Minikube/Kind cluster up in running to run this target, and run container-builder before the test test-e2e: install-operator-sdk - go test ./test/e2e/* -v -ginkgo.v -timeout 30m + go test ./test/e2e/* -v -ginkgo.v -ginkgo.no-color -ginkgo.json-report=./e2e-test-report.json -timeout 60m .PHONY: before-pr before-pr: test generate-all diff --git a/test/e2e/helpers.go b/test/e2e/helpers.go index 28e747df3..b8f8bd853 100644 --- a/test/e2e/helpers.go +++ b/test/e2e/helpers.go @@ -84,11 +84,12 @@ func verifyHealthStatusInPod(name string, namespace string) { Expect(h.Status).To(Equal(upStatus)) return } - if len(errs.Error()) > 0 { + errs = err + /*if len(errs.Error()) > 0 { errs = fmt.Errorf("%v; %w", err, errs) } else { errs = err - } + }*/ } Expect(errs).NotTo(HaveOccurred(), fmt.Sprintf("No container was found that could respond to the health endpoint %v", errs)) @@ -99,7 +100,12 @@ func getHealthStatusInContainer(podName string, containerName string, ns string) cmd := exec.Command("kubectl", "exec", "-t", podName, "-n", ns, "-c", containerName, "--", "curl", "-s", "localhost:8080/q/health") output, err := utils.Run(cmd) Expect(err).NotTo(HaveOccurred()) - err = json.Unmarshal(output, &h) + stringOutput := string(output) + startIndex := strings.Index(stringOutput, "{") + endIndex := strings.LastIndex(stringOutput, "}") + stringOutput = stringOutput[startIndex-1:endIndex+1] + fmt.Printf("Parsed following JSON object from health Endpoint response: %v\n", stringOutput) + err = json.Unmarshal([]byte(stringOutput), &h) if err != nil { return nil, fmt.Errorf("failed to execute curl command against health endpoint in container %s:%v with output %s", containerName, err, output) } diff --git a/test/e2e/platform_test.go b/test/e2e/platform_test.go index 49afd5aed..b57201f6f 100644 --- a/test/e2e/platform_test.go +++ b/test/e2e/platform_test.go @@ -87,7 +87,7 @@ var _ = Describe("Validate the persistence", Ordered, func() { cmd = exec.Command("kubectl", "wait", "pod", "-n", targetNamespace, "-l", "app=sonataflow-platform", "--for", "condition=Ready", "--timeout=5s") _, err = utils.Run(cmd) return err - }, 10*time.Minute, 5).Should(Succeed()) + }, 20*time.Minute, 5).Should(Succeed()) By("Evaluate status of service's health endpoint") cmd = exec.Command("kubectl", "get", "pod", "-l", "app=sonataflow-platform", "-n", targetNamespace, "-ojsonpath={.items[*].metadata.name}") output, err := utils.Run(cmd) @@ -113,7 +113,7 @@ var _ = Describe("Validate the persistence", Ordered, func() { Expect(sf).NotTo(BeEmpty(), "sonataflow name is empty") EventuallyWithOffset(1, func() bool { return verifyWorkflowIsInRunningStateInNamespace(sf, targetNamespace) - }, 5*time.Minute, 5).Should(BeTrue()) + },10*time.Minute, 5).Should(BeTrue()) } }, Entry("with both Job Service and Data Index and ephemeral persistence and the workflow in a dev profile", test.GetSonataFlowE2EPlatformServicesDirectory(), dev, ephemeral), diff --git a/test/e2e/workflow_test.go b/test/e2e/workflow_test.go index e05c843f7..cb646981c 100644 --- a/test/e2e/workflow_test.go +++ b/test/e2e/workflow_test.go @@ -68,7 +68,7 @@ var _ = Describe("SonataFlow Operator", Ordered, func() { "test/testdata/"+test.SonataFlowSimpleOpsYamlCR), "-n", targetNamespace) _, err := utils.Run(cmd) return err - }, 2*time.Minute, time.Second).Should(Succeed()) + }, 3*time.Minute, time.Second).Should(Succeed()) By("check the workflow is in running state") EventuallyWithOffset(1, func() bool { return verifyWorkflowIsInRunningState("simple", targetNamespace) }, 15*time.Minute, 30*time.Second).Should(BeTrue()) @@ -78,7 +78,7 @@ var _ = Describe("SonataFlow Operator", Ordered, func() { "test/testdata/"+test.SonataFlowSimpleOpsYamlCR), "-n", targetNamespace) _, err := utils.Run(cmd) return err - }, 2*time.Minute, time.Second).Should(Succeed()) + }, 3*time.Minute, time.Second).Should(Succeed()) }) It("should successfully deploy the Greeting Workflow in prod mode and verify if it's running", func() { @@ -88,7 +88,7 @@ var _ = Describe("SonataFlow Operator", Ordered, func() { "test/testdata/"+test.SonataFlowGreetingsDataInputSchemaConfig), "-n", targetNamespace) _, err := utils.Run(cmd) return err - }, 2*time.Minute, time.Second).Should(Succeed()) + }, 3*time.Minute, time.Second).Should(Succeed()) By("creating an instance of the SonataFlow Operand(CR)") EventuallyWithOffset(1, func() error { @@ -96,7 +96,7 @@ var _ = Describe("SonataFlow Operator", Ordered, func() { "test/testdata/"+test.SonataFlowGreetingsWithDataInputSchemaCR), "-n", targetNamespace) _, err := utils.Run(cmd) return err - }, 2*time.Minute, time.Second).Should(Succeed()) + }, 3*time.Minute, time.Second).Should(Succeed()) By("check the workflow is in running state") EventuallyWithOffset(1, func() bool { return verifyWorkflowIsInRunningState("greeting", targetNamespace) }, 15*time.Minute, 30*time.Second).Should(BeTrue()) @@ -106,7 +106,7 @@ var _ = Describe("SonataFlow Operator", Ordered, func() { "test/testdata/"+test.SonataFlowGreetingsWithDataInputSchemaCR), "-n", targetNamespace) _, err := utils.Run(cmd) return err - }, 2*time.Minute, time.Second).Should(Succeed()) + }, 3*time.Minute, time.Second).Should(Succeed()) }) It("should successfully deploy the orderprocessing workflow in devmode and verify if it's running", func() { @@ -117,7 +117,7 @@ var _ = Describe("SonataFlow Operator", Ordered, func() { test.GetSonataFlowE2eOrderProcessingFolder()), "-n", targetNamespace) _, err := utils.Run(cmd) return err - }, 2*time.Minute, time.Second).Should(Succeed()) + }, 3*time.Minute, time.Second).Should(Succeed()) By("check the workflow is in running state") EventuallyWithOffset(1, func() bool { return verifyWorkflowIsInRunningState("orderprocessing", targetNamespace) }, 10*time.Minute, 30*time.Second).Should(BeTrue()) @@ -135,7 +135,7 @@ var _ = Describe("SonataFlow Operator", Ordered, func() { test.GetSonataFlowE2eOrderProcessingFolder()), "-n", targetNamespace) _, err := utils.Run(cmd) return err - }, 2*time.Minute, time.Second).Should(Succeed()) + }, 3*time.Minute, time.Second).Should(Succeed()) }) }) @@ -184,7 +184,7 @@ var _ = Describe("Validate the persistence ", Ordered, func() { out, err := utils.Run(cmd) GinkgoWriter.Printf("%s\n", string(out)) return err - }, 10*time.Minute, 5).Should(Succeed()) + }, 12*time.Minute, 5).Should(Succeed()) By("Evaluate status of the workflow's pod database connection health endpoint") cmd = exec.Command("kubectl", "get", "pod", "-l", "sonataflow.org/workflow-app=callbackstatetimeouts", "-n", ns, "-ojsonpath={.items[*].metadata.name}") @@ -205,7 +205,7 @@ var _ = Describe("Validate the persistence ", Ordered, func() { } } return false - }, 10*time.Minute).Should(BeTrue()) + }, 12*time.Minute).Should(BeTrue()) }, Entry("defined in the workflow from an existing kubernetes service as a reference", test.GetSonataFlowE2EWorkflowPersistenceSampleDataDirectory("by_service")), )