Skip to content

Commit

Permalink
Bootstrap e2e test cluster as a github action
Browse files Browse the repository at this point in the history 
Signed-off-by: Jordi Gil <[email protected]>
  • Loading branch information
@jordigilh
jordigilh committed Jan 19, 2024
1 parent 2b45c51 commit 865aa8e
Show file tree
Hide file tree
Showing 3 changed files with 35 additions and 167 deletions.
6 changes: 6 additions & 0 deletions .github/workflows/e2e.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,12 @@ jobs:
kubectl get pods -A
# TODO: install the operator-sdk first, then cache the installation

- name: Deploy operator
run: |
make deploy IMG=${{ env.OPERATOR_IMAGE_NAME }}
kubectl wait pod -A -l control-plane=controller-manager --for condition=Ready
- name: Run tests
run: |
make test-e2e
Expand Down
36 changes: 3 additions & 33 deletions test/e2e/platform_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ const (
ephemeral = "ephemeral"
postgreSQL = "postgreSQL"
dev = "dev"
production = "production"
production = "prod"
)

var _ = Describe("Validate the persistence", Ordered, func() {
Expand All @@ -34,49 +34,19 @@ var _ = Describe("Validate the persistence", Ordered, func() {
targetNamespace string
)

BeforeAll(func() {

operatorImageName, err := utils.GetOperatorImageName()
ExpectWithOffset(1, err).NotTo(HaveOccurred())

By("deploying the controller-manager")
cmd := exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", operatorImageName))

outputMake, err := utils.Run(cmd)
fmt.Println(string(outputMake))
ExpectWithOffset(1, err).NotTo(HaveOccurred())

By("validating that the controller-manager pod is running as expected")

By("Wait for SonatatFlowPlatform CR to complete deployment")
// wait for service deployments to be ready
EventuallyWithOffset(1, func() error {
cmd = exec.Command("kubectl", "wait", "pod", "-n", sonataflowOperatorNamespace, "-l", "control-plane=controller-manager", "--for", "condition=Ready", "--timeout=5s")
_, err = utils.Run(cmd)
return err
}, 10*time.Minute, 5).Should(Succeed())
})

AfterAll(func() {
By("removing manager namespace")
cmd := exec.Command("make", "undeploy")
_, _ = utils.Run(cmd)
})

BeforeEach(func() {
targetNamespace = fmt.Sprintf("test-%d", rand.Intn(1024)+1)
cmd := exec.Command("kubectl", "create", "namespace", targetNamespace)
_, err := utils.Run(cmd)
Expect(err).NotTo(HaveOccurred())
})
AfterEach(func() {
// Remove resources in test namespace
if len(targetNamespace) > 0 {
// Remove resources in test namespace with no failure
if !CurrentSpecReport().Failed() && len(targetNamespace) > 0 {
cmd := exec.Command("kubectl", "delete", "namespace", targetNamespace, "--wait")
_, err := utils.Run(cmd)
Expect(err).NotTo(HaveOccurred())
}

})
var _ = Context("with platform services", func() {

Expand Down
160 changes: 26 additions & 134 deletions test/e2e/workflow_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ package e2e

import (
"fmt"
"math/rand"
"os/exec"
"path/filepath"
"time"
Expand All @@ -37,131 +38,22 @@ import (
. "github.com/onsi/gomega"
)

// sonataflowOperatorNamespace store the ns where the Operator and Operand will be executed
const sonataflowOperatorNamespace = "sonataflow-operator-system"
var _ = Describe("SonataFlow Operator", func() {

var _ = Describe("SonataFlow Operator", Ordered, func() {

BeforeAll(func() {

// Now, let's ensure that all namespaces can raise a Warn when we apply the manifests
// and that the namespace where the Operator and Operand will run are enforced as
// restricted so that we can ensure that both can be admitted and run with the enforcement

// See: https://kubernetes.io/docs/tutorials/security/seccomp/

/*
TODO: Uncomment to enable when https://issues.redhat.com/browse/KOGITO-9110 will be available
By("labeling all namespaces to warn when we apply the manifest if would violate the PodStandards")
cmd = exec.Command("kubectl", "label", "--overwrite", "ns", "--all",
"pod-security.kubernetes.io/audit=restricted",
"pod-security.kubernetes.io/enforce-version=v1.22",
"pod-security.kubernetes.io/warn=restricted")
_, err := utils.Run(cmd)
ExpectWithOffset(1, err).NotTo(HaveOccurred())
By("labeling enforce the namespace where the Operator and Operand(s) will run")
cmd = exec.Command("kubectl", "label", "--overwrite", "ns", namespace,
"pod-security.kubernetes.io/audit=restricted",
"pod-security.kubernetes.io/enforce-version=v1.22",
"pod-security.kubernetes.io/enforce=restricted")
_, err = utils.Run(cmd)
Expect(err).To(Not(HaveOccurred()))
*/

var controllerPodName string
operatorImageName, err := utils.GetOperatorImageName()
ExpectWithOffset(1, err).NotTo(HaveOccurred())

By("deploying the controller-manager")
cmd := exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", operatorImageName))

outputMake, err := utils.Run(cmd)
fmt.Println(string(outputMake))
ExpectWithOffset(1, err).NotTo(HaveOccurred())

/* // TODO: Uncomment to enable when https://issues.redhat.com/browse/KOGITO-9110 will be available
By("validating that manager Pod/container(s) are restricted")
// Get Podsecurity violation lines
lines, err := utils.StringToLines(string(outputMake))
ExpectWithOffset(1, err).NotTo(HaveOccurred())
var violationLines []string
applySeccompProfilePatch := false
for _, line := range lines {
if strings.Contains(line, "Warning: would violate PodSecurity") {
if strings.Contains(line, "must set securityContext.seccompProfile.type to") {
// Ignore this violation as it is expected
applySeccompProfilePatch = true
} else {
violationLines = append(violationLines, line)
}
}
}
Expect(violationLines).To(BeEmpty())
if applySeccompProfilePatch {
By("Applying seccompProfile")
cmd = exec.Command("kubectl", "patch", "deployment", "sonataflow-operator-controller-manager", "-p", `{"spec":{"template":{"spec":{"securityContext":{"seccompProfile":{"type":"RuntimeDefault"}}}}}}`, "-n", namespace)
var targetNamespace string
BeforeEach(func() {
targetNamespace = fmt.Sprintf("test-%d", rand.Intn(1024)+1)
cmd := exec.Command("kubectl", "create", "namespace", targetNamespace)
_, err := utils.Run(cmd)
Expect(err).NotTo(HaveOccurred())
})
AfterEach(func() {
// Remove resources in test namespace
if !CurrentGinkgoTestDescription().Failed && len(targetNamespace) > 0 {
cmd := exec.Command("kubectl", "delete", "namespace", targetNamespace, "--wait")
_, err := utils.Run(cmd)
if utils.IsDebugEnabled() {
err = utils.OutputDeployment(namespace, "sonataflow-operator-controller-manager")
}
ExpectWithOffset(1, err).NotTo(HaveOccurred())
Expect(err).NotTo(HaveOccurred())
}
*/

By("validating that the controller-manager pod is running as expected")
verifyControllerUp := func() error {
var podOutput []byte
var err error

if utils.IsDebugEnabled() {
Expect(utils.OutputAllPods()).NotTo(HaveOccurred())
Expect(utils.OutputAllEvents(sonataflowOperatorNamespace)).NotTo(HaveOccurred())
}

// Get pod name
cmd = exec.Command("kubectl", "get",
"pods", "-l", "control-plane=controller-manager",
"-o", "go-template={{ range .items }}{{ if not .metadata.deletionTimestamp }}{{ .metadata.name }}"+
"{{ \"\\n\" }}{{ end }}{{ end }}",
"-n", sonataflowOperatorNamespace,
)
podOutput, err = utils.Run(cmd)
fmt.Println(string(podOutput))
ExpectWithOffset(2, err).NotTo(HaveOccurred())
podNames := utils.GetNonEmptyLines(string(podOutput))
if len(podNames) != 1 {
return fmt.Errorf("expect 1 controller pods running, but got %d", len(podNames))
}
controllerPodName = podNames[0]
ExpectWithOffset(2, controllerPodName).Should(ContainSubstring("controller-manager"))

// Validate pod status
cmd = exec.Command("kubectl", "get",
"pods", controllerPodName, "-o", "jsonpath={.status.phase}",
"-n", sonataflowOperatorNamespace,
)
status, err := utils.Run(cmd)
fmt.Println(string(status))
ExpectWithOffset(2, err).NotTo(HaveOccurred())
if string(status) != "Running" {
return fmt.Errorf("controller pod in %s status", status)
}
return nil
}
EventuallyWithOffset(1, verifyControllerUp, 2*time.Minute, time.Second).Should(Succeed())
})

AfterAll(func() {
By("removing manager namespace")
cmd := exec.Command("make", "undeploy")
_, _ = utils.Run(cmd)
By("uninstalling CRDs")
cmd = exec.Command("make", "uninstall")
_, _ = utils.Run(cmd)
})

Describe("ensure that Operator and Operand(s) can run in restricted namespaces", func() {
Expand All @@ -171,17 +63,17 @@ var _ = Describe("SonataFlow Operator", Ordered, func() {
By("creating an instance of the SonataFlow Operand(CR)")
EventuallyWithOffset(1, func() error {
cmd := exec.Command("kubectl", "apply", "-f", filepath.Join(projectDir,
"test/testdata/"+test.SonataFlowSimpleOpsYamlCR), "-n", sonataflowOperatorNamespace)
"test/testdata/"+test.SonataFlowSimpleOpsYamlCR), "-n", targetNamespace)
_, err := utils.Run(cmd)
return err
}, 2*time.Minute, time.Second).Should(Succeed())

By("check the workflow is in running state")
EventuallyWithOffset(1, func() bool { return verifyWorkflowIsInRunningState("simple", sonataflowOperatorNamespace) }, 15*time.Minute, 30*time.Second).Should(BeTrue())
EventuallyWithOffset(1, func() bool { return verifyWorkflowIsInRunningState("simple", targetNamespace) }, 15*time.Minute, 30*time.Second).Should(BeTrue())

EventuallyWithOffset(1, func() error {
cmd := exec.Command("kubectl", "delete", "-f", filepath.Join(projectDir,
"test/testdata/"+test.SonataFlowSimpleOpsYamlCR), "-n", sonataflowOperatorNamespace)
"test/testdata/"+test.SonataFlowSimpleOpsYamlCR), "-n", targetNamespace)
_, err := utils.Run(cmd)
return err
}, 2*time.Minute, time.Second).Should(Succeed())
Expand All @@ -191,25 +83,25 @@ var _ = Describe("SonataFlow Operator", Ordered, func() {
By("creating external resources DataInputSchema configMap")
EventuallyWithOffset(1, func() error {
cmd := exec.Command("kubectl", "apply", "-f", filepath.Join(projectDir,
"test/testdata/"+test.SonataFlowGreetingsDataInputSchemaConfig), "-n", sonataflowOperatorNamespace)
"test/testdata/"+test.SonataFlowGreetingsDataInputSchemaConfig), "-n", targetNamespace)
_, err := utils.Run(cmd)
return err
}, 2*time.Minute, time.Second).Should(Succeed())

By("creating an instance of the SonataFlow Operand(CR)")
EventuallyWithOffset(1, func() error {
cmd := exec.Command("kubectl", "apply", "-f", filepath.Join(projectDir,
"test/testdata/"+test.SonataFlowGreetingsWithDataInputSchemaCR), "-n", sonataflowOperatorNamespace)
"test/testdata/"+test.SonataFlowGreetingsWithDataInputSchemaCR), "-n", targetNamespace)
_, err := utils.Run(cmd)
return err
}, 2*time.Minute, time.Second).Should(Succeed())

By("check the workflow is in running state")
EventuallyWithOffset(1, func() bool { return verifyWorkflowIsInRunningState("greeting", sonataflowOperatorNamespace) }, 15*time.Minute, 30*time.Second).Should(BeTrue())
EventuallyWithOffset(1, func() bool { return verifyWorkflowIsInRunningState("greeting", targetNamespace) }, 15*time.Minute, 30*time.Second).Should(BeTrue())

EventuallyWithOffset(1, func() error {
cmd := exec.Command("kubectl", "delete", "-f", filepath.Join(projectDir,
"test/testdata/"+test.SonataFlowGreetingsWithDataInputSchemaCR), "-n", sonataflowOperatorNamespace)
"test/testdata/"+test.SonataFlowGreetingsWithDataInputSchemaCR), "-n", targetNamespace)
_, err := utils.Run(cmd)
return err
}, 2*time.Minute, time.Second).Should(Succeed())
Expand All @@ -220,25 +112,25 @@ var _ = Describe("SonataFlow Operator", Ordered, func() {
By("creating an instance of the SonataFlow Workflow in DevMode")
EventuallyWithOffset(1, func() error {
cmd := exec.Command("kubectl", "apply", "-f", filepath.Join(projectDir,
test.GetSonataFlowE2eOrderProcessingFolder()), "-n", sonataflowOperatorNamespace)
test.GetSonataFlowE2eOrderProcessingFolder()), "-n", targetNamespace)
_, err := utils.Run(cmd)
return err
}, 2*time.Minute, time.Second).Should(Succeed())

By("check the workflow is in running state")
EventuallyWithOffset(1, func() bool { return verifyWorkflowIsInRunningState("orderprocessing", sonataflowOperatorNamespace) }, 10*time.Minute, 30*time.Second).Should(BeTrue())
EventuallyWithOffset(1, func() bool { return verifyWorkflowIsInRunningState("orderprocessing", targetNamespace) }, 10*time.Minute, 30*time.Second).Should(BeTrue())

cmdLog := exec.Command("kubectl", "logs", "orderprocessing", "-n", sonataflowOperatorNamespace)
cmdLog := exec.Command("kubectl", "logs", "orderprocessing", "-n", targetNamespace)
if responseLog, errLog := utils.Run(cmdLog); errLog == nil {
GinkgoWriter.Println(fmt.Sprintf("devmode podlog %s", responseLog))
}

By("check that the workflow is addressable")
EventuallyWithOffset(1, func() bool { return verifyWorkflowIsAddressable("orderprocessing", sonataflowOperatorNamespace) }, 10*time.Minute, 30*time.Second).Should(BeTrue())
EventuallyWithOffset(1, func() bool { return verifyWorkflowIsAddressable("orderprocessing", targetNamespace) }, 10*time.Minute, 30*time.Second).Should(BeTrue())

EventuallyWithOffset(1, func() error {
cmd := exec.Command("kubectl", "delete", "-f", filepath.Join(projectDir,
test.GetSonataFlowE2eOrderProcessingFolder()), "-n", sonataflowOperatorNamespace)
test.GetSonataFlowE2eOrderProcessingFolder()), "-n", targetNamespace)
_, err := utils.Run(cmd)
return err
}, 2*time.Minute, time.Second).Should(Succeed())
Expand Down

0 comments on commit 865aa8e

Please sign in to comment.