From ea8919a6819c12517620988ddafca149d1d5b1da Mon Sep 17 00:00:00 2001 From: Brandon Palm Date: Fri, 4 Oct 2024 11:51:42 -0500 Subject: [PATCH] PoC: Add hosted integration testing --- .../actions/setup-partner-cluster/action.yml | 67 +++ .github/workflows/integration-testing.yml | 31 + Makefile | 3 + integration/deployment_test.go | 547 ++++++++++++++++++ integration/integration.go | 30 + 5 files changed, 678 insertions(+) create mode 100644 .github/actions/setup-partner-cluster/action.yml create mode 100644 .github/workflows/integration-testing.yml create mode 100644 integration/deployment_test.go create mode 100644 integration/integration.go diff --git a/.github/actions/setup-partner-cluster/action.yml b/.github/actions/setup-partner-cluster/action.yml new file mode 100644 index 000000000..a842cb46a --- /dev/null +++ b/.github/actions/setup-partner-cluster/action.yml @@ -0,0 +1,67 @@ +name: setup-partner-cluster +description: 'Setup a partner cluster for testing.' + +inputs: + make-command: + description: 'The make command to run.' + required: true + +runs: + using: 'composite' + steps: + - name: Check out `certsuite-sample-workload` + uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + with: + repository: redhat-best-practices-for-k8s/certsuite-sample-workload + path: certsuite-sample-workload + + - name: Check if /etc/docker/daemon.json exists and if not, create it. + shell: bash + run: | + if [ ! -f /etc/docker/daemon.json ]; then + echo '{}' | sudo tee /etc/docker/daemon.json + fi + + - name: Bootstrap cluster and docker + uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0 + with: + timeout_minutes: 90 + max_attempts: 3 + command: cd ${GITHUB_WORKSPACE}/certsuite-sample-workload; make bootstrap-cluster; make bootstrap-docker-ubuntu-local + + - name: Mount docker volume to /mnt/sdb + shell: bash + run: | + df -h + lsblk + if [ ! -d /mnt/docker-storage ]; then + sudo mkdir /mnt/docker-storage + fi + sudo jq '. +={"data-root" : "/mnt/docker-storage"}' < /etc/docker/daemon.json > /tmp/docker-daemon.json + sudo cp /tmp/docker-daemon.json /etc/docker/daemon.json + cat /etc/docker/daemon.json + sudo systemctl restart docker + sudo ls -la /mnt/docker-storage + + - name: Run 'make rebuild-cluster' + uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0 + with: + timeout_minutes: 90 + max_attempts: 3 + command: cd ${GITHUB_WORKSPACE}/certsuite-sample-workload; make rebuild-cluster + + - name: Run 'make ${{inputs.make-command}}' + uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0 + with: + timeout_minutes: 90 + max_attempts: 3 + command: cd ${GITHUB_WORKSPACE}/certsuite-sample-workload; python3 -m venv .venv; source .venv/bin/activate; pip install --upgrade pip; pip install jinjanator; cp .venv/bin/jinjanate .venv/bin/j2; make ${{ inputs.make-command }} + + - name: Show pods + shell: bash + run: oc get pods -A + + - name: Wait for all pods to be ready + run: ./scripts/wait-for-all-pods-running.sh + working-directory: certsuite-sample-workload + timeout-minutes: 10 diff --git a/.github/workflows/integration-testing.yml b/.github/workflows/integration-testing.yml new file mode 100644 index 000000000..741e2dd4c --- /dev/null +++ b/.github/workflows/integration-testing.yml @@ -0,0 +1,31 @@ +name: Integration Testing + +on: + workflow_dispatch: + pull_request: + branches: + - main + +jobs: + integration: + runs-on: ubuntu-22.04 + env: + SHELL: /bin/bash + SKIP_PRELOAD_IMAGES: true + KUBECONFIG: '/home/runner/.kube/config' + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Go 1.23 + uses: actions/setup-go@v5 + with: + go-version: 1.23.2 + + - name: Setup partner cluster + uses: ./.github/actions/setup-partner-cluster + with: + make-command: 'install-for-qe' + + - name: Run integration tests + run: make integration-test diff --git a/Makefile b/Makefile index c6d83446d..44e5e4632 100644 --- a/Makefile +++ b/Makefile @@ -23,6 +23,9 @@ install: deps-update test: go test ./... +integration-test: + go test -tags=integration ./... + coverage-html: go test -coverprofile=coverage.out ./... go tool cover -html=coverage.out diff --git a/integration/deployment_test.go b/integration/deployment_test.go new file mode 100644 index 000000000..fe4a9706b --- /dev/null +++ b/integration/deployment_test.go @@ -0,0 +1,547 @@ +//go:build integration +// +build integration + +package integration + +import ( + "testing" + "time" + + "github.com/openshift-kni/eco-goinfra/pkg/clients" + "github.com/openshift-kni/eco-goinfra/pkg/deployment" + "github.com/openshift-kni/eco-goinfra/pkg/namespace" + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" +) + +const ( + namespacePrefix = "ecogoinfra-deployment" + containerImage = "nginx:latest" + timeoutDuration = time.Duration(60) * time.Second +) + +func TestDeploymentCreate(t *testing.T) { + t.Parallel() + client := clients.New("") + + // Create a random namespace for the test + randomNamespace := generateRandomNamespace(namespacePrefix) + + // Create the namespace in the cluster using the namespaces package + _, err := namespace.NewBuilder(client, randomNamespace).Create() + assert.Nil(t, err) + + // Defer the deletion of the namespace + defer func() { + // Delete the namespace + err := namespace.NewBuilder(client, randomNamespace).Delete() + assert.Nil(t, err) + }() + + var deploymentName = "create-test" + + builder := deployment.NewBuilder(client, deploymentName, randomNamespace, map[string]string{ + "app": "test", + }, corev1.Container{ + Name: "test", + Image: containerImage, + Command: []string{ + "sleep", + "3600", + }, + }) + + // Create a deployment in the namespace + _, err = builder.CreateAndWaitUntilReady(timeoutDuration) + assert.Nil(t, err) + + // Check if the deployment was created + builder, err = deployment.Pull(client, deploymentName, randomNamespace) + assert.Nil(t, err) + assert.NotNil(t, builder.Object) +} + +func TestDeploymentDelete(t *testing.T) { + t.Parallel() + client := clients.New("") + + // Create a random namespace for the test + randomNamespace := generateRandomNamespace(namespacePrefix) + + // Create the namespace in the cluster using the namespaces package + _, err := namespace.NewBuilder(client, randomNamespace).Create() + assert.Nil(t, err) + + // Defer the deletion of the namespace + defer func() { + // Delete the namespace + err := namespace.NewBuilder(client, randomNamespace).Delete() + assert.Nil(t, err) + }() + + var deploymentName = "delete-test" + + builder := deployment.NewBuilder(client, deploymentName, randomNamespace, map[string]string{ + "app": "test", + }, corev1.Container{ + Name: "test", + Image: containerImage, + Command: []string{ + "sleep", + "3600", + }, + }) + + // Create a deployment in the namespace + _, err = builder.CreateAndWaitUntilReady(timeoutDuration) + assert.Nil(t, err) + + // Check if the deployment was created + builder, err = deployment.Pull(client, deploymentName, randomNamespace) + assert.Nil(t, err) + assert.NotNil(t, builder.Object) + + // Delete the deployment + err = builder.DeleteAndWait(timeoutDuration) + assert.Nil(t, err) + + // Check if the deployment was deleted + builder, err = deployment.Pull(client, deploymentName, randomNamespace) + assert.Equal(t, "deployment object delete-test does not exist in namespace "+randomNamespace, err.Error()) +} + +func TestDeploymentWithNodeSelector(t *testing.T) { + t.Parallel() + client := clients.New("") + + // Create a random namespace for the test + randomNamespace := generateRandomNamespace(namespacePrefix) + + // Create the namespace in the cluster using the namespaces package + _, err := namespace.NewBuilder(client, randomNamespace).Create() + assert.Nil(t, err) + + // Defer the deletion of the namespace + defer func() { + // Delete the namespace + err := namespace.NewBuilder(client, randomNamespace).Delete() + assert.Nil(t, err) + }() + + var deploymentName = "node-selector-test" + + builder := deployment.NewBuilder(client, deploymentName, randomNamespace, map[string]string{ + "app": "test", + }, corev1.Container{ + Name: "test", + Image: containerImage, + Command: []string{ + "sleep", + "3600", + }, + }).WithNodeSelector(map[string]string{ + "node-role.kubernetes.io/worker": "worker", + }) + + // Create a deployment in the namespace + _, err = builder.CreateAndWaitUntilReady(timeoutDuration) + assert.Nil(t, err) + + // Check if the deployment was created + builder, err = deployment.Pull(client, deploymentName, randomNamespace) + assert.Nil(t, err) + assert.NotNil(t, builder.Object) + + // Check if the deployment has the node selector + assert.Equal(t, map[string]string{ + "node-role.kubernetes.io/worker": "worker", + }, builder.Object.Spec.Template.Spec.NodeSelector) +} + +func TestDeploymentWithReplicas(t *testing.T) { + t.Parallel() + client := clients.New("") + + // Create a random namespace for the test + randomNamespace := generateRandomNamespace(namespacePrefix) + + // Create the namespace in the cluster using the namespaces package + _, err := namespace.NewBuilder(client, randomNamespace).Create() + assert.Nil(t, err) + + // Defer the deletion of the namespace + defer func() { + // Delete the namespace + err := namespace.NewBuilder(client, randomNamespace).Delete() + assert.Nil(t, err) + }() + + var deploymentName = "replicas-test" + + builder := deployment.NewBuilder(client, deploymentName, randomNamespace, map[string]string{ + "app": "test", + }, corev1.Container{ + Name: "test", + Image: containerImage, + Command: []string{ + "sleep", + "3600", + }, + }).WithReplicas(2) + + // Create a deployment in the namespace + _, err = builder.CreateAndWaitUntilReady(timeoutDuration) + assert.Nil(t, err) + + // Check if the deployment was created + builder, err = deployment.Pull(client, deploymentName, randomNamespace) + assert.Nil(t, err) + assert.NotNil(t, builder.Object) + + // Check if the deployment has the correct number of replicas + assert.Equal(t, int32(2), *builder.Object.Spec.Replicas) +} + +func TestDeploymentWithAdditionalContainerSpecs(t *testing.T) { + t.Parallel() + client := clients.New("") + + // Create a random namespace for the test + randomNamespace := generateRandomNamespace(namespacePrefix) + + // Create the namespace in the cluster using the namespaces package + _, err := namespace.NewBuilder(client, randomNamespace).Create() + assert.Nil(t, err) + + // Defer the deletion of the namespace + defer func() { + // Delete the namespace + err := namespace.NewBuilder(client, randomNamespace).Delete() + assert.Nil(t, err) + }() + + var deploymentName = "additional-container-specs-test" + + builder := deployment.NewBuilder(client, deploymentName, randomNamespace, map[string]string{ + "app": "test", + }, corev1.Container{ + Name: "test", + Image: containerImage, + Command: []string{ + "sleep", + "3600", + }, + }).WithAdditionalContainerSpecs([]corev1.Container{ + { + Name: "additional-container", + Image: containerImage, + Command: []string{ + "sleep", + "3600", + }, + }, + }) + + // Create a deployment in the namespace + _, err = builder.CreateAndWaitUntilReady(timeoutDuration) + assert.Nil(t, err) + + // Check if the deployment was created + builder, err = deployment.Pull(client, deploymentName, randomNamespace) + assert.Nil(t, err) + assert.NotNil(t, builder.Object) + + // Check if the deployment has the additional container + assert.Len(t, builder.Object.Spec.Template.Spec.Containers, 2) +} + +func TestDeploymentWithSecurityContext(t *testing.T) { + t.Parallel() + client := clients.New("") + + // Create a random namespace for the test + randomNamespace := generateRandomNamespace(namespacePrefix) + + // Create the namespace in the cluster using the namespaces package + _, err := namespace.NewBuilder(client, randomNamespace).Create() + assert.Nil(t, err) + + // Defer the deletion of the namespace + defer func() { + // Delete the namespace + err := namespace.NewBuilder(client, randomNamespace).Delete() + assert.Nil(t, err) + }() + + var deploymentName = "security-context-test" + + var falseVar = false + + builder := deployment.NewBuilder(client, deploymentName, randomNamespace, map[string]string{ + "app": "test", + }, corev1.Container{ + Name: "test", + Image: containerImage, + Command: []string{ + "sleep", + "3600", + }, + }).WithSecurityContext(&corev1.PodSecurityContext{ + RunAsNonRoot: &falseVar, + }) + + // Create a deployment in the namespace + _, err = builder.Create() + assert.Nil(t, err) + + // Check if the deployment was created + builder, err = deployment.Pull(client, deploymentName, randomNamespace) + assert.Nil(t, err) + assert.NotNil(t, builder.Object) + + // Check if the deployment has the security context + assert.NotNil(t, builder.Object.Spec.Template.Spec.SecurityContext) + assert.Equal(t, false, *builder.Object.Spec.Template.Spec.SecurityContext.RunAsNonRoot) + + // NOTE: The container will not spawn on OCP because of the security context warning. +} + +func TestDeploymentWithLabel(t *testing.T) { + t.Parallel() + client := clients.New("") + + // Create a random namespace for the test + randomNamespace := generateRandomNamespace(namespacePrefix) + + // Create the namespace in the cluster using the namespaces package + _, err := namespace.NewBuilder(client, randomNamespace).Create() + assert.Nil(t, err) + + // Defer the deletion of the namespace + defer func() { + // Delete the namespace + err := namespace.NewBuilder(client, randomNamespace).Delete() + assert.Nil(t, err) + }() + + var deploymentName = "label-test" + + builder := deployment.NewBuilder(client, deploymentName, randomNamespace, map[string]string{ + "app": "test", + }, corev1.Container{ + Name: "test", + Image: containerImage, + Command: []string{ + "sleep", + "3600", + }, + }).WithLabel("key", "value") + + // Create a deployment in the namespace + _, err = builder.Create() + assert.Nil(t, err) + + // Check if the deployment was created + builder, err = deployment.Pull(client, deploymentName, randomNamespace) + assert.Nil(t, err) + assert.NotNil(t, builder.Object) + + // Check if the deployment has the label + assert.NotNil(t, builder.Object.Spec.Template.Labels) + for key, value := range builder.Object.Spec.Template.Labels { + if key == "key" { + assert.Equal(t, "value", value) + } + } +} + +func TestDeploymentWithServiceAccountName(t *testing.T) { + t.Parallel() + client := clients.New("") + + // Create a random namespace for the test + randomNamespace := generateRandomNamespace(namespacePrefix) + + // Create the namespace in the cluster using the namespaces package + _, err := namespace.NewBuilder(client, randomNamespace).Create() + assert.Nil(t, err) + + // Defer the deletion of the namespace + defer func() { + // Delete the namespace + err := namespace.NewBuilder(client, randomNamespace).Delete() + assert.Nil(t, err) + }() + + var deploymentName = "service-account-name-test" + + builder := deployment.NewBuilder(client, deploymentName, randomNamespace, map[string]string{ + "app": "test", + }, corev1.Container{ + Name: "test", + Image: containerImage, + Command: []string{ + "sleep", + "3600", + }, + }).WithServiceAccountName("default") + + // Create a deployment in the namespace + _, err = builder.Create() + assert.Nil(t, err) + + // Check if the deployment was created + builder, err = deployment.Pull(client, deploymentName, randomNamespace) + assert.Nil(t, err) + assert.NotNil(t, builder.Object) + + // Check if the deployment has the service account name + assert.Equal(t, "default", builder.Object.Spec.Template.Spec.ServiceAccountName) +} + +func TestDeploymentWithVolume(t *testing.T) { + t.Parallel() + client := clients.New("") + + // Create a random namespace for the test + randomNamespace := generateRandomNamespace(namespacePrefix) + + // Create the namespace in the cluster using the namespaces package + _, err := namespace.NewBuilder(client, randomNamespace).Create() + assert.Nil(t, err) + + // Defer the deletion of the namespace + defer func() { + // Delete the namespace + err := namespace.NewBuilder(client, randomNamespace).Delete() + assert.Nil(t, err) + }() + + var deploymentName = "volume-test" + + builder := deployment.NewBuilder(client, deploymentName, randomNamespace, map[string]string{ + "app": "test", + }, corev1.Container{ + Name: "test", + Image: containerImage, + Command: []string{ + "sleep", + "3600", + }, + }).WithVolume(corev1.Volume{ + Name: "test-volume", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }) + + // Create a deployment in the namespace + _, err = builder.Create() + assert.Nil(t, err) + + // Check if the deployment was created + builder, err = deployment.Pull(client, deploymentName, randomNamespace) + assert.Nil(t, err) + assert.NotNil(t, builder.Object) + + // Check if the deployment has the volume + assert.Len(t, builder.Object.Spec.Template.Spec.Volumes, 1) + assert.Equal(t, "test-volume", builder.Object.Spec.Template.Spec.Volumes[0].Name) +} + +func TestDeploymentWithSchedulerName(t *testing.T) { + t.Parallel() + client := clients.New("") + + // Create a random namespace for the test + randomNamespace := generateRandomNamespace(namespacePrefix) + + // Create the namespace in the cluster using the namespaces package + _, err := namespace.NewBuilder(client, randomNamespace).Create() + assert.Nil(t, err) + + // Defer the deletion of the namespace + defer func() { + // Delete the namespace + err := namespace.NewBuilder(client, randomNamespace).Delete() + assert.Nil(t, err) + }() + + var deploymentName = "scheduler-name-test" + + builder := deployment.NewBuilder(client, deploymentName, randomNamespace, map[string]string{ + "app": "test", + }, corev1.Container{ + Name: "test", + Image: containerImage, + Command: []string{ + "sleep", + "3600", + }, + }).WithSchedulerName("default-scheduler") + + // Create a deployment in the namespace + _, err = builder.Create() + assert.Nil(t, err) + + // Check if the deployment was created + builder, err = deployment.Pull(client, deploymentName, randomNamespace) + assert.Nil(t, err) + assert.NotNil(t, builder.Object) + + // Check if the deployment has the scheduler name + assert.Equal(t, "default-scheduler", builder.Object.Spec.Template.Spec.SchedulerName) +} + +func TestDeploymentWithToleration(t *testing.T) { + t.Parallel() + client := clients.New("") + + // Create a random namespace for the test + randomNamespace := generateRandomNamespace(namespacePrefix) + + // Create the namespace in the cluster using the namespaces package + _, err := namespace.NewBuilder(client, randomNamespace).Create() + assert.Nil(t, err) + + // Defer the deletion of the namespace + defer func() { + // Delete the namespace + err := namespace.NewBuilder(client, randomNamespace).Delete() + assert.Nil(t, err) + }() + + var deploymentName = "toleration-test" + + builder := deployment.NewBuilder(client, deploymentName, randomNamespace, map[string]string{ + "app": "test", + }, corev1.Container{ + Name: "test", + Image: containerImage, + Command: []string{ + "sleep", + "3600", + }, + }).WithToleration(corev1.Toleration{ + Key: "key", + Operator: corev1.TolerationOpEqual, + Value: "value", + Effect: corev1.TaintEffectNoSchedule, + }) + + // Create a deployment in the namespace + _, err = builder.Create() + assert.Nil(t, err) + + // Check if the deployment was created + builder, err = deployment.Pull(client, deploymentName, randomNamespace) + assert.Nil(t, err) + assert.NotNil(t, builder.Object) + + // Check if the deployment has the toleration + assert.Len(t, builder.Object.Spec.Template.Spec.Tolerations, 1) + assert.Equal(t, "key", builder.Object.Spec.Template.Spec.Tolerations[0].Key) + assert.Equal(t, corev1.TolerationOpEqual, builder.Object.Spec.Template.Spec.Tolerations[0].Operator) + assert.Equal(t, "value", builder.Object.Spec.Template.Spec.Tolerations[0].Value) + assert.Equal(t, corev1.TaintEffectNoSchedule, builder.Object.Spec.Template.Spec.Tolerations[0].Effect) +} diff --git a/integration/integration.go b/integration/integration.go new file mode 100644 index 000000000..3e73e2a8b --- /dev/null +++ b/integration/integration.go @@ -0,0 +1,30 @@ +//go:build integration +// +build integration + +package integration + +import ( + "math/rand" + "time" +) + +var seededRand *rand.Rand = rand.New( + rand.NewSource(time.Now().UnixNano())) + +func generateRandomString(length int) string { + charset := "abcdefghijklmnopqrstuvwxyz" + + //nolint:varnamelen + b := make([]byte, length) + + for i := range b { + b[i] = charset[seededRand.Intn(len(charset))] + } + + return string(b) +} + +func generateRandomNamespace(prefix string) string { + // Generate a random 12-character string + return prefix + "-" + generateRandomString(12) +}