feat(test): add end to end tests skeleton and some basic tests

CONTRIBUTING.md

@@ -97,6 +97,8 @@ api` to regenerate the model, client and server code.
 
 ### Testing End to End
 
+`make e2e` can be used run all the integration tests in the repo.
+
 For details on how to test VMClarity end to end please see the End to End
 testing guide [here](docs/test_e2e.md).
 

Makefile

@@ -17,7 +17,7 @@ LICENSEI_VERSION = 0.5.0
 # thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
 .PHONY: help
 help: ## This help.
-	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
+	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_0-9-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
 
 .DEFAULT_GOAL := help
 
@@ -124,6 +124,14 @@ push-docker-uibackend: docker-uibackend ## Build and Push UI Backend Docker imag
 test: ## Run Unit Tests
 	@go test ./...
 
+.PHONY: e2e
+e2e: docker-apiserver docker-cli docker-orchestrator ## Run go e2e test against code
+	@cd e2e && \
+	export APIServerContainerImage=${DOCKER_REGISTRY}/vmclarity-apiserver:${DOCKER_TAG} && \
+	export OrchestratorContainerImage=${DOCKER_REGISTRY}/vmclarity-orchestrator:${DOCKER_TAG} && \
+	export ScannerContainerImage=${DOCKER_REGISTRY}/vmclarity-cli:${DOCKER_TAG} && \
+    go test -v -failfast -test.v -test.paniconexit0 -timeout 2h -ginkgo.v .
+
 .PHONY: clean-ui
 clean-ui:
 	@(rm -rf ui/build ; echo "UI cleanup done" )

e2e/abort_scan_test.go

@@ -0,0 +1,79 @@
+// Copyright © 2023 Cisco Systems, Inc. and its affiliates.
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package e2e
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/onsi/ginkgo/v2"
+	"github.com/onsi/gomega"
+
+	"github.com/openclarity/vmclarity/api/models"
+	"github.com/openclarity/vmclarity/pkg/shared/utils"
+)
+
+var _ = ginkgo.Describe("Aborting a scan", func() {
+	ginkgo.Context("which is running", func() {
+		ginkgo.It("should stop successfully", func(ctx ginkgo.SpecContext) {
+			ginkgo.By("applying a scan configuration")
+			apiScanConfig, err := client.PostScanConfig(ctx, GetDefaultScanConfig())
+			gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+			ginkgo.By("updating scan configuration to run now")
+			updateScanConfig := UpdateScanConfigToStartNow(apiScanConfig)
+			err = client.PatchScanConfig(ctx, *apiScanConfig.Id, updateScanConfig)
+			gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+			ginkgo.By("waiting until scan starts")
+			params := models.GetScansParams{
+				Filter: utils.PointerTo(fmt.Sprintf(
+					"scanConfig/id eq '%s' and state ne '%s' and state ne '%s'",
+					*apiScanConfig.Id,
+					models.ScanStateDone,
+					models.ScanStateFailed,
+				)),
+			}
+			var scans *models.Scans
+			gomega.Eventually(func() bool {
+				scans, err = client.GetScans(ctx, params)
+				gomega.Expect(err).NotTo(gomega.HaveOccurred())
+				return len(*scans.Items) == 1
+			}, DefaultTimeout, time.Second).Should(gomega.BeTrue())
+
+			ginkgo.By("aborting a scan")
+			err = client.PatchScan(ctx, *(*scans.Items)[0].Id, &models.Scan{
+				State: utils.PointerTo(models.ScanStateAborted),
+			})
+			gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+			ginkgo.By("waiting until scan state changes to failed with aborted as state reason")
+			params = models.GetScansParams{
+				Filter: utils.PointerTo(fmt.Sprintf(
+					"scanConfig/id eq '%s' and state eq '%s' and stateReason eq '%s'",
+					*apiScanConfig.Id,
+					models.ScanStateFailed,
+					models.ScanStateReasonAborted,
+				)),
+			}
+			gomega.Eventually(func() bool {
+				scans, err = client.GetScans(ctx, params)
+				gomega.Expect(err).NotTo(gomega.HaveOccurred())
+				return len(*scans.Items) == 1
+			}, DefaultTimeout, time.Second).Should(gomega.BeTrue())
+		})
+	})
+})

e2e/basic_scan_test.go

@@ -0,0 +1,92 @@
+// Copyright © 2023 Cisco Systems, Inc. and its affiliates.
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package e2e
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/onsi/ginkgo/v2"
+	"github.com/onsi/gomega"
+
+	"github.com/openclarity/vmclarity/api/models"
+	"github.com/openclarity/vmclarity/pkg/shared/utils"
+)
+
+var _ = ginkgo.Describe("Running a basic scan (only SBOM)", func() {
+	ginkgo.Context("which scans a docker container", func() {
+		ginkgo.It("should finish successfully", func(ctx ginkgo.SpecContext) {
+			ginkgo.By("waiting until test asset is found")
+			assetsParams := models.GetAssetsParams{
+				Filter: utils.PointerTo(fmt.Sprintf("terminatedOn eq null and %s", DefaultScope)),
+			}
+			gomega.Eventually(func() bool {
+				assets, err := client.GetAssets(ctx, assetsParams)
+				gomega.Expect(err).NotTo(gomega.HaveOccurred())
+				return len(*assets.Items) == 1
+			}, DefaultTimeout, time.Second).Should(gomega.BeTrue())
+
+			ginkgo.By("applying a scan configuration")
+			apiScanConfig, err := client.PostScanConfig(
+				ctx,
+				GetCustomScanConfig(
+					&models.ScanFamiliesConfig{
+						Sbom: &models.SBOMConfig{
+							Enabled: utils.PointerTo(true),
+						},
+					},
+					DefaultScope,
+					600,
+				))
+			gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+			ginkgo.By("updating scan configuration to run now")
+			updateScanConfig := UpdateScanConfigToStartNow(apiScanConfig)
+			err = client.PatchScanConfig(ctx, *apiScanConfig.Id, updateScanConfig)
+			gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+			ginkgo.By("waiting until scan starts")
+			scanParams := models.GetScansParams{
+				Filter: utils.PointerTo(fmt.Sprintf(
+					"scanConfig/id eq '%s' and state ne '%s' and state ne '%s'",
+					*apiScanConfig.Id,
+					models.ScanStateDone,
+					models.ScanStateFailed,
+				)),
+			}
+			var scans *models.Scans
+			gomega.Eventually(func() bool {
+				scans, err = client.GetScans(ctx, scanParams)
+				gomega.Expect(err).NotTo(gomega.HaveOccurred())
+				return len(*scans.Items) == 1
+			}, DefaultTimeout, time.Second).Should(gomega.BeTrue())
+
+			ginkgo.By("waiting until scan state changes to done")
+			scanParams = models.GetScansParams{
+				Filter: utils.PointerTo(fmt.Sprintf(
+					"scanConfig/id eq '%s' and state eq '%s'",
+					*apiScanConfig.Id,
+					models.ScanStateDone,
+				)),
+			}
+			gomega.Eventually(func() bool {
+				scans, err = client.GetScans(ctx, scanParams)
+				gomega.Expect(err).NotTo(gomega.HaveOccurred())
+				return len(*scans.Items) == 1
+			}, time.Second*120, time.Second).Should(gomega.BeTrue())
+		})
+	})
+})

e2e/default_scan_test.go

@@ -0,0 +1,72 @@
+// Copyright © 2023 Cisco Systems, Inc. and its affiliates.
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package e2e
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/onsi/ginkgo/v2"
+	"github.com/onsi/gomega"
+
+	"github.com/openclarity/vmclarity/api/models"
+	"github.com/openclarity/vmclarity/pkg/shared/utils"
+)
+
+var _ = ginkgo.Describe("Running a default scan (SBOM, vulnerabilities and exploits)", func() {
+	ginkgo.Context("which scans a docker container", func() {
+		ginkgo.It("should finish successfully", func(ctx ginkgo.SpecContext) {
+			ginkgo.By("applying a scan configuration")
+			apiScanConfig, err := client.PostScanConfig(ctx, GetDefaultScanConfig())
+			gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+			ginkgo.By("updating scan configuration to run now")
+			updateScanConfig := UpdateScanConfigToStartNow(apiScanConfig)
+			err = client.PatchScanConfig(ctx, *apiScanConfig.Id, updateScanConfig)
+			gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+			ginkgo.By("waiting until scan starts")
+			scanParams := models.GetScansParams{
+				Filter: utils.PointerTo(fmt.Sprintf(
+					"scanConfig/id eq '%s' and state ne '%s' and state ne '%s'",
+					*apiScanConfig.Id,
+					models.ScanStateDone,
+					models.ScanStateFailed,
+				)),
+			}
+			var scans *models.Scans
+			gomega.Eventually(func() bool {
+				scans, err = client.GetScans(ctx, scanParams)
+				gomega.Expect(err).NotTo(gomega.HaveOccurred())
+				return len(*scans.Items) == 1
+			}, DefaultTimeout, time.Second).Should(gomega.BeTrue())
+
+			ginkgo.By("waiting until scan state changes to done")
+			scanParams = models.GetScansParams{
+				Filter: utils.PointerTo(fmt.Sprintf(
+					"scanConfig/id eq '%s' and state eq '%s'",
+					*apiScanConfig.Id,
+					models.ScanStateDone,
+				)),
+			}
+			gomega.Eventually(func() bool {
+				scans, err = client.GetScans(ctx, scanParams)
+				gomega.Expect(err).NotTo(gomega.HaveOccurred())
+				return len(*scans.Items) == 1
+			}, time.Second*120, time.Second).Should(gomega.BeTrue())
+		})
+	})
+})

e2e/docker-compose.override.yml

@@ -0,0 +1,33 @@
+services:
+  orchestrator:
+    # ...
+    volumes:
+      - type: bind
+        source: /var/run/docker.sock
+        target: /var/run/docker.sock
+      - type: bind
+        source: /tmp
+        target: /tmp
+
+  # Note(paralta): Override uibackend healthcheck is required because if server background recalculation fails the
+  # recalculation interval (backgroundRecalculationInterval) will be 15min. Plus, the uibackend is currently not
+  # covered by end-to-end tests.
+  uibackend:
+    ports:
+      - "8890:8890"
+    healthcheck:
+      test: ["CMD", "nc", "-z", "127.0.0.1", "8890"]
+      interval: 10s
+      retries: 60
+
+  alpine:
+    image: alpine:3.18.2
+    command: ["sleep", "infinity"]
+    networks:
+      - asset
+    labels:
+      scanconfig: test
+
+networks:
+  asset:
+    name: ${COMPOSE_PROJECT_NAME}-asset