From 3eef08cfc3a49405c2c1de22dea31d4733fe7ef0 Mon Sep 17 00:00:00 2001 From: Catarina Paralta <46568597+paralta@users.noreply.github.com> Date: Tue, 12 Sep 2023 15:49:18 +0100 Subject: [PATCH] refactor: improve e2e documentation and small clean up (#683) * refactor(test): remove documentation to test scan abort manually since this is now covered by e2e tests * refactor(test): remove init services from test environment * refactor(test): rename docker compose file in documentation * docs(test): add short e2e test readme file * ci(test): rename test job to e2e * docs(test): mention new e2e readme file in contributing --- .github/workflows/ci.yml | 4 +- CONTRIBUTING.md | 5 +- api/openapi.yaml | 1 - docs/testplans/abort-scan.md | 87 ----------------------------------- e2e/README.md | 41 +++++++++++++++++ e2e/testenv/testenv.go | 16 +------ installation/docker/README.md | 4 +- 7 files changed, 48 insertions(+), 110 deletions(-) delete mode 100644 docs/testplans/abort-scan.md create mode 100644 e2e/README.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 534124ae7..393a2ec7a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -77,9 +77,9 @@ jobs: push: false upload: true - test: + e2e: needs: build - name: Test + name: E2E Test runs-on: ubuntu-latest steps: - name: Checkout diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 952c766a9..ecdaeb93c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -97,10 +97,9 @@ api` to regenerate the model, client and server code. ### Testing End to End -`make e2e` can be used run all the integration tests in the repo. +`make e2e` can be used run the end-to-end tests in the repository. -For details on how to test VMClarity end to end please see the End to End -testing guide [here](docs/test_e2e.md). +For details on how to test VMClarity, please check the testing guide [here](docs/test_e2e.md) on how to perform a test on AWS and the instructions [here](e2e/README.md) on how to run and add new tests. ## Sending Pull Requests diff --git a/api/openapi.yaml b/api/openapi.yaml index 116966ae0..c3ee36e72 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -1965,7 +1965,6 @@ components: type: string location: type: string - # TODO(paralta) Check if image needs to be a required property image: $ref: '#/components/schemas/ContainerImageInfo' id: diff --git a/docs/testplans/abort-scan.md b/docs/testplans/abort-scan.md deleted file mode 100644 index 8bb38871d..000000000 --- a/docs/testplans/abort-scan.md +++ /dev/null @@ -1,87 +0,0 @@ -# Test Plan - -## Scan configuration - -Create Scan Configuration file - -```shell -cat < scanconfig.json -{ - "name": "test", - "scanTemplate": { - "scope": "contains(assetInfo.tags, '{\"key\":\"scanconfig\",\"value\":\"test\"}')", - "assetScanTemplate": { - "scanFamiliesConfig": { - "sbom": { - "enabled": true - }, - "vulnerabilities": { - "enabled": true - }, - "exploits": { - "enabled": true - } - } - } - }, - "scheduled": { - "cronLine": "0 */4 * * *", - "operationTime": "2023-01-20T15:46:18+00:00" - } -} -EOF -``` - -Apply Scan Configuration to API - -```shell -curl -sSf -X POST 'http://localhost:8080/api/scanConfigs' -H 'Content-Type: application/json' \ - -d @scanconfig.json \ -| jq -r -e '.id' > scanconfig.id -``` - -Get Scan Configuration object from API - -```shell -curl -sSf -X GET 'http://localhost:8080/api/scanConfigs/'"$(cat scanconfig.id)"'' \ -| jq -r -e '.' > scanconfig.api.json -``` - -## Start Scan - -Start Scan using Scan Config - -```shell -jq -r -e '{maxParallelScanners, name, scanTemplate, scheduled} | .scheduled.operationTime = (now|todate)' \ - scanconfig.api.json \ -| curl -sSf -X PUT -H 'Content-Type: application/json' 'http://localhost:8080/api/scanConfigs/'"$(cat scanconfig.id)"'' \ - -d @- -``` - -**Wait until the Scan object is created on the backend/API side** - -Get ongoing Scan from API using ScanConfig id - -```shell -curl -sSf -G 'http://localhost:8080/api/scans' \ - --data-urlencode "\$filter=scanConfig/id eq '$(cat scanconfig.id)' and state ne 'Done' and state ne 'Failed'" \ -| jq -r -e '.items | first' > scan.api.json -``` - -## Abort Scan in progress - -```shell -cat < scan-aborted.json -{ - "state": "Aborted" -} -EOF -``` - -```shell -jq -r -e '.id' scan.api.json > scan.id \ -&& curl -sSf -X PATCH -H 'Content-Type: application/json' \ - "http://localhost:8080/api/scans/$(cat scan.id)" \ - -d @scan-aborted.json \ -| jq -r -e '.' > scan-aborted.api.json -``` diff --git a/e2e/README.md b/e2e/README.md new file mode 100644 index 000000000..2902b2ef1 --- /dev/null +++ b/e2e/README.md @@ -0,0 +1,41 @@ +# E2E tests + +VMClarity's end-to-end tests use the [Ginkgo](https://onsi.github.io/ginkgo/) test framework and the [Gomega](https://onsi.github.io/gomega/) matcher/assertion library to leverage behavior driven testing (BDD). + +This test module is composed by: + +* a `testenv.go` file where the test environment is configured, +* a `docker-compose.override.yml` file which will merge and override the installation compose file to run end-to-end tests, +* a `suite_test.go` file where the steps to perform the setup and cleanup of the Ginkgo test suite are defined, +* a `helper.go` file where common methods are defined, +* and test case files. + +## Run test + +To run all end-to-end tests, use the following command: + +``` +go test -v -failfast -test.v -test.paniconexit0 -timeout 2h -ginkgo.v . +``` + +To run a particular end-to-end test file, use: + +``` +go test -v -failfast -test.v -test.paniconexit0 -timeout 2h -ginkgo.v --ginkgo.focus-file . +``` + +## Write a new test + +To add a new test, create a new `_test.go` file in the current directory and use the following template: + +```go +var _ = ginkgo.Describe("", func() { + ginkgo.Context("", func() { + ginkgo.It("", func(ctx ginkgo.SpecContext) { + + }) + }) +}) +``` + +Additionally, check the available test cases (e.g. `basic_scan_test.go`) to get started. \ No newline at end of file diff --git a/e2e/testenv/testenv.go b/e2e/testenv/testenv.go index 99cf0a37f..ad88b7eaf 100644 --- a/e2e/testenv/testenv.go +++ b/e2e/testenv/testenv.go @@ -136,9 +136,7 @@ func (e *Environment) ServicesReady(ctx context.Context) (bool, error) { return false, err } - // TODO(paralta) remove when grype-server-init service is removed. - initServices := e.InitServices() - if len(services)-len(initServices) != len(ps) { + if len(services) != len(ps) { return false, nil } @@ -159,18 +157,6 @@ func (e *Environment) Services() []string { return services } -// TODO(paralta) remove when grype-server-init service is removed. -func (e *Environment) InitServices() []string { - services := make([]string, 0, len(e.project.Services)) - for _, srv := range e.project.Services { - val, ok := srv.Labels["type"] - if ok && val == "init" { - services = append(services, srv.Name) - } - } - return services -} - func (e *Environment) VMClarityURL() (*url.URL, error) { var vmClarityBackend types.ServiceConfig var ok bool diff --git a/installation/docker/README.md b/installation/docker/README.md index d8502d052..760d274fd 100644 --- a/installation/docker/README.md +++ b/installation/docker/README.md @@ -15,10 +15,10 @@ copied for this. ## Starting VMClarity ``` -docker compose --project-name vmclarity --file dockercompose.yml up -d --wait --remove-orphans +docker compose --project-name vmclarity --file docker-compose.yml up -d --wait --remove-orphans ``` ## Stopping VMClarity ``` -docker compose --project-name vmclarity --file dockercompose.yml down --remove-orphans +docker compose --project-name vmclarity --file docker-compose.yml down --remove-orphans ```