test(scorecard): scorecard tests for recording management (#698)

* test(scorecard): scorecard tests for recording management Signed-off-by: Thuan Vo <[email protected]> * fixup(scorecard): fix cr cleanup func * test(scorecard): registry recording test to suite * chore(scorecard): reorganize client def * chore(scorecard): clean up common setup func * chore(bundle): regenerate bundle with scorecard tag * chore(bundle): correct image tag in bundle * fix(bundle): add missing scorecard test config patch * feat(scorecard): scaffold cryostat API client * chore(scorecard): clean up API client * test(scorecard): implement recording scorecard test * fixup(scorecard): correctly add scorecard test via hack templates * fix(client): ignore unverified tls certs and base64 oauth token * chore(bundle): split cryostat tests to separate stage * fix(scorecard): extend default transport instead of overwriting * chore(scorecard): refactor client to support multi-part * fixup(client): fix request verb * fix(client): fix recording create form format * fix(scorecard): create stored credentials for target JVM * fix(scorecard): fix 502 status error * chore(scorecard): simplify client def * chore(scorecard): fetch recordings to ensure action is correctly performed * test(scorecard): test generating report for a recording * chore(scorecard): clean up * test(scorecard): list archives in tests * ci(scorecard): reconfigure ingress for kind * ci(k8s): correct cluster name * test(scorecard): use role instead of clusterrole for oauth rules * test(scorecard): parse health response for additional checks * chore(scorecard): add missing newline in logs * chore(scorecard): check status code before parsing body in health check * test(scorecard): add custom target discovery to recording scorecard test * add EOF wait and resp headers * add resp headers * chore(client): configure all clients to send safe requests * fix(clients): add missing content-type header * fix(scorecard): add missing test name in help message * chore(client): create new http requests when retrying * chore(bundle): update scorecard image tags --------- Signed-off-by: Thuan Vo <[email protected]> Co-authored-by: Ming Yu Wang <[email protected]> Co-authored-by: Ming Wang <[email protected]>
cryostatio · Mar 5, 2024 · cfcbfc7 · cfcbfc7
1 parent 2af4362
commit cfcbfc7
Show file tree

Hide file tree

Showing 14 changed files with 1,258 additions and 306 deletions.
diff --git a/.github/workflows/test-ci-reusable.yml b/.github/workflows/test-ci-reusable.yml
@@ -118,11 +118,29 @@ jobs:
         username: ${{ github.repository_owner }}
         password: ${{ secrets.GITHUB_TOKEN }}
     - name: Set up Kind cluster
+      uses: helm/[email protected]
+      with:
+        config: .github/kind-config.yaml
+        cluster_name: ci-${{ github.run_id }}
+        wait: 1m
+        ignore_failed_clean: true
+    - name: Set up Ingress Controller
       run: |
-        kind create cluster --config=".github/kind-config.yaml" -n ci-${{ github.run_id }}
-        # Enabling Ingress
+        # Install nginx ingress controller
         kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/kind/deploy.yaml
-        kubectl rollout status -w deployment/ingress-nginx-controller -n ingress-nginx --timeout 5m
+        kubectl rollout status -w \
+          deployment/ingress-nginx-controller \
+          -n ingress-nginx --timeout 5m
+        
+        # Lower the number of worker processes
+        kubectl patch cm/ingress-nginx-controller \
+          --type merge \
+          -p '{"data":{"worker-processes":"1"}}' \
+          -n ingress-nginx
+
+        # Modify /etc/hosts to resolve hostnames
+        ip_address=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ci-${{ github.run_id }}-control-plane)
+        echo "$ip_address testing.cryostat" | sudo tee -a /etc/hosts
     - name: Install Operator Lifecycle Manager
       run: curl -sL https://github.com/operator-framework/operator-lifecycle-manager/releases/download/v0.24.0/install.sh | bash -s v0.24.0
     - name: Install Cert Manager
@@ -140,8 +158,6 @@ jobs:
         SCORECARD_REGISTRY_PASSWORD="${{ secrets.GITHUB_TOKEN }}" \
         BUNDLE_IMG="${{ steps.push-bundle-to-ghcr.outputs.registry-path }}" \
         make test-scorecard
-    - name: Clean up Kind cluster
-      run: kind delete cluster -n ci-${{ github.run_id }}
     - name: Set latest commit status as ${{ job.status }}
       uses: myrotvorets/set-commit-status-action@master
       if: always()

diff --git a/bundle/manifests/cryostat-operator.clusterserviceversion.yaml b/bundle/manifests/cryostat-operator.clusterserviceversion.yaml
@@ -54,7 +54,7 @@ metadata:
     capabilities: Seamless Upgrades
     categories: Monitoring, Developer Tools
     containerImage: quay.io/cryostat/cryostat-operator:2.5.0-dev
-    createdAt: "2024-02-04T06:09:16Z"
+    createdAt: "2024-03-05T02:05:10Z"
     description: JVM monitoring and profiling tool
     operatorframework.io/initialization-resource: |-
       {

diff --git a/bundle/tests/scorecard/config.yaml b/bundle/tests/scorecard/config.yaml
@@ -66,10 +66,11 @@ stages:
     storage:
       spec:
         mountPath: {}
+- tests:
   - entrypoint:
     - cryostat-scorecard-tests
     - operator-install
-    image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20231011144522
+    image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240305020416
     labels:
       suite: cryostat
       test: operator-install
@@ -79,13 +80,23 @@ stages:
   - entrypoint:
     - cryostat-scorecard-tests
     - cryostat-cr
-    image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20231011144522
+    image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240305020416
     labels:
       suite: cryostat
       test: cryostat-cr
     storage:
       spec:
         mountPath: {}
+  - entrypoint:
+    - cryostat-scorecard-tests
+    - cryostat-recording
+    image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240305020416
+    labels:
+      suite: cryostat
+      test: cryostat-recording
+    storage:
+      spec:
+        mountPath: {}
 storage:
   spec:
     mountPath: {}
diff --git a/config/rbac/oauth_client.yaml b/config/rbac/oauth_client.yaml
@@ -3,7 +3,6 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
-  creationTimestamp: null
   name: oauth-client
 rules:
 - apiGroups:

diff --git a/config/scorecard/bases/config.yaml b/config/scorecard/bases/config.yaml
@@ -3,5 +3,7 @@ kind: Configuration
 metadata:
   name: config
 stages:
-- parallel: true
+- parallel: true # Build-in Tests
+  tests: []
+- parallel: false # Cryostat Custom Tests
   tests: []
diff --git a/config/scorecard/patches/custom.config.yaml b/config/scorecard/patches/custom.config.yaml
@@ -3,22 +3,32 @@
   path: /serviceaccount
   value: cryostat-scorecard
 - op: add
-  path: /stages/0/tests/-
+  path: /stages/1/tests/-
   value:
     entrypoint:
     - cryostat-scorecard-tests
     - operator-install
-    image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20231011144522"
+    image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240305020416"
     labels:
       suite: cryostat
       test: operator-install
 - op: add
-  path: /stages/0/tests/-
+  path: /stages/1/tests/-
   value:
     entrypoint:
     - cryostat-scorecard-tests
     - cryostat-cr
-    image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20231011144522"
+    image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240305020416"
     labels:
       suite: cryostat
       test: cryostat-cr
+- op: add
+  path: /stages/1/tests/-
+  value:
+    entrypoint:
+    - cryostat-scorecard-tests
+    - cryostat-recording
+    image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240305020416"
+    labels:
+      suite: cryostat
+      test: cryostat-recording
diff --git a/hack/custom.config.yaml.in b/hack/custom.config.yaml.in
@@ -2,7 +2,7 @@
   path: /serviceaccount
   value: cryostat-scorecard
 - op: add
-  path: /stages/0/tests/-
+  path: /stages/1/tests/-
   value:
     entrypoint:
     - cryostat-scorecard-tests
@@ -12,7 +12,7 @@
       suite: cryostat
       test: operator-install
 - op: add
-  path: /stages/0/tests/-
+  path: /stages/1/tests/-
   value:
     entrypoint:
     - cryostat-scorecard-tests
@@ -21,3 +21,13 @@
     labels:
       suite: cryostat
       test: cryostat-cr
+- op: add
+  path: /stages/1/tests/-
+  value:
+    entrypoint:
+    - cryostat-scorecard-tests
+    - cryostat-recording
+    image: "${CUSTOM_SCORECARD_IMG}"
+    labels:
+      suite: cryostat
+      test: cryostat-recording
diff --git a/internal/images/custom-scorecard-tests/main.go b/internal/images/custom-scorecard-tests/main.go
@@ -79,6 +79,7 @@ func printValidTests() []scapiv1alpha3.TestResult {
 	str := fmt.Sprintf("valid tests for this image include: %s", strings.Join([]string{
 		tests.OperatorInstallTestName,
 		tests.CryostatCRTestName,
+		tests.CryostatRecordingTestName,
 	}, ","))
 	result.Errors = append(result.Errors, str)
 
@@ -90,6 +91,7 @@ func validateTests(testNames []string) bool {
 		switch testName {
 		case tests.OperatorInstallTestName:
 		case tests.CryostatCRTestName:
+		case tests.CryostatRecordingTestName:
 		default:
 			return false
 		}
@@ -108,6 +110,8 @@ func runTests(testNames []string, bundle *apimanifests.Bundle, namespace string,
 			results = append(results, tests.OperatorInstallTest(bundle, namespace))
 		case tests.CryostatCRTestName:
 			results = append(results, tests.CryostatCRTest(bundle, namespace, openShiftCertManager))
+		case tests.CryostatRecordingTestName:
+			results = append(results, tests.CryostatRecordingTest(bundle, namespace, openShiftCertManager))
 		default:
 			log.Fatalf("unknown test found: %s", testName)
 		}

diff --git a/internal/images/custom-scorecard-tests/rbac/scorecard_role.yaml b/internal/images/custom-scorecard-tests/rbac/scorecard_role.yaml
@@ -53,6 +53,55 @@ rules:
   - cryostats/status
   verbs:
   - get
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get
+# Permissions for default OAuth configurations
+- apiGroups:
+  - operator.cryostat.io
+  resources:
+  - cryostats
+  verbs:
+  - create
+  - patch
+  - delete
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  - pods/exec
+  - services
+  verbs:
+  - create
+  - patch
+  - delete
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - replicationcontrollers
+  - endpoints
+  verbs:
+  - get
+- apiGroups:
+  - apps
+  resources:
+  - deployments
+  verbs:
+  - create
+  - get
+- apiGroups:
+  - apps
+  resources:
+  - daemonsets
+  - replicasets
+  - statefulsets
+  verbs:
+  - get
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole