test(scorecard): scorecard test for report generator (#753)

* deploy reports sidecar

* report scorecard test

* update

* rebase fix

* query health

bundle/manifests/cryostat-operator.clusterserviceversion.yaml

@@ -54,7 +54,7 @@ metadata:
     capabilities: Seamless Upgrades
     categories: Monitoring, Developer Tools
     containerImage: quay.io/cryostat/cryostat-operator:2.5.0-dev
-    createdAt: "2024-03-07T15:43:22Z"
+    createdAt: "2024-03-13T15:52:10Z"
     description: JVM monitoring and profiling tool
     operatorframework.io/initialization-resource: |-
       {

bundle/tests/scorecard/config.yaml

@@ -70,7 +70,7 @@ stages:
   - entrypoint:
     - cryostat-scorecard-tests
     - operator-install
-    image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240307153901
+    image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240313145612
     labels:
       suite: cryostat
       test: operator-install
@@ -80,7 +80,7 @@ stages:
   - entrypoint:
     - cryostat-scorecard-tests
     - cryostat-cr
-    image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240307153901
+    image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240313145612
     labels:
       suite: cryostat
       test: cryostat-cr
@@ -90,7 +90,7 @@ stages:
   - entrypoint:
     - cryostat-scorecard-tests
     - cryostat-recording
-    image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240307153901
+    image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240313145612
     labels:
       suite: cryostat
       test: cryostat-recording
@@ -100,13 +100,23 @@ stages:
   - entrypoint:
     - cryostat-scorecard-tests
     - cryostat-config-change
-    image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240307153901
+    image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240313145612
     labels:
       suite: cryostat
       test: cryostat-config-change
     storage:
       spec:
         mountPath: {}
+  - entrypoint:
+    - cryostat-scorecard-tests
+    - cryostat-report
+    image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240313145612
+    labels:
+      suite: cryostat
+      test: cryostat-report
+    storage:
+      spec:
+        mountPath: {}
 storage:
   spec:
     mountPath: {}

config/scorecard/patches/custom.config.yaml

@@ -8,7 +8,7 @@
     entrypoint:
     - cryostat-scorecard-tests
     - operator-install
-    image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240307154322"
+    image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240313155212"
     labels:
       suite: cryostat
       test: operator-install
@@ -18,7 +18,7 @@
     entrypoint:
     - cryostat-scorecard-tests
     - cryostat-cr
-    image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240307154322"
+    image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240313155212"
     labels:
       suite: cryostat
       test: cryostat-cr
@@ -28,7 +28,7 @@
     entrypoint:
     - cryostat-scorecard-tests
     - cryostat-recording
-    image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240307154322"
+    image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240313155212"
     labels:
       suite: cryostat
       test: cryostat-recording
@@ -38,7 +38,17 @@
     entrypoint:
     - cryostat-scorecard-tests
     - cryostat-config-change
-    image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240307154322"
+    image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240313155212"
     labels:
       suite: cryostat
       test: cryostat-config-change
+- op: add
+  path: /stages/1/tests/-
+  value:
+    entrypoint:
+    - cryostat-scorecard-tests
+    - cryostat-report
+    image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240313155212"
+    labels:
+      suite: cryostat
+      test: cryostat-report

hack/custom.config.yaml.in

@@ -41,3 +41,13 @@
     labels:
       suite: cryostat
       test: cryostat-config-change
+- op: add
+  path: /stages/1/tests/-
+  value:
+    entrypoint:
+    - cryostat-scorecard-tests
+    - cryostat-report
+    image: "${CUSTOM_SCORECARD_IMG}"
+    labels:
+      suite: cryostat
+      test: cryostat-report

internal/images/custom-scorecard-tests/main.go

@@ -81,6 +81,7 @@ func printValidTests() []scapiv1alpha3.TestResult {
 		tests.CryostatCRTestName,
 		tests.CryostatRecordingTestName,
 		tests.CryostatConfigChangeTestName,
+		tests.CryostatReportTestName,
 	}, ","))
 	result.Errors = append(result.Errors, str)
 
@@ -94,6 +95,7 @@ func validateTests(testNames []string) bool {
 		case tests.CryostatCRTestName:
 		case tests.CryostatRecordingTestName:
 		case tests.CryostatConfigChangeTestName:
+		case tests.CryostatReportTestName:
 		default:
 			return false
 		}
@@ -116,6 +118,8 @@ func runTests(testNames []string, bundle *apimanifests.Bundle, namespace string,
 			results = append(results, tests.CryostatRecordingTest(bundle, namespace, openShiftCertManager))
 		case tests.CryostatConfigChangeTestName:
 			results = append(results, tests.CryostatConfigChangeTest(bundle, namespace, openShiftCertManager))
+		case tests.CryostatReportTestName:
+			results = append(results, tests.CryostatReportTest(bundle, namespace, openShiftCertManager))
 		default:
 			log.Fatalf("unknown test found: %s", testName)
 		}

internal/test/scorecard/clients.go

@@ -587,7 +587,7 @@ func SendRequest(ctx context.Context, httpClient *http.Client, method string, ur
 		// Create a new request
 		req, err := NewHttpRequest(ctx, method, url, body, header)
 		if err != nil {
-			return false, fmt.Errorf("failed to create a Cryostat REST request: %s", err.Error())
+			return false, fmt.Errorf("failed to create an http request: %s", err.Error())
 		}
 
 		resp, err := httpClient.Do(req)

internal/test/scorecard/common_utils.go

@@ -17,7 +17,9 @@ package scorecard
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	"io"
 	"net/http"
 	"net/url"
 	"time"
@@ -338,6 +340,48 @@ func createAndWaitTillCryostatAvailable(cr *operatorv1beta1.Cryostat, resources
 }
 
 func waitTillCryostatReady(base *url.URL, resources *TestResources) error {
+	return sendHealthRequest(base, resources, func(resp *http.Response, r *scapiv1alpha3.TestResult) (done bool, err error) {
+		health := &HealthResponse{}
+		err = ReadJSON(resp, health)
+		if err != nil {
+			return false, fmt.Errorf("failed to read response body: %s", err.Error())
+		}
+
+		if err = health.Ready(); err != nil {
+			r.Log += fmt.Sprintf("application is not yet ready: %s\n", err.Error())
+			return false, nil // Try again
+		}
+
+		r.Log += fmt.Sprintf("application is ready at %s\n", base.String())
+		return true, nil
+	})
+}
+
+func waitTillReportReady(name string, namespace string, port int32, resources *TestResources) error {
+	client := resources.Client
+	r := resources.TestResult
+
+	ctx, cancel := context.WithTimeout(context.Background(), testTimeout)
+	defer cancel()
+
+	err := waitForDeploymentAvailability(ctx, client, namespace, name, r)
+	if err != nil {
+		return fmt.Errorf("report sidecar deployment did not become available: %s", err.Error())
+	}
+
+	reportsUrl := fmt.Sprintf("https://%s.%s.svc.cluster.local:%d", name, namespace, port)
+	base, err := url.Parse(reportsUrl)
+	if err != nil {
+		return fmt.Errorf("application URL is invalid: %s", err.Error())
+	}
+
+	return sendHealthRequest(base, resources, func(resp *http.Response, r *scapiv1alpha3.TestResult) (done bool, err error) {
+		r.Log += fmt.Sprintf("reports sidecar is ready at %s\n", base.String())
+		return true, nil
+	})
+}
+
+func sendHealthRequest(base *url.URL, resources *TestResources, healthCheck func(resp *http.Response, r *scapiv1alpha3.TestResult) (done bool, err error)) error {
 	client := NewHttpClient()
 	r := resources.TestResult
 
@@ -348,12 +392,15 @@ func waitTillCryostatReady(base *url.URL, resources *TestResources) error {
 		url := base.JoinPath("/health")
 		req, err := NewHttpRequest(ctx, http.MethodGet, url.String(), nil, make(http.Header))
 		if err != nil {
-			return false, fmt.Errorf("failed to create a Cryostat REST request: %s", err.Error())
+			return false, fmt.Errorf("failed to create a an http request: %s", err.Error())
 		}
 		req.Header.Add("Accept", "*/*")
 
 		resp, err := client.Do(req)
 		if err != nil {
+			if errors.Is(err, io.EOF) {
+				return false, nil // Retry
+			}
 			return false, err
 		}
 		defer resp.Body.Close()
@@ -365,22 +412,8 @@ func waitTillCryostatReady(base *url.URL, resources *TestResources) error {
 			}
 			return false, fmt.Errorf("API request failed with status code %d: %s", resp.StatusCode, ReadError(resp))
 		}
-
-		health := &HealthResponse{}
-		err = ReadJSON(resp, health)
-		if err != nil {
-			return false, fmt.Errorf("failed to read response body: %s", err.Error())
-		}
-
-		if err = health.Ready(); err != nil {
-			r.Log += fmt.Sprintf("application is not yet ready: %s\n", err.Error())
-			return false, nil // Try again
-		}
-
-		r.Log += fmt.Sprintf("application is ready at %s\n", base.String())
-		return true, nil
+		return healthCheck(resp, r)
 	})
-
 	return err
 }
 

internal/test/scorecard/tests.go

@@ -33,6 +33,7 @@ const (
 	CryostatCRTestName           string = "cryostat-cr"
 	CryostatRecordingTestName    string = "cryostat-recording"
 	CryostatConfigChangeTestName string = "cryostat-config-change"
+	CryostatReportTestName       string = "cryostat-report"
 )
 
 // OperatorInstallTest checks that the operator installed correctly
@@ -97,7 +98,7 @@ func CryostatConfigChangeTest(bundle *apimanifests.Bundle, namespace string, ope
 	if err != nil {
 		return fail(*r, fmt.Sprintf("failed to determine application URL: %s", err.Error()))
 	}
-	defer cleanupCryostat(r, tr.Client, CryostatRecordingTestName, namespace)
+	defer cleanupCryostat(r, tr.Client, CryostatConfigChangeTestName, namespace)
 
 	// Switch Cryostat CR to PVC for redeployment
 	ctx, cancel := context.WithTimeout(context.Background(), testTimeout)
@@ -126,7 +127,7 @@ func CryostatConfigChangeTest(bundle *apimanifests.Bundle, namespace string, ope
 	if err != nil {
 		return fail(*r, fmt.Sprintf("Cryostat redeployment did not become available: %s", err.Error()))
 	}
-	r.Log += "Cryostat deployment has successfully updated with new spec template"
+	r.Log += "Cryostat deployment has successfully updated with new spec template\n"
 
 	base, err := url.Parse(cr.Status.ApplicationURL)
 	if err != nil {
@@ -278,3 +279,39 @@ func CryostatRecordingTest(bundle *apimanifests.Bundle, namespace string, openSh
 
 	return *r
 }
+
+func CryostatReportTest(bundle *apimanifests.Bundle, namespace string, openShiftCertManager bool) scapiv1alpha3.TestResult {
+	tr := newTestResources(CryostatReportTestName)
+	r := tr.TestResult
+
+	err := setupCRTestResources(tr, openShiftCertManager)
+	if err != nil {
+		return fail(*r, fmt.Sprintf("failed to set up %s test: %s", CryostatReportTestName, err.Error()))
+	}
+
+	port := int32(10000)
+	cr := newCryostatCR(CryostatReportTestName, namespace, !tr.OpenShift)
+	cr.Spec.ReportOptions = &operatorv1beta1.ReportConfiguration{
+		Replicas: 1,
+	}
+	cr.Spec.ServiceOptions = &operatorv1beta1.ServiceConfigList{
+		ReportsConfig: &operatorv1beta1.ReportsServiceConfig{
+			HTTPPort: &port,
+		},
+	}
+
+	// Create a default Cryostat CR
+	cr, err = createAndWaitTillCryostatAvailable(cr, tr)
+	if err != nil {
+		return fail(*r, fmt.Sprintf("%s test failed: %s", CryostatReportTestName, err.Error()))
+	}
+	defer cleanupCryostat(r, tr.Client, CryostatReportTestName, namespace)
+
+	// Query health of report sidecar
+	err = waitTillReportReady(cr.Name+"-reports", cr.Namespace, port, tr)
+	if err != nil {
+		return fail(*r, fmt.Sprintf("failed to reach the application: %s", err.Error()))
+	}
+
+	return *r
+}