Skip to content

Commit

Permalink
test(scorecard): add container logs to scorecard results (#758)
Browse files Browse the repository at this point in the history
* test(scorecard): add container logs to scorecard results

* build(bundle): regenerate bundle with new scorecard tags

* chore(scorecard): refactor to remove duplicate codes
  • Loading branch information
tthvo authored Mar 19, 2024
1 parent 2201704 commit d01e0d2
Show file tree
Hide file tree
Showing 7 changed files with 159 additions and 12 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ metadata:
capabilities: Seamless Upgrades
categories: Monitoring, Developer Tools
containerImage: quay.io/cryostat/cryostat-operator:2.5.0-dev
createdAt: "2024-03-13T15:52:10Z"
createdAt: "2024-03-18T06:48:08Z"
description: JVM monitoring and profiling tool
operatorframework.io/initialization-resource: |-
{
Expand Down
10 changes: 5 additions & 5 deletions bundle/tests/scorecard/config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ stages:
- entrypoint:
- cryostat-scorecard-tests
- operator-install
image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240313145612
image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240318064753
labels:
suite: cryostat
test: operator-install
Expand All @@ -80,7 +80,7 @@ stages:
- entrypoint:
- cryostat-scorecard-tests
- cryostat-cr
image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240313145612
image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240318064753
labels:
suite: cryostat
test: cryostat-cr
Expand All @@ -90,7 +90,7 @@ stages:
- entrypoint:
- cryostat-scorecard-tests
- cryostat-recording
image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240313145612
image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240318064753
labels:
suite: cryostat
test: cryostat-recording
Expand All @@ -100,7 +100,7 @@ stages:
- entrypoint:
- cryostat-scorecard-tests
- cryostat-config-change
image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240313145612
image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240318064753
labels:
suite: cryostat
test: cryostat-config-change
Expand All @@ -110,7 +110,7 @@ stages:
- entrypoint:
- cryostat-scorecard-tests
- cryostat-report
image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240313145612
image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240318064753
labels:
suite: cryostat
test: cryostat-report
Expand Down
10 changes: 5 additions & 5 deletions config/scorecard/patches/custom.config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
entrypoint:
- cryostat-scorecard-tests
- operator-install
image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240313155212"
image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240318064753"
labels:
suite: cryostat
test: operator-install
Expand All @@ -18,7 +18,7 @@
entrypoint:
- cryostat-scorecard-tests
- cryostat-cr
image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240313155212"
image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240318064753"
labels:
suite: cryostat
test: cryostat-cr
Expand All @@ -28,7 +28,7 @@
entrypoint:
- cryostat-scorecard-tests
- cryostat-recording
image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240313155212"
image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240318064753"
labels:
suite: cryostat
test: cryostat-recording
Expand All @@ -38,7 +38,7 @@
entrypoint:
- cryostat-scorecard-tests
- cryostat-config-change
image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240313155212"
image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240318064753"
labels:
suite: cryostat
test: cryostat-config-change
Expand All @@ -48,7 +48,7 @@
entrypoint:
- cryostat-scorecard-tests
- cryostat-report
image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240313155212"
image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240318064753"
labels:
suite: cryostat
test: cryostat-report
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,13 @@ rules:
- statefulsets
verbs:
- get
# Permissions to retrieve container logs
- apiGroups:
- ""
resources:
- pods/log
verbs:
- get
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
Expand Down
28 changes: 28 additions & 0 deletions internal/test/scorecard/common_utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,10 @@ import (
netv1 "k8s.io/api/networking/v1"
kerrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/scheme"
)

Expand Down Expand Up @@ -497,3 +499,29 @@ func cleanupCryostat(r *scapiv1alpha3.TestResult, client *CryostatClientset, nam
r.Log += fmt.Sprintf("failed to delete Cryostat: %s\n", err.Error())
}
}

func getCryostatPodNameForCR(clientset *kubernetes.Clientset, cr *operatorv1beta1.Cryostat) (string, error) {
selector := metav1.LabelSelector{
MatchLabels: map[string]string{
"app": cr.Name,
"component": "cryostat",
},
}
opts := metav1.ListOptions{
LabelSelector: labels.Set(selector.MatchLabels).String(),
}

ctx, cancel := context.WithTimeout(context.TODO(), testTimeout)
defer cancel()

pods, err := clientset.CoreV1().Pods(cr.Namespace).List(ctx, opts)
if err != nil {
return "", err
}

if len(pods.Items) == 0 {
return "", fmt.Errorf("no matching cryostat pods for cr: %s", cr.Name)
}

return pods.Items[0].ObjectMeta.Name, nil
}
106 changes: 106 additions & 0 deletions internal/test/scorecard/logger.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
// Copyright The Cryostat Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package scorecard

import (
"context"
"fmt"
"io"
"strings"

operatorv1beta1 "github.com/cryostatio/cryostat-operator/api/v1beta1"
scapiv1alpha3 "github.com/operator-framework/api/pkg/apis/scorecard/v1alpha3"
v1 "k8s.io/api/core/v1"
"k8s.io/client-go/kubernetes"
)

type ContainerLog struct {
Container string
Log string
}

func LogContainer(clientset *kubernetes.Clientset, namespace, podName, containerName string, ch chan *ContainerLog) {
containerLog := &ContainerLog{
Container: containerName,
}
buf := &strings.Builder{}

err := GetContainerLogs(clientset, namespace, podName, containerName, buf)
if err != nil {
buf.WriteString(fmt.Sprintf("%s\n", err.Error()))
}

containerLog.Log = buf.String()
ch <- containerLog
}

func GetContainerLogs(clientset *kubernetes.Clientset, namespace, podName, containerName string, dest io.Writer) error {
ctx, cancel := context.WithTimeout(context.TODO(), testTimeout)
defer cancel()

logOptions := &v1.PodLogOptions{
Follow: true,
Container: containerName,
}
stream, err := clientset.CoreV1().Pods(namespace).GetLogs(podName, logOptions).Stream(ctx)
if err != nil {
return fmt.Errorf("failed to get logs for container %s in pod %s: %s", containerName, podName, err.Error())
}
defer stream.Close()

_, err = io.Copy(dest, stream)
if err != nil {
return fmt.Errorf("failed to store logs for container %s in pod %s: %s", containerName, podName, err.Error())
}
return nil
}

func CollectLogs(ch chan *ContainerLog) []*ContainerLog {
logs := make([]*ContainerLog, 0)
for i := 0; i < cap(ch); i++ {
logs = append(logs, <-ch)
}
return logs
}

func CollectContainersLogsToResult(result *scapiv1alpha3.TestResult, ch chan *ContainerLog) {
logs := CollectLogs(ch)
for _, log := range logs {
if log != nil {
result.Log += fmt.Sprintf("%s CONTAINER LOG:\n\n\t%s\n", strings.ToUpper(log.Container), log.Log)
}
}
}

func StartLogs(clientset *kubernetes.Clientset, cr *operatorv1beta1.Cryostat) (chan *ContainerLog, error) {
podName, err := getCryostatPodNameForCR(clientset, cr)
if err != nil {
return nil, fmt.Errorf("failed to get pod name for CR: %s", err.Error())
}

containerNames := []string{
cr.Name,
cr.Name + "-grafana",
cr.Name + "-jfr-datasource",
}

ch := make(chan *ContainerLog, len(containerNames))

for _, containerName := range containerNames {
go LogContainer(clientset, cr.Namespace, podName, containerName, ch)
}

return ch, nil
}
8 changes: 7 additions & 1 deletion internal/test/scorecard/tests.go
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ func CryostatConfigChangeTest(bundle *apimanifests.Bundle, namespace string, ope
}

// TODO add a built in discovery test too
func CryostatRecordingTest(bundle *apimanifests.Bundle, namespace string, openShiftCertManager bool) scapiv1alpha3.TestResult {
func CryostatRecordingTest(bundle *apimanifests.Bundle, namespace string, openShiftCertManager bool) (result scapiv1alpha3.TestResult) {
tr := newTestResources(CryostatRecordingTestName)
r := tr.TestResult

Expand All @@ -157,6 +157,12 @@ func CryostatRecordingTest(bundle *apimanifests.Bundle, namespace string, openSh
if err != nil {
return fail(*r, fmt.Sprintf("failed to determine application URL: %s", err.Error()))
}
ch, err := StartLogs(tr.Client.Clientset, cr)
if err != nil {
return fail(*r, fmt.Sprintf("failed to retrieve logs for the application: %s", err.Error()))
}
defer CollectContainersLogsToResult(&result, ch)

defer cleanupCryostat(r, tr.Client, CryostatRecordingTestName, namespace)

base, err := url.Parse(cr.Status.ApplicationURL)
Expand Down

0 comments on commit d01e0d2

Please sign in to comment.