Deleting a CSV removes related CSV metrics

Problem: Bug 1819308: When creating a CSV fails, deleting it did
not clear the CSV metrics. So when the same CSV was created
again and allowed to succeed, the olm pod still retained the
old CSV metrics

Solution: Deleting a CSV will now clear its related metrics
from the olm pod

Signed-off-by: Harish <[email protected]>

Author: harishsurf

pkg/controller/operators/olm/operator.go

@@ -856,6 +856,8 @@ func (a *Operator) handleClusterServiceVersionDeletion(obj interface{}) {
 		"phase":     clusterServiceVersion.Status.Phase,
 	})
 
+	metrics.DeleteCSVMetric(clusterServiceVersion)
+
 	defer func(csv v1alpha1.ClusterServiceVersion) {
 		if clusterServiceVersion.IsCopied() {
 			logger.Debug("deleted csv is copied. skipping operatorgroup requeue")

pkg/metrics/metrics.go

@@ -2,7 +2,6 @@ package metrics
 
 import (
 	"context"
-
 	"github.com/prometheus/client_golang/prometheus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
@@ -189,6 +188,12 @@ func CounterForSubscription(name, installedCSV, channelName, packageName string)
 	return SubscriptionSyncCount.WithLabelValues(name, installedCSV, channelName, packageName)
 }
 
+func DeleteCSVMetric(oldCSV *olmv1alpha1.ClusterServiceVersion) {
+	// Delete the old CSV metrics
+	csvAbnormal.DeleteLabelValues(oldCSV.Namespace, oldCSV.Name, oldCSV.Spec.Version.String(), string(oldCSV.Status.Phase), string(oldCSV.Status.Reason))
+	csvSucceeded.DeleteLabelValues(oldCSV.Namespace, oldCSV.Name, oldCSV.Spec.Version.String())
+}
+
 func EmitCSVMetric(oldCSV *olmv1alpha1.ClusterServiceVersion, newCSV *olmv1alpha1.ClusterServiceVersion) {
 	if oldCSV == nil || newCSV == nil {
 		return

test/e2e/metrics_e2e_test.go

@@ -44,7 +44,6 @@ var _ = Describe("Metrics", func() {
 
 		cleanupCSV, err := createCSV(GinkgoT(), c, crc, failingCSV, testNamespace, false, false)
 		Expect(err).ToNot(HaveOccurred())
-		defer cleanupCSV()
 
 		_, err = fetchCSV(GinkgoT(), crc, failingCSV.Name, testNamespace, csvFailedChecker)
 		Expect(err).ToNot(HaveOccurred())
@@ -58,6 +57,14 @@ var _ = Describe("Metrics", func() {
 			ContainSubstring("version=\"0.0.0\""),
 			ContainSubstring("csv_succeeded"),
 		))
+
+		cleanupCSV()
+
+		// Verify that when the csv has been deleted, it deletes the corresponding CSV metrics
+		Expect(getMetricsFromPod(c, getOLMPod(c), "8081")).ToNot(And(
+			ContainSubstring("csv_abnormal{name=\"%s\"", failingCSV.Name),
+			ContainSubstring("csv_succeeded{name=\"%s\"", failingCSV.Name),
+		))
 	})
 })