stats: Add a check and count for CI infra failures

This check looks for any failed e2e lanes that did not get as far as
running the tests and creating a junit.functext.xml. This is a good
indicator that a failure has occurred in kubevirtci.

Signed-off-by: Brian Carey <[email protected]>

Author: brianmcarey

pkg/constants/main.go

@@ -45,6 +45,7 @@ const (
 	SIGNetworkRetestBadgeFileName  = "sig-network-retests.svg"
 	SIGStorageRetestBadgeFileName  = "sig-storage-retests.svg"
 	SIGOperatorRetestBadgeFileName = "sig-operator-retests.svg"
+	SIGCIRetestBadgeFileName       = "sig-ci-retests.svg"
 	JSONResultsFileName            = "results.json"
 	PlotFileName                   = "plot.png"
 	MetricsFileName                = "metrics"
@@ -58,6 +59,7 @@ const (
 	SIGNetworkRetestBadgeName  = "SIG Network"
 	SIGOperatorRetestBadgeName = "SIG Operator"
 	SIGStorageRetestBadgeName  = "SIG Storage"
+	SIGCIRetestBadgeName       = "SIG CI"
 	BadgeDataFormat            = "%.2f ± std %.2f"
 	NoRetestBadgeDataFormat    = "%.0f / %.0f | %.0f%s"
 

pkg/output/output.go

@@ -159,6 +159,13 @@ func (b *Handler) writeBadges(results *types.Results) error {
 		b.options.SIGRetestsLevels,
 	)
 
+	err = b.writeSIGRetestBadge(
+		constants.SIGCIRetestBadgeName,
+		filepath.Join(basePath, constants.SIGCIRetestBadgeFileName),
+		results.Data[constants.SIGRetests],
+		b.options.SIGRetestsLevels,
+	)
+
 	err = b.writeJobFailureBadges(
 		results.Data[constants.SIGRetests],
 		b.options.SIGRetestsLevels,
@@ -209,6 +216,9 @@ func (b *Handler) writeSIGRetestBadge(name, filePath string, data types.RunningA
 	case constants.SIGOperatorRetestBadgeName:
 		value = data.SIGOperatorRetest
 		total = data.SIGOperatorTotal
+	case constants.SIGCIRetestBadgeName:
+		value = data.SIGCIRetest
+		total = data.SIGCITotal
 	}
 
 	color := BadgeColor(((value / total) * 100), levels)

pkg/sigretests/main.go

@@ -14,13 +14,15 @@ const org = "kubevirt"
 const repo = "kubevirt"
 
 type job struct {
-	jobName     string
-	buildNumber string
-	buildURL    string
-	failure     bool
+	jobName      string
+	buildNumber  string
+	buildURL     string
+	failure      bool
+	artifactsURL string
 }
 
 type SigRetests struct {
+	SigCIFailure       int
 	SigComputeFailure  int
 	SigNetworkFailure  int
 	SigStorageFailure  int
@@ -48,6 +50,20 @@ func finishedJSONURL(org string, repo string, prNumber string, jobName string, b
 	return fmt.Sprintf("%s/%s/%s/finished.json", prStorageURL(org, repo, prNumber), jobName, buildNumber)
 }
 
+func checkSIGCIFailure(job job) bool {
+	if job.failure {
+		junitURL := fmt.Sprintf("%s/junit.functest.xml", job.artifactsURL)
+		resp, err := http.Get(junitURL)
+		if err != nil {
+			return true
+		}
+		if resp.StatusCode != http.StatusOK {
+			return true
+		}
+	}
+	return false
+}
+
 func filterJobs(node *html.Node) (jobs []job) {
 	var e2eJob job
 	if node.Type == html.ElementNode && node.Data == "td" {
@@ -60,6 +76,9 @@ func filterJobs(node *html.Node) (jobs []job) {
 						buildLogUrl := strings.Split(href.Val, "/")
 						e2eJob.jobName = buildLogUrl[len(buildLogUrl)-2]
 						e2eJob.buildNumber = buildLogUrl[len(buildLogUrl)-1]
+						prNumber := buildLogUrl[len(buildLogUrl)-3]
+						e2eJob.artifactsURL = fmt.Sprintf("https://gcsweb.ci.kubevirt.io/gcs/kubevirt-prow/pr-logs/pull/kubevirt_kubevirt/%s/%s/%s/artifacts",
+							prNumber, e2eJob.jobName, e2eJob.buildNumber)
 						prowjobs = append(prowjobs, e2eJob)
 						continue
 					}
@@ -172,6 +191,8 @@ func FilterJobsPerSigs(jobs []job) (prSigRetests SigRetests) {
 	prSigRetests = SigRetests{}
 	for _, job := range jobs {
 		switch {
+		case checkSIGCIFailure(job):
+			prSigRetests.SigCIFailure += 1
 		case strings.Contains(job.jobName, "sig-compute") || strings.Contains(job.jobName, "vgpu"):
 			if job.failure {
 				prSigRetests.SigComputeFailure += 1

pkg/stats/stats.go

@@ -281,6 +281,7 @@ func (h *Handler) sigRetestsProcessor(results *types.Results) (*types.Results, e
 		dataItem.SIGNetworkRetest = dataItem.SIGNetworkRetest + float64(jobsPerSIG.SigNetworkFailure)
 		dataItem.SIGStorageRetest = dataItem.SIGStorageRetest + float64(jobsPerSIG.SigStorageFailure)
 		dataItem.SIGOperatorRetest = dataItem.SIGOperatorRetest + float64(jobsPerSIG.SigOperatorFailure)
+		dataItem.SIGCIRetest = dataItem.SIGCIRetest + float64(jobsPerSIG.SigCIFailure)
 		dataItem.SIGComputeTotal = dataItem.SIGComputeTotal + float64(jobsPerSIG.SigComputeFailure) + float64(jobsPerSIG.SigComputeSuccess)
 		dataItem.SIGNetworkTotal = dataItem.SIGNetworkTotal + float64(jobsPerSIG.SigNetworkFailure) + float64(jobsPerSIG.SigNetworkSuccess)
 		dataItem.SIGStorageTotal = dataItem.SIGStorageTotal + float64(jobsPerSIG.SigStorageFailure) + float64(jobsPerSIG.SigStorageSuccess)
@@ -294,6 +295,7 @@ func (h *Handler) sigRetestsProcessor(results *types.Results) (*types.Results, e
 		successJobNames = slices.Concat(successJobNames, jobsPerSIG.SuccessJobNames)
 		failedJobURLs = slices.Concat(failedJobURLs, jobsPerSIG.FailedJobURLs)
 	}
+	dataItem.SIGCITotal = dataItem.SIGComputeTotal + dataItem.SIGStorageTotal + dataItem.SIGNetworkTotal + dataItem.SIGOperatorTotal + dataItem.SIGCIRetest
 	sortedFailedJobs := types.SortByMostFailed(countFailedJobs(failedJobNames))
 	for i, job := range sortedFailedJobs {
 		for _, success := range successJobNames {

pkg/types/main.go

@@ -246,10 +246,12 @@ type RunningAverageDataItem struct {
 	SIGStorageRetest     float64
 	SIGNetworkRetest     float64
 	SIGOperatorRetest    float64
+	SIGCIRetest          float64
 	SIGComputeTotal      float64
 	SIGStorageTotal      float64
 	SIGNetworkTotal      float64
 	SIGOperatorTotal     float64
+	SIGCITotal           float64
 	FailedJobLeaderBoard FailedJobs
 	DataPoints           []DataPoint
 }