Skip to content

Commit c5b8028

Browse files
jianlinliujianlinliu
committed
extract Operator Progressing / Degraded Counts and Timing
1 parent 5d46d82 commit c5b8028

File tree

2 files changed

+257
-0
lines changed

2 files changed

+257
-0
lines changed

‎pkg/monitortests/clusterversionoperator/operatorstateanalyzer/monitortest.go‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,14 @@ package operatorstateanalyzer
22

33
import (
44
"context"
5+
"fmt"
6+
"path/filepath"
7+
"sort"
58
"time"
69

710
"github.com/openshift/origin/pkg/monitortestframework"
811

12+
"github.com/openshift/origin/pkg/dataloader"
913
"github.com/openshift/origin/pkg/monitor/monitorapi"
1014
"github.com/openshift/origin/pkg/test/ginkgo/junitapi"
1115
"k8s.io/client-go/rest"
@@ -14,6 +18,16 @@ import (
1418
type operatorStateChecker struct {
1519
}
1620

21+
type OperatorStateMetrics struct {
22+
OperatorName string
23+
ProgressingCount int
24+
TotalProgressingSeconds float64
25+
ProgressingDurations []float64
26+
DegradedCount int
27+
TotalDegradedSeconds float64
28+
DegradedDurations []float64
29+
}
30+
1731
func NewAnalyzer() monitortestframework.MonitorTest {
1832
return &operatorStateChecker{}
1933
}
@@ -44,9 +58,103 @@ func (*operatorStateChecker) EvaluateTestsFromConstructedIntervals(ctx context.C
4458
}
4559

4660
func (*operatorStateChecker) WriteContentToStorage(ctx context.Context, storageDir, timeSuffix string, finalIntervals monitorapi.Intervals, finalResourceState monitorapi.ResourcesMap) error {
61+
metrics := calculateOperatorStateMetrics(finalIntervals)
62+
if len(metrics) > 0 {
63+
rows := generateRowsFromMetrics(metrics)
64+
dataFile := dataloader.DataFile{
65+
TableName: "operator_state_metrics",
66+
Schema: map[string]dataloader.DataType{
67+
"Operator": dataloader.DataTypeString,
68+
"State": dataloader.DataTypeString,
69+
"Metric": dataloader.DataTypeString,
70+
"Value": dataloader.DataTypeFloat64,
71+
},
72+
Rows: rows,
73+
}
74+
fileName := filepath.Join(storageDir, fmt.Sprintf("operator-state-metrics%s-%s", timeSuffix, dataloader.AutoDataLoaderSuffix))
75+
if err := dataloader.WriteDataFile(fileName, dataFile); err != nil {
76+
return fmt.Errorf("failed to write operator state metrics: %w", err)
77+
}
78+
fmt.Printf("--->Write operator state metrics to %s successfully.\n", fileName)
79+
}
80+
4781
return nil
4882
}
4983

84+
// calculateOperatorStateMetrics processes raw intervals and aggregates them into a metrics summary map.
85+
func calculateOperatorStateMetrics(finalIntervals monitorapi.Intervals) map[string]*OperatorStateMetrics {
86+
metrics := make(map[string]*OperatorStateMetrics)
87+
88+
for _, interval := range finalIntervals {
89+
if interval.Source != monitorapi.SourceOperatorState {
90+
continue
91+
}
92+
if interval.Locator.Type != monitorapi.LocatorTypeClusterOperator {
93+
continue
94+
}
95+
operatorName := interval.Locator.Keys[monitorapi.LocatorClusterOperatorKey]
96+
if _, ok := metrics[operatorName]; !ok {
97+
metrics[operatorName] = &OperatorStateMetrics{OperatorName: operatorName}
98+
}
99+
100+
duration := interval.To.Sub(interval.From).Seconds()
101+
condition := interval.Message.Annotations[monitorapi.AnnotationCondition]
102+
103+
switch condition {
104+
case "Progressing":
105+
metrics[operatorName].ProgressingCount++
106+
metrics[operatorName].TotalProgressingSeconds += duration
107+
metrics[operatorName].ProgressingDurations = append(metrics[operatorName].ProgressingDurations, duration)
108+
case "Degraded":
109+
metrics[operatorName].DegradedCount++
110+
metrics[operatorName].TotalDegradedSeconds += duration
111+
metrics[operatorName].DegradedDurations = append(metrics[operatorName].DegradedDurations, duration)
112+
}
113+
}
114+
return metrics
115+
}
116+
117+
// generateRowsFromMetrics converts the aggregated metrics map into a slice of rows for the dataloader.
118+
func generateRowsFromMetrics(metrics map[string]*OperatorStateMetrics) []map[string]string {
119+
rows := []map[string]string{}
120+
121+
// Sort operator names for consistent output order in tests
122+
operatorNames := make([]string, 0, len(metrics))
123+
for name := range metrics {
124+
operatorNames = append(operatorNames, name)
125+
}
126+
sort.Strings(operatorNames)
127+
128+
for _, operator := range operatorNames {
129+
metric := metrics[operator]
130+
// Add summary rows:
131+
if metric.ProgressingCount > 0 {
132+
rows = append(rows, createMetricRow(operator, "Progressing", "Count", float64(metric.ProgressingCount)))
133+
rows = append(rows, createMetricRow(operator, "Progressing", "TotalSeconds", metric.TotalProgressingSeconds))
134+
for _, duration := range metric.ProgressingDurations {
135+
rows = append(rows, createMetricRow(operator, "Progressing", "IndividualDurationSeconds", duration))
136+
}
137+
}
138+
if metric.DegradedCount > 0 {
139+
rows = append(rows, createMetricRow(operator, "Degraded", "Count", float64(metric.DegradedCount)))
140+
rows = append(rows, createMetricRow(operator, "Degraded", "TotalSeconds", metric.TotalDegradedSeconds))
141+
for _, duration := range metric.DegradedDurations {
142+
rows = append(rows, createMetricRow(operator, "Degraded", "IndividualDurationSeconds", duration))
143+
}
144+
}
145+
}
146+
return rows
147+
}
148+
149+
func createMetricRow(operator, state, metric string, value float64) map[string]string {
150+
return map[string]string{
151+
"Operator": operator,
152+
"State": state,
153+
"Metric": metric,
154+
"Value": fmt.Sprintf("%f", value),
155+
}
156+
}
157+
50158
func (*operatorStateChecker) Cleanup(ctx context.Context) error {
51159
// TODO wire up the start to a context we can kill here
52160
return nil

‎pkg/monitortests/clusterversionoperator/operatorstateanalyzer/monitortest_test.go‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
package operatorstateanalyzer
2+
3+
import (
4+
"testing"
5+
"time"
6+
7+
"github.com/openshift/origin/pkg/monitor/monitorapi"
8+
"github.com/stretchr/testify/assert"
9+
"github.com/stretchr/testify/require"
10+
)
11+
12+
func TestOperatorStateAnalyzer(t *testing.T) {
13+
tests := []struct {
14+
name string
15+
intervals monitorapi.Intervals
16+
expectedMetrics map[string]*OperatorStateMetrics
17+
expectedRows []map[string]string
18+
}{
19+
{
20+
name: "single operator, progressing and degraded",
21+
intervals: monitorapi.Intervals{
22+
makeTestInterval("operator-a", "Progressing", 10),
23+
makeTestInterval("operator-a", "Progressing", 5),
24+
makeTestInterval("operator-a", "Degraded", 15),
25+
},
26+
expectedMetrics: map[string]*OperatorStateMetrics{
27+
"operator-a": {
28+
OperatorName: "operator-a",
29+
ProgressingCount: 2,
30+
TotalProgressingSeconds: 15,
31+
ProgressingDurations: []float64{10, 5},
32+
DegradedCount: 1,
33+
TotalDegradedSeconds: 15,
34+
DegradedDurations: []float64{15},
35+
},
36+
},
37+
expectedRows: []map[string]string{
38+
{"Operator": "operator-a", "State": "Progressing", "Metric": "Count", "Value": "2.000000"},
39+
{"Operator": "operator-a", "State": "Progressing", "Metric": "TotalSeconds", "Value": "15.000000"},
40+
{"Operator": "operator-a", "State": "Progressing", "Metric": "IndividualDurationSeconds", "Value": "10.000000"},
41+
{"Operator": "operator-a", "State": "Progressing", "Metric": "IndividualDurationSeconds", "Value": "5.000000"},
42+
{"Operator": "operator-a", "State": "Degraded", "Metric": "Count", "Value": "1.000000"},
43+
{"Operator": "operator-a", "State": "Degraded", "Metric": "TotalSeconds", "Value": "15.000000"},
44+
{"Operator": "operator-a", "State": "Degraded", "Metric": "IndividualDurationSeconds", "Value": "15.000000"},
45+
},
46+
},
47+
{
48+
name: "multiple operators",
49+
intervals: monitorapi.Intervals{
50+
makeTestInterval("operator-a", "Progressing", 10),
51+
makeTestInterval("operator-b", "Degraded", 20),
52+
},
53+
expectedMetrics: map[string]*OperatorStateMetrics{
54+
"operator-a": {
55+
OperatorName: "operator-a",
56+
ProgressingCount: 1,
57+
TotalProgressingSeconds: 10,
58+
ProgressingDurations: []float64{10},
59+
},
60+
"operator-b": {
61+
OperatorName: "operator-b",
62+
DegradedCount: 1,
63+
TotalDegradedSeconds: 20,
64+
DegradedDurations: []float64{20},
65+
},
66+
},
67+
expectedRows: []map[string]string{
68+
{"Operator": "operator-a", "State": "Progressing", "Metric": "Count", "Value": "1.000000"},
69+
{"Operator": "operator-a", "State": "Progressing", "Metric": "TotalSeconds", "Value": "10.000000"},
70+
{"Operator": "operator-a", "State": "Progressing", "Metric": "IndividualDurationSeconds", "Value": "10.000000"},
71+
{"Operator": "operator-b", "State": "Degraded", "Metric": "Count", "Value": "1.000000"},
72+
{"Operator": "operator-b", "State": "Degraded", "Metric": "TotalSeconds", "Value": "20.000000"},
73+
{"Operator": "operator-b", "State": "Degraded", "Metric": "IndividualDurationSeconds", "Value": "20.000000"},
74+
},
75+
},
76+
{
77+
name: "no relevant intervals",
78+
intervals: monitorapi.Intervals{},
79+
expectedMetrics: map[string]*OperatorStateMetrics{},
80+
expectedRows: []map[string]string{},
81+
},
82+
{
83+
name: "operator with only degraded state",
84+
intervals: monitorapi.Intervals{
85+
makeTestInterval("operator-c", "Degraded", 30),
86+
},
87+
expectedMetrics: map[string]*OperatorStateMetrics{
88+
"operator-c": {
89+
OperatorName: "operator-c",
90+
DegradedCount: 1,
91+
TotalDegradedSeconds: 30,
92+
DegradedDurations: []float64{30},
93+
},
94+
},
95+
expectedRows: []map[string]string{
96+
{"Operator": "operator-c", "State": "Degraded", "Metric": "Count", "Value": "1.000000"},
97+
{"Operator": "operator-c", "State": "Degraded", "Metric": "TotalSeconds", "Value": "30.000000"},
98+
{"Operator": "operator-c", "State": "Degraded", "Metric": "IndividualDurationSeconds", "Value": "30.000000"},
99+
},
100+
},
101+
}
102+
103+
for _, tc := range tests {
104+
t.Run(tc.name, func(t *testing.T) {
105+
// Test calculateOperatorStateMetrics
106+
metrics := calculateOperatorStateMetrics(tc.intervals)
107+
require.Equal(t, len(tc.expectedMetrics), len(metrics), "number of operators should match")
108+
for op, expected := range tc.expectedMetrics {
109+
actual, ok := metrics[op]
110+
require.True(t, ok, "operator %s not found in metrics", op)
111+
assert.Equal(t, expected.OperatorName, actual.OperatorName, "OperatorName should match")
112+
assert.Equal(t, expected.ProgressingCount, actual.ProgressingCount, "ProgressingCount should match")
113+
assert.InDelta(t, expected.TotalProgressingSeconds, actual.TotalProgressingSeconds, 0.001, "TotalProgressingSeconds should match")
114+
assert.ElementsMatch(t, expected.ProgressingDurations, actual.ProgressingDurations, "ProgressingDurations should match")
115+
assert.Equal(t, expected.DegradedCount, actual.DegradedCount, "DegradedCount should match")
116+
assert.InDelta(t, expected.TotalDegradedSeconds, actual.TotalDegradedSeconds, 0.001, "TotalDegradedSeconds should match")
117+
assert.ElementsMatch(t, expected.DegradedDurations, actual.DegradedDurations, "DegradedDurations should match")
118+
}
119+
120+
// Test generateRowsFromMetrics
121+
rows := generateRowsFromMetrics(metrics)
122+
assert.ElementsMatch(t, tc.expectedRows, rows, "generated rows should match expected rows")
123+
})
124+
}
125+
}
126+
127+
// Helper function to create intervals for testing
128+
func makeTestInterval(operatorName, condition string, durationSeconds float64) monitorapi.Interval {
129+
from := time.Unix(1, 0)
130+
to := from.Add(time.Duration(durationSeconds * float64(time.Second)))
131+
return monitorapi.Interval{
132+
Source: monitorapi.SourceOperatorState,
133+
Condition: monitorapi.Condition{
134+
Locator: monitorapi.Locator{
135+
Type: monitorapi.LocatorTypeClusterOperator,
136+
Keys: map[monitorapi.LocatorKey]string{
137+
monitorapi.LocatorClusterOperatorKey: operatorName,
138+
},
139+
},
140+
Message: monitorapi.Message{
141+
Annotations: map[monitorapi.AnnotationKey]string{
142+
monitorapi.AnnotationCondition: condition,
143+
},
144+
},
145+
},
146+
From: from,
147+
To: to,
148+
}
149+
}

0 commit comments

Comments
 (0)