Skip to content

Commit 4580205

Browse files
duanwei33duanwei33
committed
Add e2e tests for storage network policy labels
This commit adds comprehensive e2e tests to verify that storage-related operators and controllers have the required network policy labels and that NetworkPolicy resources exist with correct pod selectors. Changes: - Add namespace constants to helpers.go for reuse across storage tests - Add storage_networkpolicy.go with tests for CSO and CSI operators - Verify required network policy labels on deployments - Validate NetworkPolicy resources in storage namespaces - Skip these tests on MicroShift clusters where they are not applicable The tests check: 1. CSO operators: cluster-storage-operator, vsphere-problem-detector-operator, csi-snapshot-controller-operator, and csi-snapshot-controller 2. CSI operators: AWS EBS/EFS, Azure Disk/File, GCP PD/Filestore, vSphere, IBM VPC Block, OpenStack Cinder/Manila, and SMB drivers 3. NetworkPolicy resources in openshift-cluster-storage-operator and openshift-cluster-csi-drivers namespaces
1 parent 816619b commit 4580205

File tree

2 files changed

+385
-0
lines changed

2 files changed

+385
-0
lines changed

‎test/extended/storage/helpers.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,13 @@ const (
2121
defaultPollingTime = 2 * time.Second
2222
)
2323

24+
// Storage operator and CSI driver namespace constants
25+
const (
26+
CSONamespace = "openshift-cluster-storage-operator" // Cluster Storage Operator namespace
27+
CSINamespace = "openshift-cluster-csi-drivers" // Default CSI driver operators namespace
28+
ManilaCSINamespace = "openshift-manila-csi-driver" // Manila CSI driver namespace (OpenStack only)
29+
)
30+
2431
// IsCSOHealthy checks whether the Cluster Storage Operator is healthy
2532
func IsCSOHealthy(oc *exutil.CLI) (bool, error) {
2633
// CSO healthyStatus:[degradedStatus:False, progressingStatus:False, availableStatus:True, upgradeableStatus:True]

‎test/extended/storage/storage_networkpolicy.go‎

Lines changed: 378 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,378 @@
1+
package storage
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"strings"
7+
8+
g "github.com/onsi/ginkgo/v2"
9+
o "github.com/onsi/gomega"
10+
exutil "github.com/openshift/origin/test/extended/util"
11+
"k8s.io/apimachinery/pkg/api/errors"
12+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
13+
e2e "k8s.io/kubernetes/test/e2e/framework"
14+
)
15+
16+
// ResourceType defines the type of Kubernetes workload resource
17+
type ResourceType string
18+
19+
const (
20+
ResourceTypeDeployment ResourceType = "Deployment"
21+
ResourceTypeDaemonSet ResourceType = "DaemonSet"
22+
)
23+
24+
// resourceCheck defines a check for a workload resource (Deployment, DaemonSet, etc.)
25+
type resourceCheck struct {
26+
ResourceType ResourceType
27+
Namespace string
28+
Name string
29+
Platform string
30+
RequiredLabels map[string]string
31+
}
32+
33+
type npCheck struct {
34+
Namespace string
35+
Optional bool
36+
Policies map[string]map[string]string
37+
}
38+
39+
var (
40+
npLabelAPI = map[string]string{"openshift.storage.network-policy.api-server": "allow"}
41+
npLabelDNS = map[string]string{"openshift.storage.network-policy.dns": "allow"}
42+
npLabelOperatorMetrics = map[string]string{"openshift.storage.network-policy.operator-metrics": "allow"}
43+
npLabelOperatorMetricsRange = map[string]string{"openshift.storage.network-policy.operator-metrics-range": "allow"}
44+
npLabelMetricsRange = map[string]string{"openshift.storage.network-policy.metrics-range": "allow"}
45+
)
46+
47+
func mergeLabels(maps ...map[string]string) map[string]string {
48+
out := map[string]string{}
49+
for _, m := range maps {
50+
for k, v := range m {
51+
out[k] = v
52+
}
53+
}
54+
return out
55+
}
56+
57+
var (
58+
csoOperatorRequiredLabels = mergeLabels(npLabelAPI, npLabelDNS, npLabelOperatorMetrics)
59+
csoControllerRequiredLabels = mergeLabels(npLabelAPI, npLabelDNS)
60+
csiOperatorRequiredLabels = mergeLabels(npLabelAPI, npLabelDNS, npLabelOperatorMetricsRange)
61+
csiControllerRequiredLabels = mergeLabels(npLabelAPI, npLabelDNS, npLabelMetricsRange)
62+
)
63+
64+
var (
65+
npNameAPI = "allow-egress-to-api-server"
66+
npNameDNS = "allow-to-dns"
67+
npNameOperatorMetrics = "allow-ingress-to-operator-metrics"
68+
npNameMetricsRange = "allow-ingress-to-metrics-range"
69+
npNameOperatorMetricsRange = "allow-ingress-to-operator-metrics-range"
70+
)
71+
72+
var networkPolicyChecks = []npCheck{
73+
{
74+
Namespace: CSONamespace,
75+
Policies: map[string]map[string]string{
76+
npNameAPI: npLabelAPI,
77+
npNameDNS: npLabelDNS,
78+
npNameOperatorMetrics: npLabelOperatorMetrics,
79+
},
80+
},
81+
{
82+
Namespace: CSINamespace,
83+
Policies: map[string]map[string]string{
84+
npNameAPI: npLabelAPI,
85+
npNameDNS: npLabelDNS,
86+
npNameMetricsRange: npLabelMetricsRange,
87+
npNameOperatorMetricsRange: npLabelOperatorMetricsRange,
88+
},
89+
},
90+
{
91+
Namespace: ManilaCSINamespace,
92+
Optional: true,
93+
Policies: map[string]map[string]string{
94+
npNameAPI: npLabelAPI,
95+
npNameDNS: npLabelDNS,
96+
npNameMetricsRange: npLabelMetricsRange,
97+
npNameOperatorMetricsRange: npLabelOperatorMetricsRange,
98+
},
99+
},
100+
}
101+
102+
var _ = g.Describe("[sig-storage][OCPFeature:StorageNetworkPolicy] Storage Network Policy", func() {
103+
defer g.GinkgoRecover()
104+
var (
105+
oc = exutil.NewCLI("storage-network-policy")
106+
currentPlatform = e2e.TestContext.Provider
107+
)
108+
109+
g.BeforeEach(func() {
110+
isMicroShift, err := exutil.IsMicroShiftCluster(oc.AdminKubeClient())
111+
o.Expect(err).NotTo(o.HaveOccurred())
112+
if isMicroShift {
113+
g.Skip("Storage Network Policy tests are not supported on MicroShift")
114+
}
115+
})
116+
117+
g.It("should verify required labels for CSO related Operators", func() {
118+
CSOResourcesToCheck := []resourceCheck{
119+
{
120+
ResourceType: ResourceTypeDeployment,
121+
Namespace: CSONamespace,
122+
Name: "cluster-storage-operator",
123+
Platform: "all",
124+
RequiredLabels: csoOperatorRequiredLabels,
125+
},
126+
{
127+
ResourceType: ResourceTypeDeployment,
128+
Namespace: CSONamespace,
129+
Name: "vsphere-problem-detector-operator",
130+
Platform: "vsphere",
131+
RequiredLabels: csoOperatorRequiredLabels,
132+
},
133+
{
134+
ResourceType: ResourceTypeDeployment,
135+
Namespace: CSONamespace,
136+
Name: "csi-snapshot-controller-operator",
137+
Platform: "all",
138+
RequiredLabels: csoOperatorRequiredLabels,
139+
},
140+
{
141+
ResourceType: ResourceTypeDeployment,
142+
Namespace: CSONamespace,
143+
Name: "csi-snapshot-controller",
144+
Platform: "all",
145+
RequiredLabels: csoControllerRequiredLabels,
146+
},
147+
}
148+
runResourceChecks(oc, CSOResourcesToCheck, currentPlatform)
149+
})
150+
151+
g.It("should verify required labels for CSI related Operators", func() {
152+
CSIResourcesToCheck := []resourceCheck{
153+
{
154+
ResourceType: ResourceTypeDeployment,
155+
Namespace: CSINamespace,
156+
Name: "aws-ebs-csi-driver-operator",
157+
Platform: "aws",
158+
RequiredLabels: csiOperatorRequiredLabels,
159+
},
160+
{
161+
ResourceType: ResourceTypeDeployment,
162+
Namespace: CSINamespace,
163+
Name: "aws-ebs-csi-driver-controller",
164+
Platform: "aws",
165+
RequiredLabels: csiControllerRequiredLabels,
166+
},
167+
{
168+
ResourceType: ResourceTypeDeployment,
169+
Namespace: CSINamespace,
170+
Name: "aws-efs-csi-driver-operator",
171+
Platform: "aws",
172+
RequiredLabels: csiOperatorRequiredLabels,
173+
},
174+
{
175+
ResourceType: ResourceTypeDeployment,
176+
Namespace: CSINamespace,
177+
Name: "azure-disk-csi-driver-operator",
178+
Platform: "azure",
179+
RequiredLabels: csiOperatorRequiredLabels,
180+
},
181+
{
182+
ResourceType: ResourceTypeDeployment,
183+
Namespace: CSINamespace,
184+
Name: "azure-disk-csi-driver-controller",
185+
Platform: "azure",
186+
RequiredLabels: csiControllerRequiredLabels,
187+
},
188+
{
189+
ResourceType: ResourceTypeDeployment,
190+
Namespace: CSINamespace,
191+
Name: "azure-file-csi-driver-operator",
192+
Platform: "azure",
193+
RequiredLabels: csiOperatorRequiredLabels,
194+
},
195+
{
196+
ResourceType: ResourceTypeDeployment,
197+
Namespace: CSINamespace,
198+
Name: "azure-file-csi-driver-controller",
199+
Platform: "azure",
200+
RequiredLabels: csiControllerRequiredLabels,
201+
},
202+
{
203+
ResourceType: ResourceTypeDeployment,
204+
Namespace: CSINamespace,
205+
Name: "gcp-pd-csi-driver-operator",
206+
Platform: "gcp",
207+
RequiredLabels: csiOperatorRequiredLabels,
208+
},
209+
{
210+
ResourceType: ResourceTypeDeployment,
211+
Namespace: CSINamespace,
212+
Name: "gcp-filestore-csi-driver-operator",
213+
Platform: "gcp",
214+
RequiredLabels: csiOperatorRequiredLabels,
215+
},
216+
{
217+
ResourceType: ResourceTypeDeployment,
218+
Namespace: CSINamespace,
219+
Name: "vmware-vsphere-csi-driver-operator",
220+
Platform: "vsphere",
221+
RequiredLabels: csiOperatorRequiredLabels,
222+
},
223+
{
224+
ResourceType: ResourceTypeDeployment,
225+
Namespace: CSINamespace,
226+
Name: "vmware-vsphere-csi-driver-controller",
227+
Platform: "vsphere",
228+
RequiredLabels: csiControllerRequiredLabels,
229+
},
230+
{
231+
ResourceType: ResourceTypeDeployment,
232+
Namespace: CSINamespace,
233+
Name: "ibm-vpc-block-csi-driver-operator",
234+
Platform: "ibmcloud",
235+
RequiredLabels: csiOperatorRequiredLabels,
236+
},
237+
{
238+
ResourceType: ResourceTypeDeployment,
239+
Namespace: CSINamespace,
240+
Name: "ibm-vpc-block-csi-controller",
241+
Platform: "ibmcloud",
242+
RequiredLabels: csiControllerRequiredLabels,
243+
},
244+
{
245+
ResourceType: ResourceTypeDeployment,
246+
Namespace: CSINamespace,
247+
Name: "openstack-cinder-csi-driver-operator",
248+
Platform: "openstack",
249+
RequiredLabels: csiOperatorRequiredLabels,
250+
},
251+
{
252+
ResourceType: ResourceTypeDeployment,
253+
Namespace: CSINamespace,
254+
Name: "openstack-cinder-csi-driver-controller",
255+
Platform: "openstack",
256+
RequiredLabels: csiControllerRequiredLabels,
257+
},
258+
{
259+
ResourceType: ResourceTypeDeployment,
260+
Namespace: CSINamespace,
261+
Name: "manila-csi-driver-operator",
262+
Platform: "openstack",
263+
RequiredLabels: csiOperatorRequiredLabels,
264+
},
265+
{
266+
ResourceType: ResourceTypeDeployment,
267+
Namespace: ManilaCSINamespace,
268+
Name: "openstack-manila-csi-controllerplugin",
269+
Platform: "openstack",
270+
RequiredLabels: csiControllerRequiredLabels,
271+
},
272+
{
273+
ResourceType: ResourceTypeDeployment,
274+
Namespace: CSINamespace,
275+
Name: "smb-csi-driver-operator",
276+
Platform: "all",
277+
RequiredLabels: csiOperatorRequiredLabels,
278+
},
279+
{
280+
ResourceType: ResourceTypeDeployment,
281+
Namespace: CSINamespace,
282+
Name: "smb-csi-driver-controller",
283+
Platform: "all",
284+
RequiredLabels: csiControllerRequiredLabels,
285+
},
286+
}
287+
runResourceChecks(oc, CSIResourcesToCheck, currentPlatform)
288+
})
289+
290+
g.It("should ensure required NetworkPolicies exist with correct labels", func() {
291+
for _, c := range networkPolicyChecks {
292+
_, err := oc.AdminKubeClient().CoreV1().Namespaces().Get(context.TODO(), c.Namespace, metav1.GetOptions{})
293+
if err != nil {
294+
if c.Optional {
295+
g.By(fmt.Sprintf("Skipping optional namespace %s (not found)", c.Namespace))
296+
continue
297+
}
298+
o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("namespace %s should exist", c.Namespace))
299+
}
300+
301+
for npName, labels := range c.Policies {
302+
np, err := oc.AdminKubeClient().NetworkingV1().NetworkPolicies(c.Namespace).Get(context.TODO(), npName, metav1.GetOptions{})
303+
o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("NetworkPolicy %s/%s should exist", c.Namespace, npName))
304+
305+
for key, val := range labels {
306+
gotVal, ok := np.Spec.PodSelector.MatchLabels[key]
307+
o.Expect(ok).To(o.BeTrue(), fmt.Sprintf("NetworkPolicy %s/%s missing label %s", c.Namespace, npName, key))
308+
o.Expect(gotVal).To(o.Equal(val), fmt.Sprintf("NetworkPolicy %s/%s label %s mismatch (got=%s, want=%s)", c.Namespace, npName, key, gotVal, val))
309+
}
310+
}
311+
}
312+
})
313+
})
314+
315+
func runResourceChecks(oc *exutil.CLI, resources []resourceCheck, currentPlatform string) {
316+
results := []string{}
317+
hasFail := false
318+
for _, res := range resources {
319+
if res.Platform != "" && res.Platform != currentPlatform && res.Platform != "all" {
320+
results = append(results, fmt.Sprintf("[SKIP] %s %s/%s (platform mismatch: %s)", res.ResourceType, res.Namespace, res.Name, res.Platform))
321+
continue
322+
}
323+
324+
var podTemplateLabels map[string]string
325+
var err error
326+
resourceName := fmt.Sprintf("%s %s/%s", res.ResourceType, res.Namespace, res.Name)
327+
328+
switch res.ResourceType {
329+
case ResourceTypeDeployment:
330+
deployment, err := oc.AdminKubeClient().AppsV1().Deployments(res.Namespace).Get(context.TODO(), res.Name, metav1.GetOptions{})
331+
if err != nil {
332+
if errors.IsNotFound(err) {
333+
results = append(results, fmt.Sprintf("[SKIP] %s not found", resourceName))
334+
continue
335+
}
336+
g.Fail(fmt.Sprintf("Error fetching %s: %v", resourceName, err))
337+
}
338+
podTemplateLabels = deployment.Spec.Template.Labels
339+
340+
case ResourceTypeDaemonSet:
341+
daemonset, err := oc.AdminKubeClient().AppsV1().DaemonSets(res.Namespace).Get(context.TODO(), res.Name, metav1.GetOptions{})
342+
if err != nil {
343+
if errors.IsNotFound(err) {
344+
results = append(results, fmt.Sprintf("[SKIP] %s not found", resourceName))
345+
continue
346+
}
347+
g.Fail(fmt.Sprintf("Error fetching %s: %v", resourceName, err))
348+
}
349+
podTemplateLabels = daemonset.Spec.Template.Labels
350+
351+
default:
352+
g.Fail(fmt.Sprintf("Unsupported resource type: %s", res.ResourceType))
353+
}
354+
355+
if err != nil {
356+
continue
357+
}
358+
359+
missingLabels := []string{}
360+
for key, val := range res.RequiredLabels {
361+
if podTemplateLabels[key] != val {
362+
missingLabels = append(missingLabels, fmt.Sprintf("%s=%s", key, val))
363+
}
364+
}
365+
366+
if len(missingLabels) > 0 {
367+
results = append(results, fmt.Sprintf("[FAIL] %s missing labels: %s", resourceName, strings.Join(missingLabels, ", ")))
368+
hasFail = true
369+
} else {
370+
results = append(results, fmt.Sprintf("[PASS] %s", resourceName))
371+
}
372+
}
373+
374+
if hasFail {
375+
summary := strings.Join(results, "\n")
376+
g.Fail(fmt.Sprintf("Some resources are missing required labels:\n\n%s\n", summary))
377+
}
378+
}

0 commit comments

Comments
 (0)