From 2f9ce6d9232727b0566ad1371c2f143d2a5ec456 Mon Sep 17 00:00:00 2001 From: Lawrence Gadban Date: Wed, 20 Nov 2024 10:26:17 -0600 Subject: [PATCH 01/19] fix upstream status flicker --- projects/gloo/pkg/syncer/envoy_translator_syncer.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/projects/gloo/pkg/syncer/envoy_translator_syncer.go b/projects/gloo/pkg/syncer/envoy_translator_syncer.go index 25cd7518ca7..f92c9fc13d7 100644 --- a/projects/gloo/pkg/syncer/envoy_translator_syncer.go +++ b/projects/gloo/pkg/syncer/envoy_translator_syncer.go @@ -142,12 +142,17 @@ func (s *translatorSyncer) syncEnvoy(ctx context.Context, snap *v1snap.ApiSnapsh } } - allReports.Accept(snap.Upstreams.AsInputResources()...) - allReports.Accept(snap.UpstreamGroups.AsInputResources()...) // Only mark non-kube gateways as accepted // Regardless, kube gw proxies are filtered out of these reports before reporting in translator_syncer.go allReports.Accept(nonKubeProxies.AsInputResources()...) + // report Upstream[Group]s as Accepted initially but only if we at least 1 edge proxy + // otherwise, we won't actually translate them, and so if there is an error, we will incorrectly report Accepted + if len(nonKubeProxies) > 0 { + allReports.Accept(snap.Upstreams.AsInputResources()...) + allReports.Accept(snap.UpstreamGroups.AsInputResources()...) + } + // sync non-kube gw proxies for _, proxy := range nonKubeProxies { proxyCtx := ctx From a87a6adceba3a314a075ad937803bce97e6e91bc Mon Sep 17 00:00:00 2001 From: Lawrence Gadban Date: Wed, 20 Nov 2024 10:45:07 -0600 Subject: [PATCH 02/19] changelog --- changelog/v1.18.0-rc2/fix-us-status-flicker.yaml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changelog/v1.18.0-rc2/fix-us-status-flicker.yaml diff --git a/changelog/v1.18.0-rc2/fix-us-status-flicker.yaml b/changelog/v1.18.0-rc2/fix-us-status-flicker.yaml new file mode 100644 index 00000000000..644c30cb232 --- /dev/null +++ b/changelog/v1.18.0-rc2/fix-us-status-flicker.yaml @@ -0,0 +1,4 @@ +changelog: + - type: NON_USER_FACING + description: >- + Fix flicker on Upstream status when kube gw syncer rejects a resource and no edge proxies are present From 5060f08ea61f62394783c481e7e33f60b7fd18be Mon Sep 17 00:00:00 2001 From: Lawrence Gadban Date: Wed, 20 Nov 2024 16:29:57 -0600 Subject: [PATCH 03/19] track obj status in krt collections the status reporter compares the desired status with the existing status in the solo-kit object to determine if it should actually UPDATE the resource. the current proxy_syncer will do a once per second status sync and relies on this status comparison to be functional to prevent endless object UPDATEs. this commit fixes the solo-kit objects (really wrappers) in the krt collections to contain the status so an accurate comparison can take place. --- projects/gateway2/proxy_syncer/proxy_syncer.go | 2 ++ projects/gateway2/setup/ggv2setup.go | 5 ++++- .../translator/plugins/routeoptions/route_options_plugin.go | 1 + .../plugins/virtualhostoptions/virtualhost_options_plugin.go | 1 + 4 files changed, 8 insertions(+), 1 deletion(-) diff --git a/projects/gateway2/proxy_syncer/proxy_syncer.go b/projects/gateway2/proxy_syncer/proxy_syncer.go index 56787aadc8a..01992ea4ac6 100644 --- a/projects/gateway2/proxy_syncer/proxy_syncer.go +++ b/projects/gateway2/proxy_syncer/proxy_syncer.go @@ -371,6 +371,7 @@ func (s *ProxySyncer) Init(ctx context.Context, dbg *krt.DebugHandler) error { Namespace: u.GetNamespace(), } glooUs.SetMetadata(&md) + glooUs.NamespacedStatuses = &u.Status us := &krtcollections.UpstreamWrapper{Inner: glooUs} return us }, krt.WithName("GlooUpstreams"), withDebug) @@ -715,6 +716,7 @@ func (s *ProxySyncer) translateProxy( Namespace: kac.GetNamespace(), } gac.SetMetadata(&md) + gac.NamespacedStatuses = &kac.Status acfgs = append(acfgs, gac) } latestSnap.AuthConfigs = acfgs diff --git a/projects/gateway2/setup/ggv2setup.go b/projects/gateway2/setup/ggv2setup.go index e0bea7e7661..82ba60edf31 100644 --- a/projects/gateway2/setup/ggv2setup.go +++ b/projects/gateway2/setup/ggv2setup.go @@ -227,7 +227,10 @@ func (g *genericStatusReporter) WriteReports(ctx context.Context, resourceErrs r resourceStatus := g.statusClient.GetStatus(resource) if status.Equal(resourceStatus) { - logger.Debugf("skipping report for %v as it has not changed", resource.GetMetadata().Ref()) + // TODO: find a way to log this but it is noisy currently due to once per second status sync + // see: projects/gateway2/proxy_syncer/kube_gw_translator_syncer.go#syncStatus(...) + // and it's call site in projects/gateway2/proxy_syncer/proxy_syncer.go + // logger.Debugf("skipping report for %v as it has not changed", resource.GetMetadata().Ref()) continue } diff --git a/projects/gateway2/translator/plugins/routeoptions/route_options_plugin.go b/projects/gateway2/translator/plugins/routeoptions/route_options_plugin.go index 11058fb0cfb..8bb543e5f41 100644 --- a/projects/gateway2/translator/plugins/routeoptions/route_options_plugin.go +++ b/projects/gateway2/translator/plugins/routeoptions/route_options_plugin.go @@ -178,6 +178,7 @@ func (p *plugin) ApplyStatusPlugin(ctx context.Context, statusCtx *plugins.Statu roObj.Spec.GetMetadata().Name = roObj.GetName() roObj.Spec.GetMetadata().Namespace = roObj.GetNamespace() roObjSk := &roObj.Spec + roObjSk.NamespacedStatuses = &roObj.Status // mark this object to be processed routeOptionReport.Accept(roObjSk) diff --git a/projects/gateway2/translator/plugins/virtualhostoptions/virtualhost_options_plugin.go b/projects/gateway2/translator/plugins/virtualhostoptions/virtualhost_options_plugin.go index 647a31614a2..d06ca5148c7 100644 --- a/projects/gateway2/translator/plugins/virtualhostoptions/virtualhost_options_plugin.go +++ b/projects/gateway2/translator/plugins/virtualhostoptions/virtualhost_options_plugin.go @@ -241,6 +241,7 @@ func (p *plugin) ApplyStatusPlugin(ctx context.Context, statusCtx *plugins.Statu vhOptObj.Spec.GetMetadata().Name = vhOptObj.GetName() vhOptObj.Spec.GetMetadata().Namespace = vhOptObj.GetNamespace() vhOptObjSk := &vhOptObj.Spec + vhOptObjSk.NamespacedStatuses = &vhOptObj.Status // mark this object to be processed virtualHostOptionReport.Accept(vhOptObjSk) From 3a9bb337f5e79d4a6b9bc8bdc8f6c71eb0d63b3e Mon Sep 17 00:00:00 2001 From: Lawrence Gadban Date: Wed, 20 Nov 2024 16:36:00 -0600 Subject: [PATCH 04/19] pr feedback re: commentary --- projects/gloo/pkg/syncer/envoy_translator_syncer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/gloo/pkg/syncer/envoy_translator_syncer.go b/projects/gloo/pkg/syncer/envoy_translator_syncer.go index f92c9fc13d7..aaf947f3331 100644 --- a/projects/gloo/pkg/syncer/envoy_translator_syncer.go +++ b/projects/gloo/pkg/syncer/envoy_translator_syncer.go @@ -146,7 +146,7 @@ func (s *translatorSyncer) syncEnvoy(ctx context.Context, snap *v1snap.ApiSnapsh // Regardless, kube gw proxies are filtered out of these reports before reporting in translator_syncer.go allReports.Accept(nonKubeProxies.AsInputResources()...) - // report Upstream[Group]s as Accepted initially but only if we at least 1 edge proxy + // mark Upstream[Group]s as Accepted initially, but only if we have at least 1 edge proxy; // otherwise, we won't actually translate them, and so if there is an error, we will incorrectly report Accepted if len(nonKubeProxies) > 0 { allReports.Accept(snap.Upstreams.AsInputResources()...) From 91add9515bd906d261c7e2b464ab35bccb15e8a8 Mon Sep 17 00:00:00 2001 From: Lawrence Gadban Date: Fri, 22 Nov 2024 10:06:10 -0600 Subject: [PATCH 05/19] update tests to handle missing us status --- ...ps-gw-and-route-with-attached-options.yaml | 114 ++++++++++++++++++ projects/gateway2/setup/ggv2setup.go | 2 +- .../{input_resources.go => resources.go} | 13 ++ test/kube2e/gloo/happypath_test.go | 10 +- 4 files changed, 135 insertions(+), 4 deletions(-) create mode 100644 projects/gateway2/examples/example-https-gw-and-route-with-attached-options.yaml rename test/helpers/{input_resources.go => resources.go} (91%) diff --git a/projects/gateway2/examples/example-https-gw-and-route-with-attached-options.yaml b/projects/gateway2/examples/example-https-gw-and-route-with-attached-options.yaml new file mode 100644 index 00000000000..afafdf7bea9 --- /dev/null +++ b/projects/gateway2/examples/example-https-gw-and-route-with-attached-options.yaml @@ -0,0 +1,114 @@ +kind: Gateway +apiVersion: gateway.networking.k8s.io/v1 +metadata: + name: https-gw +spec: + gatewayClassName: gloo-gateway + listeners: + - protocol: HTTPS + port: 8443 + name: https + tls: + mode: Terminate + certificateRefs: + - name: https + kind: Secret + allowedRoutes: + namespaces: + from: Same +--- +apiVersion: v1 +kind: Secret +type: kubernetes.io/tls +metadata: + name: https +data: + tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUN0akNDQVo0Q0FRQXdEUVlKS29aSWh2Y05BUUVMQlFBd0lURVRNQkVHQTFVRUNnd0tZVzU1SUdSdmJXRnAKYmpFS01BZ0dBMVVFQXd3QktqQWVGdzB5TkRBNU1qSXhOak00TURaYUZ3MHlOVEE1TWpJeE5qTTRNRFphTUNFeApDakFJQmdOVkJBTU1BU294RXpBUkJnTlZCQW9NQ21GdWVTQmtiMjFoYVc0d2dnRWlNQTBHQ1NxR1NJYjNEUUVCCkFRVUFBNElCRHdBd2dnRUtBb0lCQVFEc2REdmY0MmgyYzdSb2JOb0tSZWFRYzFqdUpaWGtwZzFOWER5aFZOK3cKbFdkVDg0UlFhS1RYVVYxZTkwUUVyb2VrbFZnQld5d3A5WlNsUUhzSTc3MTBVVm16N3RhUFRXbWd0WlhHRy9hTApXSDhiMklsdTUrVXhpajB2K2NheUJqWEVuVWJTVFJQYXlkYU11N1hqODNKV0tKSDZ6TzNVK3pwbCt1SXFVc0p3Ck9GOWpWeFNvMWpEMEQvdllObFNiMTFjNWVjWHUrdk9WMzkrUEpmTDFmUjVQNSt0Vnk1cFlKYWtjNDhJMmlWVzAKNmkvcUdtb3lHMGdCdVZZMkZETzFlcndmUmxHMTRSNEdwdXVWbUg2TEFMcWs2bEZSbHpmbHRYVTBJZ0o5dVNvNApjTjFLMlFHTUhmYUF2RHJqL1JReStKdTQ3TENRcTFYRUU1RzJqdlZOMlNhekFnTUJBQUV3RFFZSktvWklodmNOCkFRRUxCUUFEZ2dFQkFKNkNnVXg3d3huLyswK1hJT2FsclZacnh2SXQvc1J4YnVvQ3NvM04zbm50V0g4TmpVRHIKdURIYUgvRktBVkdsSTFPT0NTa2VhVmlHdGpZdXBkL0ZUbUI2b0pEM2F4cUYzMWVHTDNFTWR4aVZkMTJEbEZsbwp2bkJXZGs4d2xLNk9EcHdUTkd2eEgxOVA5WEN6UjZYRXU0SGlXYnNRZ29YaWN6VC9vQzdCM3NqaG5HYkFmUGx5CkdIRWVhNWV6ay9zU053YmhTcmFabzBZOFNuUzFqSHFKSlhuWk1PbmpCLzlqRGJ6VTZoaVN3a1hiNFZSU3B3ZkIKSE43Q2t3d3FjT0JGWUQ3bGg5bDMzVDN1L0RSWDJWY2FOSWRXSlErRUdTSnowYngrMVhGZXh4R3NjNXNtUWVwZwpIUEEwTVU1ZGNnQkxGSUlFUU1BNHYrSXRKN3BpN2pvN3c3dz0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + tls.key: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2Z0lCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktnd2dnU2tBZ0VBQW9JQkFRRHNkRHZmNDJoMmM3Um8KYk5vS1JlYVFjMWp1SlpYa3BnMU5YRHloVk4rd2xXZFQ4NFJRYUtUWFVWMWU5MFFFcm9la2xWZ0JXeXdwOVpTbApRSHNJNzcxMFVWbXo3dGFQVFdtZ3RaWEdHL2FMV0g4YjJJbHU1K1V4aWowditjYXlCalhFblViU1RSUGF5ZGFNCnU3WGo4M0pXS0pINnpPM1UrenBsK3VJcVVzSndPRjlqVnhTbzFqRDBEL3ZZTmxTYjExYzVlY1h1K3ZPVjM5K1AKSmZMMWZSNVA1K3RWeTVwWUpha2M0OEkyaVZXMDZpL3FHbW95RzBnQnVWWTJGRE8xZXJ3ZlJsRzE0UjRHcHV1VgptSDZMQUxxazZsRlJsemZsdFhVMElnSjl1U280Y04xSzJRR01IZmFBdkRyai9SUXkrSnU0N0xDUXExWEVFNUcyCmp2Vk4yU2F6QWdNQkFBRUNnZ0VBT1lvZmQrQVM3NkJBYXBqeWhrVjBVZnAvelpQLzRlQTJwSGlwbUdqYTlsR2wKRDF6VytlbzlFdGlON0NvbnR5dnhmaStKZHVzRTJFRVdweEFGNEtyV1k1UFlURnoyZ1hESkgzNlFyK0RtSWdxcgpBOCt4d2ZkVVlyWE5KZnVXU29RUVdZUWNVOWpGMkJ2OXhjbDMvYnlrT2lzdE91YmJpVlNKWlgrandjRFo2QXlZCkM1Zm1CeXpCaERaWTV2NTg0VzZaMUxTUzZFR3ErQWVmd1hDNnoxcVluV2dlYXBrVG5jbEVESk4vVU9tQnJUeW0KaCtxRjdKTUZBWk1xbGVOQ1U1Wk1aUG43TWRPWkt5c20zaTRSUVNXL0NLMEFzWU9hNmx2TEJkZGxEN2k3THdORQpmRzZnUjhiVXZhQWtETTNEY3B3a0QyeURtUVhwSHFDRmZGUWRjT0p4SVFLQmdRRDJTajNtNHRmdFhUT3JvRDZNClM0N24xTVAzdU40WkJHUnZrcnZQbmdIckRQRG43aEhiZDg2MVJJRnN3dG0zWjRzTmk3RnVsWGU5YUdsek1OWHQKQ1NBYkd3UTJLbFZjeEpIYXZsYWlqUWgwVkZtZVlGSHpMNVMzeGxyc3RzNURnMnp2YU5Jd1VqNVpJMUY0a01YVgpHbFFHNU43dldvTjFIQlJ5NTFoN0VJV3dkd0tCZ1FEMXhyZ3ltcHVRYm5tOE1qTHJzYUd5Y0RxbzFCNzhMN2NHCnk0dEVCY2tGaTRKSTVrbjhxYy8wMTlCQ3dyYTBON0lvNkxMYmhhTVJDdFNLRXhVdDE5QXlBdStZd1U3V1JpKysKY0pLVUo3aTBoYXBMMk4zU1FwS3dMUFBLdXpQc2pkOTNmS2ZHNmdNZElnS0ZnUmlNK2JOdlZtTmNVS1pWa2xUMwpMZFNveGxsbXBRS0JnUUNaOVg4TXZmRk5LRjRNZzNlV0xBV0JWcmZLTzM1YkZTdlRzMTVFUXZuZi9ZNzY3UStzCnFoQkZzRFZYejlaWm1CaU10eTZMZWxRSHl5VmdKTDhXRmdaRDYzZU9oSmdvRFprL3JJeHJ3TTN3S21wN1hxcmIKNG5xTkJ5bi9uNmNBL1Q4enljcmErZzZQbWt4cm1kU1Jpb3ZNWTNZWkdmUGpXVkpQSWQ5bHpzWkRzUUtCZ1FDNwpyYVNvMTlFbEtJZURRNTRJeDFzZmhIMzJ3QXh0TFE2SmZOTDYvYVAwdlFTZklHT3RNZ3NmLzlFRHVlYUVsZWRSClAraWNvMTdUTzc3ek5RRStRWWUxT1BLM1poNEttQXdMVTdGS3UwNWJZNXFZVXFHSTF0ZG0rdHlybWVwYm9EYmQKNm0zQUFxQ2dGWUZ4YXhSUlNyaFBzOXNwK2xnYURpVWRuM3k5Nk1FS0ZRS0JnQXZFeDVYYnZ2RDB3d3F6cjcveApTUmM5aDRiQWFuTW8vUnFxaFd6MU9tU2VMaUh4NlhNc0ROSW01bTBjUlRwNUo1UWRicDdycDB2RS9lQU5BekhWCjdJSE5vaVNDeWs2NnU0anJKL3UvRXpPMkx1VmtGNWdSL1RvdU03eVBNVzJGL004U3gvRWc1M1k0NFBSZGpvT3oKQWY5QmUzYjZxMTdDb3ByU3ZVb2xNU09tCi0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: example-route +spec: + parentRefs: + - name: https-gw + hostnames: + - "example.com" + rules: + - backendRefs: + - name: example-svc + port: 8080 + - matches: + - path: + type: PathPrefix + value: /timeout + backendRefs: + - name: example-svc + port: 8080 +--- +apiVersion: gateway.solo.io/v1 +kind: RouteOption +metadata: + name: policy-attached +spec: + targetRefs: + - group: gateway.networking.k8s.io + kind: HTTPRoute + name: example-route + options: + faults: + abort: + httpStatus: 500 + timeout: 11s + # extauth: + # configRef: + # name: basic-auth + # namespace: httpbin +--- +apiVersion: v1 +kind: Service +metadata: + name: example-svc +spec: + selector: + app.kubernetes.io/name: nginx + ports: + - protocol: TCP + port: 8080 + targetPort: http-web-svc +# --- +# apiVersion: v1 +# kind: Pod +# metadata: +# name: nginx +# labels: +# app.kubernetes.io/name: nginx +# spec: +# containers: +# - name: nginx +# image: nginx:stable +# ports: +# - containerPort: 80 +# name: http-web-svc +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: nginx + template: + metadata: + labels: + app.kubernetes.io/name: nginx + spec: + containers: + - name: nginx + image: nginx:stable + ports: + - containerPort: 80 + name: http-web-svc diff --git a/projects/gateway2/setup/ggv2setup.go b/projects/gateway2/setup/ggv2setup.go index 82ba60edf31..5d780061164 100644 --- a/projects/gateway2/setup/ggv2setup.go +++ b/projects/gateway2/setup/ggv2setup.go @@ -229,7 +229,7 @@ func (g *genericStatusReporter) WriteReports(ctx context.Context, resourceErrs r if status.Equal(resourceStatus) { // TODO: find a way to log this but it is noisy currently due to once per second status sync // see: projects/gateway2/proxy_syncer/kube_gw_translator_syncer.go#syncStatus(...) - // and it's call site in projects/gateway2/proxy_syncer/proxy_syncer.go + // and its call site in projects/gateway2/proxy_syncer/proxy_syncer.go // logger.Debugf("skipping report for %v as it has not changed", resource.GetMetadata().Ref()) continue } diff --git a/test/helpers/input_resources.go b/test/helpers/resources.go similarity index 91% rename from test/helpers/input_resources.go rename to test/helpers/resources.go index 5f39c015397..5dbddaca98c 100644 --- a/test/helpers/input_resources.go +++ b/test/helpers/resources.go @@ -21,6 +21,19 @@ const ( defaultEventuallyPollingInterval = 1 * time.Second ) +type ResourceGetter func() (resources.Resource, error) + +func EventuallyResourceExists(getter ResourceGetter, intervals ...interface{}) { + timeoutInterval, pollingInterval := getTimeoutAndPollingIntervalsOrDefault(intervals...) + gomega.Eventually(func() (bool, error) { + _, err := getter() + if err != nil { + return false, err + } + return true, nil + }, timeoutInterval, pollingInterval).Should(gomega.BeTrue()) +} + type InputResourceGetter func() (resources.InputResource, error) type InputResourceListGetter func() (resources.InputResourceList, error) diff --git a/test/kube2e/gloo/happypath_test.go b/test/kube2e/gloo/happypath_test.go index 7a2c1ea7c6a..6aefb1ddb18 100644 --- a/test/kube2e/gloo/happypath_test.go +++ b/test/kube2e/gloo/happypath_test.go @@ -157,7 +157,9 @@ var _ = Describe("Happy path", func() { err := envoyInstance.RunWithRole(role, testClients.GlooPort) Expect(err).NotTo(HaveOccurred()) - testhelpers.EventuallyResourceAccepted(func() (resources.InputResource, error) { + // Upstreams no longer report status if they have not been translated at all to avoid conflicting with + // other syncers that have translated them, so we can only detect that the objects exist here + testhelpers.EventuallyResourceExists(func() (resources.Resource, error) { return getUpstream() }, "20s", ".5s") }) @@ -239,9 +241,11 @@ var _ = Describe("Happy path", func() { }) It("watch all namespaces", func() { - testhelpers.EventuallyResourceAccepted(func() (resources.InputResource, error) { + // Upstreams no longer report status if they have not been translated at all to avoid conflicting with + // other syncers that have translated them, so we can only detect that the objects exist here + testhelpers.EventuallyResourceExists(func() (resources.Resource, error) { return getUpstream() - }) + }, "20s", ".5s") up, err := getUpstream() Expect(err).NotTo(HaveOccurred()) From 2b464a991ce3891758362d799450dc07c0d23088 Mon Sep 17 00:00:00 2001 From: Lawrence Gadban Date: Fri, 22 Nov 2024 10:53:15 -0600 Subject: [PATCH 06/19] fix more assertions --- .../validation_allow_warnings/suite.go | 6 ++---- .../validation_always_accept/suite.go | 9 ++++---- .../testutils/assertions/resources.go | 21 +++++++++++++++++++ 3 files changed, 27 insertions(+), 9 deletions(-) create mode 100644 test/kubernetes/testutils/assertions/resources.go diff --git a/test/kubernetes/e2e/features/validation/validation_allow_warnings/suite.go b/test/kubernetes/e2e/features/validation/validation_allow_warnings/suite.go index b0e77d65f4d..5d59f3ad944 100644 --- a/test/kubernetes/e2e/features/validation/validation_allow_warnings/suite.go +++ b/test/kubernetes/e2e/features/validation/validation_allow_warnings/suite.go @@ -115,12 +115,10 @@ func (s *testingSuite) TestInvalidUpstreamMissingPort() { // Upstream is only rejected when the upstream plugin is run when a valid cluster is present err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, validation.ExampleUpstream, "-n", s.testInstallation.Metadata.InstallNamespace) s.Assert().NoError(err, "can apply valid upstream") - s.testInstallation.Assertions.EventuallyResourceStatusMatchesState( - func() (resources.InputResource, error) { + s.testInstallation.Assertions.EventuallyResourceExsits( + func() (resources.Resource, error) { return s.testInstallation.ResourceClients.UpstreamClient().Read(s.testInstallation.Metadata.InstallNamespace, validation.ExampleUpstreamName, clients.ReadOpts{Ctx: s.ctx}) }, - core.Status_Accepted, - gloo_defaults.GlooReporter, ) err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, validation.ExampleVS, "-n", s.testInstallation.Metadata.InstallNamespace) s.Assert().NoError(err, "can apply valid virtual service") diff --git a/test/kubernetes/e2e/features/validation/validation_always_accept/suite.go b/test/kubernetes/e2e/features/validation/validation_always_accept/suite.go index 8b473f0309d..d2ebccfd55e 100644 --- a/test/kubernetes/e2e/features/validation/validation_always_accept/suite.go +++ b/test/kubernetes/e2e/features/validation/validation_always_accept/suite.go @@ -161,15 +161,14 @@ func (s *testingSuite) TestVirtualServiceWithSecretDeletion() { err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, validation.UnusedSecret, "-n", s.testInstallation.Metadata.InstallNamespace) s.Assert().NoError(err) - // Upstream should be accepted + // Upstreams no longer report status if they have not been translated at all to avoid conflicting with + // other syncers that have translated them, so we can only detect that the objects exist here err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, validation.ExampleUpstream, "-n", s.testInstallation.Metadata.InstallNamespace) s.Assert().NoError(err) - s.testInstallation.Assertions.EventuallyResourceStatusMatchesState( - func() (resources.InputResource, error) { + s.testInstallation.Assertions.EventuallyResourceExsits( + func() (resources.Resource, error) { return s.testInstallation.ResourceClients.UpstreamClient().Read(s.testInstallation.Metadata.InstallNamespace, validation.ExampleUpstreamName, clients.ReadOpts{Ctx: s.ctx}) }, - core.Status_Accepted, - gloo_defaults.GlooReporter, ) // Apply VS with secret after Upstream and Secret exist err = s.testInstallation.Actions.Kubectl().Apply(s.ctx, []byte(substitutedSecretVS)) diff --git a/test/kubernetes/testutils/assertions/resources.go b/test/kubernetes/testutils/assertions/resources.go new file mode 100644 index 00000000000..88fa2cae32f --- /dev/null +++ b/test/kubernetes/testutils/assertions/resources.go @@ -0,0 +1,21 @@ +package assertions + +import ( + "time" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + "github.com/solo-io/gloo/test/helpers" + "github.com/solo-io/gloo/test/kube2e/helper" +) + +// Checks GetNamespacedStatuses status for gloo installation namespace +func (p *Provider) EventuallyResourceExsits(getter helpers.ResourceGetter, timeout ...time.Duration) { + ginkgo.GinkgoHelper() + + currentTimeout, pollingInterval := helper.GetTimeouts(timeout...) + gomega.Eventually(func(g gomega.Gomega) { + _, err := getter() + g.Expect(err).NotTo(gomega.HaveOccurred(), "failed to get resource") + }, currentTimeout, pollingInterval).Should(gomega.Succeed()) +} From 2bc1bacc3904b1378e7d26273c047a7e69c02d6a Mon Sep 17 00:00:00 2001 From: Lawrence Gadban Date: Fri, 22 Nov 2024 10:58:38 -0600 Subject: [PATCH 07/19] another one --- .../features/validation/validation_strict_warnings/suite.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/kubernetes/e2e/features/validation/validation_strict_warnings/suite.go b/test/kubernetes/e2e/features/validation/validation_strict_warnings/suite.go index 1c8ca461b6d..85015fe1b42 100644 --- a/test/kubernetes/e2e/features/validation/validation_strict_warnings/suite.go +++ b/test/kubernetes/e2e/features/validation/validation_strict_warnings/suite.go @@ -154,12 +154,10 @@ func (s *testingSuite) TestInvalidUpstreamMissingPort() { // Upstream is only rejected when the upstream plugin is run when a valid cluster is present err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, validation.ExampleUpstream, "-n", s.testInstallation.Metadata.InstallNamespace) s.Assert().NoError(err, "can apply valid upstream") - s.testInstallation.Assertions.EventuallyResourceStatusMatchesState( - func() (resources.InputResource, error) { + s.testInstallation.Assertions.EventuallyResourceExsits( + func() (resources.Resource, error) { return s.testInstallation.ResourceClients.UpstreamClient().Read(s.testInstallation.Metadata.InstallNamespace, validation.ExampleUpstreamName, clients.ReadOpts{Ctx: s.ctx}) }, - core.Status_Accepted, - gloo_defaults.GlooReporter, ) err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, validation.ExampleVS, "-n", s.testInstallation.Metadata.InstallNamespace) s.Assert().NoError(err, "can apply valid virtual service") From c73cbdd35397253417163aefc2a5734ee807b554 Mon Sep 17 00:00:00 2001 From: Lawrence Gadban Date: Fri, 22 Nov 2024 11:40:48 -0600 Subject: [PATCH 08/19] anotha one --- test/e2e/aws_test.go | 4 +++- test/kube2e/gateway/gateway_test.go | 4 +++- .../e2e/features/basicrouting/edge_suite.go | 8 ++++---- .../discovery_watchlabels_suite.go | 18 ++++++++---------- 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/test/e2e/aws_test.go b/test/e2e/aws_test.go index fc8ddc52db1..4a7eb718c1f 100644 --- a/test/e2e/aws_test.go +++ b/test/e2e/aws_test.go @@ -221,7 +221,9 @@ var _ = Describe("AWS Lambda", func() { Expect(err).NotTo(HaveOccurred()) // wait for the upstream to be created - helpers.EventuallyResourceAccepted(func() (resources.InputResource, error) { + // Upstreams no longer report status if they have not been translated at all to avoid conflicting with + // other syncers that have translated them, so we can only detect that the objects exist here + helpers.EventuallyResourceExists(func() (resources.Resource, error) { return testClients.UpstreamClient.Read(upstream.Metadata.Namespace, upstream.Metadata.Name, clients.ReadOpts{}) }, "30s", "1s") } diff --git a/test/kube2e/gateway/gateway_test.go b/test/kube2e/gateway/gateway_test.go index 5a280939827..faba896171f 100644 --- a/test/kube2e/gateway/gateway_test.go +++ b/test/kube2e/gateway/gateway_test.go @@ -1290,7 +1290,9 @@ var _ = Describe("Kube2e: gateway", func() { upstreamName = kubernetesplugin.UpstreamName(testHelper.InstallNamespace, service.Name, 5678) // wait for upstream to get created by discovery - helpers.EventuallyResourceAccepted(func() (resources.InputResource, error) { + // Upstreams no longer report status if they have not been translated at all to avoid conflicting with + // other syncers that have translated them, so we can only detect that the objects exist here + helpers.EventuallyResourceExists(func() (resources.Resource, error) { return resourceClientset.UpstreamClient().Read(testHelper.InstallNamespace, upstreamName, clients.ReadOpts{Ctx: ctx}) }) // add subset spec to upstream diff --git a/test/kubernetes/e2e/features/basicrouting/edge_suite.go b/test/kubernetes/e2e/features/basicrouting/edge_suite.go index 35031017cdf..852706cc446 100644 --- a/test/kubernetes/e2e/features/basicrouting/edge_suite.go +++ b/test/kubernetes/e2e/features/basicrouting/edge_suite.go @@ -71,14 +71,14 @@ func (s *edgeBasicRoutingSuite) TestBasicVirtualServiceRouting() { }) // Upstream is only rejected when the upstream plugin is run when a valid cluster is present + // Upstreams no longer report status if they have not been translated at all to avoid conflicting with + // other syncers that have translated them, so we can only detect that the objects exist here err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, ossvalidation.ExampleUpstream, "-n", s.testInstallation.Metadata.InstallNamespace) s.Assert().NoError(err, "can apply valid upstream") - s.testInstallation.Assertions.EventuallyResourceStatusMatchesState( - func() (resources.InputResource, error) { + s.testInstallation.Assertions.EventuallyResourceExsits( + func() (resources.Resource, error) { return s.testInstallation.ResourceClients.UpstreamClient().Read(s.testInstallation.Metadata.InstallNamespace, ossvalidation.ExampleUpstreamName, clients.ReadOpts{Ctx: s.ctx}) }, - core.Status_Accepted, - defaults.GlooReporter, ) err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, ossvalidation.ExampleVS, "-n", s.testInstallation.Metadata.InstallNamespace) s.Assert().NoError(err, "can apply valid virtual service") diff --git a/test/kubernetes/e2e/features/discovery_watchlabels/discovery_watchlabels_suite.go b/test/kubernetes/e2e/features/discovery_watchlabels/discovery_watchlabels_suite.go index 82992d027d5..2e8585b0b7d 100644 --- a/test/kubernetes/e2e/features/discovery_watchlabels/discovery_watchlabels_suite.go +++ b/test/kubernetes/e2e/features/discovery_watchlabels/discovery_watchlabels_suite.go @@ -10,11 +10,9 @@ import ( "github.com/solo-io/gloo/projects/gloo/pkg/plugins/kubernetes" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/solo-io/gloo/projects/gloo/pkg/defaults" "github.com/solo-io/gloo/test/kubernetes/e2e" "github.com/solo-io/solo-kit/pkg/api/v1/clients" "github.com/solo-io/solo-kit/pkg/api/v1/resources" - "github.com/solo-io/solo-kit/pkg/api/v1/resources/core" "github.com/stretchr/testify/suite" ) @@ -64,13 +62,13 @@ func (s *discoveryWatchlabelsSuite) TestDiscoverUpstreamMatchingWatchLabels() { s.Assert().NoError(err, "can apply service") // eventually an Upstream should be created for the Service with matching labels + // Upstreams no longer report status if they have not been translated at all to avoid conflicting with + // other syncers that have translated them, so we can only detect that the objects exist here labeledUsName := kubernetes.UpstreamName(s.testInstallation.Metadata.InstallNamespace, "example-svc", 8000) - s.testInstallation.Assertions.EventuallyResourceStatusMatchesState( - func() (resources.InputResource, error) { + s.testInstallation.Assertions.EventuallyResourceExsits( + func() (resources.Resource, error) { return s.testInstallation.ResourceClients.UpstreamClient().Read(s.testInstallation.Metadata.InstallNamespace, labeledUsName, clients.ReadOpts{Ctx: s.ctx}) }, - core.Status_Accepted, - defaults.GlooReporter, ) // the Upstream should have DiscoveryMetadata labels matching the parent Service @@ -129,13 +127,13 @@ func (s *discoveryWatchlabelsSuite) TestDiscoverySpecPreserved() { s.Assert().NoError(err, "can apply service") // eventually an Upstream should be created for the Service with matching labels + // Upstreams no longer report status if they have not been translated at all to avoid conflicting with + // other syncers that have translated them, so we can only detect that the objects exist here labeledUsName := kubernetes.UpstreamName(s.testInstallation.Metadata.InstallNamespace, "example-svc", 8000) - s.testInstallation.Assertions.EventuallyResourceStatusMatchesState( - func() (resources.InputResource, error) { + s.testInstallation.Assertions.EventuallyResourceExsits( + func() (resources.Resource, error) { return s.testInstallation.ResourceClients.UpstreamClient().Read(s.testInstallation.Metadata.InstallNamespace, labeledUsName, clients.ReadOpts{Ctx: s.ctx}) }, - core.Status_Accepted, - defaults.GlooReporter, ) // the Upstream should have DiscoveryMetadata labels matching the parent Service From 409520e6533025ab7aea9b3f48c36f3c4f9195fc Mon Sep 17 00:00:00 2001 From: Lawrence Gadban Date: Fri, 22 Nov 2024 12:45:07 -0600 Subject: [PATCH 09/19] more fixes --- .../kubernetes/e2e/features/basicrouting/edge_suite.go | 2 +- .../discovery_watchlabels_suite.go | 4 ++-- test/kubernetes/e2e/features/tracing/suite.go | 8 ++++---- .../features/validation/full_envoy_validation/suite.go | 10 ++++------ .../validation/validation_allow_warnings/suite.go | 2 +- .../validation/validation_always_accept/suite.go | 8 +++----- .../validation/validation_strict_warnings/suite.go | 2 +- test/kubernetes/testutils/assertions/resources.go | 2 +- 8 files changed, 17 insertions(+), 21 deletions(-) diff --git a/test/kubernetes/e2e/features/basicrouting/edge_suite.go b/test/kubernetes/e2e/features/basicrouting/edge_suite.go index 852706cc446..1961f78161a 100644 --- a/test/kubernetes/e2e/features/basicrouting/edge_suite.go +++ b/test/kubernetes/e2e/features/basicrouting/edge_suite.go @@ -75,7 +75,7 @@ func (s *edgeBasicRoutingSuite) TestBasicVirtualServiceRouting() { // other syncers that have translated them, so we can only detect that the objects exist here err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, ossvalidation.ExampleUpstream, "-n", s.testInstallation.Metadata.InstallNamespace) s.Assert().NoError(err, "can apply valid upstream") - s.testInstallation.Assertions.EventuallyResourceExsits( + s.testInstallation.Assertions.EventuallyResourceExists( func() (resources.Resource, error) { return s.testInstallation.ResourceClients.UpstreamClient().Read(s.testInstallation.Metadata.InstallNamespace, ossvalidation.ExampleUpstreamName, clients.ReadOpts{Ctx: s.ctx}) }, diff --git a/test/kubernetes/e2e/features/discovery_watchlabels/discovery_watchlabels_suite.go b/test/kubernetes/e2e/features/discovery_watchlabels/discovery_watchlabels_suite.go index 2e8585b0b7d..79ef3f5f08d 100644 --- a/test/kubernetes/e2e/features/discovery_watchlabels/discovery_watchlabels_suite.go +++ b/test/kubernetes/e2e/features/discovery_watchlabels/discovery_watchlabels_suite.go @@ -65,7 +65,7 @@ func (s *discoveryWatchlabelsSuite) TestDiscoverUpstreamMatchingWatchLabels() { // Upstreams no longer report status if they have not been translated at all to avoid conflicting with // other syncers that have translated them, so we can only detect that the objects exist here labeledUsName := kubernetes.UpstreamName(s.testInstallation.Metadata.InstallNamespace, "example-svc", 8000) - s.testInstallation.Assertions.EventuallyResourceExsits( + s.testInstallation.Assertions.EventuallyResourceExists( func() (resources.Resource, error) { return s.testInstallation.ResourceClients.UpstreamClient().Read(s.testInstallation.Metadata.InstallNamespace, labeledUsName, clients.ReadOpts{Ctx: s.ctx}) }, @@ -130,7 +130,7 @@ func (s *discoveryWatchlabelsSuite) TestDiscoverySpecPreserved() { // Upstreams no longer report status if they have not been translated at all to avoid conflicting with // other syncers that have translated them, so we can only detect that the objects exist here labeledUsName := kubernetes.UpstreamName(s.testInstallation.Metadata.InstallNamespace, "example-svc", 8000) - s.testInstallation.Assertions.EventuallyResourceExsits( + s.testInstallation.Assertions.EventuallyResourceExists( func() (resources.Resource, error) { return s.testInstallation.ResourceClients.UpstreamClient().Read(s.testInstallation.Metadata.InstallNamespace, labeledUsName, clients.ReadOpts{Ctx: s.ctx}) }, diff --git a/test/kubernetes/e2e/features/tracing/suite.go b/test/kubernetes/e2e/features/tracing/suite.go index c1d2a3034e2..174f594ca68 100644 --- a/test/kubernetes/e2e/features/tracing/suite.go +++ b/test/kubernetes/e2e/features/tracing/suite.go @@ -101,13 +101,13 @@ func (s *testingSuite) BeforeTest(string, string) { err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, tracingConfigManifest) s.NoError(err, "can apply gloo tracing resources") // accept the upstream - s.testInstallation.Assertions.EventuallyResourceStatusMatchesState( - func() (resources.InputResource, error) { + // Upstreams no longer report status if they have not been translated at all to avoid conflicting with + // other syncers that have translated them, so we can only detect that the objects exist here + s.testInstallation.Assertions.EventuallyResourceExists( + func() (resources.Resource, error) { return s.testInstallation.ResourceClients.UpstreamClient().Read( otelcolUpstream.Namespace, otelcolUpstream.Name, clients.ReadOpts{Ctx: s.ctx}) }, - core.Status_Accepted, - gloo_defaults.GlooReporter, ) // accept the virtual service s.testInstallation.Assertions.EventuallyResourceStatusMatchesState( diff --git a/test/kubernetes/e2e/features/validation/full_envoy_validation/suite.go b/test/kubernetes/e2e/features/validation/full_envoy_validation/suite.go index 6fa18a103f3..92a3f868284 100644 --- a/test/kubernetes/e2e/features/validation/full_envoy_validation/suite.go +++ b/test/kubernetes/e2e/features/validation/full_envoy_validation/suite.go @@ -3,13 +3,11 @@ package full_envoy_validation import ( "context" - gloo_defaults "github.com/solo-io/gloo/projects/gloo/pkg/defaults" "github.com/solo-io/gloo/test/kubernetes/e2e" testdefaults "github.com/solo-io/gloo/test/kubernetes/e2e/defaults" "github.com/solo-io/gloo/test/kubernetes/e2e/features/validation" "github.com/solo-io/solo-kit/pkg/api/v1/clients" "github.com/solo-io/solo-kit/pkg/api/v1/resources" - "github.com/solo-io/solo-kit/pkg/api/v1/resources/core" "github.com/stretchr/testify/suite" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -54,12 +52,12 @@ func (s *testingSuite) TestRejectInvalidTransformation() { err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, validation.ExampleUpstream, "-n", s.testInstallation.Metadata.InstallNamespace) s.Assert().NoError(err) - s.testInstallation.Assertions.EventuallyResourceStatusMatchesState( - func() (resources.InputResource, error) { + // Upstreams no longer report status if they have not been translated at all to avoid conflicting with + // other syncers that have translated them, so we can only detect that the objects exist here + s.testInstallation.Assertions.EventuallyResourceExists( + func() (resources.Resource, error) { return s.testInstallation.ResourceClients.UpstreamClient().Read(s.testInstallation.Metadata.InstallNamespace, "nginx-upstream", clients.ReadOpts{Ctx: s.ctx}) }, - core.Status_Accepted, - gloo_defaults.GlooReporter, ) s.T().Cleanup(func() { diff --git a/test/kubernetes/e2e/features/validation/validation_allow_warnings/suite.go b/test/kubernetes/e2e/features/validation/validation_allow_warnings/suite.go index 5d59f3ad944..82a22ae58d0 100644 --- a/test/kubernetes/e2e/features/validation/validation_allow_warnings/suite.go +++ b/test/kubernetes/e2e/features/validation/validation_allow_warnings/suite.go @@ -115,7 +115,7 @@ func (s *testingSuite) TestInvalidUpstreamMissingPort() { // Upstream is only rejected when the upstream plugin is run when a valid cluster is present err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, validation.ExampleUpstream, "-n", s.testInstallation.Metadata.InstallNamespace) s.Assert().NoError(err, "can apply valid upstream") - s.testInstallation.Assertions.EventuallyResourceExsits( + s.testInstallation.Assertions.EventuallyResourceExists( func() (resources.Resource, error) { return s.testInstallation.ResourceClients.UpstreamClient().Read(s.testInstallation.Metadata.InstallNamespace, validation.ExampleUpstreamName, clients.ReadOpts{Ctx: s.ctx}) }, diff --git a/test/kubernetes/e2e/features/validation/validation_always_accept/suite.go b/test/kubernetes/e2e/features/validation/validation_always_accept/suite.go index d2ebccfd55e..6f0091f7649 100644 --- a/test/kubernetes/e2e/features/validation/validation_always_accept/suite.go +++ b/test/kubernetes/e2e/features/validation/validation_always_accept/suite.go @@ -165,7 +165,7 @@ func (s *testingSuite) TestVirtualServiceWithSecretDeletion() { // other syncers that have translated them, so we can only detect that the objects exist here err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, validation.ExampleUpstream, "-n", s.testInstallation.Metadata.InstallNamespace) s.Assert().NoError(err) - s.testInstallation.Assertions.EventuallyResourceExsits( + s.testInstallation.Assertions.EventuallyResourceExists( func() (resources.Resource, error) { return s.testInstallation.ResourceClients.UpstreamClient().Read(s.testInstallation.Metadata.InstallNamespace, validation.ExampleUpstreamName, clients.ReadOpts{Ctx: s.ctx}) }, @@ -245,12 +245,10 @@ func (s *testingSuite) TestPersistInvalidVirtualService() { // First apply Upstream err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, validation.ExampleUpstream, "-n", s.testInstallation.Metadata.InstallNamespace) s.Assert().NoError(err, "can apply "+validation.ExampleUpstream) - s.testInstallation.Assertions.EventuallyResourceStatusMatchesState( - func() (resources.InputResource, error) { + s.testInstallation.Assertions.EventuallyResourceExists( + func() (resources.Resource, error) { return s.testInstallation.ResourceClients.UpstreamClient().Read(s.testInstallation.Metadata.InstallNamespace, validation.ExampleUpstreamName, clients.ReadOpts{Ctx: s.ctx}) }, - core.Status_Accepted, - gloo_defaults.GlooReporter, ) // Then apply VirtualService diff --git a/test/kubernetes/e2e/features/validation/validation_strict_warnings/suite.go b/test/kubernetes/e2e/features/validation/validation_strict_warnings/suite.go index 85015fe1b42..60f1976009b 100644 --- a/test/kubernetes/e2e/features/validation/validation_strict_warnings/suite.go +++ b/test/kubernetes/e2e/features/validation/validation_strict_warnings/suite.go @@ -154,7 +154,7 @@ func (s *testingSuite) TestInvalidUpstreamMissingPort() { // Upstream is only rejected when the upstream plugin is run when a valid cluster is present err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, validation.ExampleUpstream, "-n", s.testInstallation.Metadata.InstallNamespace) s.Assert().NoError(err, "can apply valid upstream") - s.testInstallation.Assertions.EventuallyResourceExsits( + s.testInstallation.Assertions.EventuallyResourceExists( func() (resources.Resource, error) { return s.testInstallation.ResourceClients.UpstreamClient().Read(s.testInstallation.Metadata.InstallNamespace, validation.ExampleUpstreamName, clients.ReadOpts{Ctx: s.ctx}) }, diff --git a/test/kubernetes/testutils/assertions/resources.go b/test/kubernetes/testutils/assertions/resources.go index 88fa2cae32f..76f64701528 100644 --- a/test/kubernetes/testutils/assertions/resources.go +++ b/test/kubernetes/testutils/assertions/resources.go @@ -10,7 +10,7 @@ import ( ) // Checks GetNamespacedStatuses status for gloo installation namespace -func (p *Provider) EventuallyResourceExsits(getter helpers.ResourceGetter, timeout ...time.Duration) { +func (p *Provider) EventuallyResourceExists(getter helpers.ResourceGetter, timeout ...time.Duration) { ginkgo.GinkgoHelper() currentTimeout, pollingInterval := helper.GetTimeouts(timeout...) From b908568fedeca65bb286001c8c51b301948bfd06 Mon Sep 17 00:00:00 2001 From: Lawrence Gadban Date: Fri, 22 Nov 2024 13:55:53 -0600 Subject: [PATCH 10/19] more fixes --- .../validation/full_envoy_validation/suite.go | 7 +++++++ .../validation/validation_reject_invalid/suite.go | 8 ++++---- .../validation/validation_strict_warnings/suite.go | 10 ++++++---- test/kubernetes/testutils/assertions/resources.go | 12 ++++++++++++ 4 files changed, 29 insertions(+), 8 deletions(-) diff --git a/test/kubernetes/e2e/features/validation/full_envoy_validation/suite.go b/test/kubernetes/e2e/features/validation/full_envoy_validation/suite.go index 92a3f868284..e37422f36c8 100644 --- a/test/kubernetes/e2e/features/validation/full_envoy_validation/suite.go +++ b/test/kubernetes/e2e/features/validation/full_envoy_validation/suite.go @@ -60,6 +60,13 @@ func (s *testingSuite) TestRejectInvalidTransformation() { }, ) + // we need to make sure Gloo has had a chance to process it + s.testInstallation.Assertions.ConsistentlyResourceExists( + s.ctx, + func() (resources.Resource, error) { + return s.testInstallation.ResourceClients.UpstreamClient().Read(s.testInstallation.Metadata.InstallNamespace, "nginx-upstream", clients.ReadOpts{Ctx: s.ctx}) + }, + ) s.T().Cleanup(func() { err := s.testInstallation.Actions.Kubectl().DeleteFileSafe(s.ctx, validation.VSTransformationHeaderText, "-n", s.testInstallation.Metadata.InstallNamespace) s.Assert().NoError(err) diff --git a/test/kubernetes/e2e/features/validation/validation_reject_invalid/suite.go b/test/kubernetes/e2e/features/validation/validation_reject_invalid/suite.go index 510bcee937e..31af89e7ca0 100644 --- a/test/kubernetes/e2e/features/validation/validation_reject_invalid/suite.go +++ b/test/kubernetes/e2e/features/validation/validation_reject_invalid/suite.go @@ -94,14 +94,14 @@ func (s *testingSuite) TestVirtualServiceWithSecretDeletion() { s.Assert().NoError(err) // Upstream should be accepted + // Upstreams no longer report status if they have not been translated at all to avoid conflicting with + // other syncers that have translated them, so we can only detect that the objects exist here err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, validation.ExampleUpstream, "-n", s.testInstallation.Metadata.InstallNamespace) s.Assert().NoError(err) - s.testInstallation.Assertions.EventuallyResourceStatusMatchesState( - func() (resources.InputResource, error) { + s.testInstallation.Assertions.EventuallyResourceExists( + func() (resources.Resource, error) { return s.testInstallation.ResourceClients.UpstreamClient().Read(s.testInstallation.Metadata.InstallNamespace, validation.ExampleUpstreamName, clients.ReadOpts{Ctx: s.ctx}) }, - core.Status_Accepted, - gloo_defaults.GlooReporter, ) // Apply VS with secret after Upstream and Secret exist err = s.testInstallation.Actions.Kubectl().Apply(s.ctx, []byte(substitutedSecretVS)) diff --git a/test/kubernetes/e2e/features/validation/validation_strict_warnings/suite.go b/test/kubernetes/e2e/features/validation/validation_strict_warnings/suite.go index 60f1976009b..1f9d69e012a 100644 --- a/test/kubernetes/e2e/features/validation/validation_strict_warnings/suite.go +++ b/test/kubernetes/e2e/features/validation/validation_strict_warnings/suite.go @@ -93,14 +93,14 @@ func (s *testingSuite) TestVirtualServiceWithSecretDeletion() { s.Assert().NoError(err) // Upstream should be accepted + // Upstreams no longer report status if they have not been translated at all to avoid conflicting with + // other syncers that have translated them, so we can only detect that the objects exist here err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, validation.ExampleUpstream, "-n", s.testInstallation.Metadata.InstallNamespace) s.Assert().NoError(err) - s.testInstallation.Assertions.EventuallyResourceStatusMatchesState( - func() (resources.InputResource, error) { + s.testInstallation.Assertions.EventuallyResourceExists( + func() (resources.Resource, error) { return s.testInstallation.ResourceClients.UpstreamClient().Read(s.testInstallation.Metadata.InstallNamespace, validation.ExampleUpstreamName, clients.ReadOpts{Ctx: s.ctx}) }, - core.Status_Accepted, - gloo_defaults.GlooReporter, ) // Apply VS with secret after Upstream and Secret exist err = s.testInstallation.Actions.Kubectl().Apply(s.ctx, []byte(substitutedSecretVS)) @@ -152,6 +152,8 @@ func (s *testingSuite) TestInvalidUpstreamMissingPort() { }) // Upstream is only rejected when the upstream plugin is run when a valid cluster is present + // Upstreams no longer report status if they have not been translated at all to avoid conflicting with + // other syncers that have translated them, so we can only detect that the objects exist here err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, validation.ExampleUpstream, "-n", s.testInstallation.Metadata.InstallNamespace) s.Assert().NoError(err, "can apply valid upstream") s.testInstallation.Assertions.EventuallyResourceExists( diff --git a/test/kubernetes/testutils/assertions/resources.go b/test/kubernetes/testutils/assertions/resources.go index 76f64701528..4f87f9a236f 100644 --- a/test/kubernetes/testutils/assertions/resources.go +++ b/test/kubernetes/testutils/assertions/resources.go @@ -1,6 +1,7 @@ package assertions import ( + "context" "time" "github.com/onsi/ginkgo/v2" @@ -19,3 +20,14 @@ func (p *Provider) EventuallyResourceExists(getter helpers.ResourceGetter, timeo g.Expect(err).NotTo(gomega.HaveOccurred(), "failed to get resource") }, currentTimeout, pollingInterval).Should(gomega.Succeed()) } + +func (p *Provider) ConsistentlyResourceExists(ctx context.Context, getter helpers.ResourceGetter) { + p.Gomega.Consistently(ctx, func(innerG gomega.Gomega) { + _, err := getter() + innerG.Expect(err).NotTo(gomega.HaveOccurred(), "failed to get resource") + }). + WithContext(ctx). + WithTimeout(time.Second*5). + WithPolling(time.Second*1). + Should(gomega.Succeed(), "resource should be found in cluster") +} From 411c838c6baa742b4a758d1c158b3ceebedd7a14 Mon Sep 17 00:00:00 2001 From: Lawrence Gadban Date: Fri, 22 Nov 2024 13:57:43 -0600 Subject: [PATCH 11/19] revert add of example --- ...ps-gw-and-route-with-attached-options.yaml | 114 ------------------ 1 file changed, 114 deletions(-) delete mode 100644 projects/gateway2/examples/example-https-gw-and-route-with-attached-options.yaml diff --git a/projects/gateway2/examples/example-https-gw-and-route-with-attached-options.yaml b/projects/gateway2/examples/example-https-gw-and-route-with-attached-options.yaml deleted file mode 100644 index afafdf7bea9..00000000000 --- a/projects/gateway2/examples/example-https-gw-and-route-with-attached-options.yaml +++ /dev/null @@ -1,114 +0,0 @@ -kind: Gateway -apiVersion: gateway.networking.k8s.io/v1 -metadata: - name: https-gw -spec: - gatewayClassName: gloo-gateway - listeners: - - protocol: HTTPS - port: 8443 - name: https - tls: - mode: Terminate - certificateRefs: - - name: https - kind: Secret - allowedRoutes: - namespaces: - from: Same ---- -apiVersion: v1 -kind: Secret -type: kubernetes.io/tls -metadata: - name: https -data: - tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUN0akNDQVo0Q0FRQXdEUVlKS29aSWh2Y05BUUVMQlFBd0lURVRNQkVHQTFVRUNnd0tZVzU1SUdSdmJXRnAKYmpFS01BZ0dBMVVFQXd3QktqQWVGdzB5TkRBNU1qSXhOak00TURaYUZ3MHlOVEE1TWpJeE5qTTRNRFphTUNFeApDakFJQmdOVkJBTU1BU294RXpBUkJnTlZCQW9NQ21GdWVTQmtiMjFoYVc0d2dnRWlNQTBHQ1NxR1NJYjNEUUVCCkFRVUFBNElCRHdBd2dnRUtBb0lCQVFEc2REdmY0MmgyYzdSb2JOb0tSZWFRYzFqdUpaWGtwZzFOWER5aFZOK3cKbFdkVDg0UlFhS1RYVVYxZTkwUUVyb2VrbFZnQld5d3A5WlNsUUhzSTc3MTBVVm16N3RhUFRXbWd0WlhHRy9hTApXSDhiMklsdTUrVXhpajB2K2NheUJqWEVuVWJTVFJQYXlkYU11N1hqODNKV0tKSDZ6TzNVK3pwbCt1SXFVc0p3Ck9GOWpWeFNvMWpEMEQvdllObFNiMTFjNWVjWHUrdk9WMzkrUEpmTDFmUjVQNSt0Vnk1cFlKYWtjNDhJMmlWVzAKNmkvcUdtb3lHMGdCdVZZMkZETzFlcndmUmxHMTRSNEdwdXVWbUg2TEFMcWs2bEZSbHpmbHRYVTBJZ0o5dVNvNApjTjFLMlFHTUhmYUF2RHJqL1JReStKdTQ3TENRcTFYRUU1RzJqdlZOMlNhekFnTUJBQUV3RFFZSktvWklodmNOCkFRRUxCUUFEZ2dFQkFKNkNnVXg3d3huLyswK1hJT2FsclZacnh2SXQvc1J4YnVvQ3NvM04zbm50V0g4TmpVRHIKdURIYUgvRktBVkdsSTFPT0NTa2VhVmlHdGpZdXBkL0ZUbUI2b0pEM2F4cUYzMWVHTDNFTWR4aVZkMTJEbEZsbwp2bkJXZGs4d2xLNk9EcHdUTkd2eEgxOVA5WEN6UjZYRXU0SGlXYnNRZ29YaWN6VC9vQzdCM3NqaG5HYkFmUGx5CkdIRWVhNWV6ay9zU053YmhTcmFabzBZOFNuUzFqSHFKSlhuWk1PbmpCLzlqRGJ6VTZoaVN3a1hiNFZSU3B3ZkIKSE43Q2t3d3FjT0JGWUQ3bGg5bDMzVDN1L0RSWDJWY2FOSWRXSlErRUdTSnowYngrMVhGZXh4R3NjNXNtUWVwZwpIUEEwTVU1ZGNnQkxGSUlFUU1BNHYrSXRKN3BpN2pvN3c3dz0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= - tls.key: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2Z0lCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktnd2dnU2tBZ0VBQW9JQkFRRHNkRHZmNDJoMmM3Um8KYk5vS1JlYVFjMWp1SlpYa3BnMU5YRHloVk4rd2xXZFQ4NFJRYUtUWFVWMWU5MFFFcm9la2xWZ0JXeXdwOVpTbApRSHNJNzcxMFVWbXo3dGFQVFdtZ3RaWEdHL2FMV0g4YjJJbHU1K1V4aWowditjYXlCalhFblViU1RSUGF5ZGFNCnU3WGo4M0pXS0pINnpPM1UrenBsK3VJcVVzSndPRjlqVnhTbzFqRDBEL3ZZTmxTYjExYzVlY1h1K3ZPVjM5K1AKSmZMMWZSNVA1K3RWeTVwWUpha2M0OEkyaVZXMDZpL3FHbW95RzBnQnVWWTJGRE8xZXJ3ZlJsRzE0UjRHcHV1VgptSDZMQUxxazZsRlJsemZsdFhVMElnSjl1U280Y04xSzJRR01IZmFBdkRyai9SUXkrSnU0N0xDUXExWEVFNUcyCmp2Vk4yU2F6QWdNQkFBRUNnZ0VBT1lvZmQrQVM3NkJBYXBqeWhrVjBVZnAvelpQLzRlQTJwSGlwbUdqYTlsR2wKRDF6VytlbzlFdGlON0NvbnR5dnhmaStKZHVzRTJFRVdweEFGNEtyV1k1UFlURnoyZ1hESkgzNlFyK0RtSWdxcgpBOCt4d2ZkVVlyWE5KZnVXU29RUVdZUWNVOWpGMkJ2OXhjbDMvYnlrT2lzdE91YmJpVlNKWlgrandjRFo2QXlZCkM1Zm1CeXpCaERaWTV2NTg0VzZaMUxTUzZFR3ErQWVmd1hDNnoxcVluV2dlYXBrVG5jbEVESk4vVU9tQnJUeW0KaCtxRjdKTUZBWk1xbGVOQ1U1Wk1aUG43TWRPWkt5c20zaTRSUVNXL0NLMEFzWU9hNmx2TEJkZGxEN2k3THdORQpmRzZnUjhiVXZhQWtETTNEY3B3a0QyeURtUVhwSHFDRmZGUWRjT0p4SVFLQmdRRDJTajNtNHRmdFhUT3JvRDZNClM0N24xTVAzdU40WkJHUnZrcnZQbmdIckRQRG43aEhiZDg2MVJJRnN3dG0zWjRzTmk3RnVsWGU5YUdsek1OWHQKQ1NBYkd3UTJLbFZjeEpIYXZsYWlqUWgwVkZtZVlGSHpMNVMzeGxyc3RzNURnMnp2YU5Jd1VqNVpJMUY0a01YVgpHbFFHNU43dldvTjFIQlJ5NTFoN0VJV3dkd0tCZ1FEMXhyZ3ltcHVRYm5tOE1qTHJzYUd5Y0RxbzFCNzhMN2NHCnk0dEVCY2tGaTRKSTVrbjhxYy8wMTlCQ3dyYTBON0lvNkxMYmhhTVJDdFNLRXhVdDE5QXlBdStZd1U3V1JpKysKY0pLVUo3aTBoYXBMMk4zU1FwS3dMUFBLdXpQc2pkOTNmS2ZHNmdNZElnS0ZnUmlNK2JOdlZtTmNVS1pWa2xUMwpMZFNveGxsbXBRS0JnUUNaOVg4TXZmRk5LRjRNZzNlV0xBV0JWcmZLTzM1YkZTdlRzMTVFUXZuZi9ZNzY3UStzCnFoQkZzRFZYejlaWm1CaU10eTZMZWxRSHl5VmdKTDhXRmdaRDYzZU9oSmdvRFprL3JJeHJ3TTN3S21wN1hxcmIKNG5xTkJ5bi9uNmNBL1Q4enljcmErZzZQbWt4cm1kU1Jpb3ZNWTNZWkdmUGpXVkpQSWQ5bHpzWkRzUUtCZ1FDNwpyYVNvMTlFbEtJZURRNTRJeDFzZmhIMzJ3QXh0TFE2SmZOTDYvYVAwdlFTZklHT3RNZ3NmLzlFRHVlYUVsZWRSClAraWNvMTdUTzc3ek5RRStRWWUxT1BLM1poNEttQXdMVTdGS3UwNWJZNXFZVXFHSTF0ZG0rdHlybWVwYm9EYmQKNm0zQUFxQ2dGWUZ4YXhSUlNyaFBzOXNwK2xnYURpVWRuM3k5Nk1FS0ZRS0JnQXZFeDVYYnZ2RDB3d3F6cjcveApTUmM5aDRiQWFuTW8vUnFxaFd6MU9tU2VMaUh4NlhNc0ROSW01bTBjUlRwNUo1UWRicDdycDB2RS9lQU5BekhWCjdJSE5vaVNDeWs2NnU0anJKL3UvRXpPMkx1VmtGNWdSL1RvdU03eVBNVzJGL004U3gvRWc1M1k0NFBSZGpvT3oKQWY5QmUzYjZxMTdDb3ByU3ZVb2xNU09tCi0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K ---- -apiVersion: gateway.networking.k8s.io/v1 -kind: HTTPRoute -metadata: - name: example-route -spec: - parentRefs: - - name: https-gw - hostnames: - - "example.com" - rules: - - backendRefs: - - name: example-svc - port: 8080 - - matches: - - path: - type: PathPrefix - value: /timeout - backendRefs: - - name: example-svc - port: 8080 ---- -apiVersion: gateway.solo.io/v1 -kind: RouteOption -metadata: - name: policy-attached -spec: - targetRefs: - - group: gateway.networking.k8s.io - kind: HTTPRoute - name: example-route - options: - faults: - abort: - httpStatus: 500 - timeout: 11s - # extauth: - # configRef: - # name: basic-auth - # namespace: httpbin ---- -apiVersion: v1 -kind: Service -metadata: - name: example-svc -spec: - selector: - app.kubernetes.io/name: nginx - ports: - - protocol: TCP - port: 8080 - targetPort: http-web-svc -# --- -# apiVersion: v1 -# kind: Pod -# metadata: -# name: nginx -# labels: -# app.kubernetes.io/name: nginx -# spec: -# containers: -# - name: nginx -# image: nginx:stable -# ports: -# - containerPort: 80 -# name: http-web-svc ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: nginx -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: nginx - template: - metadata: - labels: - app.kubernetes.io/name: nginx - spec: - containers: - - name: nginx - image: nginx:stable - ports: - - containerPort: 80 - name: http-web-svc From c6fa36c3e572d4b041c8359e39ce3fcff2927aa7 Mon Sep 17 00:00:00 2001 From: Lawrence Gadban Date: Fri, 22 Nov 2024 13:58:52 -0600 Subject: [PATCH 12/19] cleanup --- test/kubernetes/testutils/assertions/resources.go | 1 - 1 file changed, 1 deletion(-) diff --git a/test/kubernetes/testutils/assertions/resources.go b/test/kubernetes/testutils/assertions/resources.go index 4f87f9a236f..442dddd66e9 100644 --- a/test/kubernetes/testutils/assertions/resources.go +++ b/test/kubernetes/testutils/assertions/resources.go @@ -10,7 +10,6 @@ import ( "github.com/solo-io/gloo/test/kube2e/helper" ) -// Checks GetNamespacedStatuses status for gloo installation namespace func (p *Provider) EventuallyResourceExists(getter helpers.ResourceGetter, timeout ...time.Duration) { ginkgo.GinkgoHelper() From d93ee03a55f73cca796f41491f1087cce1f7e14e Mon Sep 17 00:00:00 2001 From: Lawrence Gadban Date: Fri, 22 Nov 2024 14:42:03 -0600 Subject: [PATCH 13/19] more test fix --- test/kubernetes/e2e/features/basicrouting/edge_suite.go | 7 +++++++ .../features/validation/validation_reject_invalid/suite.go | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/test/kubernetes/e2e/features/basicrouting/edge_suite.go b/test/kubernetes/e2e/features/basicrouting/edge_suite.go index 1961f78161a..89cb219a67b 100644 --- a/test/kubernetes/e2e/features/basicrouting/edge_suite.go +++ b/test/kubernetes/e2e/features/basicrouting/edge_suite.go @@ -80,6 +80,13 @@ func (s *edgeBasicRoutingSuite) TestBasicVirtualServiceRouting() { return s.testInstallation.ResourceClients.UpstreamClient().Read(s.testInstallation.Metadata.InstallNamespace, ossvalidation.ExampleUpstreamName, clients.ReadOpts{Ctx: s.ctx}) }, ) + // we need to make sure Gloo has had a chance to process it + s.testInstallation.Assertions.ConsistentlyResourceExists( + s.ctx, + func() (resources.Resource, error) { + return s.testInstallation.ResourceClients.UpstreamClient().Read(s.testInstallation.Metadata.InstallNamespace, "nginx-upstream", clients.ReadOpts{Ctx: s.ctx}) + }, + ) err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, ossvalidation.ExampleVS, "-n", s.testInstallation.Metadata.InstallNamespace) s.Assert().NoError(err, "can apply valid virtual service") s.testInstallation.Assertions.EventuallyResourceStatusMatchesState( diff --git a/test/kubernetes/e2e/features/validation/validation_reject_invalid/suite.go b/test/kubernetes/e2e/features/validation/validation_reject_invalid/suite.go index 31af89e7ca0..39f41541563 100644 --- a/test/kubernetes/e2e/features/validation/validation_reject_invalid/suite.go +++ b/test/kubernetes/e2e/features/validation/validation_reject_invalid/suite.go @@ -103,6 +103,13 @@ func (s *testingSuite) TestVirtualServiceWithSecretDeletion() { return s.testInstallation.ResourceClients.UpstreamClient().Read(s.testInstallation.Metadata.InstallNamespace, validation.ExampleUpstreamName, clients.ReadOpts{Ctx: s.ctx}) }, ) + // we need to make sure Gloo has had a chance to process it + s.testInstallation.Assertions.ConsistentlyResourceExists( + s.ctx, + func() (resources.Resource, error) { + return s.testInstallation.ResourceClients.UpstreamClient().Read(s.testInstallation.Metadata.InstallNamespace, "nginx-upstream", clients.ReadOpts{Ctx: s.ctx}) + }, + ) // Apply VS with secret after Upstream and Secret exist err = s.testInstallation.Actions.Kubectl().Apply(s.ctx, []byte(substitutedSecretVS)) s.Assert().NoError(err) From 82b31ff0b6ed37891d65c70b41331fdc1a17cc1d Mon Sep 17 00:00:00 2001 From: Lawrence Gadban Date: Fri, 22 Nov 2024 15:16:58 -0600 Subject: [PATCH 14/19] last fix? --- .../validation/validation_strict_warnings/suite.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/kubernetes/e2e/features/validation/validation_strict_warnings/suite.go b/test/kubernetes/e2e/features/validation/validation_strict_warnings/suite.go index 1f9d69e012a..39cc891e13a 100644 --- a/test/kubernetes/e2e/features/validation/validation_strict_warnings/suite.go +++ b/test/kubernetes/e2e/features/validation/validation_strict_warnings/suite.go @@ -161,6 +161,13 @@ func (s *testingSuite) TestInvalidUpstreamMissingPort() { return s.testInstallation.ResourceClients.UpstreamClient().Read(s.testInstallation.Metadata.InstallNamespace, validation.ExampleUpstreamName, clients.ReadOpts{Ctx: s.ctx}) }, ) + // we need to make sure Gloo has had a chance to process it + s.testInstallation.Assertions.ConsistentlyResourceExists( + s.ctx, + func() (resources.Resource, error) { + return s.testInstallation.ResourceClients.UpstreamClient().Read(s.testInstallation.Metadata.InstallNamespace, "nginx-upstream", clients.ReadOpts{Ctx: s.ctx}) + }, + ) err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, validation.ExampleVS, "-n", s.testInstallation.Metadata.InstallNamespace) s.Assert().NoError(err, "can apply valid virtual service") s.testInstallation.Assertions.EventuallyResourceStatusMatchesState( From 31adabd4a2ce41aba39c17a4971fccb643ee70b3 Mon Sep 17 00:00:00 2001 From: Lawrence Gadban Date: Fri, 22 Nov 2024 15:52:15 -0600 Subject: [PATCH 15/19] anotha timing fix --- .../validation/validation_strict_warnings/suite.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/kubernetes/e2e/features/validation/validation_strict_warnings/suite.go b/test/kubernetes/e2e/features/validation/validation_strict_warnings/suite.go index 39cc891e13a..d9459026be2 100644 --- a/test/kubernetes/e2e/features/validation/validation_strict_warnings/suite.go +++ b/test/kubernetes/e2e/features/validation/validation_strict_warnings/suite.go @@ -102,6 +102,13 @@ func (s *testingSuite) TestVirtualServiceWithSecretDeletion() { return s.testInstallation.ResourceClients.UpstreamClient().Read(s.testInstallation.Metadata.InstallNamespace, validation.ExampleUpstreamName, clients.ReadOpts{Ctx: s.ctx}) }, ) + // we need to make sure Gloo has had a chance to process it + s.testInstallation.Assertions.ConsistentlyResourceExists( + s.ctx, + func() (resources.Resource, error) { + return s.testInstallation.ResourceClients.UpstreamClient().Read(s.testInstallation.Metadata.InstallNamespace, "nginx-upstream", clients.ReadOpts{Ctx: s.ctx}) + }, + ) // Apply VS with secret after Upstream and Secret exist err = s.testInstallation.Actions.Kubectl().Apply(s.ctx, []byte(substitutedSecretVS)) s.Assert().NoError(err) From 30622cd47ed2a46a6c6178dddddb2f605b9e05b2 Mon Sep 17 00:00:00 2001 From: Lawrence Gadban Date: Fri, 22 Nov 2024 16:53:30 -0600 Subject: [PATCH 16/19] another us check --- .../features/validation/split_webhook/suite.go | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/test/kubernetes/e2e/features/validation/split_webhook/suite.go b/test/kubernetes/e2e/features/validation/split_webhook/suite.go index e15676b90d5..aa146688bb4 100644 --- a/test/kubernetes/e2e/features/validation/split_webhook/suite.go +++ b/test/kubernetes/e2e/features/validation/split_webhook/suite.go @@ -8,13 +8,11 @@ import ( "time" "github.com/onsi/gomega" - gloo_defaults "github.com/solo-io/gloo/projects/gloo/pkg/defaults" "github.com/solo-io/gloo/test/kubernetes/e2e" "github.com/solo-io/gloo/test/kubernetes/e2e/features/validation" "github.com/solo-io/gloo/test/kubernetes/testutils/helper" "github.com/solo-io/solo-kit/pkg/api/v1/clients" "github.com/solo-io/solo-kit/pkg/api/v1/resources" - "github.com/solo-io/solo-kit/pkg/api/v1/resources/core" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/stretchr/testify/suite" @@ -196,12 +194,19 @@ var ( } validateUpstreamCreated = func(s *testingSuite) { - s.testInstallation.Assertions.EventuallyResourceStatusMatchesState( - func() (resources.InputResource, error) { + // Upstreams no longer report status if they have not been translated at all to avoid conflicting with + // other syncers that have translated them, so we can only detect that the objects exist here + s.testInstallation.Assertions.EventuallyResourceExists( + func() (resources.Resource, error) { return s.testInstallation.ResourceClients.UpstreamClient().Read(s.testInstallation.Metadata.InstallNamespace, "json-upstream", clients.ReadOpts{Ctx: s.ctx}) }, - core.Status_Accepted, - gloo_defaults.GlooReporter, + ) + // we need to make sure Gloo has had a chance to process it + s.testInstallation.Assertions.ConsistentlyResourceExists( + s.ctx, + func() (resources.Resource, error) { + return s.testInstallation.ResourceClients.UpstreamClient().Read(s.testInstallation.Metadata.InstallNamespace, "nginx-upstream", clients.ReadOpts{Ctx: s.ctx}) + }, ) } From edf781df992199f482d284cb5b5ed08d986c2716 Mon Sep 17 00:00:00 2001 From: Lawrence Gadban Date: Fri, 22 Nov 2024 20:06:44 -0600 Subject: [PATCH 17/19] test cleanup --- .../e2e/features/validation/split_webhook/suite.go | 6 ++++-- test/kubernetes/e2e/features/validation/types.go | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/test/kubernetes/e2e/features/validation/split_webhook/suite.go b/test/kubernetes/e2e/features/validation/split_webhook/suite.go index aa146688bb4..6a25ce53a20 100644 --- a/test/kubernetes/e2e/features/validation/split_webhook/suite.go +++ b/test/kubernetes/e2e/features/validation/split_webhook/suite.go @@ -198,14 +198,16 @@ var ( // other syncers that have translated them, so we can only detect that the objects exist here s.testInstallation.Assertions.EventuallyResourceExists( func() (resources.Resource, error) { - return s.testInstallation.ResourceClients.UpstreamClient().Read(s.testInstallation.Metadata.InstallNamespace, "json-upstream", clients.ReadOpts{Ctx: s.ctx}) + uc := s.testInstallation.ResourceClients.UpstreamClient() + return uc.Read(s.testInstallation.Metadata.InstallNamespace, validation.SplitWebhookBasicUpstreamName, clients.ReadOpts{Ctx: s.ctx}) }, ) // we need to make sure Gloo has had a chance to process it s.testInstallation.Assertions.ConsistentlyResourceExists( s.ctx, func() (resources.Resource, error) { - return s.testInstallation.ResourceClients.UpstreamClient().Read(s.testInstallation.Metadata.InstallNamespace, "nginx-upstream", clients.ReadOpts{Ctx: s.ctx}) + uc := s.testInstallation.ResourceClients.UpstreamClient() + return uc.Read(s.testInstallation.Metadata.InstallNamespace, validation.SplitWebhookBasicUpstreamName, clients.ReadOpts{Ctx: s.ctx}) }, ) } diff --git a/test/kubernetes/e2e/features/validation/types.go b/test/kubernetes/e2e/features/validation/types.go index 720c00ea9a0..8a6138aecac 100644 --- a/test/kubernetes/e2e/features/validation/types.go +++ b/test/kubernetes/e2e/features/validation/types.go @@ -10,8 +10,9 @@ import ( ) const ( - ExampleVsName = "example-vs" - ExampleUpstreamName = "nginx-upstream" + ExampleVsName = "example-vs" + ExampleUpstreamName = "nginx-upstream" + SplitWebhookBasicUpstreamName = "json-upstream" ValidVsName = "i-am-valid" InvalidVsName = "i-am-invalid" From d0613049fe1ff6a9a50467b9b33f3708b7a3dd15 Mon Sep 17 00:00:00 2001 From: Lawrence Gadban Date: Sat, 23 Nov 2024 10:16:47 -0600 Subject: [PATCH 18/19] changelog --- changelog/v1.18.0-rc2/fix-us-status-flicker.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/changelog/v1.18.0-rc2/fix-us-status-flicker.yaml b/changelog/v1.18.0-rc2/fix-us-status-flicker.yaml index 644c30cb232..767a050efc7 100644 --- a/changelog/v1.18.0-rc2/fix-us-status-flicker.yaml +++ b/changelog/v1.18.0-rc2/fix-us-status-flicker.yaml @@ -2,3 +2,10 @@ changelog: - type: NON_USER_FACING description: >- Fix flicker on Upstream status when kube gw syncer rejects a resource and no edge proxies are present + issueLink: https://github.com/solo-io/solo-projects/issues/7243 + resolvesIssue: true + - type: NON_USER_FACING + description: >- + Fix missing status on krt objects resulting in continuous status updates (and webhook hits) + issueLink: https://github.com/solo-io/solo-projects/issues/7257 + resolvesIssue: true From 0f2a5f62fff65bfd763dc06d9b4661204c203e3f Mon Sep 17 00:00:00 2001 From: Lawrence Gadban Date: Sun, 24 Nov 2024 10:46:16 -0600 Subject: [PATCH 19/19] changelog --- changelog/v1.18.0-rc2/fix-us-status-flicker.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/changelog/v1.18.0-rc2/fix-us-status-flicker.yaml b/changelog/v1.18.0-rc2/fix-us-status-flicker.yaml index 767a050efc7..455ca62b6f7 100644 --- a/changelog/v1.18.0-rc2/fix-us-status-flicker.yaml +++ b/changelog/v1.18.0-rc2/fix-us-status-flicker.yaml @@ -9,3 +9,8 @@ changelog: Fix missing status on krt objects resulting in continuous status updates (and webhook hits) issueLink: https://github.com/solo-io/solo-projects/issues/7257 resolvesIssue: true + - type: BREAKING_CHANGE + description: >- + Upstreams and UpstreamGroups no longer get Accepted status by default. If they have not gone through translation they will have an empty status field. + issueLink: https://github.com/solo-io/gloo/issues/10401 + resolvesIssue: true