Feat/large validation (#10417)

Co-authored-by: Ryan Old <[email protected]>
Co-authored-by: soloio-bulldozer[bot] <48420018+soloio-bulldozer[bot]@users.noreply.github.com>
Co-authored-by: changelog-bot <changelog-bot>

Makefile

@@ -287,10 +287,9 @@ run-hashicorp-e2e-tests: GINKGO_FLAGS += --label-filter="end-to-end && !performa
 run-hashicorp-e2e-tests: test
 
 .PHONY: run-kube-e2e-tests
-run-kube-e2e-tests: TEST_PKG = ./test/kube2e/$(KUBE2E_TESTS) ## Run the Kubernetes E2E Tests in the {KUBE2E_TESTS} package
+run-kube-e2e-tests: TEST_PKG = ./test/kube2e/$(KUBE2E_TESTS) ## Run the legacy Kubernetes E2E Tests in the {KUBE2E_TESTS} package
 run-kube-e2e-tests: test
 
-
 #----------------------------------------------------------------------------------
 # Go Tests
 #----------------------------------------------------------------------------------
@@ -1080,6 +1079,9 @@ endif # distroless images
 CLUSTER_NAME ?= kind
 INSTALL_NAMESPACE ?= gloo-system
 
+kind-setup:
+	VERSION=${VERSION} CLUSTER_NAME=${CLUSTER_NAME} ./ci/kind/setup-kind.sh
+
 kind-load-%-distroless:
 	kind load docker-image $(IMAGE_REGISTRY)/$*:$(VERSION)-distroless --name $(CLUSTER_NAME)
 

changelog/v1.18.0-rc3/validate-large-configs.yaml

@@ -0,0 +1,9 @@
+changelog:
+  - type: FIX
+    issueLink: https://github.com/solo-io/solo-projects/issues/7089
+    resolvesIssue: false
+    description: >-
+      Fix the validation of large configurations when using envoy validation.
+      This was rarely seen in practice but occurred more often with the new fullEnvoyConfig validation.
+      Previously if the configuration grew too large translation would be blocked.
+      

projects/envoyinit/pkg/runner/run.go

@@ -6,6 +6,7 @@ import (
 	"log"
 	"os"
 	"syscall"
+	"time"
 
 	"github.com/rotisserie/eris"
 	"github.com/solo-io/gloo/pkg/utils/cmdutils"
@@ -30,8 +31,15 @@ const (
 func RunEnvoyValidate(ctx context.Context, envoyExecutable, bootstrapConfig string) error {
 	logger := contextutils.LoggerFrom(ctx)
 
-	validateCmd := cmdutils.Command(ctx, envoyExecutable, "--mode", "validate", "--config-yaml", bootstrapConfig, "-l", "critical", "--log-format", "%v")
-	if err := validateCmd.Run(); err != nil {
+	validateCmd := cmdutils.Command(ctx, envoyExecutable, "--mode", "validate", "--config-path", "/dev/fd/0",
+		"-l", "critical", "--log-format", "%v")
+	validateCmd = validateCmd.WithStdin(bytes.NewBufferString(bootstrapConfig))
+
+	start := time.Now()
+	err := validateCmd.Run()
+	logger.Debugf("envoy validation of %d size completed in %s", len(bootstrapConfig), time.Since(start))
+
+	if err != nil {
 		if os.IsNotExist(err) {
 			// log a warning and return nil; will allow users to continue to run Gloo locally without
 			// relying on the Gloo container with Envoy already published to the expected directory

projects/gloo/api/v1/settings.proto

@@ -920,6 +920,10 @@ message GatewayOptions {
         //
         // This feature is disabled by default and is not recommended for production deployments unless
         // the performance implications are well understood and acceptable.
+        //
+        // Large configurations can take more than 10 seconds to validate, causing the validating webhook to timeout.
+        // When enabling this feature, consider increasing the timeout for the validating webhook
+        // (`.Values.gateway.validation.webhook.timeoutSeconds`).
         google.protobuf.BoolValue full_envoy_validation = 14;
     }
 

test/kube2e/README.md

@@ -2,6 +2,10 @@
 > This directory houses legacy tests. All new tests should instead be added to the `test/kubernetes/e2e` directory.
 
 # Kubernetes End-to-End tests
+
+> These are our legacy Kubernetes E2E tests. We are migrating them to `../kubernetes/e2e`. Create new E2E tests there
+> using the new framework.
+
 See the [developer kube-e2e testing guide](/devel/testing/kube-e2e-tests.md) for more information about the philosophy of these tests.
 
 *Note: All commands should be run from the root directory of the Gloo repository*
@@ -68,7 +72,7 @@ To run the regression tests, your kubeconfig file must point to a running Kubern
 
 Use the same command that CI relies on:
 ```bash
-KUBE2E_TESTS=<test-to-run> make run-kube-e2e-tests
+CLUSTER_NAME=solo-test-cluster KUBE2E_TESTS=<test-to-run> make run-kube-e2e-tests
 ```
 
 #### Test Environment Variables
@@ -81,6 +85,7 @@ The below table contains the environment variables that can be used to configure
 | WAIT_ON_FAIL     | 0       | Set to 1 to prevent Ginkgo from cleaning up the Gloo Edge installation in case of failure. Useful to exec into inspect resources created by the test. A command to resume the test run (and thus clean up resources) will be logged to the output. |
 | TEAR_DOWN        | false   | Set to true to uninstall Gloo after the test suite completes                                                                                                                                                                                       |
 | RELEASED_VERSION | ''      | Used by nightlies to tests a specific released version. 'LATEST' will find the latest release                                                                                                                                                      |
+| CLUSTER_NAME     | kind    | Used to control which Kind cluster to run the tests inside | 
 
 #### Common Test Errors
 `getting Helm chart version: expected a single entry with name [gloo], found: 5`\

test/kube2e/gateway/gateway_suite_test.go

@@ -23,6 +23,7 @@ import (
 	"github.com/solo-io/gloo/test/helpers"
 	"github.com/solo-io/gloo/test/kube2e"
 	"github.com/solo-io/gloo/test/kube2e/helper"
+	testruntime "github.com/solo-io/gloo/test/kubernetes/testutils/runtime"
 	skhelpers "github.com/solo-io/solo-kit/test/helpers"
 
 	. "github.com/onsi/ginkgo/v2"
@@ -77,7 +78,8 @@ func StartTestHelper() {
 	}
 
 	// We rely on the "new" kubernetes/e2e setup code, since it incorporates controller-runtime logging setup
-	clusterContext := cluster.MustKindContext("kind")
+	runtimeContext := testruntime.NewContext()
+	clusterContext := cluster.MustKindContext(runtimeContext.ClusterName)
 
 	resourceClientset, err = kube2e.NewKubeResourceClientSet(ctx, clusterContext.RestConfig)
 	Expect(err).NotTo(HaveOccurred(), "can create kube resource client set")

test/kube2e/gloo/gloo_suite_test.go

@@ -23,6 +23,7 @@ import (
 	"github.com/solo-io/gloo/test/helpers"
 	"github.com/solo-io/gloo/test/kube2e"
 	"github.com/solo-io/gloo/test/kube2e/helper"
+	testruntime "github.com/solo-io/gloo/test/kubernetes/testutils/runtime"
 	glootestutils "github.com/solo-io/gloo/test/testutils"
 	"github.com/solo-io/go-utils/testutils"
 
@@ -76,7 +77,8 @@ var _ = BeforeSuite(func() {
 	}
 
 	// We rely on the "new" kubernetes/e2e setup code, since it incorporates controller-runtime logging setup
-	clusterContext := cluster.MustKindContext("kind")
+	runtimeContext := testruntime.NewContext()
+	clusterContext := cluster.MustKindContext(runtimeContext.ClusterName)
 
 	resourceClientset, err = kube2e.NewKubeResourceClientSet(ctx, clusterContext.RestConfig)
 	Expect(err).NotTo(HaveOccurred(), "can create kube resource client set")

test/kubernetes/e2e/features/validation/full_envoy_validation/suite.go

@@ -2,6 +2,7 @@ package full_envoy_validation
 
 import (
 	"context"
+	"fmt"
 
 	"github.com/solo-io/gloo/test/kubernetes/e2e"
 	testdefaults "github.com/solo-io/gloo/test/kubernetes/e2e/defaults"
@@ -77,3 +78,22 @@ func (s *testingSuite) TestRejectInvalidTransformation() {
 	s.Assert().Contains(output, "Failed to parse response template: Failed to parse "+
 		"header template ':status': [inja.exception.parser_error] (at 1:92) expected statement close, got '%'")
 }
+
+// TestLargeConfiguration checks webhook accepts large configuration when fullEnvoyValidation=true
+func (s *testingSuite) TestLargeConfiguration() {
+	s.T().Cleanup(func() {
+		err := s.testInstallation.Actions.Kubectl().DeleteFileSafe(s.ctx, validation.LargeConfiguration, "-n",
+			s.testInstallation.Metadata.InstallNamespace)
+		s.Assertions.NoError(err, "can delete large configuration")
+
+		err = s.testInstallation.Actions.Kubectl().DeleteFileSafe(s.ctx, validation.ExampleUpstream)
+		s.Assertions.NoError(err, "can delete example upstream")
+	})
+
+	err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, validation.ExampleUpstream, "-n", s.testInstallation.Metadata.InstallNamespace)
+	s.Assert().NoError(err)
+
+	err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, validation.LargeConfiguration, "-n",
+		s.testInstallation.Metadata.InstallNamespace)
+	fmt.Println(err)
+}