Adds support for global circuit budget (#6013)

Solves #6001

Signed-off-by: Sotiris Nanopoulos <[email protected]>

apis/projectcontour/v1alpha1/contourconfig.go

@@ -107,6 +107,21 @@ const (
 	EnvoyServerType XDSServerType = "envoy"
 )
 
+type GlobalCircuitBreakerDefaults struct {
+	// The maximum number of connections that a single Envoy instance allows to the Kubernetes Service; defaults to 1024.
+	// +optional
+	MaxConnections uint32 `json:"maxConnections,omitempty" yaml:"max-connections,omitempty"`
+	// The maximum number of pending requests that a single Envoy instance allows to the Kubernetes Service; defaults to 1024.
+	// +optional
+	MaxPendingRequests uint32 `json:"maxPendingRequests,omitempty" yaml:"max-pending-requests,omitempty"`
+	// The maximum parallel requests a single Envoy instance allows to the Kubernetes Service; defaults to 1024
+	// +optional
+	MaxRequests uint32 `json:"maxRequests,omitempty" yaml:"max-requests,omitempty"`
+	// The maximum number of parallel retries a single Envoy instance allows to the Kubernetes Service; defaults to 3.
+	// +optional
+	MaxRetries uint32 `json:"maxRetries,omitempty" yaml:"max-retries,omitempty"`
+}
+
 // XDSServerConfig holds the config for the Contour xDS server.
 type XDSServerConfig struct {
 	// Defines the XDSServer to use for `contour serve`.
@@ -689,6 +704,12 @@ type ClusterParameters struct {
 	// +optional
 	PerConnectionBufferLimitBytes *uint32 `json:"per-connection-buffer-limit-bytes,omitempty"`
 
+	// GlobalCircuitBreakerDefaults specifies default circuit breaker budget across all services.
+	// If defined, this will be used as the default for all services.
+	//
+	// +optional
+	GlobalCircuitBreakerDefaults *GlobalCircuitBreakerDefaults `json:"circuitBreakers,omitempty"`
+
 	// UpstreamTLS contains the TLS policy parameters for upstream connections
 	//
 	// +optional

changelogs/unreleased/6013-davinci26-minor.md

@@ -0,0 +1,3 @@
+## Support for Global Circuit Breaker Policy
+
+The way [circuit-breaker-annotations](https://projectcontour.io/docs/1.27/config/annotations/) work currently is that when not present they are being defaulted to Envoy defaults. The Envoy defaults can be quite low for larger clusters with more traffic so if a user accidentally deletes them or unset them this cause an issue. With this change we are providing contour administrators the ability to provide global defaults that are good. In that case even if the user forgets to set them or deletes them they can have the safety net of good defaults. They can be configured via [cluster.circuit-breakers](https://projectcontour.io/docs/1.28/configuration/#circuit-breakers)  or via `ContourConfiguration`` CRD in [spec.envoy.cluster.circuitBreakers](https://projectcontour.io/docs/1.28/config/api/#projectcontour.io/v1alpha1.GlobalCircuitBreakerDefaults)

cmd/contour/serve.go

@@ -560,6 +560,7 @@ func (s *Server) doServe() error {
 		globalRateLimitService:             contourConfiguration.RateLimitService,
 		maxRequestsPerConnection:           contourConfiguration.Envoy.Cluster.MaxRequestsPerConnection,
 		perConnectionBufferLimitBytes:      contourConfiguration.Envoy.Cluster.PerConnectionBufferLimitBytes,
+		globalCircuitBreakerDefaults:       contourConfiguration.Envoy.Cluster.GlobalCircuitBreakerDefaults,
 		upstreamTLS: &dag.UpstreamTLS{
 			MinimumProtocolVersion: annotation.TLSVersion(contourConfiguration.Envoy.Cluster.UpstreamTLS.MinimumProtocolVersion, "1.2"),
 			MaximumProtocolVersion: annotation.TLSVersion(contourConfiguration.Envoy.Cluster.UpstreamTLS.MaximumProtocolVersion, "1.3"),
@@ -1122,6 +1123,7 @@ type dagBuilderConfig struct {
 	maxRequestsPerConnection           *uint32
 	perConnectionBufferLimitBytes      *uint32
 	globalRateLimitService             *contour_api_v1alpha1.RateLimitServiceConfig
+	globalCircuitBreakerDefaults       *contour_api_v1alpha1.GlobalCircuitBreakerDefaults
 	upstreamTLS                        *dag.UpstreamTLS
 }
 
@@ -1192,6 +1194,7 @@ func (s *Server) getDAGBuilder(dbc dagBuilderConfig) *dag.Builder {
 			ConnectTimeout:                dbc.connectTimeout,
 			MaxRequestsPerConnection:      dbc.maxRequestsPerConnection,
 			PerConnectionBufferLimitBytes: dbc.perConnectionBufferLimitBytes,
+			GlobalCircuitBreakerDefaults:  dbc.globalCircuitBreakerDefaults,
 			SetSourceMetadataOnRoutes:     true,
 			UpstreamTLS:                   dbc.upstreamTLS,
 		},
@@ -1217,6 +1220,7 @@ func (s *Server) getDAGBuilder(dbc dagBuilderConfig) *dag.Builder {
 			GlobalRateLimitService:        dbc.globalRateLimitService,
 			PerConnectionBufferLimitBytes: dbc.perConnectionBufferLimitBytes,
 			SetSourceMetadataOnRoutes:     true,
+			GlobalCircuitBreakerDefaults:  dbc.globalCircuitBreakerDefaults,
 			UpstreamTLS:                   dbc.upstreamTLS,
 		},
 	}
@@ -1229,6 +1233,7 @@ func (s *Server) getDAGBuilder(dbc dagBuilderConfig) *dag.Builder {
 			MaxRequestsPerConnection:      dbc.maxRequestsPerConnection,
 			PerConnectionBufferLimitBytes: dbc.perConnectionBufferLimitBytes,
 			SetSourceMetadataOnRoutes:     true,
+			GlobalCircuitBreakerDefaults:  dbc.globalCircuitBreakerDefaults,
 		})
 	}
 

cmd/contour/serve_test.go

@@ -123,6 +123,31 @@ func TestGetDAGBuilder(t *testing.T) {
 		assert.ElementsMatch(t, map[string]string(nil), ingressProcessor.ResponseHeadersPolicy.Remove)
 	})
 
+	t.Run("GlobalCircuitBreakerDefaults specified for all processors", func(t *testing.T) {
+		g := contour_api_v1alpha1.GlobalCircuitBreakerDefaults{
+			MaxConnections: 100,
+		}
+
+		serve := &Server{
+			log: logrus.StandardLogger(),
+		}
+		got := serve.getDAGBuilder(dagBuilderConfig{
+			gatewayControllerName:        "projectcontour.io/gateway-controller",
+			rootNamespaces:               []string{},
+			dnsLookupFamily:              contour_api_v1alpha1.AutoClusterDNSFamily,
+			globalCircuitBreakerDefaults: &g,
+		})
+
+		iProcessor := mustGetIngressProcessor(t, got)
+		assert.EqualValues(t, iProcessor.GlobalCircuitBreakerDefaults, &g)
+
+		hProcessor := mustGetHTTPProxyProcessor(t, got)
+		assert.EqualValues(t, hProcessor.GlobalCircuitBreakerDefaults, &g)
+
+		gProcessor := mustGetGatewayAPIProcessor(t, got)
+		assert.EqualValues(t, gProcessor.GlobalCircuitBreakerDefaults, &g)
+	})
+
 	t.Run("request and response headers policy specified for ingress", func(t *testing.T) {
 		policy := &contour_api_v1alpha1.PolicyConfig{
 			RequestHeadersPolicy: &contour_api_v1alpha1.HeadersPolicy{
@@ -192,6 +217,19 @@ func TestGetDAGBuilder(t *testing.T) {
 	// TODO(3453): test additional properties of the DAG builder (processor fields, cache fields, Gateway tests (requires a client fake))
 }
 
+func mustGetGatewayAPIProcessor(t *testing.T, builder *dag.Builder) *dag.GatewayAPIProcessor {
+	t.Helper()
+	for i := range builder.Processors {
+		found, ok := builder.Processors[i].(*dag.GatewayAPIProcessor)
+		if ok {
+			return found
+		}
+	}
+
+	require.FailNow(t, "GatewayAPIProcessor not found in list of DAG builder's processors")
+	return nil
+}
+
 func mustGetHTTPProxyProcessor(t *testing.T, builder *dag.Builder) *dag.HTTPProxyProcessor {
 	t.Helper()
 	for i := range builder.Processors {

cmd/contour/servecontext.go

@@ -572,6 +572,7 @@ func (ctx *serveContext) convertToContourConfigurationSpec() contour_api_v1alpha
 				DNSLookupFamily:               dnsLookupFamily,
 				MaxRequestsPerConnection:      ctx.Config.Cluster.MaxRequestsPerConnection,
 				PerConnectionBufferLimitBytes: ctx.Config.Cluster.PerConnectionBufferLimitBytes,
+				GlobalCircuitBreakerDefaults:  ctx.Config.Cluster.GlobalCircuitBreakerDefaults,
 				UpstreamTLS: &contour_api_v1alpha1.EnvoyTLS{
 					MinimumProtocolVersion: ctx.Config.Cluster.UpstreamTLS.MinimumProtocolVersion,
 					MaximumProtocolVersion: ctx.Config.Cluster.UpstreamTLS.MaximumProtocolVersion,

cmd/contour/servecontext_test.go

@@ -473,7 +473,8 @@ func TestConvertServeContext(t *testing.T) {
 					ConnectTimeout:        ref.To("2s"),
 				},
 				Cluster: &contour_api_v1alpha1.ClusterParameters{
-					DNSLookupFamily: contour_api_v1alpha1.AutoClusterDNSFamily,
+					DNSLookupFamily:              contour_api_v1alpha1.AutoClusterDNSFamily,
+					GlobalCircuitBreakerDefaults: nil,
 					UpstreamTLS: &contour_api_v1alpha1.EnvoyTLS{
 						MinimumProtocolVersion: "",
 						MaximumProtocolVersion: "",
@@ -776,6 +777,26 @@ func TestConvertServeContext(t *testing.T) {
 				return cfg
 			},
 		},
+		"global circuit breaker defaults": {
+			getServeContext: func(ctx *serveContext) *serveContext {
+				ctx.Config.Cluster.GlobalCircuitBreakerDefaults = &contour_api_v1alpha1.GlobalCircuitBreakerDefaults{
+					MaxConnections:     4,
+					MaxPendingRequests: 5,
+					MaxRequests:        6,
+					MaxRetries:         7,
+				}
+				return ctx
+			},
+			getContourConfiguration: func(cfg contour_api_v1alpha1.ContourConfigurationSpec) contour_api_v1alpha1.ContourConfigurationSpec {
+				cfg.Envoy.Cluster.GlobalCircuitBreakerDefaults = &contour_api_v1alpha1.GlobalCircuitBreakerDefaults{
+					MaxConnections:     4,
+					MaxPendingRequests: 5,
+					MaxRequests:        6,
+					MaxRetries:         7,
+				}
+				return cfg
+			},
+		},
 		"global external authorization": {
 			getServeContext: func(ctx *serveContext) *serveContext {
 				ctx.Config.GlobalExternalAuthorization = config.GlobalExternalAuthorization{

examples/contour/01-crds.yaml

@@ -78,6 +78,36 @@ spec:
                     description: Cluster holds various configurable Envoy cluster
                       values that can be set in the config file.
                     properties:
+                      circuitBreakers:
+                        description: GlobalCircuitBreakerDefaults specifies default
+                          circuit breaker budget across all services. If defined,
+                          this will be used as the default for all services.
+                        properties:
+                          maxConnections:
+                            description: The maximum number of connections that a
+                              single Envoy instance allows to the Kubernetes Service;
+                              defaults to 1024.
+                            format: int32
+                            type: integer
+                          maxPendingRequests:
+                            description: The maximum number of pending requests that
+                              a single Envoy instance allows to the Kubernetes Service;
+                              defaults to 1024.
+                            format: int32
+                            type: integer
+                          maxRequests:
+                            description: The maximum parallel requests a single Envoy
+                              instance allows to the Kubernetes Service; defaults
+                              to 1024
+                            format: int32
+                            type: integer
+                          maxRetries:
+                            description: The maximum number of parallel retries a
+                              single Envoy instance allows to the Kubernetes Service;
+                              defaults to 3.
+                            format: int32
+                            type: integer
+                        type: object
                       dnsLookupFamily:
                         description: "DNSLookupFamily defines how external names are
                           looked up When configured as V4, the DNS resolver will only
@@ -3684,6 +3714,36 @@ spec:
                         description: Cluster holds various configurable Envoy cluster
                           values that can be set in the config file.
                         properties:
+                          circuitBreakers:
+                            description: GlobalCircuitBreakerDefaults specifies default
+                              circuit breaker budget across all services. If defined,
+                              this will be used as the default for all services.
+                            properties:
+                              maxConnections:
+                                description: The maximum number of connections that
+                                  a single Envoy instance allows to the Kubernetes
+                                  Service; defaults to 1024.
+                                format: int32
+                                type: integer
+                              maxPendingRequests:
+                                description: The maximum number of pending requests
+                                  that a single Envoy instance allows to the Kubernetes
+                                  Service; defaults to 1024.
+                                format: int32
+                                type: integer
+                              maxRequests:
+                                description: The maximum parallel requests a single
+                                  Envoy instance allows to the Kubernetes Service;
+                                  defaults to 1024
+                                format: int32
+                                type: integer
+                              maxRetries:
+                                description: The maximum number of parallel retries
+                                  a single Envoy instance allows to the Kubernetes
+                                  Service; defaults to 3.
+                                format: int32
+                                type: integer
+                            type: object
                           dnsLookupFamily:
                             description: "DNSLookupFamily defines how external names
                               are looked up When configured as V4, the DNS resolver

examples/render/contour-deployment.yaml

@@ -297,6 +297,36 @@ spec:
                     description: Cluster holds various configurable Envoy cluster
                       values that can be set in the config file.
                     properties:
+                      circuitBreakers:
+                        description: GlobalCircuitBreakerDefaults specifies default
+                          circuit breaker budget across all services. If defined,
+                          this will be used as the default for all services.
+                        properties:
+                          maxConnections:
+                            description: The maximum number of connections that a
+                              single Envoy instance allows to the Kubernetes Service;
+                              defaults to 1024.
+                            format: int32
+                            type: integer
+                          maxPendingRequests:
+                            description: The maximum number of pending requests that
+                              a single Envoy instance allows to the Kubernetes Service;
+                              defaults to 1024.
+                            format: int32
+                            type: integer
+                          maxRequests:
+                            description: The maximum parallel requests a single Envoy
+                              instance allows to the Kubernetes Service; defaults
+                              to 1024
+                            format: int32
+                            type: integer
+                          maxRetries:
+                            description: The maximum number of parallel retries a
+                              single Envoy instance allows to the Kubernetes Service;
+                              defaults to 3.
+                            format: int32
+                            type: integer
+                        type: object
                       dnsLookupFamily:
                         description: "DNSLookupFamily defines how external names are
                           looked up When configured as V4, the DNS resolver will only
@@ -3903,6 +3933,36 @@ spec:
                         description: Cluster holds various configurable Envoy cluster
                           values that can be set in the config file.
                         properties:
+                          circuitBreakers:
+                            description: GlobalCircuitBreakerDefaults specifies default
+                              circuit breaker budget across all services. If defined,
+                              this will be used as the default for all services.
+                            properties:
+                              maxConnections:
+                                description: The maximum number of connections that
+                                  a single Envoy instance allows to the Kubernetes
+                                  Service; defaults to 1024.
+                                format: int32
+                                type: integer
+                              maxPendingRequests:
+                                description: The maximum number of pending requests
+                                  that a single Envoy instance allows to the Kubernetes
+                                  Service; defaults to 1024.
+                                format: int32
+                                type: integer
+                              maxRequests:
+                                description: The maximum parallel requests a single
+                                  Envoy instance allows to the Kubernetes Service;
+                                  defaults to 1024
+                                format: int32
+                                type: integer
+                              maxRetries:
+                                description: The maximum number of parallel retries
+                                  a single Envoy instance allows to the Kubernetes
+                                  Service; defaults to 3.
+                                format: int32
+                                type: integer
+                            type: object
                           dnsLookupFamily:
                             description: "DNSLookupFamily defines how external names
                               are looked up When configured as V4, the DNS resolver