Add max-connections-per-listener config option (#6058)

Setting Max Connections Per Listener sets the limit on the number of active connections to a listener. Signed-off-by: Edwin Xie <[email protected]> Co-authored-by: Christian Ang <[email protected]>
projectcontour · Jan 10, 2024 · 9e8e129 · 9e8e129
1 parent d82528d
commit 9e8e129
Show file tree

Hide file tree

Showing 16 changed files with 395 additions and 8 deletions.
diff --git a/apis/projectcontour/v1alpha1/contourconfig.go b/apis/projectcontour/v1alpha1/contourconfig.go
@@ -438,6 +438,13 @@ type EnvoyListenerConfig struct {
 	// +kubebuilder:validation:Minimum=1
 	// +optional
 	HTTP2MaxConcurrentStreams *uint32 `json:"httpMaxConcurrentStreams,omitempty"`
+
+	// Defines the limit on number of active connections to a listener. The limit is applied
+	// per listener. The default value when this is not set is unlimited.
+	//
+	// +kubebuilder:validation:Minimum=1
+	// +optional
+	MaxConnectionsPerListener *uint32 `json:"maxConnectionsPerListener,omitempty"`
 }
 
 // SocketOptions defines configurable socket options for Envoy listeners.

diff --git a/apis/projectcontour/v1alpha1/zz_generated.deepcopy.go b/apis/projectcontour/v1alpha1/zz_generated.deepcopy.go
diff --git a/changelogs/unreleased/6058-flawedmatrix-minor.md b/changelogs/unreleased/6058-flawedmatrix-minor.md
@@ -0,0 +1,5 @@
+## Allow setting connection limit per listener
+
+Adds a `listeners.max-connections-per-listener` config option to Contour config file and `spec.envoy.listener.maxConnectionsPerListener` to the ContourConfiguration CRD.
+
+Setting the max connection limit per listener field limits the number of active connections to a listener. The default, if unset, is unlimited.
diff --git a/cmd/contour/serve.go b/cmd/contour/serve.go
@@ -478,7 +478,8 @@ func (s *Server) doServe() error {
 		&xdscache_v3.ClusterCache{},
 		endpointHandler,
 		xdscache_v3.NewRuntimeCache(xdscache_v3.ConfigurableRuntimeSettings{
-			MaxRequestsPerIOCycle: contourConfiguration.Envoy.Listener.MaxRequestsPerIOCycle,
+			MaxRequestsPerIOCycle:     contourConfiguration.Envoy.Listener.MaxRequestsPerIOCycle,
+			MaxConnectionsPerListener: contourConfiguration.Envoy.Listener.MaxConnectionsPerListener,
 		}),
 	}
 

diff --git a/cmd/contour/servecontext.go b/cmd/contour/servecontext.go
@@ -530,6 +530,7 @@ func (ctx *serveContext) convertToContourConfigurationSpec() contour_api_v1alpha
 				MaxRequestsPerConnection:      ctx.Config.Listener.MaxRequestsPerConnection,
 				MaxRequestsPerIOCycle:         ctx.Config.Listener.MaxRequestsPerIOCycle,
 				HTTP2MaxConcurrentStreams:     ctx.Config.Listener.HTTP2MaxConcurrentStreams,
+				MaxConnectionsPerListener:     ctx.Config.Listener.MaxConnectionsPerListener,
 				TLS: &contour_api_v1alpha1.EnvoyTLS{
 					MinimumProtocolVersion: ctx.Config.TLS.MinimumProtocolVersion,
 					MaximumProtocolVersion: ctx.Config.TLS.MaximumProtocolVersion,

diff --git a/cmd/contour/servecontext_test.go b/cmd/contour/servecontext_test.go
@@ -903,11 +903,13 @@ func TestConvertServeContext(t *testing.T) {
 			getServeContext: func(ctx *serveContext) *serveContext {
 				ctx.Config.Listener.MaxRequestsPerIOCycle = ref.To(uint32(10))
 				ctx.Config.Listener.HTTP2MaxConcurrentStreams = ref.To(uint32(30))
+				ctx.Config.Listener.MaxConnectionsPerListener = ref.To(uint32(50))
 				return ctx
 			},
 			getContourConfiguration: func(cfg contour_api_v1alpha1.ContourConfigurationSpec) contour_api_v1alpha1.ContourConfigurationSpec {
 				cfg.Envoy.Listener.MaxRequestsPerIOCycle = ref.To(uint32(10))
 				cfg.Envoy.Listener.HTTP2MaxConcurrentStreams = ref.To(uint32(30))
+				cfg.Envoy.Listener.MaxConnectionsPerListener = ref.To(uint32(50))
 				return cfg
 			},
 		},

diff --git a/examples/contour/01-crds.yaml b/examples/contour/01-crds.yaml
@@ -280,6 +280,13 @@ spec:
                         format: int32
                         minimum: 1
                         type: integer
+                      maxConnectionsPerListener:
+                        description: Defines the limit on number of active connections
+                          to a listener. The limit is applied per listener. The default
+                          value when this is not set is unlimited.
+                        format: int32
+                        minimum: 1
+                        type: integer
                       maxRequestsPerConnection:
                         description: Defines the maximum requests for downstream connections.
                           If not specified, there is no limit. see https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/protocol.proto#envoy-v3-api-msg-config-core-v3-httpprotocoloptions
@@ -3925,6 +3932,13 @@ spec:
                             format: int32
                             minimum: 1
                             type: integer
+                          maxConnectionsPerListener:
+                            description: Defines the limit on number of active connections
+                              to a listener. The limit is applied per listener. The
+                              default value when this is not set is unlimited.
+                            format: int32
+                            minimum: 1
+                            type: integer
                           maxRequestsPerConnection:
                             description: Defines the maximum requests for downstream
                               connections. If not specified, there is no limit. see

diff --git a/examples/render/contour-deployment.yaml b/examples/render/contour-deployment.yaml
@@ -499,6 +499,13 @@ spec:
                         format: int32
                         minimum: 1
                         type: integer
+                      maxConnectionsPerListener:
+                        description: Defines the limit on number of active connections
+                          to a listener. The limit is applied per listener. The default
+                          value when this is not set is unlimited.
+                        format: int32
+                        minimum: 1
+                        type: integer
                       maxRequestsPerConnection:
                         description: Defines the maximum requests for downstream connections.
                           If not specified, there is no limit. see https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/protocol.proto#envoy-v3-api-msg-config-core-v3-httpprotocoloptions
@@ -4144,6 +4151,13 @@ spec:
                             format: int32
                             minimum: 1
                             type: integer
+                          maxConnectionsPerListener:
+                            description: Defines the limit on number of active connections
+                              to a listener. The limit is applied per listener. The
+                              default value when this is not set is unlimited.
+                            format: int32
+                            minimum: 1
+                            type: integer
                           maxRequestsPerConnection:
                             description: Defines the maximum requests for downstream
                               connections. If not specified, there is no limit. see

diff --git a/examples/render/contour-gateway-provisioner.yaml b/examples/render/contour-gateway-provisioner.yaml
@@ -291,6 +291,13 @@ spec:
                         format: int32
                         minimum: 1
                         type: integer
+                      maxConnectionsPerListener:
+                        description: Defines the limit on number of active connections
+                          to a listener. The limit is applied per listener. The default
+                          value when this is not set is unlimited.
+                        format: int32
+                        minimum: 1
+                        type: integer
                       maxRequestsPerConnection:
                         description: Defines the maximum requests for downstream connections.
                           If not specified, there is no limit. see https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/protocol.proto#envoy-v3-api-msg-config-core-v3-httpprotocoloptions
@@ -3936,6 +3943,13 @@ spec:
                             format: int32
                             minimum: 1
                             type: integer
+                          maxConnectionsPerListener:
+                            description: Defines the limit on number of active connections
+                              to a listener. The limit is applied per listener. The
+                              default value when this is not set is unlimited.
+                            format: int32
+                            minimum: 1
+                            type: integer
                           maxRequestsPerConnection:
                             description: Defines the maximum requests for downstream
                               connections. If not specified, there is no limit. see

diff --git a/examples/render/contour-gateway.yaml b/examples/render/contour-gateway.yaml
@@ -502,6 +502,13 @@ spec:
                         format: int32
                         minimum: 1
                         type: integer
+                      maxConnectionsPerListener:
+                        description: Defines the limit on number of active connections
+                          to a listener. The limit is applied per listener. The default
+                          value when this is not set is unlimited.
+                        format: int32
+                        minimum: 1
+                        type: integer
                       maxRequestsPerConnection:
                         description: Defines the maximum requests for downstream connections.
                           If not specified, there is no limit. see https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/protocol.proto#envoy-v3-api-msg-config-core-v3-httpprotocoloptions
@@ -4147,6 +4154,13 @@ spec:
                             format: int32
                             minimum: 1
                             type: integer
+                          maxConnectionsPerListener:
+                            description: Defines the limit on number of active connections
+                              to a listener. The limit is applied per listener. The
+                              default value when this is not set is unlimited.
+                            format: int32
+                            minimum: 1
+                            type: integer
                           maxRequestsPerConnection:
                             description: Defines the maximum requests for downstream
                               connections. If not specified, there is no limit. see

diff --git a/examples/render/contour.yaml b/examples/render/contour.yaml
@@ -499,6 +499,13 @@ spec:
                         format: int32
                         minimum: 1
                         type: integer
+                      maxConnectionsPerListener:
+                        description: Defines the limit on number of active connections
+                          to a listener. The limit is applied per listener. The default
+                          value when this is not set is unlimited.
+                        format: int32
+                        minimum: 1
+                        type: integer
                       maxRequestsPerConnection:
                         description: Defines the maximum requests for downstream connections.
                           If not specified, there is no limit. see https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/protocol.proto#envoy-v3-api-msg-config-core-v3-httpprotocoloptions
@@ -4144,6 +4151,13 @@ spec:
                             format: int32
                             minimum: 1
                             type: integer
+                          maxConnectionsPerListener:
+                            description: Defines the limit on number of active connections
+                              to a listener. The limit is applied per listener. The
+                              default value when this is not set is unlimited.
+                            format: int32
+                            minimum: 1
+                            type: integer
                           maxRequestsPerConnection:
                             description: Defines the maximum requests for downstream
                               connections. If not specified, there is no limit. see

diff --git a/internal/xdscache/v3/runtime.go b/internal/xdscache/v3/runtime.go
@@ -14,6 +14,9 @@
 package v3
 
 import (
+	"fmt"
+	"sync"
+
 	resource "github.com/envoyproxy/go-control-plane/pkg/resource/v3"
 	"github.com/projectcontour/contour/internal/contour"
 	"github.com/projectcontour/contour/internal/dag"
@@ -24,42 +27,84 @@ import (
 )
 
 type ConfigurableRuntimeSettings struct {
-	MaxRequestsPerIOCycle *uint32
+	MaxRequestsPerIOCycle     *uint32
+	MaxConnectionsPerListener *uint32
 }
 
 // RuntimeCache manages the contents of the gRPC RTDS cache.
 type RuntimeCache struct {
 	contour.Cond
 	runtimeKV map[string]*structpb.Value
+
+	dynamicRuntimeKV map[string]*structpb.Value
+	mu               sync.Mutex
+
+	maxConnectionsPerListener *uint32
 }
 
 // NewRuntimeCache builds a RuntimeCache with the provided runtime
 // settings that will be set in the runtime layer configured by Contour.
 func NewRuntimeCache(runtimeSettings ConfigurableRuntimeSettings) *RuntimeCache {
 	runtimeKV := make(map[string]*structpb.Value)
+	dynamicRuntimeKV := make(map[string]*structpb.Value)
 	if runtimeSettings.MaxRequestsPerIOCycle != nil && *runtimeSettings.MaxRequestsPerIOCycle > 0 {
 		runtimeKV["http.max_requests_per_io_cycle"] = structpb.NewNumberValue(float64(*runtimeSettings.MaxRequestsPerIOCycle))
 	}
-	return &RuntimeCache{runtimeKV: runtimeKV}
+	return &RuntimeCache{
+		runtimeKV:                 runtimeKV,
+		dynamicRuntimeKV:          dynamicRuntimeKV,
+		maxConnectionsPerListener: runtimeSettings.MaxConnectionsPerListener,
+	}
+}
+
+func (c *RuntimeCache) buildDynamicLayer() []proto.Message {
+	values := make(map[string]*structpb.Value)
+	for k, v := range c.runtimeKV {
+		values[k] = v
+	}
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	for k, v := range c.dynamicRuntimeKV {
+		values[k] = v
+	}
+	return protobuf.AsMessages(envoy_v3.RuntimeLayers(values))
 }
 
-// Contents returns all Runtime layers.
+// Contents returns all Runtime layers (currently only the dynamic layer).
 func (c *RuntimeCache) Contents() []proto.Message {
-	return protobuf.AsMessages(envoy_v3.RuntimeLayers(c.runtimeKV))
+	return c.buildDynamicLayer()
 }
 
 // Query returns only the "dynamic" layer if requested, otherwise empty.
 func (c *RuntimeCache) Query(names []string) []proto.Message {
 	for _, name := range names {
 		if name == envoy_v3.DynamicRuntimeLayerName {
-			return protobuf.AsMessages(envoy_v3.RuntimeLayers(c.runtimeKV))
+			return c.buildDynamicLayer()
 		}
 	}
 	return []proto.Message{}
 }
 
 func (*RuntimeCache) TypeURL() string { return resource.RuntimeType }
 
-func (c *RuntimeCache) OnChange(_ *dag.DAG) {
-	// DAG changes do not affect runtime layers at the moment.
+// Update replaces the contents of the cache with the supplied map.
+func (c *RuntimeCache) Update(v map[string]*structpb.Value) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	c.dynamicRuntimeKV = v
+	c.Cond.Notify()
+}
+
+func (c *RuntimeCache) OnChange(root *dag.DAG) {
+	dynamicRuntimeKV := make(map[string]*structpb.Value)
+	if c.maxConnectionsPerListener != nil && *c.maxConnectionsPerListener > 0 {
+		for _, listener := range root.Listeners {
+			fieldName := fmt.Sprintf("envoy.resource_limits.listener.%s.connection_limit", listener.Name)
+
+			dynamicRuntimeKV[fieldName] = structpb.NewNumberValue(float64(*c.maxConnectionsPerListener))
+		}
+	}
+
+	c.Update(dynamicRuntimeKV)
 }