Lb maxconcurrent req (#15)

* update k8s api version to v0.27.2 * loadbalancer max concurrent requests annotation * Add test Signed-off-by: Haardik Dharma <[email protected]> * Update unit tests Signed-off-by: Haardik Dharma <[email protected]> * Add note about downtime in README Signed-off-by: Haardik Dharma <[email protected]> --------- Signed-off-by: Haardik Dharma <[email protected]> Co-authored-by: Haardik Dharma <[email protected]>
civo · Jun 14, 2023 · 4b2f3e7 · 4b2f3e7
1 parent 665e252
commit 4b2f3e7
Show file tree

Hide file tree

Showing 6 changed files with 586 additions and 423 deletions.
diff --git a/README.md b/README.md
@@ -19,6 +19,7 @@ Read More: https://www.civo.com/learn/managing-external-load-balancers-on-civo
 | Annotation | Description | Example Values |
 |------------|-------------|----------------|
 | kubernetes.civo.com/firewall-id | If provided, an existing Firewall will be used. | 03093EF6-31E6-48B1-AB1D-152AC3A8C90A |
+| kubernetes.civo.com/max-concurrent-requests | If provided, the user can specify the max number of concurrent requests a loadbalancer could afford. Note: Defaults to 10000. Whenever user updates this value, there will be a downtime for the loadbalancer (usually < 1 minute)  | 10000, 20000, 30000, 40000, 50000 |
 | kubernetes.civo.com/loadbalancer-enable-proxy-protocol | If set, a proxy-protocol header will be sent via the load balancer. <br /><br />NB: This requires support from the Service End Points within the cluster. | send-proxy<br />send-proxy-v2 |
 | kubernetes.civo.com/loadbalancer-algorithm | Custom the algorithm the external load balancer uses | round_robin<br />least_connections |
 | kubernetes.civo.com/ipv4-address | If set, LoadBalancer will have the mentioned IP as the public IP. Please note: the reserved IP should be present in the account before claiming it. | 10.0.0.20<br/> my-reserved-ip |

diff --git a/cloud-controller-manager/civo/loadbalancer.go b/cloud-controller-manager/civo/loadbalancer.go
@@ -3,6 +3,7 @@ package civo
 import (
 	"context"
 	"fmt"
+	"strconv"
 	"strings"
 
 	v1 "k8s.io/api/core/v1"
@@ -23,6 +24,9 @@ const (
 	// annotationCivoFirewallID is the annotation specifying the CivoFirewall ID.
 	annotationCivoFirewallID = "kubernetes.civo.com/firewall-id"
 
+	// annotationCivoLoadBalancerMaxConcurrentRequests is the annotation specifying the max number of concurrent requests a CivoLoadBalancer can afford
+	annotationCivoLoadBalancerMaxConcurrentRequests = "kubernetes.civo.com/max-concurrent-requests"
+
 	// annotationCivoLoadBalancerEnableProxyProtocol is the annotation specifying whether PROXY protocol should be enabled.
 	annotationCivoLoadBalancerEnableProxyProtocol = "kubernetes.civo.com/loadbalancer-enable-proxy-protocol"
 
@@ -135,6 +139,15 @@ func (l *loadbalancer) updateLBConfig(civolb *civogo.LoadBalancer, service *v1.S
 		lbuc.FirewallID = firewallID
 	}
 
+	if maxConcurrentReq := getMaxConcurrentRequests(service); maxConcurrentReq != "" {
+		maxReq, err := strconv.Atoi(maxConcurrentReq)
+		if err != nil {
+			klog.Errorf("Unable to parse Loadbalancer MaxConcurrentRequests annotation, error: %s", err.Error())
+			return nil, err
+		}
+		lbuc.MaxConcurrentRequests = &maxReq
+	}
+
 	backends := []civogo.LoadBalancerBackendConfig{}
 	for _, port := range service.Spec.Ports {
 		for _, node := range nodes {
@@ -257,6 +270,9 @@ func (l *loadbalancer) UpdateLoadBalancer(ctx context.Context, clusterName strin
 	if civolb.EnableProxyProtocol != ulb.EnableProxyProtocol {
 		updateServiceAnnotation(service, annotationCivoLoadBalancerEnableProxyProtocol, ulb.EnableProxyProtocol)
 	}
+	if civolb.MaxConcurrentRequests != ulb.MaxConcurrentRequests {
+		updateServiceAnnotation(service, annotationCivoLoadBalancerMaxConcurrentRequests, fmt.Sprint(ulb.MaxConcurrentRequests))
+	}
 
 	return nil
 }
@@ -315,6 +331,7 @@ func getLoadBalancer(ctx context.Context, c civogo.Clienter, kclient kubernetes.
 			updateServiceAnnotation(service, annotationCivoLoadBalancerName, civolb.Name)
 			updateServiceAnnotation(service, annotationCivoClusterID, ClusterID)
 			updateServiceAnnotation(service, annotationCivoFirewallID, civolb.FirewallID)
+			updateServiceAnnotation(service, annotationCivoLoadBalancerMaxConcurrentRequests, fmt.Sprint(civolb.MaxConcurrentRequests))
 			updateServiceAnnotation(service, annotationCivoLoadBalancerAlgorithm, civolb.Algorithm)
 		}
 	}
@@ -346,6 +363,15 @@ func createLoadBalancer(ctx context.Context, clusterName string, service *v1.Ser
 		lbc.FirewallID = firewallID
 	}
 
+	if maxConcurrentReq := getMaxConcurrentRequests(service); maxConcurrentReq != "" {
+		maxReq, err := strconv.Atoi(maxConcurrentReq)
+		if err != nil {
+			klog.Errorf("Unable to parse Loadbalancer MaxConcurrentRequests annotation, error: %s", err.Error())
+			return err
+		}
+		lbc.MaxConcurrentRequests = &maxReq
+	}
+
 	backends := []civogo.LoadBalancerBackendConfig{}
 	for _, port := range service.Spec.Ports {
 		for _, node := range nodes {
@@ -373,6 +399,7 @@ func createLoadBalancer(ctx context.Context, clusterName string, service *v1.Ser
 	updateServiceAnnotation(service, annotationCivoLoadBalancerID, lb.ID)
 	updateServiceAnnotation(service, annotationCivoLoadBalancerName, lb.Name)
 	updateServiceAnnotation(service, annotationCivoLoadBalancerAlgorithm, lb.Algorithm)
+	updateServiceAnnotation(service, annotationCivoLoadBalancerMaxConcurrentRequests, fmt.Sprint(lb.MaxConcurrentRequests))
 
 	if lb.EnableProxyProtocol != "" {
 		updateServiceAnnotation(service, annotationCivoLoadBalancerEnableProxyProtocol, lb.EnableProxyProtocol)
@@ -413,6 +440,11 @@ func getFirewallID(service *v1.Service) string {
 	return service.Annotations[annotationCivoFirewallID]
 }
 
+// getMaxConcurrentRequests returns the max concurrent requests a CivoLoadbalancer could afford from the service annotation.
+func getMaxConcurrentRequests(service *v1.Service) string {
+	return service.Annotations[annotationCivoLoadBalancerMaxConcurrentRequests]
+}
+
 func getProtocol(svc *v1.Service, port v1.ServicePort) string {
 	if svc.Annotations[annotationCivoProtocol] != "" {
 		return strings.ToUpper(svc.Annotations[annotationCivoProtocol])

diff --git a/cloud-controller-manager/civo/loadbalancer_test.go b/cloud-controller-manager/civo/loadbalancer_test.go
@@ -216,6 +216,26 @@ func TestUpdateServiceAnnotation(t *testing.T) {
 				},
 			},
 		},
+		{
+			"Annotation for max concurrent requests set to 20000",
+			&corev1.Service{
+				Spec: corev1.ServiceSpec{
+					Type: corev1.ServiceTypeLoadBalancer,
+				},
+			},
+			"kubernetes.civo.com/max-concurrent-requests",
+			"20000",
+			&corev1.Service{
+				ObjectMeta: metav1.ObjectMeta{
+					Annotations: map[string]string{
+						"kubernetes.civo.com/max-concurrent-requests": "20000",
+					},
+				},
+				Spec: corev1.ServiceSpec{
+					Type: corev1.ServiceTypeLoadBalancer,
+				},
+			},
+		},
 	}
 
 	for _, test := range tests {
@@ -261,10 +281,11 @@ func TestGetLoadBalanacer(t *testing.T) {
 					Name:      "test",
 					Namespace: corev1.NamespaceDefault,
 					Annotations: map[string]string{
-						annotationCivoLoadBalancerID:   "6dc1c87d-b8a1-42cd-8fc6-8293378e5715",
-						annotationCivoFirewallID:       "fc91d382-2609-4f5c-a875-1491776fab8c",
-						annotationCivoClusterID:        "a32fe5eb-1922-43e8-81bc-7f83b4011334",
-						annotationCivoLoadBalancerName: "civo-lb-test",
+						annotationCivoLoadBalancerID:                    "6dc1c87d-b8a1-42cd-8fc6-8293378e5715",
+						annotationCivoFirewallID:                        "fc91d382-2609-4f5c-a875-1491776fab8c",
+						annotationCivoClusterID:                         "a32fe5eb-1922-43e8-81bc-7f83b4011334",
+						annotationCivoLoadBalancerName:                  "civo-lb-test",
+						annotationCivoLoadBalancerMaxConcurrentRequests: "10000",
 					},
 				},
 				Spec: corev1.ServiceSpec{
@@ -281,13 +302,14 @@ func TestGetLoadBalanacer(t *testing.T) {
 			},
 			store: []civogo.LoadBalancer{
 				{
-					ID:         "6dc1c87d-b8a1-42cd-8fc6-8293378e5715",
-					Name:       "civo-lb-test",
-					Algorithm:  "round-robin",
-					PublicIP:   "192.168.11.11",
-					FirewallID: "fc91d382-2609-4f5c-a875-1491776fab8c",
-					ClusterID:  "a32fe5eb-1922-43e8-81bc-7f83b4011334",
-					State:      statusAvailable,
+					ID:                    "6dc1c87d-b8a1-42cd-8fc6-8293378e5715",
+					Name:                  "civo-lb-test",
+					Algorithm:             "round-robin",
+					PublicIP:              "192.168.11.11",
+					FirewallID:            "fc91d382-2609-4f5c-a875-1491776fab8c",
+					ClusterID:             "a32fe5eb-1922-43e8-81bc-7f83b4011334",
+					State:                 statusAvailable,
+					MaxConcurrentRequests: 10000,
 				},
 			},
 			expected: &corev1.LoadBalancerStatus{
@@ -683,10 +705,11 @@ func TestUpdateLoadBalancer(t *testing.T) {
 					Name:      "test-update",
 					Namespace: corev1.NamespaceDefault,
 					Annotations: map[string]string{
-						annotationCivoLoadBalancerID:                  "6dc1c87d-b8a1-42cd-8fc6-8293378e5715",
-						annotationCivoClusterID:                       "a32fe5eb-1922-43e8-81bc-7f83b4011334",
-						annotationCivoLoadBalancerAlgorithm:           "least-connections",
-						annotationCivoLoadBalancerEnableProxyProtocol: "send-proxy",
+						annotationCivoLoadBalancerID:                    "6dc1c87d-b8a1-42cd-8fc6-8293378e5715",
+						annotationCivoClusterID:                         "a32fe5eb-1922-43e8-81bc-7f83b4011334",
+						annotationCivoLoadBalancerAlgorithm:             "least-connections",
+						annotationCivoLoadBalancerEnableProxyProtocol:   "send-proxy",
+						annotationCivoLoadBalancerMaxConcurrentRequests: "30000",
 					},
 				},
 				Spec: corev1.ServiceSpec{
@@ -746,6 +769,7 @@ func TestUpdateLoadBalancer(t *testing.T) {
 							IP: "192.168.1.2",
 						},
 					},
+					MaxConcurrentRequests: 30000,
 				},
 			},
 			err: nil,
@@ -791,6 +815,10 @@ func TestUpdateLoadBalancer(t *testing.T) {
 			if svc.Annotations[annotationCivoLoadBalancerAlgorithm] != "" {
 				g.Expect(svc.Annotations[annotationCivoLoadBalancerAlgorithm]).To(Equal("least-connections"))
 			}
+
+			if svc.Annotations[annotationCivoLoadBalancerMaxConcurrentRequests] != "" {
+				g.Expect(svc.Annotations[annotationCivoLoadBalancerMaxConcurrentRequests]).To(Equal("30000"))
+			}
 		})
 	}
 }

diff --git a/e2e/suite_test.go b/e2e/suite_test.go
@@ -239,7 +239,7 @@ func run(secret *corev1.Secret, kubeconfig string) {
 
 	// Set the provider name
 	opts.KubeCloudShared.CloudProvider.Name = civo.ProviderName
-	opts.Kubeconfig = "./kubeconfig"
+	opts.Generic.ClientConnection.Kubeconfig = "./kubeconfig"
 	// Disable Cloud Routes
 	opts.KubeCloudShared.ConfigureCloudRoutes = false
 

diff --git a/go.mod b/go.mod
@@ -1,33 +1,115 @@
 module github.com/civo/civo-cloud-controller-manager
 
-go 1.16
+go 1.20
 
 require (
-	github.com/civo/civogo v0.3.19
-	github.com/emicklei/go-restful v2.11.1+incompatible // indirect
-	github.com/go-openapi/swag v0.19.7 // indirect
-	github.com/google/go-cmp v0.5.6 // indirect
-	github.com/google/uuid v1.3.0 // indirect
+	github.com/civo/civogo v0.3.35
 	github.com/joho/godotenv v1.4.0
-	github.com/onsi/gomega v1.19.0
-	github.com/sirupsen/logrus v1.8.1 // indirect
-	k8s.io/api v0.21.1
-	k8s.io/apimachinery v0.21.1
-	k8s.io/client-go v12.0.0+incompatible
-	k8s.io/cloud-provider v0.21.1
-	k8s.io/component-base v0.21.1
-	k8s.io/klog/v2 v2.9.0
-	sigs.k8s.io/controller-runtime v0.0.0-00010101000000-000000000000
+	github.com/onsi/gomega v1.27.7
+	k8s.io/api v0.27.2
+	k8s.io/apimachinery v0.27.2
+	k8s.io/client-go v0.27.2
+	k8s.io/cloud-provider v0.27.2
+	k8s.io/component-base v0.27.2
+	k8s.io/klog/v2 v2.90.1
+	sigs.k8s.io/controller-runtime v0.15.0
 )
 
-replace (
-	github.com/googleapis/gnostic => github.com/googleapis/gnostic v0.4.1
-	github.com/openshift/api => github.com/openshift/api v0.0.0-20200526144822-34f54f12813a
-	github.com/openshift/client-go => github.com/openshift/client-go v0.0.0-20200521150516-05eb9880269c
-	github.com/operator-framework/operator-lifecycle-manager => github.com/operator-framework/operator-lifecycle-manager v0.0.0-20190128024246-5eb7ae5bdb7a
-	k8s.io/client-go => k8s.io/client-go v0.21.1
-	k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.21.1
-	k8s.io/kubectl => k8s.io/kubectl v0.21.1
-	sigs.k8s.io/controller-runtime => sigs.k8s.io/controller-runtime v0.9.0
-	sigs.k8s.io/structured-merge-diff => sigs.k8s.io/structured-merge-diff v1.0.0
+require (
+	cloud.google.com/go/compute/metadata v0.2.1 // indirect
+	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
+	github.com/NYTimes/gziphandler v1.1.1 // indirect
+	github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 // indirect
+	github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/blang/semver/v4 v4.0.0 // indirect
+	github.com/cenkalti/backoff/v4 v4.1.3 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/coreos/go-semver v0.3.0 // indirect
+	github.com/coreos/go-systemd/v22 v22.4.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/emicklei/go-restful/v3 v3.9.0 // indirect
+	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
+	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
+	github.com/felixge/httpsnoop v1.0.3 // indirect
+	github.com/fsnotify/fsnotify v1.6.0 // indirect
+	github.com/go-logr/logr v1.2.4 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-openapi/jsonpointer v0.19.6 // indirect
+	github.com/go-openapi/jsonreference v0.20.1 // indirect
+	github.com/go-openapi/swag v0.22.3 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/cel-go v0.12.6 // indirect
+	github.com/google/gnostic v0.5.7-v3refs // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/google/go-querystring v1.1.0 // indirect
+	github.com/google/gofuzz v1.1.0 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
+	github.com/imdario/mergo v0.3.6 // indirect
+	github.com/inconshreveable/mousetrap v1.0.1 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/mitchellh/mapstructure v1.4.1 // indirect
+	github.com/moby/term v0.0.0-20221205130635-1aeaba878587 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/prometheus/client_golang v1.15.1 // indirect
+	github.com/prometheus/client_model v0.4.0 // indirect
+	github.com/prometheus/common v0.42.0 // indirect
+	github.com/prometheus/procfs v0.9.0 // indirect
+	github.com/spf13/cobra v1.6.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/stoewer/go-strcase v1.2.0 // indirect
+	go.etcd.io/etcd/api/v3 v3.5.7 // indirect
+	go.etcd.io/etcd/client/pkg/v3 v3.5.7 // indirect
+	go.etcd.io/etcd/client/v3 v3.5.7 // indirect
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.35.1 // indirect
+	go.opentelemetry.io/otel v1.10.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.10.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.10.0 // indirect
+	go.opentelemetry.io/otel/metric v0.31.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.10.0 // indirect
+	go.opentelemetry.io/otel/trace v1.10.0 // indirect
+	go.opentelemetry.io/proto/otlp v0.19.0 // indirect
+	go.uber.org/atomic v1.7.0 // indirect
+	go.uber.org/multierr v1.6.0 // indirect
+	go.uber.org/zap v1.24.0 // indirect
+	golang.org/x/crypto v0.1.0 // indirect
+	golang.org/x/net v0.10.0 // indirect
+	golang.org/x/oauth2 v0.5.0 // indirect
+	golang.org/x/sync v0.1.0 // indirect
+	golang.org/x/sys v0.8.0 // indirect
+	golang.org/x/term v0.8.0 // indirect
+	golang.org/x/text v0.9.0 // indirect
+	golang.org/x/time v0.3.0 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect
+	google.golang.org/grpc v1.51.0 // indirect
+	google.golang.org/protobuf v1.30.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/apiserver v0.27.2 // indirect
+	k8s.io/component-helpers v0.27.2 // indirect
+	k8s.io/controller-manager v0.27.2 // indirect
+	k8s.io/kms v0.27.2 // indirect
+	k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect
+	k8s.io/utils v0.0.0-20230209194617-a36077c30491 // indirect
+	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2 // indirect
+	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
+	sigs.k8s.io/yaml v1.3.0 // indirect
 )
+
+replace cloud.google.com/go/compute/metadata => cloud.google.com/go/compute/metadata v0.2.1