Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

External Network: support allocating static LoadBalancerIP and NodePort #2105

Merged
merged 1 commit into from
Nov 14, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions apis/networking/v1alpha1/gatewayserver_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,13 @@ type Endpoint struct {
// +kubebuilder:default=ClusterIP
// +kubebuilder:validation:Enum=ClusterIP;NodePort;LoadBalancer;ExternalName
ServiceType corev1.ServiceType `json:"serviceType,omitempty"`
// NodePort allocates a static port for the NodePort service.
// +optional
NodePort *int32 `json:"nodePort,omitempty"`
// LoadBalancerIP override the LoadBalancer IP to use a specific IP address (e.g., static LB). It is used only if service type is LoadBalancer.
// LoadBalancer provider must support this feature.
// +optional
LoadBalancerIP *string `json:"loadBalancerIP,omitempty"`
}

// GatewayServerSpec defines the desired state of GatewayServer.
Expand Down
14 changes: 12 additions & 2 deletions apis/networking/v1alpha1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 4 additions & 0 deletions cmd/liqoctl/cmd/network.go
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,10 @@ func newNetworkConnectCommand(ctx context.Context, options *network.Options) *co
fmt.Sprintf("Service type of the Gateway Server. Default: %s", gatewayserver.DefaultServiceType))
cmd.Flags().Int32Var(&options.ServerPort, "server-port", gatewayserver.DefaultPort,
fmt.Sprintf("Port of the Gateway Server. Default: %d", gatewayserver.DefaultPort))
cmd.Flags().Int32Var(&options.ServerNodePort, "node-port", 0,
"Force the NodePort of the Gateway Server. Leave empty to let Kubernetes allocate a random NodePort")
cmd.Flags().StringVar(&options.ServerLoadBalancerIP, "load-balancer-ip", "",
"Force LoadBalancer IP of the Gateway Server. Leave empty to use the one provided by the LoadBalancer provider")

// Client flags
cmd.Flags().StringVar(&options.ClientGatewayType, "client-type", gatewayclient.DefaultGatewayType,
Expand Down
5 changes: 4 additions & 1 deletion deployments/liqo/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -115,12 +115,15 @@
| networkManager.pod.labels | object | `{}` | Labels for the networkManager pod. |
| networkManager.pod.resources | object | `{"limits":{},"requests":{}}` | Resource requests and limits (https://kubernetes.io/docs/user-guide/compute-resources/) for the networkManager pod. |
| networking.clientResources | list | `[{"apiVersion":"networking.liqo.io/v1alpha1","resource":"wggatewayclients"}]` | Set the list of resources that implement the GatewayClient |
| networking.gateway | object | `{"ping":{"interval":"2s","lossThreshold":5,"updateStatusInterval":"10s"},"replicas":1}` | Set the options for gateway templates |
| networking.gateway | object | `{"ping":{"interval":"2s","lossThreshold":5,"updateStatusInterval":"10s"},"replicas":1,"server":{"service":{"allocateLoadBalancerNodePorts":""}}}` | Set the options for gateway templates |
| networking.gateway.ping | object | `{"interval":"2s","lossThreshold":5,"updateStatusInterval":"10s"}` | Set the options to configure the gateway ping used to check connection |
| networking.gateway.ping.interval | string | `"2s"` | Set the interval between two consecutive pings |
| networking.gateway.ping.lossThreshold | int | `5` | Set the number of consecutive pings that must fail to consider the connection as lost |
| networking.gateway.ping.updateStatusInterval | string | `"10s"` | Set the interval at which the connection resource status is updated |
| networking.gateway.replicas | int | `1` | Set the number of replicas for the gateway deployments |
| networking.gateway.server | object | `{"service":{"allocateLoadBalancerNodePorts":""}}` | Set the options to configure the gateway server |
| networking.gateway.server.service | object | `{"allocateLoadBalancerNodePorts":""}` | Set the options to configure the server service |
| networking.gateway.server.service.allocateLoadBalancerNodePorts | string | `""` | Set to "false" if you expose the gateway service as LoadBalancer and you do not want to create also a NodePort associated to it (Note: this setting is useful only on cloud providers that support this feature). |
| networking.internal | bool | `true` | Use the default Liqo network manager. |
| networking.iptables | object | `{"mode":"nf_tables"}` | Iptables configuration tuning. |
| networking.iptables.mode | string | `"nf_tables"` | Select the iptables mode to use. Possible values are "legacy" and "nf_tables". |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,17 @@ spec:
endpoint:
description: Endpoint specifies the endpoint of the tunnel.
properties:
loadBalancerIP:
description: LoadBalancerIP override the LoadBalancer IP to use
a specific IP address (e.g., static LB). It is used only if
service type is LoadBalancer. LoadBalancer provider must support
this feature.
type: string
nodePort:
description: NodePort allocates a static port for the NodePort
service.
format: int32
type: integer
port:
description: Port specifies the port of the endpoint.
format: int32
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,9 @@ spec:
- port: "{{"{{ .Spec.Endpoint.Port }}"}}"
protocol: UDP
targetPort: "{{"{{ .Spec.Endpoint.Port }}"}}"
{{- if .Values.networking.gateway.server.service.allocateLoadBalancerNodePorts }}
allocateLoadBalancerNodePorts: {{ .Values.networking.gateway.server.service.allocateLoadBalancerNodePorts }}
{{- end }}
deployment:
metadata:
{{- include "liqo.metadataTemplate" $templateConfig | nindent 10 }}
Expand Down
6 changes: 6 additions & 0 deletions deployments/liqo/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,12 @@ networking:
interval: 2s
# -- Set the interval at which the connection resource status is updated
updateStatusInterval: 10s
# -- Set the options to configure the gateway server
server:
# -- Set the options to configure the server service
service:
# -- Set to "false" if you expose the gateway service as LoadBalancer and you do not want to create also a NodePort associated to it (Note: this setting is useful only on cloud providers that support this feature).
allocateLoadBalancerNodePorts: ""

peering:
# -- Set the default configuration for the networking resources created during the peering process
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ import (

networkingv1alpha1 "github.com/liqotech/liqo/apis/networking/v1alpha1"
"github.com/liqotech/liqo/pkg/consts"
"github.com/liqotech/liqo/pkg/gateway"
enutils "github.com/liqotech/liqo/pkg/liqo-controller-manager/external-network/utils"
mapsutil "github.com/liqotech/liqo/pkg/utils/maps"
)
Expand Down Expand Up @@ -86,7 +87,7 @@ func (r *WgGatewayClientReconciler) Reconcile(ctx context.Context, req ctrl.Requ
}

// Ensure deployment (create or update)
deployNsName := types.NamespacedName{Namespace: wgClient.Namespace, Name: wgClient.Name}
deployNsName := types.NamespacedName{Namespace: wgClient.Namespace, Name: gateway.GenerateResourceName(wgClient.Name)}
_, err = r.ensureDeployment(ctx, wgClient, deployNsName)
if err != nil {
return ctrl.Result{}, err
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ import (
networkingv1alpha1 "github.com/liqotech/liqo/apis/networking/v1alpha1"
"github.com/liqotech/liqo/pkg/consts"
"github.com/liqotech/liqo/pkg/discovery"
"github.com/liqotech/liqo/pkg/gateway"
enutils "github.com/liqotech/liqo/pkg/liqo-controller-manager/external-network/utils"
"github.com/liqotech/liqo/pkg/utils"
mapsutil "github.com/liqotech/liqo/pkg/utils/maps"
Expand Down Expand Up @@ -96,14 +97,14 @@ func (r *WgGatewayServerReconciler) Reconcile(ctx context.Context, req ctrl.Requ
}

// Ensure deployment (create or update)
deployNsName := types.NamespacedName{Namespace: wgServer.Namespace, Name: wgServer.Name}
deployNsName := types.NamespacedName{Namespace: wgServer.Namespace, Name: gateway.GenerateResourceName(wgServer.Name)}
deploy, err := r.ensureDeployment(ctx, wgServer, deployNsName)
if err != nil {
return ctrl.Result{}, err
}

// Ensure service (create or update)
svcNsName := types.NamespacedName{Namespace: wgServer.Namespace, Name: wgServer.Name}
svcNsName := types.NamespacedName{Namespace: wgServer.Namespace, Name: gateway.GenerateResourceName(wgServer.Name)}
_, err = r.ensureService(ctx, wgServer, svcNsName)
if err != nil {
return ctrl.Result{}, err
Expand Down
42 changes: 42 additions & 0 deletions pkg/liqoctl/network/cluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,14 @@ import (
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"

discoveryv1alpha1 "github.com/liqotech/liqo/apis/discovery/v1alpha1"
networkingv1alpha1 "github.com/liqotech/liqo/apis/networking/v1alpha1"
"github.com/liqotech/liqo/pkg/consts"
"github.com/liqotech/liqo/pkg/gateway"
"github.com/liqotech/liqo/pkg/liqoctl/factory"
"github.com/liqotech/liqo/pkg/liqoctl/output"
"github.com/liqotech/liqo/pkg/liqoctl/rest/configuration"
Expand Down Expand Up @@ -211,6 +213,26 @@ func (c *Cluster) GetGatewayClient(ctx context.Context, name string) (*networkin
return gwClient, nil
}

func endpointHasChanged(endpoint *networkingv1alpha1.Endpoint, service *corev1.Service) bool {
if endpoint.ServiceType != service.Spec.Type {
return true
}
if len(service.Spec.Ports) > 0 {
if endpoint.Port != service.Spec.Ports[0].Port {
return true
}

if endpoint.NodePort != nil && *endpoint.NodePort != service.Spec.Ports[0].NodePort {
return true
}
}
if endpoint.LoadBalancerIP != nil && *endpoint.LoadBalancerIP != service.Spec.LoadBalancerIP ||
endpoint.LoadBalancerIP == nil && service.Spec.LoadBalancerIP != "" {
return true
}
return false
}

// EnsureGatewayServer create or updates a GatewayServer.
func (c *Cluster) EnsureGatewayServer(ctx context.Context, name string, opts *gatewayserver.ForgeOptions) (*networkingv1alpha1.GatewayServer, error) {
s := c.local.Printer.StartSpinner("Setting up gateway server")
Expand All @@ -219,6 +241,26 @@ func (c *Cluster) EnsureGatewayServer(ctx context.Context, name string, opts *ga
s.Fail(fmt.Sprintf("An error occurred while forging gateway server: %v", output.PrettyErr(err)))
return nil, err
}

// If the forged server endpoint has different parameters from the existing server service (if present),
// we delete the existing gateway server so that the client can correctly connect to the new endpoint.
var service corev1.Service
svcNsName := types.NamespacedName{Namespace: gwServer.Namespace, Name: gateway.GenerateResourceName(gwServer.Name)}
err = c.local.CRClient.Get(ctx, svcNsName, &service)
if client.IgnoreNotFound(err) != nil {
s.Fail(fmt.Sprintf("An error occurred while retrieving gateway server service: %v", output.PrettyErr(err)))
return nil, err
} else if err == nil {
// Server service already exists. Check if endpoint has changed parameters.
if endpointHasChanged(&gwServer.Spec.Endpoint, &service) {
if err := c.local.CRClient.Delete(ctx, gwServer); err != nil {
s.Fail(fmt.Sprintf("An error occurred while deleting gateway server: %v", output.PrettyErr(err)))
return nil, err
}
s.Success("Deleted existing gateway server")
}
}

_, err = controllerutil.CreateOrUpdate(ctx, c.local.CRClient, gwServer, func() error {
return gatewayserver.MutateGatewayServer(gwServer, opts)
})
Expand Down
5 changes: 5 additions & 0 deletions pkg/liqoctl/network/handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (

v1 "k8s.io/api/core/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/utils/ptr"
"sigs.k8s.io/controller-runtime/pkg/client"

networkingv1alpha1 "github.com/liqotech/liqo/apis/networking/v1alpha1"
Expand All @@ -44,6 +45,8 @@ type Options struct {
ServerTemplateNamespace string
ServerServiceType *argsutils.StringEnum
ServerPort int32
ServerNodePort int32
ServerLoadBalancerIP string

ClientGatewayType string
ClientTemplateName string
Expand Down Expand Up @@ -304,6 +307,8 @@ func (o *Options) newGatewayServerForgeOptions(kubeClient kubernetes.Interface,
ServiceType: v1.ServiceType(o.ServerServiceType.Value),
MTU: o.MTU,
Port: o.ServerPort,
NodePort: ptr.To(o.ServerNodePort),
LoadBalancerIP: ptr.To(o.ServerLoadBalancerIP),
Proxy: o.Proxy,
}
}
Expand Down
12 changes: 8 additions & 4 deletions pkg/liqoctl/rest/gatewayclient/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,9 @@
package gatewayclient

import (
v1 "k8s.io/api/core/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/utils/ptr"

discoveryv1alpha1 "github.com/liqotech/liqo/apis/discovery/v1alpha1"
networkingv1alpha1 "github.com/liqotech/liqo/apis/networking/v1alpha1"
Expand Down Expand Up @@ -48,6 +49,7 @@ func ForgeGatewayClient(name, namespace string, o *ForgeOptions) (*networkingv1a

// MutateGatewayClient mutates a GatewayClient.
func MutateGatewayClient(gwClient *networkingv1alpha1.GatewayClient, o *ForgeOptions) error {
// Metadata
gwClient.Kind = networkingv1alpha1.GatewayClientKind
gwClient.APIVersion = networkingv1alpha1.GroupVersion.String()

Expand All @@ -56,15 +58,17 @@ func MutateGatewayClient(gwClient *networkingv1alpha1.GatewayClient, o *ForgeOpt
}
gwClient.Labels[liqoconsts.RemoteClusterID] = o.RemoteClusterID

// MTU
gwClient.Spec.MTU = o.MTU

protocol := v1.Protocol(o.Protocol)
// Server Endpoint
gwClient.Spec.Endpoint = networkingv1alpha1.EndpointStatus{
Addresses: o.Addresses,
Port: o.Port,
Protocol: &protocol,
Protocol: ptr.To(corev1.Protocol(o.Protocol)),
}

// Client Template Reference
gvr, err := enutils.ParseGroupVersionResource(o.GatewayType)
if err != nil {
return err
Expand All @@ -73,7 +77,7 @@ func MutateGatewayClient(gwClient *networkingv1alpha1.GatewayClient, o *ForgeOpt
if err != nil {
return err
}
gwClient.Spec.ClientTemplateRef = v1.ObjectReference{
gwClient.Spec.ClientTemplateRef = corev1.ObjectReference{
Name: o.TemplateName,
Namespace: o.TemplateNamespace,
Kind: kind,
Expand Down
4 changes: 4 additions & 0 deletions pkg/liqoctl/rest/gatewayserver/create.go
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,10 @@ func (o *Options) Create(ctx context.Context, options *rest.CreateOptions) *cobr
cmd.Flags().Var(o.ServiceType, "service-type", fmt.Sprintf("Service type of Gateway Server. Default: %s", DefaultServiceType))
cmd.Flags().IntVar(&o.MTU, "mtu", DefaultMTU, "MTU of Gateway Server")
cmd.Flags().Int32Var(&o.Port, "port", DefaultPort, "Port of Gateway Server")
cmd.Flags().Int32Var(&o.NodePort, "node-port", 0,
"Force the NodePort of the Gateway Server. Leave empty to let Kubernetes allocate a random NodePort")
cmd.Flags().StringVar(&o.LoadBalancerIP, "load-balancer-ip", "",
"Force LoadBalancer IP of the Gateway Server. Leave empty to use the one provided by the LoadBalancer provider")
cmd.Flags().BoolVar(&o.Proxy, "proxy", DefaultProxy, "Enable proxy for the Gateway Server")
cmd.Flags().BoolVar(&o.Wait, "wait", DefaultWait, "Wait for the Gateway Server to be ready")

Expand Down
7 changes: 7 additions & 0 deletions pkg/liqoctl/rest/gatewayserver/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ package gatewayserver
import (
corev1 "k8s.io/api/core/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/utils/ptr"

"github.com/liqotech/liqo/pkg/liqoctl/rest"
argsutils "github.com/liqotech/liqo/pkg/utils/args"
Expand Down Expand Up @@ -46,6 +47,8 @@ type Options struct {
ServiceType *argsutils.StringEnum
MTU int
Port int32
NodePort int32
LoadBalancerIP string
Proxy bool
Wait bool
}
Expand Down Expand Up @@ -78,6 +81,8 @@ type ForgeOptions struct {
ServiceType corev1.ServiceType
MTU int
Port int32
NodePort *int32
LoadBalancerIP *string
Proxy bool
}

Expand All @@ -91,6 +96,8 @@ func (o *Options) getForgeOptions() *ForgeOptions {
ServiceType: corev1.ServiceType(o.ServiceType.Value),
MTU: o.MTU,
Port: o.Port,
NodePort: ptr.To(o.NodePort),
LoadBalancerIP: ptr.To(o.LoadBalancerIP),
Proxy: o.Proxy,
}
}
11 changes: 11 additions & 0 deletions pkg/liqoctl/rest/gatewayserver/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ func ForgeGatewayServer(name, namespace string, o *ForgeOptions) (*networkingv1a

// MutateGatewayServer mutates a GatewayServer.
func MutateGatewayServer(gwServer *networkingv1alpha1.GatewayServer, o *ForgeOptions) error {
// Metadata
gwServer.Kind = networkingv1alpha1.GatewayServerKind
gwServer.APIVersion = networkingv1alpha1.GroupVersion.String()

Expand All @@ -56,12 +57,22 @@ func MutateGatewayServer(gwServer *networkingv1alpha1.GatewayServer, o *ForgeOpt
}
gwServer.Labels[liqoconsts.RemoteClusterID] = o.RemoteClusterID

// MTU
gwServer.Spec.MTU = o.MTU

// Server Endpoint
gwServer.Spec.Endpoint = networkingv1alpha1.Endpoint{
Port: o.Port,
ServiceType: o.ServiceType,
}
if o.NodePort != nil && *o.NodePort != 0 {
gwServer.Spec.Endpoint.NodePort = o.NodePort
}
if o.LoadBalancerIP != nil && *o.LoadBalancerIP != "" {
gwServer.Spec.Endpoint.LoadBalancerIP = o.LoadBalancerIP
}

// Server Template Reference
gvr, err := enutils.ParseGroupVersionResource(o.GatewayType)
if err != nil {
return err
Expand Down