From c97ef7556d1a59335c5bf3acb881974134902f20 Mon Sep 17 00:00:00 2001 From: Alessandro Olivero Date: Tue, 19 Sep 2023 16:44:54 +0200 Subject: [PATCH] gateway server and client controllers --- cmd/liqo-controller-manager/main.go | 37 +++ .../liqo-controller-manager-ClusterRole.yaml | 24 ++ deployments/liqo/templates/_helpers.tpl | 11 + .../liqo-controller-manager-deployment.yaml | 4 + deployments/liqo/values.yaml | 8 + .../client-operator/client_controller.go | 205 +++++++++++++++++ .../server-operator/server_controller.go | 217 ++++++++++++++++++ .../external-network/utils/factory.go | 34 +++ .../external-network/utils/factorysource.go | 104 +++++++++ .../external-network/utils/getters.go | 67 ++++++ .../external-network/utils/ownerenqueuer.go | 65 ++++++ .../external-network/utils/patch.go | 70 ++++++ .../external-network/utils/template.go | 74 ++++++ pkg/utils/mapper/mapper.go | 4 + 14 files changed, 924 insertions(+) create mode 100644 pkg/liqo-controller-manager/external-network/client-operator/client_controller.go create mode 100644 pkg/liqo-controller-manager/external-network/server-operator/server_controller.go create mode 100644 pkg/liqo-controller-manager/external-network/utils/factory.go create mode 100644 pkg/liqo-controller-manager/external-network/utils/factorysource.go create mode 100644 pkg/liqo-controller-manager/external-network/utils/getters.go create mode 100644 pkg/liqo-controller-manager/external-network/utils/ownerenqueuer.go create mode 100644 pkg/liqo-controller-manager/external-network/utils/patch.go create mode 100644 pkg/liqo-controller-manager/external-network/utils/template.go diff --git a/cmd/liqo-controller-manager/main.go b/cmd/liqo-controller-manager/main.go index 24a6c4ffcd..50f68da953 100644 --- a/cmd/liqo-controller-manager/main.go +++ b/cmd/liqo-controller-manager/main.go @@ -35,6 +35,8 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/selection" utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/dynamic/dynamicinformer" "k8s.io/client-go/kubernetes" clientgoscheme "k8s.io/client-go/kubernetes/scheme" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" @@ -53,12 +55,16 @@ import ( discoveryv1alpha1 "github.com/liqotech/liqo/apis/discovery/v1alpha1" ipamv1alpha1 "github.com/liqotech/liqo/apis/ipam/v1alpha1" netv1alpha1 "github.com/liqotech/liqo/apis/net/v1alpha1" + networkingv1alpha1 "github.com/liqotech/liqo/apis/networking/v1alpha1" offloadingv1alpha1 "github.com/liqotech/liqo/apis/offloading/v1alpha1" sharingv1alpha1 "github.com/liqotech/liqo/apis/sharing/v1alpha1" virtualkubeletv1alpha1 "github.com/liqotech/liqo/apis/virtualkubelet/v1alpha1" "github.com/liqotech/liqo/cmd/virtual-kubelet/root" "github.com/liqotech/liqo/pkg/consts" identitymanager "github.com/liqotech/liqo/pkg/identityManager" + clientoperator "github.com/liqotech/liqo/pkg/liqo-controller-manager/external-network/client-operator" + serveroperator "github.com/liqotech/liqo/pkg/liqo-controller-manager/external-network/server-operator" + enutils "github.com/liqotech/liqo/pkg/liqo-controller-manager/external-network/utils" foreignclusteroperator "github.com/liqotech/liqo/pkg/liqo-controller-manager/foreign-cluster-operator" ipctrl "github.com/liqotech/liqo/pkg/liqo-controller-manager/ip-controller" mapsctrl "github.com/liqotech/liqo/pkg/liqo-controller-manager/namespacemap-controller" @@ -107,6 +113,7 @@ func init() { _ = offloadingv1alpha1.AddToScheme(scheme) _ = virtualkubeletv1alpha1.AddToScheme(scheme) _ = ipamv1alpha1.AddToScheme(scheme) + _ = networkingv1alpha1.AddToScheme(scheme) } func main() { @@ -121,6 +128,8 @@ func main() { var labelsNotReflected argsutils.StringList var annotationsNotReflected argsutils.StringList var ipamClient ipam.IpamClient + var gatewayServerResources argsutils.StringList + var gatewayClientResources argsutils.StringList webhookPort := flag.Uint("webhook-port", 9443, "The port the webhook server binds to") metricsAddr := flag.String("metrics-address", ":8080", "The address the metric endpoint binds to") @@ -203,6 +212,10 @@ func main() { // Node failure controller parameter enableNodeFailureController := flag.Bool("enable-node-failure-controller", false, "Enable the node failure controller") + // External network parameters + flag.Var(&gatewayServerResources, "gateway-server-resources", "The list of resource types that implements the gateway server. They must be in the form //") + flag.Var(&gatewayClientResources, "gateway-client-resources", "The list of resource types that implements the gateway client. They must be in the form //") + liqoerrors.InitFlags(nil) restcfg.InitFlags(nil) klog.InitFlags(nil) @@ -237,6 +250,11 @@ func main() { config := restcfg.SetRateLimiter(ctrl.GetConfigOrDie()) + dynClient := dynamic.NewForConfigOrDie(config) + factory := &enutils.RunnableFactory{ + DynamicSharedInformerFactory: dynamicinformer.NewFilteredDynamicSharedInformerFactory(dynClient, 0, corev1.NamespaceAll, nil), + } + // Create a label selector to filter only the events for pods managed by a ShadowPod (i.e., remote offloaded pods), // as those are the only ones we are interested in to implement the resiliency mechanism. reqRemoteLiqoPods, err := labels.NewRequirement(consts.ManagedByLabelKey, selection.Equals, []string{consts.ManagedByShadowPodValue}) @@ -269,6 +287,11 @@ func main() { os.Exit(1) } + if err = mgr.Add(factory); err != nil { + klog.Error(err) + os.Exit(1) + } + // Create a label selector to filter only the events for local offloaded pods reqLocalLiqoPods, err := labels.NewRequirement(consts.LocalPodLabelKey, selection.Equals, []string{consts.LocalPodLabelValue}) utilruntime.Must(err) @@ -495,6 +518,20 @@ func main() { klog.Fatal(err) } + serverReconciler := serveroperator.NewServerReconciler(ctx, mgr.GetClient(), + dynClient, factory, mgr.GetScheme(), gatewayServerResources.StringList) + if err := serverReconciler.SetupWithManager(mgr); err != nil { + klog.Error(err) + os.Exit(1) + } + + clientReconciler := clientoperator.NewClientReconciler(ctx, mgr.GetClient(), + dynClient, factory, mgr.GetScheme(), gatewayClientResources.StringList) + if err := clientReconciler.SetupWithManager(mgr); err != nil { + klog.Error(err) + os.Exit(1) + } + // Start the handler to approve the virtual kubelet certificate signing requests. csrWatcher := csr.NewWatcher(clientset, *resyncPeriod, labels.Everything(), fields.Everything()) csrWatcher.RegisterHandler(csr.ApproverHandler(clientset, "LiqoApproval", "This CSR was approved by Liqo", diff --git a/deployments/liqo/files/liqo-controller-manager-ClusterRole.yaml b/deployments/liqo/files/liqo-controller-manager-ClusterRole.yaml index 6e88c61213..050f4d9fd3 100644 --- a/deployments/liqo/files/liqo-controller-manager-ClusterRole.yaml +++ b/deployments/liqo/files/liqo-controller-manager-ClusterRole.yaml @@ -375,6 +375,30 @@ rules: - get - update - watch +- apiGroups: + - networking.liqo.io + resources: + - gatewayclients + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - networking.liqo.io + resources: + - gatewayservers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - offloading.liqo.io resources: diff --git a/deployments/liqo/templates/_helpers.tpl b/deployments/liqo/templates/_helpers.tpl index fa95945edf..34806e05d8 100644 --- a/deployments/liqo/templates/_helpers.tpl +++ b/deployments/liqo/templates/_helpers.tpl @@ -172,6 +172,17 @@ Concatenates a values list into a string in the form "--commandName=val1,val2" - {{ trimSuffix "," $res }} {{- end -}} +{{/* +Concatenates a values list of groupVersionResources into a string in the form "--commandName=group1/version1/resource1,group2/version2/resource2" +*/}} +{{- define "liqo.concatenateGroupVersionResources" -}} +{{- $res := print .commandName "=" -}} +{{- range $val := .list -}} +{{- $res = print $res $val.apiVersion "/" $val.resource "," -}} +{{- end -}} +- {{ trimSuffix "," $res }} +{{- end -}} + {{/* Get the liqo clusterID ConfigMap name */}} diff --git a/deployments/liqo/templates/liqo-controller-manager-deployment.yaml b/deployments/liqo/templates/liqo-controller-manager-deployment.yaml index b125d25d9b..a577182f0f 100644 --- a/deployments/liqo/templates/liqo-controller-manager-deployment.yaml +++ b/deployments/liqo/templates/liqo-controller-manager-deployment.yaml @@ -92,6 +92,10 @@ spec: - --configmap-reflection-type={{ .Values.reflection.configmap.type }} - --secret-reflection-type={{ .Values.reflection.secret.type }} - --event-reflection-type={{ .Values.reflection.event.type }} + {{- $d := dict "commandName" "--gateway-server-resources" "list" .Values.networking.serverResources }} + {{- include "liqo.concatenateGroupVersionResources" $d | nindent 10 }} + {{- $d := dict "commandName" "--gateway-client-resources" "list" .Values.networking.clientResources }} + {{- include "liqo.concatenateGroupVersionResources" $d | nindent 10 }} {{- if .Values.reflection.skip.labels }} {{- $d := dict "commandName" "--labels-not-reflected" "list" .Values.reflection.skip.labels }} {{- include "liqo.concatenateList" $d | nindent 10 }} diff --git a/deployments/liqo/values.yaml b/deployments/liqo/values.yaml index 3f1c38d26f..dd4f2a5f3c 100644 --- a/deployments/liqo/values.yaml +++ b/deployments/liqo/values.yaml @@ -37,6 +37,14 @@ networking: # The default value is configured to ensure correct behavior regardless of the combination of the underlying environments # (e.g., cloud providers). This guarantees improved compatibility at the cost of possible limited performance drops. mtu: 1340 + # -- Set the list of resources that implement the GatewayServer + serverResources: + - apiVersion: networking.liqo.io/v1alpha1 + resource: wggatewayservers + # -- Set the list of resources that implement the GatewayClient + clientResources: + - apiVersion: networking.liqo.io/v1alpha1 + resource: wggatewayclients reflection: skip: diff --git a/pkg/liqo-controller-manager/external-network/client-operator/client_controller.go b/pkg/liqo-controller-manager/external-network/client-operator/client_controller.go new file mode 100644 index 0000000000..2eba6ff326 --- /dev/null +++ b/pkg/liqo-controller-manager/external-network/client-operator/client_controller.go @@ -0,0 +1,205 @@ +// Copyright 2019-2023 The Liqo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clientoperator + +import ( + "context" + "fmt" + "strings" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" + "k8s.io/klog/v2" + "k8s.io/utils/pointer" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + networkingv1alpha1 "github.com/liqotech/liqo/apis/networking/v1alpha1" + enutils "github.com/liqotech/liqo/pkg/liqo-controller-manager/external-network/utils" +) + +// ClientReconciler manage GatewayClient lifecycle. +type ClientReconciler struct { + client.Client + Scheme *runtime.Scheme + DynClient dynamic.Interface + Factory *enutils.RunnableFactory + ClientResources []string +} + +// NewClientReconciler returns a new ClientReconciler. +func NewClientReconciler(ctx context.Context, + cl client.Client, dynClient dynamic.Interface, + factory *enutils.RunnableFactory, s *runtime.Scheme, + clientResources []string) *ClientReconciler { + return &ClientReconciler{ + Client: cl, + Scheme: s, + DynClient: dynClient, + Factory: factory, + ClientResources: clientResources, + } +} + +// cluster-role +// +kubebuilder:rbac:groups=networking.liqo.io,resources=gatewayclients,verbs=get;list;watch;delete;create;update;patch + +// Reconcile manage GatewayClient lifecycle. +func (r *ClientReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res ctrl.Result, err error) { + gwClient := &networkingv1alpha1.GatewayClient{} + if err = r.Get(ctx, req.NamespacedName, gwClient); err != nil { + if apierrors.IsNotFound(err) { + klog.Infof("Gateway client %q not found", req.NamespacedName) + return ctrl.Result{}, nil + } + klog.Errorf("Unable to get the gateway client %q: %s", req.NamespacedName, err) + return ctrl.Result{}, err + } + + defer func() { + newErr := r.Status().Update(ctx, gwClient) + if newErr != nil { + if err != nil { + klog.Errorf("Error reconciling the gateway client %q: %s", req.NamespacedName, err) + } + klog.Errorf("Unable to update the gateway client %q: %s", req.NamespacedName, newErr) + err = newErr + } + }() + + if err = r.EnsureGatewayClient(ctx, gwClient); err != nil { + klog.Errorf("Unable to ensure the gateway client %q: %s", req.NamespacedName, err) + return ctrl.Result{}, err + } + + return ctrl.Result{}, nil +} + +// EnsureGatewayClient ensures the GatewayClient is correctly configured. +func (r *ClientReconciler) EnsureGatewayClient(ctx context.Context, gwClient *networkingv1alpha1.GatewayClient) error { + templateGV, err := schema.ParseGroupVersion(gwClient.Spec.ClientTemplateRef.APIVersion) + if err != nil { + return fmt.Errorf("unable to parse the client template group version: %w", err) + } + + templateGVR := schema.GroupVersionResource{ + Group: templateGV.Group, + Version: templateGV.Version, + Resource: enutils.KindToResource(gwClient.Spec.ClientTemplateRef.Kind), + } + template, err := r.DynClient.Resource(templateGVR). + Namespace(gwClient.Spec.ClientTemplateRef.Namespace). + Get(ctx, gwClient.Spec.ClientTemplateRef.Name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("unable to get the client template: %w", err) + } + + templateSpec, ok := template.Object["spec"].(map[string]interface{}) + if !ok { + return fmt.Errorf("unable to get the spec of the client template") + } + objectKindInt, ok := templateSpec["objectKind"].(map[string]interface{}) + if !ok { + return fmt.Errorf("unable to get the object kind of the client template") + } + objectKind := metav1.TypeMeta{ + Kind: objectKindInt["kind"].(string), + APIVersion: objectKindInt["apiVersion"].(string), + } + objectTemplate, ok := templateSpec["template"].(map[string]interface{}) + if !ok { + return fmt.Errorf("unable to get the template of the client template") + } + objectTemplateMetadataInt, ok := objectTemplate["metadata"].(map[string]interface{}) + if !ok { + return fmt.Errorf("unable to get the metadata of the client template") + } + objectTemplateMetadata := metav1.ObjectMeta{ + Name: enutils.GetValueOrDefault(objectTemplateMetadataInt, "name", gwClient.Name), + Namespace: enutils.GetValueOrDefault(objectTemplateMetadataInt, "namespace", gwClient.Namespace), + Labels: enutils.TranslateMap(objectTemplateMetadataInt["labels"]), + Annotations: enutils.TranslateMap(objectTemplateMetadataInt["annotations"]), + } + objectTemplateSpec, ok := objectTemplate["spec"].(map[string]interface{}) + if !ok { + return fmt.Errorf("unable to get the spec of the client template") + } + + unstructuredObject, err := enutils.CreateOrPatch(ctx, r.DynClient.Resource(objectKind.GroupVersionKind().GroupVersion().WithResource(enutils.KindToResource(objectKind.Kind))). + Namespace(gwClient.Namespace), gwClient.Name, func(obj *unstructured.Unstructured) error { + obj.SetGroupVersionKind(objectKind.GroupVersionKind()) + obj.SetName(gwClient.Name) + obj.SetNamespace(gwClient.Namespace) + obj.SetLabels(objectTemplateMetadata.Labels) + obj.SetAnnotations(objectTemplateMetadata.Annotations) + obj.SetOwnerReferences([]metav1.OwnerReference{ + { + APIVersion: gwClient.APIVersion, + Kind: gwClient.Kind, + Name: gwClient.Name, + UID: gwClient.UID, + Controller: pointer.Bool(true), + }, + }) + spec, err := enutils.RenderTemplate(objectTemplateSpec, gwClient.Spec) + if err != nil { + return fmt.Errorf("unable to render the template: %w", err) + } + obj.Object["spec"] = spec + return nil + }) + if err != nil { + return fmt.Errorf("unable to update the client: %w", err) + } + + gwClient.Status.ClientRef = corev1.ObjectReference{ + APIVersion: unstructuredObject.GetAPIVersion(), + Kind: unstructuredObject.GetKind(), + Name: unstructuredObject.GetName(), + Namespace: unstructuredObject.GetNamespace(), + UID: unstructuredObject.GetUID(), + } + + return nil +} + +// SetupWithManager register the ClientReconciler to the manager. +func (r *ClientReconciler) SetupWithManager(mgr ctrl.Manager) error { + ownerEnqueuer := enutils.NewOwnerEnqueuer(networkingv1alpha1.GatewayClientKind) + factorySource := enutils.NewFactorySource(r.Factory) + + for _, resource := range r.ClientResources { + tmp := strings.Split(resource, "/") + if len(tmp) != 3 { + return fmt.Errorf("invalid resource %q", resource) + } + gvr := schema.GroupVersionResource{ + Group: tmp[0], + Version: tmp[1], + Resource: tmp[2], + } + factorySource.ForResource(gvr) + } + + return ctrl.NewControllerManagedBy(mgr). + WatchesRawSource(factorySource.Source(), ownerEnqueuer). + For(&networkingv1alpha1.GatewayClient{}). + Complete(r) +} diff --git a/pkg/liqo-controller-manager/external-network/server-operator/server_controller.go b/pkg/liqo-controller-manager/external-network/server-operator/server_controller.go new file mode 100644 index 0000000000..8984a9933e --- /dev/null +++ b/pkg/liqo-controller-manager/external-network/server-operator/server_controller.go @@ -0,0 +1,217 @@ +// Copyright 2019-2023 The Liqo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package serveroperator + +import ( + "context" + "fmt" + "strings" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" + "k8s.io/klog/v2" + "k8s.io/utils/pointer" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + networkingv1alpha1 "github.com/liqotech/liqo/apis/networking/v1alpha1" + enutils "github.com/liqotech/liqo/pkg/liqo-controller-manager/external-network/utils" +) + +// ServerReconciler manage GatewayServer lifecycle. +type ServerReconciler struct { + client.Client + Scheme *runtime.Scheme + DynClient dynamic.Interface + Factory *enutils.RunnableFactory + ServerResources []string +} + +// NewServerReconciler returns a new ServerReconciler. +func NewServerReconciler(ctx context.Context, + cl client.Client, dynClient dynamic.Interface, + factory *enutils.RunnableFactory, s *runtime.Scheme, + serverResources []string) *ServerReconciler { + return &ServerReconciler{ + Client: cl, + Scheme: s, + DynClient: dynClient, + Factory: factory, + ServerResources: serverResources, + } +} + +// cluster-role +// +kubebuilder:rbac:groups=networking.liqo.io,resources=gatewayservers,verbs=get;list;watch;delete;create;update;patch + +// Reconcile manage GatewayServer lifecycle. +func (r *ServerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res ctrl.Result, err error) { + server := &networkingv1alpha1.GatewayServer{} + if err = r.Get(ctx, req.NamespacedName, server); err != nil { + if apierrors.IsNotFound(err) { + klog.Infof("Gateway server %q not found", req.NamespacedName) + return ctrl.Result{}, nil + } + klog.Errorf("Unable to get the gateway server %q: %s", req.NamespacedName, err) + return ctrl.Result{}, err + } + + defer func() { + newErr := r.Status().Update(ctx, server) + if newErr != nil { + if err != nil { + klog.Errorf("Error reconciling the gateway server %q: %s", req.NamespacedName, err) + } + klog.Errorf("Unable to update the gateway server %q: %s", req.NamespacedName, newErr) + err = newErr + } + }() + + if err = r.EnsureGatewayServer(ctx, server); err != nil { + klog.Errorf("Unable to ensure the gateway server %q: %s", req.NamespacedName, err) + return ctrl.Result{}, err + } + + return ctrl.Result{}, nil +} + +// EnsureGatewayServer ensures the GatewayServer is correctly configured. +func (r *ServerReconciler) EnsureGatewayServer(ctx context.Context, server *networkingv1alpha1.GatewayServer) error { + templateGV, err := schema.ParseGroupVersion(server.Spec.ServerTemplateRef.APIVersion) + if err != nil { + return fmt.Errorf("unable to parse the server template group version: %w", err) + } + + templateGVR := schema.GroupVersionResource{ + Group: templateGV.Group, + Version: templateGV.Version, + Resource: enutils.KindToResource(server.Spec.ServerTemplateRef.Kind), + } + template, err := r.DynClient.Resource(templateGVR). + Namespace(server.Spec.ServerTemplateRef.Namespace). + Get(ctx, server.Spec.ServerTemplateRef.Name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("unable to get the server template: %w", err) + } + + templateSpec, ok := template.Object["spec"].(map[string]interface{}) + if !ok { + return fmt.Errorf("unable to get the spec of the server template") + } + objectKindInt, ok := templateSpec["objectKind"].(map[string]interface{}) + if !ok { + return fmt.Errorf("unable to get the object kind of the server template") + } + objectKind := metav1.TypeMeta{ + Kind: objectKindInt["kind"].(string), + APIVersion: objectKindInt["apiVersion"].(string), + } + objectTemplate, ok := templateSpec["template"].(map[string]interface{}) + if !ok { + return fmt.Errorf("unable to get the template of the server template") + } + objectTemplateMetadataInt, ok := objectTemplate["metadata"].(map[string]interface{}) + if !ok { + return fmt.Errorf("unable to get the metadata of the server template") + } + objectTemplateMetadata := metav1.ObjectMeta{ + Name: enutils.GetValueOrDefault(objectTemplateMetadataInt, "name", server.Name), + Namespace: enutils.GetValueOrDefault(objectTemplateMetadataInt, "namespace", server.Namespace), + Labels: enutils.TranslateMap(objectTemplateMetadataInt["labels"]), + Annotations: enutils.TranslateMap(objectTemplateMetadataInt["annotations"]), + } + objectTemplateSpec, ok := objectTemplate["spec"].(map[string]interface{}) + if !ok { + return fmt.Errorf("unable to get the spec of the server template") + } + + unstructuredObject, err := enutils.CreateOrPatch(ctx, r.DynClient.Resource(objectKind.GroupVersionKind().GroupVersion().WithResource(enutils.KindToResource(objectKind.Kind))). + Namespace(server.Namespace), server.Name, func(obj *unstructured.Unstructured) error { + obj.SetGroupVersionKind(objectKind.GroupVersionKind()) + obj.SetName(server.Name) + obj.SetNamespace(server.Namespace) + obj.SetLabels(objectTemplateMetadata.Labels) + obj.SetAnnotations(objectTemplateMetadata.Annotations) + obj.SetOwnerReferences([]metav1.OwnerReference{ + { + APIVersion: server.APIVersion, + Kind: server.Kind, + Name: server.Name, + UID: server.UID, + Controller: pointer.Bool(true), + }, + }) + spec, err := enutils.RenderTemplate(objectTemplateSpec, server.Spec) + if err != nil { + return fmt.Errorf("unable to render the template: %w", err) + } + obj.Object["spec"] = spec + return nil + }) + if err != nil { + return fmt.Errorf("unable to update the server: %w", err) + } + + server.Status.ServerRef = corev1.ObjectReference{ + APIVersion: unstructuredObject.GetAPIVersion(), + Kind: unstructuredObject.GetKind(), + Name: unstructuredObject.GetName(), + Namespace: unstructuredObject.GetNamespace(), + UID: unstructuredObject.GetUID(), + } + + status, ok := unstructuredObject.Object["status"].(map[string]interface{}) + if !ok { + // the object does not have a status + return nil + } + endpoint, ok := status["endpoint"].(map[string]interface{}) + if !ok { + // the object does not have an endpoint + return nil + } + server.Status.Endpoint = enutils.ParseEndpoint(endpoint) + + return nil +} + +// SetupWithManager register the ServerReconciler to the manager. +func (r *ServerReconciler) SetupWithManager(mgr ctrl.Manager) error { + ownerEnqueuer := enutils.NewOwnerEnqueuer(networkingv1alpha1.GatewayServerKind) + factorySource := enutils.NewFactorySource(r.Factory) + + for _, resource := range r.ServerResources { + tmp := strings.Split(resource, "/") + if len(tmp) != 3 { + return fmt.Errorf("invalid resource %q", resource) + } + gvr := schema.GroupVersionResource{ + Group: tmp[0], + Version: tmp[1], + Resource: tmp[2], + } + factorySource.ForResource(gvr) + } + + return ctrl.NewControllerManagedBy(mgr). + WatchesRawSource(factorySource.Source(), ownerEnqueuer). + For(&networkingv1alpha1.GatewayServer{}). + Complete(r) +} diff --git a/pkg/liqo-controller-manager/external-network/utils/factory.go b/pkg/liqo-controller-manager/external-network/utils/factory.go new file mode 100644 index 0000000000..4840eb08cf --- /dev/null +++ b/pkg/liqo-controller-manager/external-network/utils/factory.go @@ -0,0 +1,34 @@ +// Copyright 2019-2023 The Liqo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "context" + + "k8s.io/client-go/dynamic/dynamicinformer" + "sigs.k8s.io/controller-runtime/pkg/manager" +) + +var _ manager.Runnable = &RunnableFactory{} + +type RunnableFactory struct { + dynamicinformer.DynamicSharedInformerFactory +} + +func (r *RunnableFactory) Start(ctx context.Context) error { + r.DynamicSharedInformerFactory.Start(ctx.Done()) + r.DynamicSharedInformerFactory.WaitForCacheSync(ctx.Done()) + return nil +} diff --git a/pkg/liqo-controller-manager/external-network/utils/factorysource.go b/pkg/liqo-controller-manager/external-network/utils/factorysource.go new file mode 100644 index 0000000000..b2f544bfd3 --- /dev/null +++ b/pkg/liqo-controller-manager/external-network/utils/factorysource.go @@ -0,0 +1,104 @@ +// Copyright 2019-2023 The Liqo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/source" + + networkingv1alpha1 "github.com/liqotech/liqo/apis/networking/v1alpha1" +) + +func NewFactorySource(factory *RunnableFactory) *FactorySource { + c := make(chan event.GenericEvent) + handler := &factoryEventHandler{ + C: c, + } + return &FactorySource{ + handler: handler, + c: c, + factory: factory, + } +} + +type FactorySource struct { + handler *factoryEventHandler + c chan event.GenericEvent + factory *RunnableFactory +} + +func (f *FactorySource) Source() source.Source { + return &source.Channel{ + Source: f.c, + } +} + +func (f *FactorySource) ForResource(gvr schema.GroupVersionResource) { + _, err := f.factory.ForResource(gvr).Informer().AddEventHandler(f.handler) + utilruntime.Must(err) +} + +type factoryEventHandler struct { + C chan event.GenericEvent +} + +func (h *factoryEventHandler) OnAdd(obj interface{}, isInInitialList bool) { + unstructObj := obj.(*unstructured.Unstructured) + h.C <- event.GenericEvent{ + Object: &networkingv1alpha1.GatewayServer{ + ObjectMeta: metav1.ObjectMeta{ + Name: unstructObj.GetName(), + Namespace: unstructObj.GetNamespace(), + OwnerReferences: unstructObj.GetOwnerReferences(), + Labels: unstructObj.GetLabels(), + Annotations: unstructObj.GetAnnotations(), + }, + }, + } +} + +func (h *factoryEventHandler) OnUpdate(oldObj, newObj interface{}) { + unstructObj := newObj.(*unstructured.Unstructured) + h.C <- event.GenericEvent{ + Object: &networkingv1alpha1.GatewayServer{ + ObjectMeta: metav1.ObjectMeta{ + Name: unstructObj.GetName(), + Namespace: unstructObj.GetNamespace(), + OwnerReferences: unstructObj.GetOwnerReferences(), + Labels: unstructObj.GetLabels(), + Annotations: unstructObj.GetAnnotations(), + }, + }, + } +} + +func (h *factoryEventHandler) OnDelete(obj interface{}) { + unstructObj := obj.(*unstructured.Unstructured) + h.C <- event.GenericEvent{ + Object: &networkingv1alpha1.GatewayServer{ + ObjectMeta: metav1.ObjectMeta{ + Name: unstructObj.GetName(), + Namespace: unstructObj.GetNamespace(), + OwnerReferences: unstructObj.GetOwnerReferences(), + Labels: unstructObj.GetLabels(), + Annotations: unstructObj.GetAnnotations(), + }, + }, + } +} diff --git a/pkg/liqo-controller-manager/external-network/utils/getters.go b/pkg/liqo-controller-manager/external-network/utils/getters.go new file mode 100644 index 0000000000..cff3b83c03 --- /dev/null +++ b/pkg/liqo-controller-manager/external-network/utils/getters.go @@ -0,0 +1,67 @@ +// Copyright 2019-2023 The Liqo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "strings" + + corev1 "k8s.io/api/core/v1" + + networkingv1alpha1 "github.com/liqotech/liqo/apis/networking/v1alpha1" +) + +func ParseEndpoint(endpoint map[string]interface{}) *networkingv1alpha1.EndpointStatus { + res := &networkingv1alpha1.EndpointStatus{} + if value, ok := endpoint["addresses"]; ok { + res.Addresses = value.([]string) + } + if value, ok := endpoint["port"]; ok { + res.Port = value.(int32) + } + if value, ok := endpoint["protocol"]; ok { + tmp := value.(corev1.Protocol) + res.Protocol = &tmp + } + return res +} + +func GetValueOrDefault(m map[string]interface{}, key string, defaultValue string) string { + if value, ok := m[key]; ok { + return value.(string) + } + return defaultValue +} + +func TranslateMap(obj interface{}) map[string]string { + if obj == nil { + return nil + } + + m, ok := obj.(map[string]interface{}) + if !ok { + return nil + } + + res := make(map[string]string) + for k, v := range m { + res[k] = v.(string) + } + return res +} + +func KindToResource(kind string) string { + // lowercased and pluralized + return strings.ToLower(kind) + "s" +} diff --git a/pkg/liqo-controller-manager/external-network/utils/ownerenqueuer.go b/pkg/liqo-controller-manager/external-network/utils/ownerenqueuer.go new file mode 100644 index 0000000000..9127af86db --- /dev/null +++ b/pkg/liqo-controller-manager/external-network/utils/ownerenqueuer.go @@ -0,0 +1,65 @@ +// Copyright 2019-2023 The Liqo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "context" + + "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + ctrl "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +func NewOwnerEnqueuer(ownerKind string) handler.EventHandler { + return &OwnerEnqueuer{ + ownerKind: ownerKind, + } +} + +var _ handler.EventHandler = &OwnerEnqueuer{} + +type OwnerEnqueuer struct { + ownerKind string +} + +func (h *OwnerEnqueuer) Create(ctx context.Context, evt event.CreateEvent, q workqueue.RateLimitingInterface) { + panic("implement me") +} + +func (h *OwnerEnqueuer) Update(ctx context.Context, evt event.UpdateEvent, q workqueue.RateLimitingInterface) { + panic("implement me") +} + +func (h *OwnerEnqueuer) Delete(ctx context.Context, evt event.DeleteEvent, q workqueue.RateLimitingInterface) { + panic("implement me") +} + +func (h *OwnerEnqueuer) Generic(ctx context.Context, evt event.GenericEvent, q workqueue.RateLimitingInterface) { + owners := evt.Object.GetOwnerReferences() + + for _, owner := range owners { + if owner.Kind == h.ownerKind { + q.Add(ctrl.Request{ + NamespacedName: client.ObjectKey{ + Namespace: evt.Object.GetNamespace(), + Name: owner.Name, + }, + }) + return + } + } +} diff --git a/pkg/liqo-controller-manager/external-network/utils/patch.go b/pkg/liqo-controller-manager/external-network/utils/patch.go new file mode 100644 index 0000000000..0d53246432 --- /dev/null +++ b/pkg/liqo-controller-manager/external-network/utils/patch.go @@ -0,0 +1,70 @@ +// Copyright 2019-2023 The Liqo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "context" + "encoding/json" + "fmt" + + "gomodules.xyz/jsonpatch/v2" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/dynamic" +) + +func CreateOrPatch(ctx context.Context, c dynamic.ResourceInterface, objName string, mutate func(obj *unstructured.Unstructured) error) (*unstructured.Unstructured, error) { + oldObj, err := c.Get(ctx, objName, metav1.GetOptions{}) + switch { + case err == nil: + // the object already exists, patch it + newObj := oldObj.DeepCopy() + if err := mutate(newObj); err != nil { + return nil, fmt.Errorf("unable to patch the object: %w", err) + } + oldRaw, err := json.Marshal(oldObj) + if err != nil { + return nil, fmt.Errorf("unable to marshal the old object: %w", err) + } + newRaw, err := json.Marshal(newObj) + if err != nil { + return nil, fmt.Errorf("unable to marshal the new object: %w", err) + } + patch, err := jsonpatch.CreatePatch(oldRaw, newRaw) + if err != nil { + return nil, fmt.Errorf("unable to create the patch: %w", err) + } + if len(patch) == 0 { + // nothing to patch + return newObj, nil + } + patchRaw, err := json.Marshal(patch) + if err != nil { + return nil, fmt.Errorf("unable to marshal the patch: %w", err) + } + return c.Patch(ctx, newObj.GetName(), types.JSONPatchType, patchRaw, metav1.PatchOptions{}) + case apierrors.IsNotFound(err): + // the object does not exist, create it + newObj := &unstructured.Unstructured{} + if err := mutate(newObj); err != nil { + return nil, fmt.Errorf("unable to create the object: %w", err) + } + return c.Create(ctx, newObj, metav1.CreateOptions{}) + default: + return nil, fmt.Errorf("unable to get the object: %w", err) + } +} diff --git a/pkg/liqo-controller-manager/external-network/utils/template.go b/pkg/liqo-controller-manager/external-network/utils/template.go new file mode 100644 index 0000000000..b23cbec543 --- /dev/null +++ b/pkg/liqo-controller-manager/external-network/utils/template.go @@ -0,0 +1,74 @@ +// Copyright 2019-2023 The Liqo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "bytes" + "reflect" + "strconv" + "text/template" +) + +func RenderTemplate(obj interface{}, data interface{}) (interface{}, error) { + // if the object is a string, render the template + if reflect.TypeOf(obj).Kind() == reflect.String { + tmpl, err := template.New("").Parse(obj.(string)) + if err != nil { + return obj, err + } + + res := bytes.NewBufferString("") + if err := tmpl.Execute(res, data); err != nil { + return obj, err + } + + ret, err := strconv.Atoi(res.String()) + if err == nil { + return ret, nil + } + + return res.String(), nil + } + + // if the object is a map, render the template for each value + if reflect.TypeOf(obj).Kind() == reflect.Map { + for k, v := range obj.(map[string]interface{}) { + res, err := RenderTemplate(v, data) + if err != nil { + return obj, err + } + + obj.(map[string]interface{})[k] = res + } + + return obj, nil + } + + // if the object is a slice, render the template for each element + if reflect.TypeOf(obj).Kind() == reflect.Slice { + for i, v := range obj.([]interface{}) { + res, err := RenderTemplate(v, data) + if err != nil { + return obj, err + } + + obj.([]interface{})[i] = res + } + + return obj, nil + } + + return obj, nil +} diff --git a/pkg/utils/mapper/mapper.go b/pkg/utils/mapper/mapper.go index dfd2010b79..ffe589c1ad 100644 --- a/pkg/utils/mapper/mapper.go +++ b/pkg/utils/mapper/mapper.go @@ -32,6 +32,7 @@ import ( discoveryv1alpha1 "github.com/liqotech/liqo/apis/discovery/v1alpha1" ipamv1alpha1 "github.com/liqotech/liqo/apis/ipam/v1alpha1" netv1alpha1 "github.com/liqotech/liqo/apis/net/v1alpha1" + networkingv1alpha1 "github.com/liqotech/liqo/apis/networking/v1alpha1" offv1alpha1 "github.com/liqotech/liqo/apis/offloading/v1alpha1" sharingv1alpha1 "github.com/liqotech/liqo/apis/sharing/v1alpha1" virtualKubeletv1alpha1 "github.com/liqotech/liqo/apis/virtualkubelet/v1alpha1" @@ -89,6 +90,9 @@ func addDefaults(dClient *discovery.DiscoveryClient, mapper *meta.DefaultRESTMap if err = addGroup(dClient, ipamv1alpha1.GroupVersion, mapper); err != nil { return err } + if err = addGroup(dClient, networkingv1alpha1.GroupVersion, mapper); err != nil { + return err + } // Kubernetes groups if err = addGroup(dClient, corev1.SchemeGroupVersion, mapper); err != nil {