From 3efb9f190faca26dd58a8bfc4089a23dbf0cdeca Mon Sep 17 00:00:00 2001 From: Omer Schwartz Date: Mon, 25 Nov 2024 14:09:50 +0000 Subject: [PATCH] Change Mdns deployment type to StatefulSet Change the Mdns deployment type as well as updating how the predictable IP map is generated. --- ...signate.openstack.org_designatemdnses.yaml | 12 ++ .../designate.openstack.org_designates.yaml | 12 ++ api/v1beta1/designatemdns_types.go | 12 ++ api/v1beta1/zz_generated.deepcopy.go | 9 +- ...signate.openstack.org_designatemdnses.yaml | 12 ++ .../designate.openstack.org_designates.yaml | 12 ++ config/rbac/role.yaml | 19 -- controllers/designate_controller.go | 56 +++--- controllers/designateapi_controller.go | 1 - controllers/designatemdns_controller.go | 169 +++++++++++------- main.go | 2 +- .../{daemonset.go => statefulset.go} | 24 +-- 12 files changed, 205 insertions(+), 135 deletions(-) rename pkg/designatemdns/{daemonset.go => statefulset.go} (89%) diff --git a/api/bases/designate.openstack.org_designatemdnses.yaml b/api/bases/designate.openstack.org_designatemdnses.yaml index 1a716e96..4a61c787 100644 --- a/api/bases/designate.openstack.org_designatemdnses.yaml +++ b/api/bases/designate.openstack.org_designatemdnses.yaml @@ -70,6 +70,11 @@ spec: description: ContainerImage - Designate Container Image URL (will be set to environmental default if empty) type: string + controlNetworkName: + default: designate + description: ControlNetworkName - specify which network attachment + is to be used for control, notifys and zone transfers. + type: string customServiceConfig: description: CustomServiceConfig - customize the service config using this parameter to change service defaults, or overwrite rendered @@ -126,6 +131,13 @@ spec: from the Secret type: string type: object + replicas: + default: 1 + description: Replicas - Designate Mdns Replicas + format: int32 + maximum: 32 + minimum: 0 + type: integer resources: description: Resources - Compute Resources required by this service (Limits/Requests). https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ diff --git a/api/bases/designate.openstack.org_designates.yaml b/api/bases/designate.openstack.org_designates.yaml index cd2fc633..57c2ced2 100644 --- a/api/bases/designate.openstack.org_designates.yaml +++ b/api/bases/designate.openstack.org_designates.yaml @@ -827,6 +827,11 @@ spec: description: ContainerImage - Designate Container Image URL (will be set to environmental default if empty) type: string + controlNetworkName: + default: designate + description: ControlNetworkName - specify which network attachment + is to be used for control, notifys and zone transfers. + type: string customServiceConfig: description: CustomServiceConfig - customize the service config using this parameter to change service defaults, or overwrite @@ -884,6 +889,13 @@ spec: password from the Secret type: string type: object + replicas: + default: 1 + description: Replicas - Designate Mdns Replicas + format: int32 + maximum: 32 + minimum: 0 + type: integer resources: description: Resources - Compute Resources required by this service (Limits/Requests). https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ diff --git a/api/v1beta1/designatemdns_types.go b/api/v1beta1/designatemdns_types.go index 7edd4f72..8b4e31be 100644 --- a/api/v1beta1/designatemdns_types.go +++ b/api/v1beta1/designatemdns_types.go @@ -45,6 +45,13 @@ type DesignateMdnsSpecBase struct { // Common input parameters for all Designate services DesignateTemplate `json:",inline"` + // +kubebuilder:validation:Optional + // +kubebuilder:default=1 + // +kubebuilder:validation:Maximum=32 + // +kubebuilder:validation:Minimum=0 + // Replicas - Designate Mdns Replicas + Replicas *int32 `json:"replicas"` + // +kubebuilder:validation:Optional // DatabaseHostname - Designate Database Hostname DatabaseHostname string `json:"databaseHostname,omitempty"` @@ -60,6 +67,11 @@ type DesignateMdnsSpecBase struct { // +operator-sdk:csv:customresourcedefinitions:type=spec // TLS - Parameters related to the TLS TLS tls.Ca `json:"tls,omitempty"` + + // +kubebuilder:default="designate" + // +kubebuilder:validation:Optional + // ControlNetworkName - specify which network attachment is to be used for control, notifys and zone transfers. + ControlNetworkName string `json:"controlNetworkName"` } // DesignateMdnsStatus defines the observed state of DesignateMdns diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index f9d99d14..7f4a64c4 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -729,7 +729,7 @@ func (in *DesignateMdnsList) DeepCopyObject() runtime.Object { func (in *DesignateMdnsSpec) DeepCopyInto(out *DesignateMdnsSpec) { *out = *in in.DesignateServiceTemplate.DeepCopyInto(&out.DesignateServiceTemplate) - out.DesignateMdnsSpecBase = in.DesignateMdnsSpecBase + in.DesignateMdnsSpecBase.DeepCopyInto(&out.DesignateMdnsSpecBase) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DesignateMdnsSpec. @@ -746,6 +746,11 @@ func (in *DesignateMdnsSpec) DeepCopy() *DesignateMdnsSpec { func (in *DesignateMdnsSpecBase) DeepCopyInto(out *DesignateMdnsSpecBase) { *out = *in out.DesignateTemplate = in.DesignateTemplate + if in.Replicas != nil { + in, out := &in.Replicas, &out.Replicas + *out = new(int32) + **out = **in + } out.TLS = in.TLS } @@ -763,7 +768,7 @@ func (in *DesignateMdnsSpecBase) DeepCopy() *DesignateMdnsSpecBase { func (in *DesignateMdnsSpecCore) DeepCopyInto(out *DesignateMdnsSpecCore) { *out = *in in.DesignateServiceTemplateCore.DeepCopyInto(&out.DesignateServiceTemplateCore) - out.DesignateMdnsSpecBase = in.DesignateMdnsSpecBase + in.DesignateMdnsSpecBase.DeepCopyInto(&out.DesignateMdnsSpecBase) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DesignateMdnsSpecCore. diff --git a/config/crd/bases/designate.openstack.org_designatemdnses.yaml b/config/crd/bases/designate.openstack.org_designatemdnses.yaml index 1a716e96..4a61c787 100644 --- a/config/crd/bases/designate.openstack.org_designatemdnses.yaml +++ b/config/crd/bases/designate.openstack.org_designatemdnses.yaml @@ -70,6 +70,11 @@ spec: description: ContainerImage - Designate Container Image URL (will be set to environmental default if empty) type: string + controlNetworkName: + default: designate + description: ControlNetworkName - specify which network attachment + is to be used for control, notifys and zone transfers. + type: string customServiceConfig: description: CustomServiceConfig - customize the service config using this parameter to change service defaults, or overwrite rendered @@ -126,6 +131,13 @@ spec: from the Secret type: string type: object + replicas: + default: 1 + description: Replicas - Designate Mdns Replicas + format: int32 + maximum: 32 + minimum: 0 + type: integer resources: description: Resources - Compute Resources required by this service (Limits/Requests). https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ diff --git a/config/crd/bases/designate.openstack.org_designates.yaml b/config/crd/bases/designate.openstack.org_designates.yaml index cd2fc633..57c2ced2 100644 --- a/config/crd/bases/designate.openstack.org_designates.yaml +++ b/config/crd/bases/designate.openstack.org_designates.yaml @@ -827,6 +827,11 @@ spec: description: ContainerImage - Designate Container Image URL (will be set to environmental default if empty) type: string + controlNetworkName: + default: designate + description: ControlNetworkName - specify which network attachment + is to be used for control, notifys and zone transfers. + type: string customServiceConfig: description: CustomServiceConfig - customize the service config using this parameter to change service defaults, or overwrite @@ -884,6 +889,13 @@ spec: password from the Secret type: string type: object + replicas: + default: 1 + description: Replicas - Designate Mdns Replicas + format: int32 + maximum: 32 + minimum: 0 + type: integer resources: description: Resources - Compute Resources required by this service (Limits/Requests). https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 72eb5469..c5d93472 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -5,13 +5,6 @@ metadata: creationTimestamp: null name: manager-role rules: -- apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list - apiGroups: - "" resources: @@ -35,18 +28,6 @@ rules: - patch - update - watch -- apiGroups: - - apps - resources: - - daemonsets - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - apiGroups: - apps resources: diff --git a/controllers/designate_controller.go b/controllers/designate_controller.go index 7dc82c67..59ddc560 100644 --- a/controllers/designate_controller.go +++ b/controllers/designate_controller.go @@ -119,7 +119,6 @@ type DesignateReconciler struct { // +kubebuilder:rbac:groups=rabbitmq.openstack.org,resources=transporturls,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=redis.openstack.org,resources=redises,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=k8s.cni.cncf.io,resources=network-attachment-definitions,verbs=get;list;watch -// +kubebuilder:rbac:groups="",resources=nodes,verbs=get;list // +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;create;update;patch;delete;watch // service account, role, rolebinding @@ -775,25 +774,14 @@ func (r *DesignateReconciler) reconcileNormal(ctx context.Context, instance *des allocatedIPs[predIP] = true } - // Get a list of the nodes in the cluster - - // TODO(oschwart): - // * confirm whether or not this lists only the nodes we want (i.e. ones - // that will host the daemonset) - // * do we want to provide a mechanism to temporarily disabling this list - // for maintenance windows where nodes might be "coming and going" - - nodes, err := helper.GetKClient().CoreV1().Nodes().List(ctx, metav1.ListOptions{}) - if err != nil { - return ctrl.Result{}, err - } - - var nodeNames []string - for _, node := range nodes.Items { - nodeNames = append(nodeNames, fmt.Sprintf("mdns_%s", node.Name)) + // Handle Mdns predictable IPs configmap + mdnsReplicaCount := int(*instance.Spec.DesignateMdns.Replicas) + var mdnsNames []string + for i := 0; i < mdnsReplicaCount; i++ { + mdnsNames = append(mdnsNames, fmt.Sprintf("mdns_address_%d", i)) } - updatedMap, allocatedIPs, err := r.allocatePredictableIPs(ctx, predictableIPParams, nodeNames, mdnsConfigMap.Data, allocatedIPs) + updatedMap, allocatedIPs, err := r.allocatePredictableIPs(ctx, predictableIPParams, mdnsNames, mdnsConfigMap.Data, allocatedIPs) if err != nil { return ctrl.Result{}, err } @@ -959,7 +947,7 @@ func (r *DesignateReconciler) reconcileNormal(ctx context.Context, instance *des Log.Info("Deployment Worker task reconciled") // deploy designate-mdns - designateMdns, op, err := r.mdnsDaemonSetCreateOrUpdate(ctx, instance) + designateMdns, op, err := r.mdnsStatefulSetCreateOrUpdate(ctx, instance) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( designatev1beta1.DesignateMdnsReadyCondition, @@ -1585,8 +1573,8 @@ func (r *DesignateReconciler) workerDeploymentCreateOrUpdate(ctx context.Context return deployment, op, err } -func (r *DesignateReconciler) mdnsDaemonSetCreateOrUpdate(ctx context.Context, instance *designatev1beta1.Designate) (*designatev1beta1.DesignateMdns, controllerutil.OperationResult, error) { - daemonset := &designatev1beta1.DesignateMdns{ +func (r *DesignateReconciler) mdnsStatefulSetCreateOrUpdate(ctx context.Context, instance *designatev1beta1.Designate) (*designatev1beta1.DesignateMdns, controllerutil.OperationResult, error) { + statefulSet := &designatev1beta1.DesignateMdns{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-mdns", instance.Name), Namespace: instance.Namespace, @@ -1597,20 +1585,20 @@ func (r *DesignateReconciler) mdnsDaemonSetCreateOrUpdate(ctx context.Context, i instance.Spec.DesignateMdns.NodeSelector = instance.Spec.NodeSelector } - op, err := controllerutil.CreateOrUpdate(ctx, r.Client, daemonset, func() error { - daemonset.Spec = instance.Spec.DesignateMdns + op, err := controllerutil.CreateOrUpdate(ctx, r.Client, statefulSet, func() error { + statefulSet.Spec = instance.Spec.DesignateMdns // Add in transfers from umbrella Designate CR (this instance) spec // TODO: Add logic to determine when to set/overwrite, etc - daemonset.Spec.ServiceUser = instance.Spec.ServiceUser - daemonset.Spec.DatabaseHostname = instance.Status.DatabaseHostname - daemonset.Spec.DatabaseAccount = instance.Spec.DatabaseAccount - daemonset.Spec.Secret = instance.Spec.Secret - daemonset.Spec.TransportURLSecret = instance.Status.TransportURLSecret - daemonset.Spec.ServiceAccount = instance.RbacResourceName() - daemonset.Spec.TLS = instance.Spec.DesignateAPI.TLS.Ca - daemonset.Spec.NodeSelector = instance.Spec.DesignateMdns.NodeSelector - - err := controllerutil.SetControllerReference(instance, daemonset, r.Scheme) + statefulSet.Spec.ServiceUser = instance.Spec.ServiceUser + statefulSet.Spec.DatabaseHostname = instance.Status.DatabaseHostname + statefulSet.Spec.DatabaseAccount = instance.Spec.DatabaseAccount + statefulSet.Spec.Secret = instance.Spec.Secret + statefulSet.Spec.TransportURLSecret = instance.Status.TransportURLSecret + statefulSet.Spec.ServiceAccount = instance.RbacResourceName() + statefulSet.Spec.TLS = instance.Spec.DesignateAPI.TLS.Ca + statefulSet.Spec.NodeSelector = instance.Spec.DesignateMdns.NodeSelector + + err := controllerutil.SetControllerReference(instance, statefulSet, r.Scheme) if err != nil { return err } @@ -1618,7 +1606,7 @@ func (r *DesignateReconciler) mdnsDaemonSetCreateOrUpdate(ctx context.Context, i return nil }) - return daemonset, op, err + return statefulSet, op, err } func (r *DesignateReconciler) producerDeploymentCreateOrUpdate(ctx context.Context, instance *designatev1beta1.Designate) (*designatev1beta1.DesignateProducer, controllerutil.OperationResult, error) { diff --git a/controllers/designateapi_controller.go b/controllers/designateapi_controller.go index 0339f53e..499e4c57 100644 --- a/controllers/designateapi_controller.go +++ b/controllers/designateapi_controller.go @@ -104,7 +104,6 @@ var keystoneServices = []map[string]string{ // +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list; // +kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;create;update;patch;delete;watch -// +kubebuilder:rbac:groups=apps,resources=daemonsets,verbs=get;list;create;update;patch;delete;watch // +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;create;update;patch;delete;watch // +kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneservices,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneendpoints,verbs=get;list;watch;create;update;patch;delete diff --git a/controllers/designatemdns_controller.go b/controllers/designatemdns_controller.go index ed16acb7..2a5cc759 100644 --- a/controllers/designatemdns_controller.go +++ b/controllers/designatemdns_controller.go @@ -44,12 +44,12 @@ import ( designatemdns "github.com/openstack-k8s-operators/designate-operator/pkg/designatemdns" "github.com/openstack-k8s-operators/lib-common/modules/common" "github.com/openstack-k8s-operators/lib-common/modules/common/condition" - "github.com/openstack-k8s-operators/lib-common/modules/common/daemonset" "github.com/openstack-k8s-operators/lib-common/modules/common/env" "github.com/openstack-k8s-operators/lib-common/modules/common/helper" "github.com/openstack-k8s-operators/lib-common/modules/common/labels" nad "github.com/openstack-k8s-operators/lib-common/modules/common/networkattachment" "github.com/openstack-k8s-operators/lib-common/modules/common/secret" + "github.com/openstack-k8s-operators/lib-common/modules/common/statefulset" "github.com/openstack-k8s-operators/lib-common/modules/common/tls" "github.com/openstack-k8s-operators/lib-common/modules/common/util" mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" @@ -74,12 +74,13 @@ func (r *DesignateMdnsReconciler) GetScheme() *runtime.Scheme { type DesignateMdnsReconciler struct { client.Client Kclient kubernetes.Interface + Log logr.Logger Scheme *runtime.Scheme } -// GetLogger returns a logger object with a prefix of "controller.name" and additional controller context fields -func (r *DesignateMdnsReconciler) GetLogger(ctx context.Context) logr.Logger { - return log.FromContext(ctx).WithName("Controllers").WithName("DesignateMdns") +// GetLogger +func (r *DesignateMdnsReconciler) GetLogger() logr.Logger { + return r.Log } //+kubebuilder:rbac:groups=designate.openstack.org,resources=designatemdnses,verbs=get;list;watch;create;update;patch;delete @@ -102,7 +103,7 @@ func (r *DesignateMdnsReconciler) GetLogger(ctx context.Context) logr.Logger { // For more details, check Reconcile and its Result here: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.12.2/pkg/reconcile func (r *DesignateMdnsReconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ctrl.Result, _err error) { - Log := r.GetLogger(ctx) + _ = r.Log.WithValues("designatemdns", req.NamespacedName) // Fetch the DesignateMdns instance instance := &designatev1beta1.DesignateMdns{} err := r.Client.Get(ctx, req.NamespacedName, instance) @@ -122,7 +123,7 @@ func (r *DesignateMdnsReconciler) Reconcile(ctx context.Context, req ctrl.Reques r.Client, r.Kclient, r.Scheme, - Log, + r.Log, ) if err != nil { return ctrl.Result{}, err @@ -184,7 +185,7 @@ func (r *DesignateMdnsReconciler) Reconcile(ctx context.Context, req ctrl.Reques // Handle service delete if !instance.DeletionTimestamp.IsZero() { - return r.reconcileDelete(ctx, instance, helper) + return r.reconcileDelete(instance, helper) } // Handle non-deleted clusters @@ -192,11 +193,9 @@ func (r *DesignateMdnsReconciler) Reconcile(ctx context.Context, req ctrl.Reques } // SetupWithManager sets up the controller with the Manager. -func (r *DesignateMdnsReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error { +func (r *DesignateMdnsReconciler) SetupWithManager(mgr ctrl.Manager) error { // Watch for changes to any CustomServiceConfigSecrets. Global secrets // (e.g. TransportURLSecret) are handled by the top designate controller. - Log := r.GetLogger(ctx) - // index passwordSecretField if err := mgr.GetFieldIndexer().IndexField(context.Background(), &designatev1beta1.DesignateMdns{}, passwordSecretField, func(rawObj client.Object) []string { // Extract the secret name from the spec, if one is provided @@ -232,7 +231,7 @@ func (r *DesignateMdnsReconciler) SetupWithManager(ctx context.Context, mgr ctrl client.InNamespace(namespace), } if err := r.Client.List(context.Background(), apis, listOpts...); err != nil { - Log.Error(err, "Unable to retrieve Mdns CRs %v") + r.Log.Error(err, "Unable to retrieve Mdns CRs %v") return nil } for _, cr := range apis.Items { @@ -242,7 +241,7 @@ func (r *DesignateMdnsReconciler) SetupWithManager(ctx context.Context, mgr ctrl Namespace: namespace, Name: cr.Name, } - Log.Info(fmt.Sprintf("Secret %s is used by Designate CR %s", secretName, cr.Name)) + r.Log.Info(fmt.Sprintf("Secret %s is used by Designate CR %s", secretName, cr.Name)) result = append(result, reconcile.Request{NamespacedName: name}) } } @@ -263,7 +262,7 @@ func (r *DesignateMdnsReconciler) SetupWithManager(ctx context.Context, mgr ctrl client.InNamespace(o.GetNamespace()), } if err := r.Client.List(context.Background(), apis, listOpts...); err != nil { - Log.Error(err, "Unable to retrieve Mdns CRs %v") + r.Log.Error(err, "Unable to retrieve Mdns CRs %v") return nil } @@ -279,7 +278,7 @@ func (r *DesignateMdnsReconciler) SetupWithManager(ctx context.Context, mgr ctrl Namespace: o.GetNamespace(), Name: cr.Name, } - Log.Info(fmt.Sprintf("ConfigMap object %s and CR %s marked with label: %s", o.GetName(), cr.Name, l)) + r.Log.Info(fmt.Sprintf("ConfigMap object %s and CR %s marked with label: %s", o.GetName(), cr.Name, l)) result = append(result, reconcile.Request{NamespacedName: name}) } } @@ -292,7 +291,7 @@ func (r *DesignateMdnsReconciler) SetupWithManager(ctx context.Context, mgr ctrl return ctrl.NewControllerManagedBy(mgr). For(&designatev1beta1.DesignateMdns{}). - Owns(&appsv1.DaemonSet{}). + Owns(&appsv1.StatefulSet{}). Owns(&corev1.Service{}). // watch the secrets we don't own Watches(&corev1.Secret{}, @@ -347,35 +346,28 @@ func (r *DesignateMdnsReconciler) findObjectsForSrc(ctx context.Context, src cli return requests } -func (r *DesignateMdnsReconciler) reconcileDelete(ctx context.Context, instance *designatev1beta1.DesignateMdns, helper *helper.Helper) (ctrl.Result, error) { - Log := r.GetLogger(ctx) - - Log.Info(fmt.Sprintf("Reconciling Service '%s' delete", instance.Name)) +func (r *DesignateMdnsReconciler) reconcileDelete(instance *designatev1beta1.DesignateMdns, helper *helper.Helper) (ctrl.Result, error) { + r.Log.Info(fmt.Sprintf("Reconciling Service '%s' delete", instance.Name)) // We did all the cleanup on the objects we created so we can remove the // finalizer from ourselves to allow the deletion controllerutil.RemoveFinalizer(instance, helper.GetFinalizer()) - Log.Info(fmt.Sprintf("Reconciled Service '%s' delete successfully", instance.Name)) + r.Log.Info(fmt.Sprintf("Reconciled Service '%s' delete successfully", instance.Name)) return ctrl.Result{}, nil } func (r *DesignateMdnsReconciler) reconcileInit( - ctx context.Context, instance *designatev1beta1.DesignateMdns, ) (ctrl.Result, error) { - Log := r.GetLogger(ctx) + r.Log.Info(fmt.Sprintf("Reconciling Service '%s' init", instance.Name)) - Log.Info(fmt.Sprintf("Reconciling Service '%s' init", instance.Name)) - - Log.Info(fmt.Sprintf("Reconciled Service '%s' init successfully", instance.Name)) + r.Log.Info(fmt.Sprintf("Reconciled Service '%s' init successfully", instance.Name)) return ctrl.Result{}, nil } func (r *DesignateMdnsReconciler) reconcileNormal(ctx context.Context, instance *designatev1beta1.DesignateMdns, helper *helper.Helper) (ctrl.Result, error) { - Log := r.GetLogger(ctx) - - Log.Info("Reconciling Service") + r.Log.Info("Reconciling Service") // ConfigMap configMapVars := make(map[string]env.Setter) @@ -414,7 +406,7 @@ func (r *DesignateMdnsReconciler) reconcileNormal(ctx context.Context, instance // parentDesignateName := designate.GetOwningDesignateName(instance) - Log.Info(fmt.Sprintf("Reconciling Service '%s' init: parent name: %s", instance.Name, parentDesignateName)) + r.Log.Info(fmt.Sprintf("Reconciling Service '%s' init: parent name: %s", instance.Name, parentDesignateName)) ctrlResult, err = r.getSecret(ctx, helper, instance, fmt.Sprintf("%s-scripts", parentDesignateName), &configMapVars, "") if err != nil { @@ -496,7 +488,7 @@ func (r *DesignateMdnsReconciler) reconcileNormal(ctx context.Context, instance // create hash over all the different input resources to identify if any those changed // and a restart/recreate is required. // - inputHash, hashChanged, err := r.createHashOfInputHashes(ctx, instance, configMapVars) + inputHash, hashChanged, err := r.createHashOfInputHashes(instance, configMapVars) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.ServiceConfigReadyCondition, @@ -524,7 +516,7 @@ func (r *DesignateMdnsReconciler) reconcileNormal(ctx context.Context, instance nad, err := nad.GetNADWithName(ctx, helper, netAtt, instance.Namespace) if err != nil { if k8s_errors.IsNotFound(err) { - Log.Info(fmt.Sprintf("network-attachment-definition %s not found", netAtt)) + r.Log.Info(fmt.Sprintf("network-attachment-definition %s not found", netAtt)) instance.Status.Conditions.Set(condition.FalseCondition( condition.NetworkAttachmentsReadyCondition, condition.RequestedReason, @@ -554,7 +546,7 @@ func (r *DesignateMdnsReconciler) reconcileNormal(ctx context.Context, instance } // Handle service init - ctrlResult, err = r.reconcileInit(ctx, instance) + ctrlResult, err = r.reconcileInit(instance) if err != nil { return ctrlResult, err } else if (ctrlResult != ctrl.Result{}) { @@ -562,7 +554,7 @@ func (r *DesignateMdnsReconciler) reconcileNormal(ctx context.Context, instance } // Handle service update - ctrlResult, err = r.reconcileUpdate(ctx, instance) + ctrlResult, err = r.reconcileUpdate(instance) if err != nil { return ctrlResult, err } else if (ctrlResult != ctrl.Result{}) { @@ -570,7 +562,7 @@ func (r *DesignateMdnsReconciler) reconcileNormal(ctx context.Context, instance } // Handle service upgrade - ctrlResult, err = r.reconcileUpgrade(ctx, instance) + ctrlResult, err = r.reconcileUpgrade(instance) if err != nil { return ctrlResult, err } else if (ctrlResult != ctrl.Result{}) { @@ -581,17 +573,14 @@ func (r *DesignateMdnsReconciler) reconcileNormal(ctx context.Context, instance // normal reconcile tasks // - // Define a new DaemonSet object - dset := daemonset.NewDaemonSet( - designatemdns.DaemonSet( - instance, - inputHash, - serviceLabels, - serviceAnnotations), - 5, + // Define a new Mdns StatefulSet object + statefulSetDef := designatemdns.StatefulSet(instance, inputHash, serviceLabels, serviceAnnotations) + statefulSet := statefulset.NewStatefulSet( + statefulSetDef, + time.Duration(5)*time.Second, ) - ctrlResult, err = dset.CreateOrPatch(ctx, helper) + ctrlResult, err = statefulSet.CreateOrPatch(ctx, helper) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.DeploymentReadyCondition, @@ -608,11 +597,8 @@ func (r *DesignateMdnsReconciler) reconcileNormal(ctx context.Context, instance condition.DeploymentReadyRunningMessage)) return ctrlResult, nil } - - if dset.GetDaemonSet().Generation == dset.GetDaemonSet().Status.ObservedGeneration { - instance.Status.DesiredNumberScheduled = dset.GetDaemonSet().Status.DesiredNumberScheduled - // TODO(oschwart) change for NumberReady? - instance.Status.ReadyCount = dset.GetDaemonSet().Status.NumberReady + if statefulSet.GetStatefulSet().Generation == statefulSet.GetStatefulSet().Status.ObservedGeneration { + instance.Status.ReadyCount = statefulSet.GetStatefulSet().Status.ReadyReplicas // verify if network attachment matches expectations networkReady, networkAttachmentStatus, err := nad.VerifyNetworkStatusFromAnnotation( @@ -641,11 +627,11 @@ func (r *DesignateMdnsReconciler) reconcileNormal(ctx context.Context, instance return ctrl.Result{RequeueAfter: time.Duration(1) * time.Second}, nil } - if instance.Status.ReadyCount == instance.Status.DesiredNumberScheduled { + if instance.Status.ReadyCount == *instance.Spec.Replicas { instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) } } - // create DaemonSet - end + // create StatefulSet - end // We reached the end of the Reconcile, update the Ready condition based on // the sub conditions @@ -653,33 +639,29 @@ func (r *DesignateMdnsReconciler) reconcileNormal(ctx context.Context, instance instance.Status.Conditions.MarkTrue( condition.ReadyCondition, condition.ReadyMessage) } else { - Log.Info("Not all conditions are ready for Mdns controller") + r.Log.Info("Not all conditions are ready for Mdns controller") } - Log.Info("Reconciled Service successfully") + r.Log.Info("Reconciled Service successfully") return ctrl.Result{}, nil } -func (r *DesignateMdnsReconciler) reconcileUpdate(ctx context.Context, instance *designatev1beta1.DesignateMdns) (ctrl.Result, error) { - Log := r.GetLogger(ctx) - - Log.Info(fmt.Sprintf("Reconciling Service '%s' update", instance.Name)) +func (r *DesignateMdnsReconciler) reconcileUpdate(instance *designatev1beta1.DesignateMdns) (ctrl.Result, error) { + r.Log.Info(fmt.Sprintf("Reconciling Service '%s' update", instance.Name)) // TODO: should have minor update tasks if required // - delete dbsync hash from status to rerun it? - Log.Info(fmt.Sprintf("Reconciled Service '%s' update successfully", instance.Name)) + r.Log.Info(fmt.Sprintf("Reconciled Service '%s' update successfully", instance.Name)) return ctrl.Result{}, nil } -func (r *DesignateMdnsReconciler) reconcileUpgrade(ctx context.Context, instance *designatev1beta1.DesignateMdns) (ctrl.Result, error) { - Log := r.GetLogger(ctx) - - Log.Info(fmt.Sprintf("Reconciling Service '%s' upgrade", instance.Name)) +func (r *DesignateMdnsReconciler) reconcileUpgrade(instance *designatev1beta1.DesignateMdns) (ctrl.Result, error) { + r.Log.Info(fmt.Sprintf("Reconciling Service '%s' upgrade", instance.Name)) // TODO: should have major version upgrade tasks // -delete dbsync hash from status to rerun it? - Log.Info(fmt.Sprintf("Reconciled Service '%s' upgrade successfully", instance.Name)) + r.Log.Info(fmt.Sprintf("Reconciled Service '%s' upgrade successfully", instance.Name)) return ctrl.Result{}, nil } @@ -784,6 +766,64 @@ func (r *DesignateMdnsReconciler) generateServiceConfigMaps( ), } + var nadInfo *designate.NADConfig + for _, netAtt := range instance.Spec.NetworkAttachments { + nad, err := nad.GetNADWithName(ctx, h, netAtt, instance.Namespace) + if err != nil { + if k8s_errors.IsNotFound(err) { + r.Log.Info(fmt.Sprintf("network-attachment-definition %s not found, cannot configure pod", netAtt)) + instance.Status.Conditions.Set(condition.FalseCondition( + condition.NetworkAttachmentsReadyCondition, + condition.RequestedReason, + condition.SeverityWarning, // Severity is just warning because while we expect it, we will retry. + condition.NetworkAttachmentsReadyErrorMessage, + netAtt)) + return nil + } + instance.Status.Conditions.Set(condition.FalseCondition( + condition.NetworkAttachmentsReadyCondition, + condition.ErrorReason, + condition.SeverityError, // We cannot proceed with a broken network attachment. + condition.NetworkAttachmentsReadyErrorMessage, + err.Error())) + return err + } + if nad.Name == instance.Spec.ControlNetworkName { + nadInfo, err = designate.GetNADConfig(nad) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.NetworkAttachmentsReadyCondition, + condition.ErrorReason, + condition.SeverityError, // We cannot proceed with a broken network attachment. + condition.NetworkAttachmentsReadyErrorMessage, + err.Error())) + return err + } + break + } + } + if nadInfo == nil { + return fmt.Errorf("Unable to locate network attachment %s", instance.Spec.ControlNetworkName) + } + + cidr := nadInfo.IPAM.CIDR.String() + if cidr == "" { + err = fmt.Errorf("designate control network attachment not configured, check NetworkAttachments and ControlNetworkName") + instance.Status.Conditions.Set(condition.FalseCondition( + condition.NetworkAttachmentsReadyCondition, + condition.ErrorReason, + condition.SeverityError, + condition.NetworkAttachmentsReadyErrorMessage, + err)) + return err + } + if nadInfo.IPAM.CIDR.Addr().Is4() { + templateParameters["IPVersion"] = "4" + } else { + templateParameters["IPVersion"] = "6" + } + templateParameters["AllowCIDR"] = cidr + cms := []util.Template{ // ScriptsConfigMap { @@ -814,12 +854,9 @@ func (r *DesignateMdnsReconciler) generateServiceConfigMaps( // // returns the hash, whether the hash changed (as a bool) and any error func (r *DesignateMdnsReconciler) createHashOfInputHashes( - ctx context.Context, instance *designatev1beta1.DesignateMdns, envVars map[string]env.Setter, ) (string, bool, error) { - Log := r.GetLogger(ctx) - var hashMap map[string]string changed := false @@ -830,7 +867,7 @@ func (r *DesignateMdnsReconciler) createHashOfInputHashes( } if hashMap, changed = util.SetHash(instance.Status.Hash, common.InputHashName, hash); changed { instance.Status.Hash = hashMap - Log.Info(fmt.Sprintf("Input maps hash %s - %s", common.InputHashName, hash)) + r.Log.Info(fmt.Sprintf("Input maps hash %s - %s", common.InputHashName, hash)) } return hash, changed, nil } diff --git a/main.go b/main.go index d2248794..f1b9a55e 100644 --- a/main.go +++ b/main.go @@ -178,7 +178,7 @@ func main() { Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Kclient: kclient, - }).SetupWithManager(context.Background(), mgr); err != nil { + }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "DesignateMdns") os.Exit(1) } diff --git a/pkg/designatemdns/daemonset.go b/pkg/designatemdns/statefulset.go similarity index 89% rename from pkg/designatemdns/daemonset.go rename to pkg/designatemdns/statefulset.go index 67e81d69..16dcc5f0 100644 --- a/pkg/designatemdns/daemonset.go +++ b/pkg/designatemdns/statefulset.go @@ -35,13 +35,13 @@ const ( ServiceCommand = "/usr/local/bin/kolla_set_configs && /usr/local/bin/kolla_start" ) -// DaemonSet func -func DaemonSet( +// StatefulSet func +func StatefulSet( instance *designatev1beta1.DesignateMdns, configHash string, labels map[string]string, annotations map[string]string, -) *appsv1.DaemonSet { +) *appsv1.StatefulSet { rootUser := int64(0) serviceName := fmt.Sprintf("%s-mdns", designate.ServiceName) volumes := GetVolumes(designate.GetOwningDesignateName(instance)) @@ -51,13 +51,13 @@ func DaemonSet( // TODO might need tuning TimeoutSeconds: 15, PeriodSeconds: 13, - InitialDelaySeconds: 3, + InitialDelaySeconds: 15, } readinessProbe := &corev1.Probe{ // TODO might need tuning TimeoutSeconds: 15, - PeriodSeconds: 15, - InitialDelaySeconds: 5, + PeriodSeconds: 13, + InitialDelaySeconds: 10, } args := []string{"-c", ServiceCommand} @@ -78,13 +78,13 @@ func DaemonSet( volumeMounts = append(volumeMounts, instance.Spec.TLS.CreateVolumeMounts(nil)...) } - daemonset := &appsv1.DaemonSet{ + statefulSet := &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: instance.Name, Namespace: instance.Namespace, Labels: labels, }, - Spec: appsv1.DaemonSetSpec{ + Spec: appsv1.StatefulSetSpec{ Selector: &metav1.LabelSelector{ MatchLabels: labels, }, @@ -122,7 +122,7 @@ func DaemonSet( // If possible two pods of the same service should not // run on the same worker node. If this is not possible // the get still created on the same worker node. - daemonset.Spec.Template.Spec.Affinity = affinity.DistributePods( + statefulSet.Spec.Template.Spec.Affinity = affinity.DistributePods( common.AppSelector, []string{ serviceName, @@ -130,7 +130,7 @@ func DaemonSet( corev1.LabelHostname, ) if instance.Spec.NodeSelector != nil { - daemonset.Spec.Template.Spec.NodeSelector = *instance.Spec.NodeSelector + statefulSet.Spec.Template.Spec.NodeSelector = *instance.Spec.NodeSelector } initContainerDetails := designate.APIDetails{ @@ -142,7 +142,7 @@ func DaemonSet( UserPasswordSelector: instance.Spec.PasswordSelectors.Service, VolumeMounts: designate.GetInitVolumeMounts(), } - daemonset.Spec.Template.Spec.InitContainers = designate.InitContainer(initContainerDetails) + statefulSet.Spec.Template.Spec.InitContainers = designate.InitContainer(initContainerDetails) - return daemonset + return statefulSet }