From c4edae6195e5a2fa1a2ef787d98e28718c89f878 Mon Sep 17 00:00:00 2001 From: Alexandre Guitton Date: Mon, 8 Nov 2021 14:31:37 +0100 Subject: [PATCH] [Fix/Operator] Parameter context controller case spam (#152) * add feature and bump version * append changelog * fix parameter context & set requeue configuration #124 * bump version & append changelog --- CHANGELOG.md | 12 ++- config/manager/manager.yaml | 2 +- controllers/nificluster_controller.go | 29 ++++--- controllers/nificlustertask_controller.go | 25 +++--- controllers/nifidataflow_controller.go | 22 +++--- .../nifiparametercontext_controller.go | 18 +++-- controllers/nifiregistryclient_controller.go | 16 ++-- controllers/nifiuser_controller.go | 22 +++--- controllers/nifiusergroup_controller.go | 16 ++-- helm/nifikop/Chart.yaml | 4 +- helm/nifikop/README.md | 2 +- helm/nifikop/values.yaml | 2 +- main.go | 76 +++++++++++-------- .../parametercontext/parametercontext.go | 28 ++++--- pkg/common/common.go | 27 +++++++ pkg/util/util.go | 27 +++++++ site/docs/2_setup/1_getting_started.md | 4 +- .../1_customizable_install_with_helm.md | 2 +- version/version.go | 2 +- 19 files changed, 222 insertions(+), 114 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d9e15e015..11247d637 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,11 +10,21 @@ ### Fixed Bugs +## v0.7.2 + +### Added + +- [PR #152](https://github.com/Orange-OpenSource/nifikop/pull/152) - **[Operator]** Configurable requeue interval (#124) + +### Fixed Bugs + +- [PR #152](https://github.com/Orange-OpenSource/nifikop/pull/152) - **[Operator/NiFiParameterContext]** Fix is sync control in nil value case. + ## v0.7.1 ### Added -- [PR #144](https://github.com/Orange-OpenSource/nifikop/pull/144) - **[Operator/Parameter]** Support empty string and no value set. +- [PR #144](https://github.com/Orange-OpenSource/nifikop/pull/144) - **[Operator/NiFiParameterContext]** Support empty string and no value set. ## v0.7.0 diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index ed33b05a2..894b86173 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -29,7 +29,7 @@ spec: - /manager args: - --leader-elect - image: orangeopensource/nifikop:v0.7.1-release + image: orangeopensource/nifikop:v0.7.2-release name: nifikop securityContext: allowPrivilegeEscalation: false diff --git a/controllers/nificluster_controller.go b/controllers/nificluster_controller.go index 2536623c5..dfc350dd1 100644 --- a/controllers/nificluster_controller.go +++ b/controllers/nificluster_controller.go @@ -47,11 +47,13 @@ var clusterUsersFinalizer = "nificlusters.nifi.orange.com/users" // NifiClusterReconciler reconciles a NifiCluster object type NifiClusterReconciler struct { client.Client - DirectClient client.Reader - Log logr.Logger - Scheme *runtime.Scheme - Namespaces []string - Recorder record.EventRecorder + DirectClient client.Reader + Log logr.Logger + Scheme *runtime.Scheme + Namespaces []string + Recorder record.EventRecorder + RequeueIntervals map[string]int + RequeueOffset int } // +kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list;watch;create;update;patch;delete @@ -127,6 +129,8 @@ func (r *NifiClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) nifi.New(r.Client, r.DirectClient, r.Scheme, instance), } + intervalNotReady := util.GetRequeueInterval(r.RequeueIntervals["CLUSTER_TASK_NOT_READY_REQUEUE_INTERVAL"], r.RequeueOffset) + intervalRunning := util.GetRequeueInterval(r.RequeueIntervals["CLUSTER_TASK_RUNNING_REQUEUE_INTERVAL"], r.RequeueOffset) for _, rec := range reconcilers { err = rec.Reconcile(r.Log) if err != nil { @@ -134,31 +138,31 @@ func (r *NifiClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) case errorfactory.NodesUnreachable: r.Log.Info("Nodes unreachable, may still be starting up") return reconcile.Result{ - RequeueAfter: time.Duration(15) * time.Second, + RequeueAfter: intervalNotReady, }, nil case errorfactory.NodesNotReady: r.Log.Info("Nodes not ready, may still be starting up") return reconcile.Result{ - RequeueAfter: time.Duration(15) * time.Second, + RequeueAfter: intervalNotReady, }, nil case errorfactory.ResourceNotReady: r.Log.Info("A new resource was not found or may not be ready") r.Log.Info(err.Error()) return reconcile.Result{ - RequeueAfter: time.Duration(7) * time.Second, + RequeueAfter: intervalNotReady/2, }, nil case errorfactory.ReconcileRollingUpgrade: r.Log.Info("Rolling Upgrade in Progress") return reconcile.Result{ - RequeueAfter: time.Duration(15) * time.Second, + RequeueAfter: intervalRunning, }, nil case errorfactory.NifiClusterNotReady: return reconcile.Result{ - RequeueAfter: time.Duration(15) * time.Second, + RequeueAfter: intervalNotReady, }, nil case errorfactory.NifiClusterTaskRunning: return reconcile.Result{ - RequeueAfter: time.Duration(20) * time.Second, + RequeueAfter: intervalRunning, }, nil default: return RequeueWithError(r.Log, err.Error(), err) @@ -250,6 +254,7 @@ func (r *NifiClusterReconciler) checkFinalizers(ctx context.Context, // Do any necessary PKI cleanup - a PKI backend should make sure any // user finalizations are done before it does its final cleanup + interval := util.GetRequeueInterval(r.RequeueIntervals["CLUSTER_TASK_NOT_READY_REQUEUE_INTERVAL"]/3, r.RequeueOffset) r.Log.Info("Tearing down any PKI resources for the nificluster") if err = pki.GetPKIManager(r.Client, cluster).FinalizePKI(ctx, r.Log); err != nil { switch err.(type) { @@ -257,7 +262,7 @@ func (r *NifiClusterReconciler) checkFinalizers(ctx context.Context, r.Log.Info("The PKI is not ready to be torn down") return ctrl.Result{ Requeue: true, - RequeueAfter: time.Duration(5) * time.Second, + RequeueAfter: interval, }, nil default: return RequeueWithError(r.Log, "failed to finalize PKI", err) diff --git a/controllers/nificlustertask_controller.go b/controllers/nificlustertask_controller.go index 0cbdabde7..23c6a4b85 100644 --- a/controllers/nificlustertask_controller.go +++ b/controllers/nificlustertask_controller.go @@ -24,6 +24,7 @@ import ( "github.com/Orange-OpenSource/nifikop/pkg/errorfactory" "github.com/Orange-OpenSource/nifikop/pkg/k8sutil" "github.com/Orange-OpenSource/nifikop/pkg/nificlient/config" + "github.com/Orange-OpenSource/nifikop/pkg/util" "github.com/Orange-OpenSource/nifikop/pkg/util/clientconfig" nifiutil "github.com/Orange-OpenSource/nifikop/pkg/util/nifi" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -46,9 +47,11 @@ import ( // NifiClusterTaskReconciler reconciles type NifiClusterTaskReconciler struct { client.Client - Log logr.Logger - Scheme *runtime.Scheme - Recorder record.EventRecorder + Log logr.Logger + Scheme *runtime.Scheme + Recorder record.EventRecorder + RequeueIntervals map[string]int + RequeueOffset int } // Reconcile is part of the main kubernetes reconciliation loop which aims to @@ -63,6 +66,9 @@ type NifiClusterTaskReconciler struct { func (r *NifiClusterTaskReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { _ = r.Log.WithValues("nificlustertask", req.NamespacedName) + intervalNotReady := util.GetRequeueInterval(r.RequeueIntervals["CLUSTER_TASK_NOT_READY_REQUEUE_INTERVAL"], r.RequeueOffset) + intervalRunning := util.GetRequeueInterval(r.RequeueIntervals["CLUSTER_TASK_RUNNING_REQUEUE_INTERVAL"], r.RequeueOffset) + intervalTimeout := util.GetRequeueInterval(r.RequeueIntervals["CLUSTER_TASK_TIMEOUT_REQUEUE_INTERVAL"], r.RequeueOffset) // Fetch the NifiCluster instance instance := &v1alpha1.NifiCluster{} err := r.Client.Get(ctx, req.NamespacedName, instance) @@ -95,15 +101,16 @@ func (r *NifiClusterTaskReconciler) Reconcile(ctx context.Context, req ctrl.Requ switch errors.Cause(err).(type) { case errorfactory.NifiClusterNotReady, errorfactory.ResourceNotReady: return reconcile.Result{ - RequeueAfter: time.Duration(15) * time.Second, + RequeueAfter: intervalNotReady, }, nil case errorfactory.NifiClusterTaskRunning: return reconcile.Result{ - RequeueAfter: time.Duration(20) * time.Second, + RequeueAfter: intervalRunning, }, nil case errorfactory.NifiClusterTaskTimeout, errorfactory.NifiClusterTaskFailure: + return reconcile.Result{ - RequeueAfter: time.Duration(20) * time.Second, + RequeueAfter: intervalTimeout, }, nil default: return RequeueWithError(r.Log, err.Error(), err) @@ -131,15 +138,15 @@ func (r *NifiClusterTaskReconciler) Reconcile(ctx context.Context, req ctrl.Requ switch errors.Cause(err).(type) { case errorfactory.NifiClusterNotReady: return reconcile.Result{ - RequeueAfter: time.Duration(15) * time.Second, + RequeueAfter: intervalNotReady, }, nil case errorfactory.NifiClusterTaskRunning: return reconcile.Result{ - RequeueAfter: time.Duration(20) * time.Second, + RequeueAfter: intervalRunning, }, nil case errorfactory.NifiClusterTaskTimeout, errorfactory.NifiClusterTaskFailure: return reconcile.Result{ - RequeueAfter: time.Duration(20) * time.Second, + RequeueAfter: intervalTimeout, }, nil default: return RequeueWithError(r.Log, err.Error(), err) diff --git a/controllers/nifidataflow_controller.go b/controllers/nifidataflow_controller.go index 2d6078e42..7c0b23bd9 100644 --- a/controllers/nifidataflow_controller.go +++ b/controllers/nifidataflow_controller.go @@ -49,9 +49,11 @@ var dataflowFinalizer = "nifidataflows.nifi.orange.com/finalizer" // NifiDataflowReconciler reconciles a NifiDataflow object type NifiDataflowReconciler struct { client.Client - Log logr.Logger - Scheme *runtime.Scheme - Recorder record.EventRecorder + Log logr.Logger + Scheme *runtime.Scheme + Recorder record.EventRecorder + RequeueInterval int + RequeueOffset int } // +kubebuilder:rbac:groups=nifi.orange.com,resources=nifidataflows,verbs=get;list;watch;create;update;patch;delete @@ -71,7 +73,7 @@ func (r *NifiDataflowReconciler) Reconcile(ctx context.Context, req ctrl.Request _ = r.Log.WithValues("nifidataflow", req.NamespacedName) var err error - + interval := util.GetRequeueInterval(r.RequeueInterval, r.RequeueOffset) // Fetch the NifiDataflow instance instance := &v1alpha1.NifiDataflow{} if err = r.Client.Get(ctx, req.NamespacedName, instance); err != nil { @@ -239,7 +241,7 @@ func (r *NifiDataflowReconciler) Reconcile(ctx context.Context, req ctrl.Request instance.Spec.ClusterRef.Name, clusterConnect.Id())) // the cluster does not exist - should have been caught pre-flight - return RequeueAfter(time.Duration(15) * time.Second) + return RequeueAfter(interval) } // Ìn case of the cluster reference changed. @@ -258,7 +260,7 @@ func (r *NifiDataflowReconciler) Reconcile(ctx context.Context, req ctrl.Request if err := r.Client.Update(ctx, current); err != nil { return RequeueWithError(r.Log, "failed to update NifiDatafllow", err) } - return RequeueAfter(time.Duration(15) * time.Second) + return RequeueAfter(interval) } if (instance.Spec.SyncNever() && len(instance.Status.State) > 0) || @@ -346,7 +348,7 @@ func (r *NifiDataflowReconciler) Reconcile(ctx context.Context, req ctrl.Request errorfactory.NifiFlowControllerServiceScheduling, errorfactory.NifiFlowScheduling, errorfactory.NifiFlowSyncing: return reconcile.Result{ - RequeueAfter: time.Duration(5) * time.Second, + RequeueAfter: interval/3, }, nil default: r.Recorder.Event(instance, corev1.EventTypeWarning, "SynchronizingFailed", @@ -401,7 +403,7 @@ func (r *NifiDataflowReconciler) Reconcile(ctx context.Context, req ctrl.Request if err := dataflow.ScheduleDataflow(instance, clientConfig); err != nil { switch errors.Cause(err).(type) { case errorfactory.NifiFlowControllerServiceScheduling, errorfactory.NifiFlowScheduling: - return RequeueAfter(time.Duration(5) * time.Second) + return RequeueAfter(interval/3) default: r.Recorder.Event(instance, corev1.EventTypeWarning, "StartingFailed", fmt.Sprintf("Starting dataflow %s based on flow {bucketId : %s, flowId: %s, version: %s} failed.", @@ -443,7 +445,7 @@ func (r *NifiDataflowReconciler) Reconcile(ctx context.Context, req ctrl.Request return Reconciled() } - return RequeueAfter(time.Duration(5) * time.Second) + return RequeueAfter(interval/3) } // SetupWithManager sets up the controller with the Manager. @@ -485,7 +487,7 @@ func (r *NifiDataflowReconciler) checkFinalizers(ctx context.Context, flow *v1al if err = r.finalizeNifiDataflow(flow, config); err != nil { switch errors.Cause(err).(type) { case errorfactory.NifiConnectionDropping, errorfactory.NifiFlowDraining: - return RequeueAfter(time.Duration(5) * time.Second) + return RequeueAfter(util.GetRequeueInterval(r.RequeueInterval/3, r.RequeueOffset)) default: return RequeueWithError(r.Log, "failed to finalize NiFiDataflow", err) } diff --git a/controllers/nifiparametercontext_controller.go b/controllers/nifiparametercontext_controller.go index 29d4e3f5a..e4cca7816 100644 --- a/controllers/nifiparametercontext_controller.go +++ b/controllers/nifiparametercontext_controller.go @@ -48,9 +48,11 @@ var parameterContextFinalizer = "nifiparametercontexts.nifi.orange.com/finalizer // NifiParameterContextReconciler reconciles a NifiParameterContext object type NifiParameterContextReconciler struct { client.Client - Log logr.Logger - Scheme *runtime.Scheme - Recorder record.EventRecorder + Log logr.Logger + Scheme *runtime.Scheme + Recorder record.EventRecorder + RequeueInterval int + RequeueOffset int } // +kubebuilder:rbac:groups=nifi.orange.com,resources=nifiparametercontexts,verbs=get;list;watch;create;update;patch;delete @@ -68,7 +70,7 @@ type NifiParameterContextReconciler struct { // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.7.0/pkg/reconcile func (r *NifiParameterContextReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { _ = r.Log.WithValues("nifiparametercontext", req.NamespacedName) - + interval := util.GetRequeueInterval(r.RequeueInterval, r.RequeueOffset) var err error // Fetch the NifiParameterContext instance @@ -185,7 +187,7 @@ func (r *NifiParameterContextReconciler) Reconcile(ctx context.Context, req ctrl instance.Spec.ClusterRef.Name, clusterConnect.Id())) // the cluster does not exist - should have been caught pre-flight - return RequeueAfter(time.Duration(15) * time.Second) + return RequeueAfter(interval) } // Ìn case of the cluster reference changed. @@ -204,7 +206,7 @@ func (r *NifiParameterContextReconciler) Reconcile(ctx context.Context, req ctrl if err := r.Client.Update(ctx, current); err != nil { return RequeueWithError(r.Log, "failed to update NifiParameterContext", err) } - return RequeueAfter(time.Duration(15) * time.Second) + return RequeueAfter(interval) } r.Recorder.Event(instance, corev1.EventTypeNormal, "Reconciling", @@ -248,7 +250,7 @@ func (r *NifiParameterContextReconciler) Reconcile(ctx context.Context, req ctrl if err != nil { switch errors.Cause(err).(type) { case errorfactory.NifiParameterContextUpdateRequestRunning: - return RequeueAfter(time.Duration(5) * time.Second) + return RequeueAfter(interval/3) default: r.Recorder.Event(instance, corev1.EventTypeNormal, "SynchronizingFailed", fmt.Sprintf("Synchronizing parameter context %s failed", instance.Name)) @@ -280,7 +282,7 @@ func (r *NifiParameterContextReconciler) Reconcile(ctx context.Context, req ctrl r.Log.Info("Ensured Parameter Context") - return RequeueAfter(time.Duration(15) * time.Second) + return RequeueAfter(interval) } // SetupWithManager sets up the controller with the Manager. diff --git a/controllers/nifiregistryclient_controller.go b/controllers/nifiregistryclient_controller.go index 8964273a8..7871b90f4 100644 --- a/controllers/nifiregistryclient_controller.go +++ b/controllers/nifiregistryclient_controller.go @@ -46,9 +46,11 @@ var registryClientFinalizer = "nifiregistryclients.nifi.orange.com/finalizer" // NifiRegistryClientReconciler reconciles a NifiRegistryClient object type NifiRegistryClientReconciler struct { client.Client - Log logr.Logger - Scheme *runtime.Scheme - Recorder record.EventRecorder + Log logr.Logger + Scheme *runtime.Scheme + Recorder record.EventRecorder + RequeueInterval int + RequeueOffset int } // +kubebuilder:rbac:groups=nifi.orange.com,resources=nifiregistryclients,verbs=get;list;watch;create;update;patch;delete @@ -66,7 +68,7 @@ type NifiRegistryClientReconciler struct { // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.7.0/pkg/reconcile func (r *NifiRegistryClientReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { _ = r.Log.WithValues("nifiregistryclient", req.NamespacedName) - + interval := util.GetRequeueInterval(r.RequeueInterval, r.RequeueOffset) var err error // Fetch the NifiRegistryClient instance @@ -160,7 +162,7 @@ func (r *NifiRegistryClientReconciler) Reconcile(ctx context.Context, req ctrl.R fmt.Sprintf("The referenced cluster is not ready yet : %s in %s", instance.Spec.ClusterRef.Name, clusterConnect.Id())) // the cluster does not exist - should have been caught pre-flight - return RequeueAfter(time.Duration(15) * time.Second) + return RequeueAfter(interval) } // Ìn case of the cluster reference changed. @@ -179,7 +181,7 @@ func (r *NifiRegistryClientReconciler) Reconcile(ctx context.Context, req ctrl.R if err := r.Client.Update(ctx, current); err != nil { return RequeueWithError(r.Log, "failed to update NifiRegistryClient", err) } - return RequeueAfter(time.Duration(15) * time.Second) + return RequeueAfter(interval) } r.Recorder.Event(instance, corev1.EventTypeNormal, "Reconciling", @@ -254,7 +256,7 @@ func (r *NifiRegistryClientReconciler) Reconcile(ctx context.Context, req ctrl.R r.Log.Info("Ensured Registry Client") - return RequeueAfter(time.Duration(15) * time.Second) + return RequeueAfter(interval) } // SetupWithManager sets up the controller with the Manager. diff --git a/controllers/nifiuser_controller.go b/controllers/nifiuser_controller.go index 830cb4218..dc0e11289 100644 --- a/controllers/nifiuser_controller.go +++ b/controllers/nifiuser_controller.go @@ -49,9 +49,11 @@ var userFinalizer = "nifiusers.nifi.orange.com/finalizer" // NifiUserReconciler reconciles a NifiUser object type NifiUserReconciler struct { client.Client - Log logr.Logger - Scheme *runtime.Scheme - Recorder record.EventRecorder + Log logr.Logger + Scheme *runtime.Scheme + Recorder record.EventRecorder + RequeueInterval int + RequeueOffset int } // +kubebuilder:rbac:groups=nifi.orange.com,resources=nifiusers,verbs=get;list;watch;create;update;patch;delete @@ -72,7 +74,7 @@ type NifiUserReconciler struct { // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.7.0/pkg/reconcile func (r *NifiUserReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { _ = r.Log.WithValues("nifiuser", req.NamespacedName) - + interval := util.GetRequeueInterval(r.RequeueInterval, r.RequeueOffset) var err error // Fetch the NifiUser instance @@ -182,7 +184,7 @@ func (r *NifiUserReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c r.Log.Info("generated secret not found, may not be ready") return ctrl.Result{ Requeue: true, - RequeueAfter: time.Duration(5) * time.Second, + RequeueAfter: interval/3, }, nil case errorfactory.FatalReconcileError: // TODO: (tinyzimmer) - Sleep for longer for now to give user time to see the error @@ -191,14 +193,14 @@ func (r *NifiUserReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c r.Log.Error(err, "Fatal error attempting to reconcile the user certificate. If using vault perhaps a permissions issue or improperly configured PKI?") return ctrl.Result{ Requeue: true, - RequeueAfter: time.Duration(15) * time.Second, + RequeueAfter: interval, }, nil case errorfactory.VaultAPIFailure: // Same as above in terms of things that could be checked pre-flight on the cluster r.Log.Error(err, "Vault API error attempting to reconcile the user certificate. If using vault perhaps a permissions issue or improperly configured PKI?") return ctrl.Result{ Requeue: true, - RequeueAfter: time.Duration(15) * time.Second, + RequeueAfter: interval, }, nil default: return RequeueWithError(r.Log, "failed to reconcile user secret", err) @@ -239,7 +241,7 @@ func (r *NifiUserReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c fmt.Sprintf("The referenced cluster is not ready yet : %s in %s", instance.Spec.ClusterRef.Name, clusterConnect.Id())) // the cluster does not exist - should have been caught pre-flight - return RequeueAfter(time.Duration(15) * time.Second) + return RequeueAfter(interval) } // Ìn case of the cluster reference changed. @@ -258,7 +260,7 @@ func (r *NifiUserReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c if err := r.Client.Update(ctx, current); err != nil { return RequeueWithError(r.Log, "failed to update NifiUser", err) } - return RequeueAfter(time.Duration(15) * time.Second) + return RequeueAfter(interval) } r.Recorder.Event(instance, corev1.EventTypeNormal, "Reconciling", @@ -336,7 +338,7 @@ func (r *NifiUserReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c r.Log.Info("Ensured user") - return RequeueAfter(time.Duration(15) * time.Second) + return RequeueAfter(interval) // set user status //instance.Status = v1alpha1.NifiUserStatus{ diff --git a/controllers/nifiusergroup_controller.go b/controllers/nifiusergroup_controller.go index cd89d40da..35f927dfb 100644 --- a/controllers/nifiusergroup_controller.go +++ b/controllers/nifiusergroup_controller.go @@ -47,9 +47,11 @@ var userGroupFinalizer = "nifiusergroups.nifi.orange.com/finalizer" // NifiUserGroupReconciler reconciles a NifiUserGroup object type NifiUserGroupReconciler struct { client.Client - Log logr.Logger - Scheme *runtime.Scheme - Recorder record.EventRecorder + Log logr.Logger + Scheme *runtime.Scheme + Recorder record.EventRecorder + RequeueInterval int + RequeueOffset int } // +kubebuilder:rbac:groups=nifi.orange.com,resources=nifiusergroups,verbs=get;list;watch;create;update;patch;delete @@ -67,7 +69,7 @@ type NifiUserGroupReconciler struct { // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.7.0/pkg/reconcile func (r *NifiUserGroupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { _ = r.Log.WithValues("nifiusergroup", req.NamespacedName) - + interval := util.GetRequeueInterval(r.RequeueInterval, r.RequeueOffset) var err error // Fetch the NifiUserGroup instance @@ -202,7 +204,7 @@ func (r *NifiUserGroupReconciler) Reconcile(ctx context.Context, req ctrl.Reques fmt.Sprintf("The referenced cluster is not ready yet : %s in %s", instance.Spec.ClusterRef.Name, clusterConnect.Id())) // the cluster does not exist - should have been caught pre-flight - return RequeueAfter(time.Duration(15) * time.Second) + return RequeueAfter(interval) } // Ìn case of the cluster reference changed. @@ -221,7 +223,7 @@ func (r *NifiUserGroupReconciler) Reconcile(ctx context.Context, req ctrl.Reques if err := r.Client.Update(ctx, current); err != nil { return RequeueWithError(r.Log, "failed to update NifiRegistryClient", err) } - return RequeueAfter(time.Duration(15) * time.Second) + return RequeueAfter(interval) } r.Recorder.Event(instance, corev1.EventTypeNormal, "Reconciling", @@ -291,7 +293,7 @@ func (r *NifiUserGroupReconciler) Reconcile(ctx context.Context, req ctrl.Reques r.Log.Info("Ensured User Group") - return RequeueAfter(time.Duration(15) * time.Second) + return RequeueAfter(interval) } // SetupWithManager sets up the controller with the Manager. diff --git a/helm/nifikop/Chart.yaml b/helm/nifikop/Chart.yaml index 3e33a7d6a..0674e14cf 100644 --- a/helm/nifikop/Chart.yaml +++ b/helm/nifikop/Chart.yaml @@ -4,8 +4,8 @@ name: nifikop home: https://github.com/Orange-OpenSource/nifikop sources: - https://github.com/Orange-OpenSource/nifikop -version: 0.7.1 -appVersion: 0.7.1-release +version: 0.7.2 +appVersion: 0.7.2-release icon: maintainers: - name: erdrix diff --git a/helm/nifikop/README.md b/helm/nifikop/README.md index c5d1b6155..d87d90a3b 100644 --- a/helm/nifikop/README.md +++ b/helm/nifikop/README.md @@ -23,7 +23,7 @@ The following tables lists the configurable parameters of the NiFi Operator Helm | Parameter | Description | Default | | -------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------- | | `image.repository` | Image | `orangeopensource/nifikop` | -| `image.tag` | Image tag | `v0.7.1-release` | +| `image.tag` | Image tag | `v0.7.2-release` | | `image.pullPolicy` | Image pull policy | `Always` | | `image.imagePullSecrets.enabled` | Enable tue use of secret for docker image | `false` | | `image.imagePullSecrets.name` | Name of the secret to connect to docker registry | - | diff --git a/helm/nifikop/values.yaml b/helm/nifikop/values.yaml index 376201f88..08bf545c7 100644 --- a/helm/nifikop/values.yaml +++ b/helm/nifikop/values.yaml @@ -2,7 +2,7 @@ ## image: repository: orangeopensource/nifikop - tag: v0.7.1-release + tag: v0.7.2-release pullPolicy: Always imagePullSecrets: enabled: false diff --git a/main.go b/main.go index a85257ee4..91cfb98e1 100644 --- a/main.go +++ b/main.go @@ -19,6 +19,7 @@ package main import ( "flag" "fmt" + "github.com/Orange-OpenSource/nifikop/pkg/common" "github.com/Orange-OpenSource/nifikop/version" certv1 "github.com/jetstack/cert-manager/pkg/apis/certmanager/v1alpha2" "os" @@ -123,73 +124,88 @@ func main() { os.Exit(1) } + multipliers := *common.NewRequeueConfig() if err = (&controllers.NifiClusterReconciler{ - Client: mgr.GetClient(), - DirectClient: mgr.GetAPIReader(), - Log: ctrl.Log.WithName("controllers").WithName("NifiCluster"), - Scheme: mgr.GetScheme(), - Namespaces: namespaceList, - Recorder: mgr.GetEventRecorderFor("nifi-cluster"), + Client: mgr.GetClient(), + DirectClient: mgr.GetAPIReader(), + Log: ctrl.Log.WithName("controllers").WithName("NifiCluster"), + Scheme: mgr.GetScheme(), + Namespaces: namespaceList, + Recorder: mgr.GetEventRecorderFor("nifi-cluster"), + RequeueIntervals: multipliers.ClusterTaskRequeueIntervals, + RequeueOffset: multipliers.RequeueOffset, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "NifiCluster") os.Exit(1) } if err = (&controllers.NifiClusterTaskReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("NifiClusterTask"), - Scheme: mgr.GetScheme(), - Recorder: mgr.GetEventRecorderFor("nifi-cluster-task"), + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("NifiClusterTask"), + Scheme: mgr.GetScheme(), + Recorder: mgr.GetEventRecorderFor("nifi-cluster-task"), + RequeueIntervals: multipliers.ClusterTaskRequeueIntervals, + RequeueOffset: multipliers.RequeueOffset, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "NifiClusterTask") os.Exit(1) } if err = (&controllers.NifiUserReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("NifiUser"), - Scheme: mgr.GetScheme(), - Recorder: mgr.GetEventRecorderFor("nifi-user"), + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("NifiUser"), + Scheme: mgr.GetScheme(), + Recorder: mgr.GetEventRecorderFor("nifi-user"), + RequeueInterval: multipliers.UserRequeueInterval, + RequeueOffset: multipliers.RequeueOffset, }).SetupWithManager(mgr, certManagerEnabled); err != nil { setupLog.Error(err, "unable to create controller", "controller", "NifiUser") os.Exit(1) } if err = (&controllers.NifiUserGroupReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("NifiUserGroup"), - Scheme: mgr.GetScheme(), - Recorder: mgr.GetEventRecorderFor("nifi-user-group"), + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("NifiUserGroup"), + Scheme: mgr.GetScheme(), + Recorder: mgr.GetEventRecorderFor("nifi-user-group"), + RequeueInterval: multipliers.UserGroupRequeueInterval, + RequeueOffset: multipliers.RequeueOffset, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "NifiUserGroup") os.Exit(1) } if err = (&controllers.NifiDataflowReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("NifiDataflow"), - Scheme: mgr.GetScheme(), - Recorder: mgr.GetEventRecorderFor("nifi-dataflow"), + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("NifiDataflow"), + Scheme: mgr.GetScheme(), + Recorder: mgr.GetEventRecorderFor("nifi-dataflow"), + RequeueInterval: multipliers.DataFlowRequeueInterval, + RequeueOffset: multipliers.RequeueOffset, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "NifiDataflow") os.Exit(1) } if err = (&controllers.NifiParameterContextReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("NifiParameterContext"), - Scheme: mgr.GetScheme(), - Recorder: mgr.GetEventRecorderFor("nifi-parameter-context"), + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("NifiParameterContext"), + Scheme: mgr.GetScheme(), + Recorder: mgr.GetEventRecorderFor("nifi-parameter-context"), + RequeueInterval: multipliers.ParameterContextRequeueInterval, + RequeueOffset: multipliers.RequeueOffset, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "NifiParameterContext") os.Exit(1) } if err = (&controllers.NifiRegistryClientReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("NifiRegistryClient"), - Scheme: mgr.GetScheme(), - Recorder: mgr.GetEventRecorderFor("nifi-registry-client"), + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("NifiRegistryClient"), + Scheme: mgr.GetScheme(), + Recorder: mgr.GetEventRecorderFor("nifi-registry-client"), + RequeueInterval: multipliers.RegistryClientRequeueInterval, + RequeueOffset: multipliers.RequeueOffset, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "NifiRegistryClient") os.Exit(1) diff --git a/pkg/clientwrappers/parametercontext/parametercontext.go b/pkg/clientwrappers/parametercontext/parametercontext.go index cd28c8956..18995b1a8 100644 --- a/pkg/clientwrappers/parametercontext/parametercontext.go +++ b/pkg/clientwrappers/parametercontext/parametercontext.go @@ -100,11 +100,14 @@ func SyncParameterContext(parameterContext *v1alpha1.NifiParameterContext, param return ¶meterContext.Status, errorfactory.NifiParameterContextUpdateRequestRunning{} } - status := parameterContext.Status - status.Version = *entity.Revision.Version - status.Id = entity.Id + var status *v1alpha1.NifiParameterContextStatus + if parameterContext.Status.Version != *entity.Revision.Version || parameterContext.Status.Id != entity.Id { + status := ¶meterContext.Status + status.Version = *entity.Revision.Version + status.Id = entity.Id + } - return &status, nil + return status, nil } func RemoveParameterContext(parameterContext *v1alpha1.NifiParameterContext, parameterSecrets []*corev1.Secret, @@ -146,9 +149,11 @@ func parameterContextIsSync( for _, param := range entity.Component.Parameters { if expected.Parameter.Name == param.Parameter.Name { notFound = false - if (!param.Parameter.Sensitive && ((expected.Parameter.Value == nil && param.Parameter.Value == nil) || - ((expected.Parameter.Value != nil && param.Parameter.Value != nil) && - *expected.Parameter.Value != *param.Parameter.Value))) || + + if (!param.Parameter.Sensitive && + !((expected.Parameter.Value == nil && param.Parameter.Value == nil) || + ((expected.Parameter.Value != nil && param.Parameter.Value != nil) && + (*expected.Parameter.Value == *param.Parameter.Value))))|| expected.Parameter.Description != param.Parameter.Description { return false @@ -194,10 +199,11 @@ func updateRequestPrepare( for _, param := range tmp { if expected.Parameter.Name == param.Parameter.Name { notFound = false - if (!param.Parameter.Sensitive && ((expected.Parameter.Value == nil && param.Parameter.Value == nil) || - ((expected.Parameter.Value != nil && param.Parameter.Value != nil) && - *expected.Parameter.Value != *param.Parameter.Value))) || - expected.Parameter.Description != param.Parameter.Description { + if (!param.Parameter.Sensitive && + !((expected.Parameter.Value == nil && param.Parameter.Value == nil) || + ((expected.Parameter.Value != nil && param.Parameter.Value != nil) && + (*expected.Parameter.Value == *param.Parameter.Value))))|| + expected.Parameter.Description != param.Parameter.Description { notFound = false parameters = append(parameters, expected) break diff --git a/pkg/common/common.go b/pkg/common/common.go index 491488af2..4eec36d6c 100644 --- a/pkg/common/common.go +++ b/pkg/common/common.go @@ -2,6 +2,7 @@ package common import ( "github.com/Orange-OpenSource/nifikop/pkg/nificlient" + "github.com/Orange-OpenSource/nifikop/pkg/util" "github.com/Orange-OpenSource/nifikop/pkg/util/clientconfig" "github.com/go-logr/logr" ) @@ -57,3 +58,29 @@ func NewClusterConnection(log logr.Logger, config *clientconfig.NifiConfig) (nod } return } + +type RequeueConfig struct { + UserRequeueInterval int + RegistryClientRequeueInterval int + ParameterContextRequeueInterval int + UserGroupRequeueInterval int + DataFlowRequeueInterval int + ClusterTaskRequeueIntervals map[string]int + RequeueOffset int +} + +func NewRequeueConfig() *RequeueConfig { + return &RequeueConfig{ + ClusterTaskRequeueIntervals: map[string]int{ + "CLUSTER_TASK_RUNNING_REQUEUE_INTERVAL": util.MustConvertToInt(util.GetEnvWithDefault("CLUSTER_TASK_RUNNING_REQUEUE_INTERVAL", "20"), "CLUSTER_TASK_RUNNING_REQUEUE_INTERVAL"), + "CLUSTER_TASK_TIMEOUT_REQUEUE_INTERVAL": util.MustConvertToInt(util.GetEnvWithDefault("CLUSTER_TASK_TIMEOUT_REQUEUE_INTERVAL", "20"), "CLUSTER_TASK_TIMEOUT_REQUEUE_INTERVAL"), + "CLUSTER_TASK_NOT_READY_REQUEUE_INTERVAL": util.MustConvertToInt(util.GetEnvWithDefault("CLUSTER_TASK_NOT_READY_REQUEUE_INTERVAL", "15"), "CLUSTER_TASK_NODES_UNREACHABLE_REQUEUE_INTERVAL"), + }, + UserRequeueInterval: util.MustConvertToInt(util.GetEnvWithDefault("USERS_REQUEUE_INTERVAL", "15"), "USERS_REQUEUE_INTERVAL"), + RegistryClientRequeueInterval: util.MustConvertToInt(util.GetEnvWithDefault("REGISTRY_CLIENT_REQUEUE_INTERVAL", "15"), "REGISTRY_CLIENT_REQUEUE_INTERVAL"), + ParameterContextRequeueInterval: util.MustConvertToInt(util.GetEnvWithDefault("PARAMETER_CONTEXT_REQUEUE_INTERVAL", "15"), "PARAMETER_CONTEXT_REQUEUE_INTERVAL"), + UserGroupRequeueInterval: util.MustConvertToInt(util.GetEnvWithDefault("USER_GROUP_REQUEUE_INTERVAL", "15"), "USER_GROUP_REQUEUE_INTERVAL"), + DataFlowRequeueInterval: util.MustConvertToInt(util.GetEnvWithDefault("DATAFLOW_REQUEUE_INTERVAL", "15"), "DATAFLOW_REQUEUE_INTERVAL"), + RequeueOffset: util.MustConvertToInt(util.GetEnvWithDefault("REQUEUE_OFFSET", "0"), "REQUEUE_OFFSET"), + } +} diff --git a/pkg/util/util.go b/pkg/util/util.go index c89b7e525..69d32a2df 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -16,9 +16,13 @@ package util import ( "crypto/sha1" + "fmt" + "math/rand" + "os" "reflect" "strconv" "strings" + "time" "github.com/Orange-OpenSource/nifikop/api/v1alpha1" @@ -225,3 +229,26 @@ func Hash(s string) string { h.Write([]byte(s)) return string(h.Sum(nil)) } + +func GetEnvWithDefault(key string, fallback string) string { + if value, ok := os.LookupEnv(key); ok { + return value + } + return fallback +} + +func MustConvertToInt(str string, name string) int { + i, err := strconv.Atoi(str) + if err != nil { + fmt.Printf("%s", fmt.Errorf("%w problem converting string to integer (%s)", err, name)) + os.Exit(1) + } + return i +} + +func GetRequeueInterval(interval int, offset int) time.Duration { + // @TODO : check what is the expected behaviour with offset + duration := interval + rand.Intn(offset+1) - (offset / 2) + duration = Max(duration, rand.Intn(5)+1) // make sure duration does not go zero for very large offsets + return time.Duration(duration) * time.Second +} \ No newline at end of file diff --git a/site/docs/2_setup/1_getting_started.md b/site/docs/2_setup/1_getting_started.md index 4ab1dfbfb..1446229a4 100644 --- a/site/docs/2_setup/1_getting_started.md +++ b/site/docs/2_setup/1_getting_started.md @@ -116,8 +116,8 @@ Now deploy the helm chart : helm install nifikop \ orange-incubator/nifikop \ --namespace=nifi \ - --version 0.7.1 \ - --set image.tag=v0.7.1release \ + --version 0.7.2 \ + --set image.tag=v0.7.2-release \ --set resources.requests.memory=256Mi \ --set resources.requests.cpu=250m \ --set resources.limits.memory=256Mi \ diff --git a/site/docs/2_setup/3_install/1_customizable_install_with_helm.md b/site/docs/2_setup/3_install/1_customizable_install_with_helm.md index 5175983aa..002f05a55 100644 --- a/site/docs/2_setup/3_install/1_customizable_install_with_helm.md +++ b/site/docs/2_setup/3_install/1_customizable_install_with_helm.md @@ -33,7 +33,7 @@ The following tables lists the configurable parameters of the NiFi Operator Helm | Parameter | Description | Default | | -------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------- | | `image.repository` | Image | `orangeopensource/nifikop` | -| `image.tag` | Image tag | `v0.7.1-release` | +| `image.tag` | Image tag | `v0.7.2-release` | | `image.pullPolicy` | Image pull policy | `Always` | | `image.imagePullSecrets.enabled` | Enable tue use of secret for docker image | `false` | | `image.imagePullSecrets.name` | Name of the secret to connect to docker registry | - | diff --git a/version/version.go b/version/version.go index 258e65ae9..4da9f0f20 100644 --- a/version/version.go +++ b/version/version.go @@ -1,5 +1,5 @@ package version var ( - Version = "0.7.1" + Version = "0.7.2" )