From ee9bd06d10dae347c6055bacbe4da9e70974f3fd Mon Sep 17 00:00:00 2001 From: Michael H Date: Fri, 10 Jun 2022 09:19:03 +0100 Subject: [PATCH 1/2] handle case where NiFi cluster has been destroyed before resources referencing the cluster revert unnecessary change add logging for finalizer removal update logging to be clear about what resources are getting updated --- controllers/nifidataflow_controller.go | 11 +++++++++-- controllers/nifiparametercontext_controller.go | 17 ++++++++++++----- controllers/nifiregistryclient_controller.go | 11 +++++++++-- controllers/nifiuser_controller.go | 14 +++++++++++--- controllers/nifiusergroup_controller.go | 11 +++++++++-- 5 files changed, 50 insertions(+), 14 deletions(-) diff --git a/controllers/nifidataflow_controller.go b/controllers/nifidataflow_controller.go index c9ae1ad785..4235204a5a 100644 --- a/controllers/nifidataflow_controller.go +++ b/controllers/nifidataflow_controller.go @@ -235,6 +235,13 @@ func (r *NifiDataflowReconciler) Reconcile(ctx context.Context, req ctrl.Request r.Recorder.Event(instance, corev1.EventTypeWarning, "ReferenceClusterError", fmt.Sprintf("Failed to create HTTP client for the referenced cluster : %s in %s", instance.Spec.ClusterRef.Name, currentClusterRef.Namespace)) + // the cluster is gone, so just remove the finalizer + if k8sutil.IsMarkedForDeletion(instance.ObjectMeta) { + if err = r.removeFinalizer(ctx, instance); err != nil { + return RequeueWithError(r.Log, fmt.Sprintf("failed to remove finalizer from NifiDataflow %s", instance.Name), err) + } + return Reconciled() + } // the cluster does not exist - should have been caught pre-flight return RequeueWithError(r.Log, "failed to create HTTP client the for referenced cluster", err) } @@ -491,8 +498,7 @@ func (r *NifiDataflowReconciler) updateAndFetchLatest(ctx context.Context, func (r *NifiDataflowReconciler) checkFinalizers(ctx context.Context, flow *v1alpha1.NifiDataflow, config *clientconfig.NifiConfig) (reconcile.Result, error) { - - r.Log.Info("NiFi dataflow is marked for deletion") + r.Log.Info(fmt.Sprintf("NiFi dataflow %s is marked for deletion", flow.Name)) var err error if util.StringSliceContains(flow.GetFinalizers(), dataflowFinalizer) { if err = r.finalizeNifiDataflow(flow, config); err != nil { @@ -512,6 +518,7 @@ func (r *NifiDataflowReconciler) checkFinalizers(ctx context.Context, flow *v1al } func (r *NifiDataflowReconciler) removeFinalizer(ctx context.Context, flow *v1alpha1.NifiDataflow) error { + r.Log.V(5).Info(fmt.Sprintf("Removing finalizer for NifiDataflow %s", flow.Name)) flow.SetFinalizers(util.StringSliceRemove(flow.GetFinalizers(), dataflowFinalizer)) _, err := r.updateAndFetchLatest(ctx, flow) return err diff --git a/controllers/nifiparametercontext_controller.go b/controllers/nifiparametercontext_controller.go index ad281ab5a8..4a16282991 100644 --- a/controllers/nifiparametercontext_controller.go +++ b/controllers/nifiparametercontext_controller.go @@ -171,6 +171,13 @@ func (r *NifiParameterContextReconciler) Reconcile(ctx context.Context, req ctrl r.Recorder.Event(instance, corev1.EventTypeWarning, "ReferenceClusterError", fmt.Sprintf("Failed to create HTTP client for the referenced cluster : %s in %s", instance.Spec.ClusterRef.Name, clusterRef.Namespace)) + // the cluster is gone, so just remove the finalizer + if k8sutil.IsMarkedForDeletion(instance.ObjectMeta) { + if err = r.removeFinalizer(ctx, instance); err != nil { + return RequeueWithError(r.Log, fmt.Sprintf("failed to remove finalizer from NifiParameterContext %s", instance.Name), err) + } + return Reconciled() + } // the cluster does not exist - should have been caught pre-flight return RequeueWithError(r.Log, "failed to create HTTP client the for referenced cluster", err) } @@ -335,8 +342,7 @@ func (r *NifiParameterContextReconciler) checkFinalizers( parameterContext *v1alpha1.NifiParameterContext, parameterSecrets []*corev1.Secret, config *clientconfig.NifiConfig) (reconcile.Result, error) { - - r.Log.Info("NiFi parameter context is marked for deletion") + r.Log.Info(fmt.Sprintf("NiFi parameter context %s is marked for deletion", parameterContext.Name)) var err error if util.StringSliceContains(parameterContext.GetFinalizers(), parameterContextFinalizer) { if err = r.finalizeNifiParameterContext(parameterContext, parameterSecrets, config); err != nil { @@ -349,9 +355,10 @@ func (r *NifiParameterContextReconciler) checkFinalizers( return Reconciled() } -func (r *NifiParameterContextReconciler) removeFinalizer(ctx context.Context, flow *v1alpha1.NifiParameterContext) error { - flow.SetFinalizers(util.StringSliceRemove(flow.GetFinalizers(), parameterContextFinalizer)) - _, err := r.updateAndFetchLatest(ctx, flow) +func (r *NifiParameterContextReconciler) removeFinalizer(ctx context.Context, paramCtxt *v1alpha1.NifiParameterContext) error { + r.Log.V(5).Info(fmt.Sprintf("Removing finalizer for NifiParameterContext %s", paramCtxt.Name)) + paramCtxt.SetFinalizers(util.StringSliceRemove(paramCtxt.GetFinalizers(), parameterContextFinalizer)) + _, err := r.updateAndFetchLatest(ctx, paramCtxt) return err } diff --git a/controllers/nifiregistryclient_controller.go b/controllers/nifiregistryclient_controller.go index 5de1bd6f8b..a005baa924 100644 --- a/controllers/nifiregistryclient_controller.go +++ b/controllers/nifiregistryclient_controller.go @@ -146,6 +146,13 @@ func (r *NifiRegistryClientReconciler) Reconcile(ctx context.Context, req ctrl.R r.Recorder.Event(instance, corev1.EventTypeWarning, "ReferenceClusterError", fmt.Sprintf("Failed to create HTTP client for the referenced cluster : %s in %s", instance.Spec.ClusterRef.Name, clusterRef.Namespace)) + // the cluster is gone, so just remove the finalizer + if k8sutil.IsMarkedForDeletion(instance.ObjectMeta) { + if err = r.removeFinalizer(ctx, instance); err != nil { + return RequeueWithError(r.Log, fmt.Sprintf("failed to remove finalizer from NifiRegistryClient %s", instance.Name), err) + } + return Reconciled() + } // the cluster does not exist - should have been caught pre-flight return RequeueWithError(r.Log, "failed to create HTTP client the for referenced cluster", err) } @@ -291,8 +298,7 @@ func (r *NifiRegistryClientReconciler) updateAndFetchLatest(ctx context.Context, func (r *NifiRegistryClientReconciler) checkFinalizers(ctx context.Context, reqLogger logr.Logger, registryClient *v1alpha1.NifiRegistryClient, config *clientconfig.NifiConfig) (reconcile.Result, error) { - - reqLogger.Info("NiFi registry client is marked for deletion") + reqLogger.Info(fmt.Sprintf("NiFi registry client %s is marked for deletion", registryClient.Name)) var err error if util.StringSliceContains(registryClient.GetFinalizers(), registryClientFinalizer) { if err = r.finalizeNifiRegistryClient(reqLogger, registryClient, config); err != nil { @@ -306,6 +312,7 @@ func (r *NifiRegistryClientReconciler) checkFinalizers(ctx context.Context, reqL } func (r *NifiRegistryClientReconciler) removeFinalizer(ctx context.Context, registryClient *v1alpha1.NifiRegistryClient) error { + r.Log.V(5).Info(fmt.Sprintf("Removing finalizer for NifiRegistryClient %s", registryClient.Name)) registryClient.SetFinalizers(util.StringSliceRemove(registryClient.GetFinalizers(), registryClientFinalizer)) _, err := r.updateAndFetchLatest(ctx, registryClient) return err diff --git a/controllers/nifiuser_controller.go b/controllers/nifiuser_controller.go index 0c7f660b91..3013478056 100644 --- a/controllers/nifiuser_controller.go +++ b/controllers/nifiuser_controller.go @@ -225,6 +225,13 @@ func (r *NifiUserReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c r.Recorder.Event(instance, corev1.EventTypeWarning, "ReferenceClusterError", fmt.Sprintf("Failed to create HTTP client for the referenced cluster : %s in %s", instance.Spec.ClusterRef.Name, clusterRef.Namespace)) + // the cluster is gone, so just remove the finalizer + if k8sutil.IsMarkedForDeletion(instance.ObjectMeta) { + if err = r.removeFinalizer(ctx, instance); err != nil { + return RequeueWithError(r.Log, fmt.Sprintf("failed to remove finalizer from NifiUser %s", instance.Name), err) + } + return Reconciled() + } // the cluster does not exist - should have been caught pre-flight return RequeueWithError(r.Log, "failed to create HTTP client the for referenced cluster", err) } @@ -384,7 +391,7 @@ func (r *NifiUserReconciler) updateAndFetchLatest(ctx context.Context, user *v1a } func (r *NifiUserReconciler) checkFinalizers(ctx context.Context, user *v1alpha1.NifiUser, config *clientconfig.NifiConfig) (reconcile.Result, error) { - r.Log.Info("NiFi user is marked for deletion") + r.Log.Info(fmt.Sprintf("NiFi user %s is marked for deletion", user.Name)) var err error if util.StringSliceContains(user.GetFinalizers(), userFinalizer) { if err = r.finalizeNifiUser(user, config); err != nil { @@ -399,6 +406,7 @@ func (r *NifiUserReconciler) checkFinalizers(ctx context.Context, user *v1alpha1 } func (r *NifiUserReconciler) removeFinalizer(ctx context.Context, user *v1alpha1.NifiUser) error { + r.Log.V(5).Info(fmt.Sprintf("Removing finalizer for NifiUser %s", user.Name)) user.SetFinalizers(util.StringSliceRemove(user.GetFinalizers(), userFinalizer)) _, err := r.updateAndFetchLatest(ctx, user) return err @@ -408,12 +416,12 @@ func (r *NifiUserReconciler) finalizeNifiUser(user *v1alpha1.NifiUser, config *c if err := usercli.RemoveUser(user, config); err != nil { return err } - r.Log.Info("Delete user") + r.Log.Info(fmt.Sprintf("Deleted user %s", user.Name)) return nil } func (r *NifiUserReconciler) addFinalizer(user *v1alpha1.NifiUser) { - r.Log.Info("Adding Finalizer for the NifiUser") + r.Log.Info(fmt.Sprintf("Adding Finalizer for the NifiUser %s", user.Name)) user.SetFinalizers(append(user.GetFinalizers(), userFinalizer)) return } diff --git a/controllers/nifiusergroup_controller.go b/controllers/nifiusergroup_controller.go index 394b0e773d..6aae240ac6 100644 --- a/controllers/nifiusergroup_controller.go +++ b/controllers/nifiusergroup_controller.go @@ -188,6 +188,13 @@ func (r *NifiUserGroupReconciler) Reconcile(ctx context.Context, req ctrl.Reques r.Recorder.Event(instance, corev1.EventTypeWarning, "ReferenceClusterError", fmt.Sprintf("Failed to create HTTP client for the referenced cluster : %s in %s", instance.Spec.ClusterRef.Name, clusterRef.Namespace)) + // the cluster is gone, so just remove the finalizer + if k8sutil.IsMarkedForDeletion(instance.ObjectMeta) { + if err = r.removeFinalizer(ctx, instance); err != nil { + return RequeueWithError(r.Log, fmt.Sprintf("failed to remove finalizer from NifiUserGroup %s", instance.Name), err) + } + return Reconciled() + } // the cluster does not exist - should have been caught pre-flight return RequeueWithError(r.Log, "failed to create HTTP client the for referenced cluster", err) } @@ -328,8 +335,7 @@ func (r *NifiUserGroupReconciler) updateAndFetchLatest(ctx context.Context, func (r *NifiUserGroupReconciler) checkFinalizers(ctx context.Context, userGroup *v1alpha1.NifiUserGroup, users []*v1alpha1.NifiUser, config *clientconfig.NifiConfig) (reconcile.Result, error) { - - r.Log.Info("NiFi user group is marked for deletion") + r.Log.Info(fmt.Sprintf("NiFi user group %s is marked for deletion", userGroup.Name)) var err error if util.StringSliceContains(userGroup.GetFinalizers(), userGroupFinalizer) { if err = r.finalizeNifiNifiUserGroup(userGroup, users, config); err != nil { @@ -343,6 +349,7 @@ func (r *NifiUserGroupReconciler) checkFinalizers(ctx context.Context, userGroup } func (r *NifiUserGroupReconciler) removeFinalizer(ctx context.Context, userGroup *v1alpha1.NifiUserGroup) error { + r.Log.V(5).Info(fmt.Sprintf("Removing finalizer for NifiUserGroup %s", userGroup.Name)) userGroup.SetFinalizers(util.StringSliceRemove(userGroup.GetFinalizers(), userGroupFinalizer)) _, err := r.updateAndFetchLatest(ctx, userGroup) return err From 5d2073ae2f3f8dcde69c7101c20d99de4e728ea2 Mon Sep 17 00:00:00 2001 From: Michael H Date: Fri, 10 Jun 2022 09:59:16 +0100 Subject: [PATCH 2/2] update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7386ceaa7c..43a96fdb5d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,7 @@ - [PR #76](https://github.com/konpyutaika/nifikop/pull/88) - **[Operator/NiFiCluster]** Re-ordering config out of sync steps. - [PR #93](https://github.com/konpyutaika/nifikop/pull/93) - **[Documentation]** Remove serviceAnnotations mentions and fix docs. - +- [PR #101](https://github.com/konpyutaika/nifikop/pull/101) - **[Operator]** Handle finalizer removal case where `NifiCluster` is aggressively torn down and pods are no longer available to communicate with. ## v0.10.0