[Feature/ParameterContext] Support declarative sensitive value out of…

… secret (#162)

* fix operator metrics configuration

* append changelog

* fix optinal parameter context and support empty array for parameter context

* bump version and append changelog

* updgrade dependency

* bump version

* add sensitive field for parameter type

* support empty value update

* append changelog & bump version

* append changelog & bump version

* fix test

CHANGELOG.md

@@ -10,6 +10,17 @@
 
 ### Fixed Bugs
 
+## v0.7.5
+
+### Added
+
+- [PR #161](https://github.com/Orange-OpenSource/nifikop/pull/162) - **[Operator/NiFiParameterContext]** Support declarative sensitive value out of secret.
+
+### Fixed Bugs
+
+- [PR #161](https://github.com/Orange-OpenSource/nifikop/pull/162) - **[Documentation]** NiFiCluster reference.
+- [PR #161](https://github.com/Orange-OpenSource/nifikop/pull/162) - **[Operator/NiFiParameterContext]** Fix remove parameter and update set value to "no value set".
+
 ## v0.7.4
 
 ### Fixed Bugs

api/v1alpha1/nifiparametercontext_types.go

@@ -42,6 +42,8 @@ type Parameter struct {
 	Value *string `json:"value,omitempty"`
 	// the description of the Parameter.
 	Description string `json:"description,omitempty"`
+	// Whether the parameter is sensitive or not.
+	Sensitive bool `json:"sensitive,omitempty"`
 }
 
 // NifiParameterContextStatus defines the observed state of NifiParameterContext

config/crd/bases/nifi.orange.com_nifiparametercontexts.yaml

@@ -61,6 +61,9 @@ spec:
                     name:
                       description: the name of the Parameter.
                       type: string
+                    sensitive:
+                      description: Whether the parameter is sensitive or not.
+                      type: boolean
                     value:
                       description: the value of the Parameter.
                       type: string

controllers/nificluster_controller.go

@@ -149,7 +149,7 @@ func (r *NifiClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 				r.Log.Info("A new resource was not found or may not be ready")
 				r.Log.Info(err.Error())
 				return reconcile.Result{
-					RequeueAfter: intervalNotReady/2,
+					RequeueAfter: intervalNotReady / 2,
 				}, nil
 			case errorfactory.ReconcileRollingUpgrade:
 				r.Log.Info("Rolling Upgrade in Progress")

controllers/nificlustertask_controller.go

@@ -47,9 +47,9 @@ import (
 // NifiClusterTaskReconciler reconciles
 type NifiClusterTaskReconciler struct {
 	client.Client
-	Log             logr.Logger
-	Scheme          *runtime.Scheme
-	Recorder        record.EventRecorder
+	Log              logr.Logger
+	Scheme           *runtime.Scheme
+	Recorder         record.EventRecorder
 	RequeueIntervals map[string]int
 	RequeueOffset    int
 }
@@ -67,8 +67,8 @@ func (r *NifiClusterTaskReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 	_ = r.Log.WithValues("nificlustertask", req.NamespacedName)
 
 	intervalNotReady := util.GetRequeueInterval(r.RequeueIntervals["CLUSTER_TASK_NOT_READY_REQUEUE_INTERVAL"], r.RequeueOffset)
-	intervalRunning  := util.GetRequeueInterval(r.RequeueIntervals["CLUSTER_TASK_RUNNING_REQUEUE_INTERVAL"], r.RequeueOffset)
-	intervalTimeout  := util.GetRequeueInterval(r.RequeueIntervals["CLUSTER_TASK_TIMEOUT_REQUEUE_INTERVAL"], r.RequeueOffset)
+	intervalRunning := util.GetRequeueInterval(r.RequeueIntervals["CLUSTER_TASK_RUNNING_REQUEUE_INTERVAL"], r.RequeueOffset)
+	intervalTimeout := util.GetRequeueInterval(r.RequeueIntervals["CLUSTER_TASK_TIMEOUT_REQUEUE_INTERVAL"], r.RequeueOffset)
 	// Fetch the NifiCluster instance
 	instance := &v1alpha1.NifiCluster{}
 	err := r.Client.Get(ctx, req.NamespacedName, instance)

controllers/nifidataflow_controller.go

@@ -164,7 +164,7 @@ func (r *NifiDataflowReconciler) Reconcile(ctx context.Context, req ctrl.Request
 	registryClusterRef.Namespace = registryClientNamespace
 	clusterRefs = append(clusterRefs, registryClusterRef)
 
-	if parameterContext !=nil {
+	if parameterContext != nil {
 		parameterContextClusterRef := parameterContext.Spec.ClusterRef
 		parameterContextClusterRef.Namespace = parameterContextNamespace
 		clusterRefs = append(clusterRefs, parameterContextClusterRef)
@@ -354,7 +354,7 @@ func (r *NifiDataflowReconciler) Reconcile(ctx context.Context, req ctrl.Request
 				errorfactory.NifiFlowControllerServiceScheduling,
 				errorfactory.NifiFlowScheduling, errorfactory.NifiFlowSyncing:
 				return reconcile.Result{
-					RequeueAfter: interval/3,
+					RequeueAfter: interval / 3,
 				}, nil
 			default:
 				r.Recorder.Event(instance, corev1.EventTypeWarning, "SynchronizingFailed",
@@ -409,7 +409,7 @@ func (r *NifiDataflowReconciler) Reconcile(ctx context.Context, req ctrl.Request
 		if err := dataflow.ScheduleDataflow(instance, clientConfig); err != nil {
 			switch errors.Cause(err).(type) {
 			case errorfactory.NifiFlowControllerServiceScheduling, errorfactory.NifiFlowScheduling:
-				return RequeueAfter(interval/3)
+				return RequeueAfter(interval / 3)
 			default:
 				r.Recorder.Event(instance, corev1.EventTypeWarning, "StartingFailed",
 					fmt.Sprintf("Starting dataflow %s based on flow {bucketId : %s, flowId: %s, version: %s} failed.",
@@ -451,7 +451,7 @@ func (r *NifiDataflowReconciler) Reconcile(ctx context.Context, req ctrl.Request
 		return Reconciled()
 	}
 
-	return RequeueAfter(interval/3)
+	return RequeueAfter(interval / 3)
 }
 
 // SetupWithManager sets up the controller with the Manager.

controllers/nifiparametercontext_controller.go

@@ -250,7 +250,7 @@ func (r *NifiParameterContextReconciler) Reconcile(ctx context.Context, req ctrl
 	if err != nil {
 		switch errors.Cause(err).(type) {
 		case errorfactory.NifiParameterContextUpdateRequestRunning:
-			return RequeueAfter(interval/3)
+			return RequeueAfter(interval / 3)
 		default:
 			r.Recorder.Event(instance, corev1.EventTypeNormal, "SynchronizingFailed",
 				fmt.Sprintf("Synchronizing parameter context %s failed", instance.Name))

controllers/nifiuser_controller.go

@@ -184,7 +184,7 @@ func (r *NifiUserReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c
 				r.Log.Info("generated secret not found, may not be ready")
 				return ctrl.Result{
 					Requeue:      true,
-					RequeueAfter: interval/3,
+					RequeueAfter: interval / 3,
 				}, nil
 			case errorfactory.FatalReconcileError:
 				// TODO: (tinyzimmer) - Sleep for longer for now to give user time to see the error

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/antihax/optional v1.0.0
 	github.com/banzaicloud/k8s-objectmatcher v1.4.1
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
-	github.com/erdrix/nigoapi v0.0.0-20211117113337-5c8957d8be51
+	github.com/erdrix/nigoapi v0.0.0-20211122092449-0fa36e567288
 	github.com/go-logr/logr v0.3.0
 	github.com/imdario/mergo v0.3.10
 	github.com/jarcoal/httpmock v1.0.6

go.sum

@@ -161,8 +161,8 @@ github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb
 github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/erdrix/nigoapi v0.0.0-20211117113337-5c8957d8be51 h1:NRUQIRX+cezoNs12maIay9CmeDjfLBiXdNHkoaggzgo=
-github.com/erdrix/nigoapi v0.0.0-20211117113337-5c8957d8be51/go.mod h1:owY+8fs8YXnST3ENM+ulVllYjTbzGaqKA+Y7HHJ0lZA=
+github.com/erdrix/nigoapi v0.0.0-20211122092449-0fa36e567288 h1:uWJjUP6aaeorJJbY3EUB3JVE/ScyRJWW8PL8+3pOMAY=
+github.com/erdrix/nigoapi v0.0.0-20211122092449-0fa36e567288/go.mod h1:owY+8fs8YXnST3ENM+ulVllYjTbzGaqKA+Y7HHJ0lZA=
 github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses=

helm/nifikop/Chart.yaml

@@ -4,8 +4,8 @@ name: nifikop
 home: https://github.com/Orange-OpenSource/nifikop
 sources:
   - https://github.com/Orange-OpenSource/nifikop
-version: 0.7.4
-appVersion: 0.7.4-release
+version: 0.7.5
+appVersion: 0.7.5-release
 icon:
 maintainers:
   - name: erdrix

helm/nifikop/README.md

@@ -23,7 +23,7 @@ The following tables lists the configurable parameters of the NiFi Operator Helm
 | Parameter                        | Description                                                                                                                                                                          | Default                    |
 | -------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------- |
 | `image.repository`               | Image                                                                                                                                                                                | `orangeopensource/nifikop` |
-| `image.tag`                      | Image tag                                                                                                                                                                            | `v0.7.4-release`           |
+| `image.tag`                      | Image tag                                                                                                                                                                            | `v0.7.5-release`           |
 | `image.pullPolicy`               | Image pull policy                                                                                                                                                                    | `Always`                   |
 | `image.imagePullSecrets.enabled` | Enable tue use of secret for docker image                                                                                                                                            | `false`                    |
 | `image.imagePullSecrets.name`    | Name of the secret to connect to docker registry                                                                                                                                     | -                          |

helm/nifikop/crds/nifi.orange.com_nifiparametercontexts.yaml

@@ -61,6 +61,9 @@ spec:
                     name:
                       description: the name of the Parameter.
                       type: string
+                    sensitive:
+                      description: Whether the parameter is sensitive or not.
+                      type: boolean
                     value:
                       description: the value of the Parameter.
                       type: string

helm/nifikop/values.yaml

@@ -2,7 +2,7 @@
 ##
 image:
   repository: orangeopensource/nifikop
-  tag: v0.7.4-release
+  tag: v0.7.5-release
   pullPolicy: Always
   imagePullSecrets:
     enabled: false

pkg/clientwrappers/parametercontext/parametercontext.go

@@ -153,8 +153,8 @@ func parameterContextIsSync(
 				if (!param.Parameter.Sensitive &&
 					!((expected.Parameter.Value == nil && param.Parameter.Value == nil) ||
 						((expected.Parameter.Value != nil && param.Parameter.Value != nil) &&
-							(*expected.Parameter.Value == *param.Parameter.Value))))||
-					expected.Parameter.Description != param.Parameter.Description {
+							(*expected.Parameter.Value == *param.Parameter.Value)))) ||
+					*expected.Parameter.Description != *param.Parameter.Description {
 
 					return false
 				}
@@ -202,9 +202,13 @@ func updateRequestPrepare(
 				if (!param.Parameter.Sensitive &&
 					!((expected.Parameter.Value == nil && param.Parameter.Value == nil) ||
 						((expected.Parameter.Value != nil && param.Parameter.Value != nil) &&
-							(*expected.Parameter.Value == *param.Parameter.Value))))||
-					expected.Parameter.Description != param.Parameter.Description  {
+							(*expected.Parameter.Value == *param.Parameter.Value)))) ||
+					*expected.Parameter.Description != *param.Parameter.Description {
 					notFound = false
+					if expected.Parameter.Value == nil && param.Parameter.Value != nil {
+						toRemove = append(toRemove, expected.Parameter.Name)
+						break
+					}
 					parameters = append(parameters, expected)
 					break
 				}
@@ -245,13 +249,14 @@ func updateParameterContextEntity(parameterContext *v1alpha1.NifiParameterContex
 
 	parameters := make([]nigoapi.ParameterEntity, 0)
 
+	emptyString := ""
 	for _, secret := range parameterSecrets {
 		for k, v := range secret.Data {
 			value := string(v)
 			parameters = append(parameters, nigoapi.ParameterEntity{
 				Parameter: &nigoapi.ParameterDto{
 					Name:        k,
-					Description: "",
+					Description: &emptyString,
 					Sensitive:   true,
 					Value:       &value,
 				},
@@ -260,11 +265,12 @@ func updateParameterContextEntity(parameterContext *v1alpha1.NifiParameterContex
 	}
 
 	for _, parameter := range parameterContext.Spec.Parameters {
+		desc := parameter.Description
 		parameters = append(parameters, nigoapi.ParameterEntity{
 			Parameter: &nigoapi.ParameterDto{
 				Name:        parameter.Name,
-				Description: parameter.Description,
-				Sensitive:   false,
+				Description: &desc,
+				Sensitive:   parameter.Sensitive,
 				Value:       parameter.Value,
 			},
 		})

pkg/nificlient/parametercontext_test.go

@@ -255,11 +255,12 @@ func MockParameterContext(
 
 func map2Parameters(params map[string]string, sensitive bool) []nigoapi.ParameterEntity {
 	var parameters []nigoapi.ParameterEntity
+	emptyString := ""
 	for k, v := range params {
 		parameters = append(parameters, nigoapi.ParameterEntity{
 			Parameter: &nigoapi.ParameterDto{
 				Name:        k,
-				Description: "",
+				Description: &emptyString,
 				Sensitive:   sensitive,
 				Value:       &v,
 			},

pkg/resources/nifi/nifi.go

@@ -636,7 +636,6 @@ func (r *Reconciler) reconcileNifiPod(log logr.Logger, desiredPod *corev1.Pod) (
 					}
 				}
 
-
 				log.V(1).Info("resource is in sync")
 				return nil, k8sutil.PodReady(currentPod)
 			}

pkg/util/util.go

@@ -251,4 +251,4 @@ func GetRequeueInterval(interval int, offset int) time.Duration {
 	duration := interval + rand.Intn(offset+1) - (offset / 2)
 	duration = Max(duration, rand.Intn(5)+1) // make sure duration does not go zero for very large offsets
 	return time.Duration(duration) * time.Second
-}
+}

site/docs/2_setup/1_getting_started.md

@@ -116,8 +116,8 @@ Now deploy the helm chart :
 helm install nifikop \
     orange-incubator/nifikop \
     --namespace=nifi \
-    --version 0.7.4 \
-    --set image.tag=v0.7.4-release \
+    --version 0.7.5 \
+    --set image.tag=v0.7.5-release \
     --set resources.requests.memory=256Mi \
     --set resources.requests.cpu=250m \
     --set resources.limits.memory=256Mi \

site/docs/2_setup/3_install/1_customizable_install_with_helm.md

@@ -33,7 +33,7 @@ The following tables lists the configurable parameters of the NiFi Operator Helm
 | Parameter                        | Description                                                                                                                                                                          | Default                    |
 | -------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------- |
 | `image.repository`               | Image                                                                                                                                                                                | `orangeopensource/nifikop` |
-| `image.tag`                      | Image tag                                                                                                                                                                            | `v0.7.4-release`           |
+| `image.tag`                      | Image tag                                                                                                                                                                            | `v0.7.5-release`           |
 | `image.pullPolicy`               | Image pull policy                                                                                                                                                                    | `Always`                   |
 | `image.imagePullSecrets.enabled` | Enable tue use of secret for docker image                                                                                                                                            | `false`                    |
 | `image.imagePullSecrets.name`    | Name of the secret to connect to docker registry                                                                                                                                     | -                          |

site/docs/5_references/1_nifi_cluster/1_nifi_cluster.md

@@ -88,7 +88,6 @@ spec:
 | rootProcessGroupId | string                                                              | contains the uuid of the root process group for this cluster.                               | if external type | -          |
 | secretRef          | \[ \][SecretReference](../4_nifi_parameter_context#secretreference) | reference the secret containing the informations required to authentiticate to the cluster. | if external type | -          |
 | proxyUrl           | string                                                              | defines the proxy required to query the NiFi cluster.                                       | if external type | -          |
-
 |service|[ServicePolicy](#servicepolicy)| defines the policy for services owned by NiFiKop operator. |No| - |
 |pod|[PodPolicy](#podpolicy)| defines the policy for pod owned by NiFiKop operator. |No| - |
 |zkAddress|string| specifies the ZooKeeper connection string in the form hostname:port where host and port are those of a Zookeeper server.|No|""|

site/docs/5_references/4_nifi_parameter_context.md

@@ -25,6 +25,7 @@ spec:
       description: tutu
     - name: test2
       description: toto
+      sensistive: true
 ```
 
 ## NifiParameterContext
@@ -60,6 +61,7 @@ spec:
 |name|string| the name of the Parameter. |Yes| - |
 |value|string| the value of the Parameter. |No| - |
 |description|string| the description of the Parameter. |No| - |
+|sensitive|string| Whether the parameter is sensitive or not. |No| false |
 
 ## SecretReference
 

site/website/versioned_docs/version-0.7.0/5_references/1_nifi_cluster/1_nifi_cluster.md

@@ -88,7 +88,6 @@ spec:
 | rootProcessGroupId | string                                                              | contains the uuid of the root process group for this cluster.                               | if external type | -          |
 | secretRef          | \[ \][SecretReference](../4_nifi_parameter_context#secretreference) | reference the secret containing the informations required to authentiticate to the cluster. | if external type | -          |
 | proxyUrl           | string                                                              | defines the proxy required to query the NiFi cluster.                                       | if external type | -          |
-
 |service|[ServicePolicy](#servicepolicy)| defines the policy for services owned by NiFiKop operator. |No| - |
 |pod|[PodPolicy](#podpolicy)| defines the policy for pod owned by NiFiKop operator. |No| - |
 |zkAddress|string| specifies the ZooKeeper connection string in the form hostname:port where host and port are those of a Zookeeper server.|No|""|