From f04bafdc26e08a6d60bb68380715b52081774f09 Mon Sep 17 00:00:00 2001 From: Juldrixx <31806759+juldrixx@users.noreply.github.com> Date: Thu, 22 Sep 2022 17:23:45 +0200 Subject: [PATCH] [Release] Bump version 0.14.1 (#175) * Release bump version 0.14.1 * Release bump version 0.14.1 --- .github/ISSUE_TEMPLATE/bug-report.yml | 2 +- .github/ISSUE_TEMPLATE/support-question.yml | 2 +- CHANGELOG.md | 12 +- config/manager/kustomization.yaml | 2 +- config/manager/manager.yaml | 2 +- .../keycloak-example/step-1/operator.yaml | 2 +- helm/nifikop/Chart.yaml | 4 +- helm/nifikop/README.md | 2 +- helm/nifikop/values.yaml | 2 +- site/docs/2_setup/1_getting_started.md | 4 +- .../1_customizable_install_with_helm.md | 2 +- .../1_concepts/1_introduction.md | 34 +++ .../1_concepts/2_design_principes.md | 62 +++++ .../version-v0.14.1/1_concepts/3_features.md | 67 +++++ .../version-v0.14.1/1_concepts/4_roadmap.md | 95 +++++++ .../2_setup/1_getting_started.md | 152 +++++++++++ .../2_setup/2_platform_setup/1_gke.md | 42 ++++ .../2_setup/2_platform_setup/2_k3d.md | 26 ++ .../1_customizable_install_with_helm.md | 199 +++++++++++++++ .../1_nifi_cluster/1_nodes_configuration.md | 9 + .../1_nifi_cluster/2_cluster_scaling.md | 237 ++++++++++++++++++ .../3_tasks/1_nifi_cluster/3_external_dns.md | 9 + .../1_nifi_cluster/4_external_cluster.md | 93 +++++++ .../3_tasks/2_security/1_ssl.md | 159 ++++++++++++ .../2_security/2_authentication/1_oidc.md | 42 ++++ .../2_authorization/1_custom_authorizer.md | 83 ++++++ .../3_tasks/3_nifi_dataflow.md | 126 ++++++++++ .../3_tasks/4_nifi_user_group.md | 168 +++++++++++++ .../4_examples/1_simple_nifi_cluster.md | 5 + .../1_nifi_cluster/1_nifi_cluster.md | 202 +++++++++++++++ .../1_nifi_cluster/2_read_only_config.md | 222 ++++++++++++++++ .../1_nifi_cluster/3_node_config.md | 108 ++++++++ .../5_references/1_nifi_cluster/4_node.md | 60 +++++ .../1_nifi_cluster/5_node_state.md | 73 ++++++ .../1_nifi_cluster/6_listeners_config.md | 59 +++++ .../7_external_service_config.md | 56 +++++ .../5_references/2_nifi_user.md | 101 ++++++++ .../5_references/3_nifi_registry_client.md | 42 ++++ .../5_references/4_nifi_parameter_context.md | 86 +++++++ .../5_references/5_nifi_dataflow.md | 136 ++++++++++ .../5_references/6_nifi_usergroup.md | 55 ++++ .../7_nifi_nodegroup_autoscaler.md | 59 +++++ .../0_contribution_organization.md | 66 +++++ .../6_contributing/1_developer_guide.md | 144 +++++++++++ .../6_contributing/2_reporting_bugs.md | 25 ++ .../6_contributing/3_credits.md | 11 + .../7_upgrade/1_v0.7.x_to_v0.8.0.md | 165 ++++++++++++ .../version-v0.14.1-sidebars.json | 91 +++++++ site/website/versions.json | 1 + version/version.go | 2 +- 50 files changed, 3393 insertions(+), 15 deletions(-) create mode 100644 site/website/versioned_docs/version-v0.14.1/1_concepts/1_introduction.md create mode 100644 site/website/versioned_docs/version-v0.14.1/1_concepts/2_design_principes.md create mode 100644 site/website/versioned_docs/version-v0.14.1/1_concepts/3_features.md create mode 100644 site/website/versioned_docs/version-v0.14.1/1_concepts/4_roadmap.md create mode 100644 site/website/versioned_docs/version-v0.14.1/2_setup/1_getting_started.md create mode 100644 site/website/versioned_docs/version-v0.14.1/2_setup/2_platform_setup/1_gke.md create mode 100644 site/website/versioned_docs/version-v0.14.1/2_setup/2_platform_setup/2_k3d.md create mode 100644 site/website/versioned_docs/version-v0.14.1/2_setup/3_install/1_customizable_install_with_helm.md create mode 100644 site/website/versioned_docs/version-v0.14.1/3_tasks/1_nifi_cluster/1_nodes_configuration.md create mode 100644 site/website/versioned_docs/version-v0.14.1/3_tasks/1_nifi_cluster/2_cluster_scaling.md create mode 100644 site/website/versioned_docs/version-v0.14.1/3_tasks/1_nifi_cluster/3_external_dns.md create mode 100644 site/website/versioned_docs/version-v0.14.1/3_tasks/1_nifi_cluster/4_external_cluster.md create mode 100644 site/website/versioned_docs/version-v0.14.1/3_tasks/2_security/1_ssl.md create mode 100644 site/website/versioned_docs/version-v0.14.1/3_tasks/2_security/2_authentication/1_oidc.md create mode 100644 site/website/versioned_docs/version-v0.14.1/3_tasks/2_security/2_authorization/1_custom_authorizer.md create mode 100644 site/website/versioned_docs/version-v0.14.1/3_tasks/3_nifi_dataflow.md create mode 100644 site/website/versioned_docs/version-v0.14.1/3_tasks/4_nifi_user_group.md create mode 100644 site/website/versioned_docs/version-v0.14.1/4_examples/1_simple_nifi_cluster.md create mode 100644 site/website/versioned_docs/version-v0.14.1/5_references/1_nifi_cluster/1_nifi_cluster.md create mode 100644 site/website/versioned_docs/version-v0.14.1/5_references/1_nifi_cluster/2_read_only_config.md create mode 100644 site/website/versioned_docs/version-v0.14.1/5_references/1_nifi_cluster/3_node_config.md create mode 100644 site/website/versioned_docs/version-v0.14.1/5_references/1_nifi_cluster/4_node.md create mode 100644 site/website/versioned_docs/version-v0.14.1/5_references/1_nifi_cluster/5_node_state.md create mode 100644 site/website/versioned_docs/version-v0.14.1/5_references/1_nifi_cluster/6_listeners_config.md create mode 100644 site/website/versioned_docs/version-v0.14.1/5_references/1_nifi_cluster/7_external_service_config.md create mode 100644 site/website/versioned_docs/version-v0.14.1/5_references/2_nifi_user.md create mode 100644 site/website/versioned_docs/version-v0.14.1/5_references/3_nifi_registry_client.md create mode 100644 site/website/versioned_docs/version-v0.14.1/5_references/4_nifi_parameter_context.md create mode 100644 site/website/versioned_docs/version-v0.14.1/5_references/5_nifi_dataflow.md create mode 100644 site/website/versioned_docs/version-v0.14.1/5_references/6_nifi_usergroup.md create mode 100644 site/website/versioned_docs/version-v0.14.1/5_references/7_nifi_nodegroup_autoscaler.md create mode 100644 site/website/versioned_docs/version-v0.14.1/6_contributing/0_contribution_organization.md create mode 100644 site/website/versioned_docs/version-v0.14.1/6_contributing/1_developer_guide.md create mode 100644 site/website/versioned_docs/version-v0.14.1/6_contributing/2_reporting_bugs.md create mode 100644 site/website/versioned_docs/version-v0.14.1/6_contributing/3_credits.md create mode 100644 site/website/versioned_docs/version-v0.14.1/7_upgrade/1_v0.7.x_to_v0.8.0.md create mode 100644 site/website/versioned_sidebars/version-v0.14.1-sidebars.json diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml index 068ebf7e2c..df75aa3647 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.yml +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -45,7 +45,7 @@ body: attributes: label: NiFiKop version description: NiFiKop release or git SHA - placeholder: v0.14.0-release + placeholder: v0.14.1-release validations: required: true diff --git a/.github/ISSUE_TEMPLATE/support-question.yml b/.github/ISSUE_TEMPLATE/support-question.yml index 1916c81b99..4253fdad9b 100644 --- a/.github/ISSUE_TEMPLATE/support-question.yml +++ b/.github/ISSUE_TEMPLATE/support-question.yml @@ -33,7 +33,7 @@ body: attributes: label: NiFiKop version description: NiFiKop release or git SHA - placeholder: v0.14.0-release + placeholder: v0.14.1-release - type: input attributes: diff --git a/CHANGELOG.md b/CHANGELOG.md index 2447b69adc..827ef6ab19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,14 +4,22 @@ ### Changed -- [PR #160](https://github.com/konpyutaika/nifikop/pull/160) - **[Documentation]** Upgrade documentation dependencies. - ### Fixed Bugs ### Deprecated ### Removed +## v0.14.1 + +### Changed + +- [PR #160](https://github.com/konpyutaika/nifikop/pull/160) - **[Documentation]** Upgrade documentation dependencies. + +### Fixed Bugs + +- [PR #174](https://github.com/konpyutaika/nifikop/pull/174) - **[Operator]** Fix K8S version getting. + ## v0.14.0 ### Added diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index a2729f365c..6ad3cb66d8 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -13,4 +13,4 @@ kind: Kustomization images: - name: controller newName: ghcr.io/konpyutaika/docker-images/nifikop - newTag: 0.14.0-master + newTag: 0.14.1-master diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index b412dc6299..ae6b3dbf29 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -31,7 +31,7 @@ spec: - /manager args: - --leader-elect - image: ghcr.io/konpyutaika/docker-images/nifikop:v0.14.0-release + image: ghcr.io/konpyutaika/docker-images/nifikop:v0.14.1-release name: nifikop securityContext: allowPrivilegeEscalation: false diff --git a/config/samples/keycloak-example/step-1/operator.yaml b/config/samples/keycloak-example/step-1/operator.yaml index a7fb1494ae..207971d9bd 100644 --- a/config/samples/keycloak-example/step-1/operator.yaml +++ b/config/samples/keycloak-example/step-1/operator.yaml @@ -1,4 +1,4 @@ -# nifikop 0.14.0 +# nifikop 0.14.1 rbacEnable: true namespaces: - nifi diff --git a/helm/nifikop/Chart.yaml b/helm/nifikop/Chart.yaml index 835d5e06c9..c98f9e0cf4 100644 --- a/helm/nifikop/Chart.yaml +++ b/helm/nifikop/Chart.yaml @@ -4,8 +4,8 @@ name: nifikop home: https://github.com/konpyutaika/nifikop sources: - https://github.com/konpyutaika/nifikop -version: 0.14.0 -appVersion: 0.14.0-release +version: 0.14.1 +appVersion: 0.14.1-release icon: maintainers: - name: erdrix diff --git a/helm/nifikop/README.md b/helm/nifikop/README.md index 0a35b1d7ab..940ab5ce96 100644 --- a/helm/nifikop/README.md +++ b/helm/nifikop/README.md @@ -23,7 +23,7 @@ The following tables lists the configurable parameters of the NiFi Operator Helm | Parameter | Description | Default | | -------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |--------------------------| | `image.repository` | Image | `konpyutaika/nifikop` | -| `image.tag` | Image tag | `v0.14.0-release` | +| `image.tag` | Image tag | `v0.14.1-release` | | `image.pullPolicy` | Image pull policy | `Always` | | `image.imagePullSecrets.enabled` | Enable tue use of secret for docker image | `false` | | `image.imagePullSecrets.name` | Name of the secret to connect to docker registry | - | diff --git a/helm/nifikop/values.yaml b/helm/nifikop/values.yaml index 6d9c3850da..e17cc92760 100644 --- a/helm/nifikop/values.yaml +++ b/helm/nifikop/values.yaml @@ -2,7 +2,7 @@ ## image: repository: ghcr.io/konpyutaika/docker-images/nifikop - tag: v0.14.0-release + tag: v0.14.1-release pullPolicy: Always imagePullSecrets: enabled: false diff --git a/site/docs/2_setup/1_getting_started.md b/site/docs/2_setup/1_getting_started.md index 09d2b6dccc..f4e17c16e2 100644 --- a/site/docs/2_setup/1_getting_started.md +++ b/site/docs/2_setup/1_getting_started.md @@ -109,8 +109,8 @@ Now deploy the helm chart : helm install nifikop \ oci://ghcr.io/konpyutaika/helm-charts/nifikop \ --namespace=nifi \ - --version 0.14.0 \ - --set image.tag=v0.14.0-release \ + --version 0.14.1 \ + --set image.tag=v0.14.1-release \ --set resources.requests.memory=256Mi \ --set resources.requests.cpu=250m \ --set resources.limits.memory=256Mi \ diff --git a/site/docs/2_setup/3_install/1_customizable_install_with_helm.md b/site/docs/2_setup/3_install/1_customizable_install_with_helm.md index 499954c9fc..da804cf39b 100644 --- a/site/docs/2_setup/3_install/1_customizable_install_with_helm.md +++ b/site/docs/2_setup/3_install/1_customizable_install_with_helm.md @@ -32,7 +32,7 @@ The following tables lists the configurable parameters of the NiFi Operator Helm | Parameter | Description | Default | |----------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------| | `image.repository` | Image | `ghcr.io/konpyutaika/docker-images/nifikop` | -| `image.tag` | Image tag | `v0.14.0-release` | +| `image.tag` | Image tag | `v0.14.1-release` | | `image.pullPolicy` | Image pull policy | `Always` | | `image.imagePullSecrets.enabled` | Enable tue use of secret for docker image | `false` | | `image.imagePullSecrets.name` | Name of the secret to connect to docker registry | - | diff --git a/site/website/versioned_docs/version-v0.14.1/1_concepts/1_introduction.md b/site/website/versioned_docs/version-v0.14.1/1_concepts/1_introduction.md new file mode 100644 index 0000000000..d65e316acd --- /dev/null +++ b/site/website/versioned_docs/version-v0.14.1/1_concepts/1_introduction.md @@ -0,0 +1,34 @@ +--- +id: 1_introduction +title: Introduction +sidebar_label: Introduction +--- + +The Konpyūtāika NiFi operator is a Kubernetes operator to automate provisioning, management, autoscaling and operations of [Apache NiFi](https://nifi.apache.org/) clusters deployed to K8s. + +## Overview + +Apache NiFi is an open-source solution that support powerful and scalable directed graphs of data routing, transformation, and system mediation logic. +Some of the high-level capabilities and objectives of Apache NiFi include, and some of the main features of the **NiFiKop** are: + +- **Fine grained** node configuration support +- Graceful rolling upgrade +- graceful NiFi cluster **scaling** +- Encrypted communication using SSL +- the provisioning of secure NiFi clusters +- Advanced Dataflow and user management via CRD + +Some of the roadmap features : + +- Monitoring via **Prometheus** +- Automatic reaction and self healing based on alerts (plugin system, with meaningful default alert plugins) +- graceful NiFi cluster **scaling and rebalancing** + +## Motivation + +There are already some approaches to operating NiFi on Kubernetes, however, we did not find them appropriate for use in a highly dynamic environment, nor capable of meeting our needs. + +- [Helm chart](https://github.com/cetic/helm-nifi) +- [Cloudera Nifi Operator](https://blog.cloudera.com/cloudera-flow-management-goes-cloud-native-with-apache-nifi-on-red-hat-openshift-kubernetes-platform/) + +Finally, our motivation is to build an open source solution and a community which drives the innovation and features of this operator. \ No newline at end of file diff --git a/site/website/versioned_docs/version-v0.14.1/1_concepts/2_design_principes.md b/site/website/versioned_docs/version-v0.14.1/1_concepts/2_design_principes.md new file mode 100644 index 0000000000..ef9e0f6e02 --- /dev/null +++ b/site/website/versioned_docs/version-v0.14.1/1_concepts/2_design_principes.md @@ -0,0 +1,62 @@ +--- +id: 2_design_principes +title: Design Principes +sidebar_label: Design Principes +--- + +## Pod level management + +NiFi is a stateful application. The first piece of the puzzle is the Node, which is a simple server capable of createing/forming a cluster with other Nodes. Every Node has his own **unique** configuration which differs slightly from all others. + +All NiFi on Kubernetes setup use [StatefulSet](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/) to create a NiFi Cluster. Just to quickly recap from the K8s docs: + +>StatefulSet manages the deployment and scaling of a set of Pods, and provide guarantees about their ordering and uniqueness. Like a Deployment, a StatefulSet manages Pods that are based on an identical container spec. Unlike a Deployment, a StatefulSet maintains sticky identities for each of its Pods. These pods are created from the same spec, but are not interchangeable: each has a persistent identifier that is maintained across any rescheduling. + +How does this looks from the perspective of Apache NiFi ? + +With StatefulSet we get : +- unique Node IDs generated during Pod startup +- networking between Nodes with headless services +- unique Persistent Volumes for Nodes + +Using StatefulSet we **lose** the ability to : + +- modify the configuration of unique Nodes +- remove a specific Node from a cluster (StatefulSet always removes the most recently created Node) +- use multiple, different Persistent Volumes for each Node + +The NiFi Operator uses `simple` Pods, ConfigMaps, and PersistentVolumeClaims, instead of StatefulSet (based on the design used by [Banzai Cloud Kafka Operator](https://github.com/banzaicloud/kafka-operator)). +Using these resources allows us to build an Operator which is better suited to NiFi. + +With the NiFi operator we can: + +- modify the configuration of unique Nodes +- remove specific Nodes from clusters +- use multiple Persistent Volumes for each Node + +## Dataflow Lifecycle management + +The [Dataflow Lifecycle management feature](./3_features.md#dataflow-lifecycle-management-via-crd) introduces 3 new CRDs : + +- **NiFiRegistryClient :** Allowing you to declare a [NiFi registry client](https://nifi.apache.org/docs/nifi-registry-docs/html/getting-started.html#connect-nifi-to-the-registry). +- **NiFiParameterContext :** Allowing you to create parameter context, with two kinds of parameters, a simple `map[string]string` for non-sensitive parameters and a `list of secrets` which contains sensitive parameters. +- **NiFiDataflow :** Allowing you to declare a Dataflow based on a `NiFiRegistryClient` and optionally a `ParameterContext`, which will be deployed and managed by the operator on the `targeted NiFi cluster`. + +The following diagram shows the interactions between all the components : + +![dataflow lifecycle management schema](/img/1_concepts/2_design_principes/dataflow_lifecycle_management_schema.jpg) + +With each CRD comes a new controller, with a reconcile loop : + +- **NiFiRegistryClient's controller :** + +![NiFi registry client's reconcile loop](/img/1_concepts/2_design_principes/registry_client_reconcile_loop.jpeg) + +- **NiFiParameterContext's controller :** + +![NiFi parameter context's reconcile loop](/img/1_concepts/2_design_principes/parameter_context_reconcile_loop.jpeg) + +- **NiFiDataflow's controller :** + +![NiFi dataflow's reconcile loop](/img/1_concepts/2_design_principes/dataflow_reconcile_loop.jpeg) + diff --git a/site/website/versioned_docs/version-v0.14.1/1_concepts/3_features.md b/site/website/versioned_docs/version-v0.14.1/1_concepts/3_features.md new file mode 100644 index 0000000000..60c7106862 --- /dev/null +++ b/site/website/versioned_docs/version-v0.14.1/1_concepts/3_features.md @@ -0,0 +1,67 @@ +--- +id: 3_features +title: Features +sidebar_label: Features +--- + +To highligt some of the features we needed and were not possible with the operators available, please keep reading + +## Fine Grained Node Config Support + +We needed to be able to react to events in a fine-grained way for each Node - and not in the limited way StatefulSet does (which, for example, removes the most recently created Nodes). Some of the available solutions try to overcome these deficits by placing scripts inside the container to generate configs at runtime (a good example is our [Cassandra Operator](https://github.com/Orange-OpenSource/casskop)), whereas the Orange NiFi operator's configurations are deterministically placed in specific Configmaps. + +## Graceful NiFi Cluster Scaling + +Apache NiFi is a good candidate to create an operator, because everything is made to orchestrate it through REST Api calls. With this comes automation of actions such as scaling, following all required steps : https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#decommission-nodes. + +## Graceful Rolling Upgrade + +Operator supports graceful rolling upgrade. It means the operator will check if the cluster is healthy. + +## Dynamic Configuration Support + +NiFi operates with two type of configs: + +- Read-only +- PerNode + +Read only config requires node restart to update all the others may be updated dynamically. +Operator CRD distinguishes these fields, and proceed with the right action. It can be a rolling upgrade, or +a dynamic reconfiguration. + +## Dataflow lifecycle management via CRD + +In a cloud native approach, we are looking for important management features, which we have applied to NiFi Dataflow : + +- **Automated deployment :** Based on the NiFi registry, you can describe your `NiFiDataflow` resource that will be deployed and run on the targeted NiFi cluster. +- **Portability :** On kubernetes everything is a yaml file, so with NiFiKop we give you the ability to describe your clusters but also the `registry clients`, `parameter contexts` and `dataflows` of your NiFi application, so that you can redeploy the same thing in a different namespace or cluster. +- **State management :** With NiFiKop resources, you can describe what you want, and the operator deals with the NiFi Rest API to make sure the resource stays in sync (even if someone manually makes changes directly on NiFi cluster). +- **Configurations :** Based on the `Parameter Contexts`, NiFiKop allows you to associate to your `Dataflow` (= your applications) with a different configuration depending on the environment ! + +## Users and access policies management + +Without the management of users and access policies associated, it was not possible to have a fully automated NiFi cluster setup due to : + +- **Node scaling :** when a new node joins the cluster it needs to have some roles like `proxy user request`, `view data` etc., by managing users and access policies we can easily create a user for this node with the right accesses. +- **Operator admin rights :** For the operator to manage efficiently the cluster it needs a lot of rights as `deploying process groups`, `empty the queues` etc., these rights are not available by default when you set a user as [InitialAdmin](https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#initial-admin-identity). Once again by giving the ability to define users and access policies we go through this. +- **User's access :** as seen just below we need to define the operator as `InitialAdmin`, in this situation there is no more users that can access to the web UI to manually give access to other users. That's why we extend the `InitialAdmin` concept into the operator, giving the ability to define a list of users as admins. + +In addition to these requirements to have a fully automated and managed cluster, we introduced some useful features : + +- **User management :** using `NifiUser` resource, you are able to create (or bind an existing) user in NiFi cluster and apply some access policies that will be managed and continuously synced by the operator. +- **Group management :** using `NifiUserGroup` resource, you can create groups in NiFi cluster and apply access policies and a list of `NifiUser` that will be managed and continuously synced by the operator. +- **Default group :** As the definition of `NifiUser` and `NifiUserGroup` resources could be heavy for some simple use cases, we also decided to define two default groups that you can feed with a list of users that will be created and managed by the operator (no kubernetes resources to create) : + - **Admins :** a group giving access to everything on the NiFi Cluster, + - **Readers :** a group giving access as viewer on the NiFi Cluster. + +By introducing this feature we are giving you the ability to fully automate your deployment, from the NiFi Cluster to your managed NiFi Dataflow. + +## Automatic horizontal NiFi cluster scaling via CRD + +NiFiKop supports automatically horizontally scaling `NifiCluster` node groups with a `NifiNodeGroupAutoscaler` custom resource. + +- **Kubernetes native :** The `NifiNodeGroupAutoscaler` controller implements the [Kubernetes scale subresource](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#scale-subresource) and creates a Kubernetes [`HorizontalPodAutoscaler`](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) to automatically scale the pods that NiFiKop creates for `NifiCluster` deployments. +- **Metrics-driven autoscaling :** The `HorizontalPodAutoscaler` can be driven by pod usage metrics (e.g. RAM, CPU) or through NiFi application metrics scraped by Prometheus. +- **Flexible NifiCluster node group autoscaling :** The `NifiNodeGroupAutoscaler` scales specific node groups in your `NifiCluster` and you may have as many autoscalers as you like per `NifiCluster` deployment. For example, a `NifiNodeGroupAutoscaler` may manage high-memory or high-cpu sets of nodes for volume burst scenarios or it may manage every node in your cluster. + +Through this set of features, you may elect to have NiFiKop configure automatic horizontal autoscaling for any subset of nodes in your `NifiCluster` deployment. \ No newline at end of file diff --git a/site/website/versioned_docs/version-v0.14.1/1_concepts/4_roadmap.md b/site/website/versioned_docs/version-v0.14.1/1_concepts/4_roadmap.md new file mode 100644 index 0000000000..fe28367ea3 --- /dev/null +++ b/site/website/versioned_docs/version-v0.14.1/1_concepts/4_roadmap.md @@ -0,0 +1,95 @@ +--- +id: 4_roadmap +title: Roadmap +sidebar_label: Roadmap +--- + +## Available + +### NiFi cluster installation + +| | | +| --------------------- | --------- | +| Status | Done | +| Priority | High | +| Targeted Start date | Jan 2020 | + +### Graceful NiFi Cluster Scaling + +| | | +| --------------------- | --------- | +| Status | Done | +| Priority | High | +| Targeted Start date | Jan 2020 | + +Apache NiFi is a good candidate to create an operator, because everything is made to orchestrate it through REST Api calls. With this comes automation of actions such as scaling, following all required steps : https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#decommission-nodes. + +### Communication via SSL + +| | | +| --------------------- | -------- | +| Status | Done | +| Priority | High | +| Targeted Start date | May 2020 | + + +The operator fully automates NiFi's SSL support. +The operator can provision the required secrets and certificates for you, or you can provide your own. + +### Dataflow lifecycle management via CRD + +| | | +| --------------------- | --------- | +| Status | Done | +| Priority | High | +| Targeted Start date | Aug 2020 | + +### Users & access policies management + +| | | +| --------------------- | ----- | +| Status | Done| +| Priority | High | +| Targeted Start date | November 2020 | + +The operator fully automates NiFi's user and access policies management. + +## Backlog + +### Monitoring via Prometheus + +| | | +| --------------------- | -------- | +| Status | To Do | +| Priority | High | +| Targeted Start date | Oct 2020 | + +The NiFi operator exposes NiFi JMX metrics to Prometheus. + +### Reacting on Alerts + +| | | +| --------------------- | ----- | +| Status | To Do | +| Priority | Low | +| Targeted Start date | - | + +The NiFi Operator acts as a **Prometheus Alert Manager**. It receives alerts defined in Prometheus, and creates actions based on Prometheus alert annotations. + +Currently, there are three actions expected : +- upscale cluster (add a new Node) +- downscale cluster (remove a Node) +- add additional disk to a Node + +### Seamless Istio mesh support + +| | | +| --------------------- | ----- | +| Status | To Do | +| Priority | Low | +| Targeted Start date | - | + +- Operator allows to use ClusterIP services instead of Headless, which still works better in case of Service meshes. +- To avoid too early nifi initialization, which might lead to unready sidecar container. The operator will use a small script to +mitigate this behaviour. All NiFi image can be used the only one requirement is an available **wget** command. +- To access a NiFi cluster which runs inside the mesh. Operator will supports creating Istio ingress gateways. \ No newline at end of file diff --git a/site/website/versioned_docs/version-v0.14.1/2_setup/1_getting_started.md b/site/website/versioned_docs/version-v0.14.1/2_setup/1_getting_started.md new file mode 100644 index 0000000000..f4e17c16e2 --- /dev/null +++ b/site/website/versioned_docs/version-v0.14.1/2_setup/1_getting_started.md @@ -0,0 +1,152 @@ +--- +id: 1_getting_started +title: Getting Started +sidebar_label: Getting Started +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +The operator installs the 1.12.1 version of Apache NiFi, can run on Minikube v0.33.1+ and **Kubernetes 1.21.0+**, and require **Helm 3**. + +:::info +The operator supports NiFi 1.11.0+ +::: + +As a pre-requisite it needs a Kubernetes cluster. Also, NiFi requires Zookeeper so you need to first have a Zookeeper cluster if you don't already have one. + +> We believe in the `separation of concerns` principle, thus the NiFi operator does not install nor manage Zookeeper. + +## Prerequisites + +### Install Zookeeper + +To install Zookeeper we recommend using the [Bitnami's Zookeeper chart](https://github.com/bitnami/charts/tree/master/bitnami/zookeeper). + +```bash +helm repo add bitnami https://charts.bitnami.com/bitnami +``` + +```bash +# You have to create the namespace before executing following command +helm install zookeeper bitnami/zookeeper \ + --set resources.requests.memory=256Mi \ + --set resources.requests.cpu=250m \ + --set resources.limits.memory=256Mi \ + --set resources.limits.cpu=250m \ + --set global.storageClass=standard \ + --set networkPolicy.enabled=true \ + --set replicaCount=3 +``` + +:::warning +Replace the `storageClass` parameter value with your own. +::: + +### Install cert-manager + +The NiFiKop operator uses `cert-manager` for issuing certificates to users and and nodes, so you'll need to have it setup in case you want to deploy a secured cluster with authentication enabled. The minimum supported cert-manager version is v1.0. + + + + +```bash +# Install the CustomResourceDefinitions and cert-manager itself +kubectl apply -f \ + https://github.com/jetstack/cert-manager/releases/download/v1.7.2/cert-manager.yaml +``` + + + + +```bash +# Install CustomResourceDefinitions first +kubectl apply --validate=false -f \ + https://github.com/jetstack/cert-manager/releases/download/v1.7.2/cert-manager.crds.yaml + +# Add the jetstack helm repo +helm repo add jetstack https://charts.jetstack.io +helm repo update + +# You have to create the namespace before executing following command +helm install cert-manager \ + --namespace cert-manager \ + --version v1.7.2 jetstack/cert-manager +``` + + + + +## Installation + +## Installing with Helm + +You can deploy the operator using a Helm chart [Helm chart](https://github.com/konpyutaika/nifikop/tree/master/helm): + +> To install an other version of the operator use `helm install --name=nifikop --namespace=nifi --set operator.image.tag=x.y.z konpyutaika-incubator/nifikop` + +In the case where you don't want to deploy the crds using helm (`--skip-crds`), you have to deploy manually the crds : + +```bash +kubectl apply -f https://raw.githubusercontent.com/konpyutaika/nifikop/master/config/crd/bases/nifi.konpyutaika.com_nificlusters.yaml +kubectl apply -f https://raw.githubusercontent.com/konpyutaika/nifikop/master/config/crd/bases/nifi.konpyutaika.com_nifiusers.yaml +kubectl apply -f https://raw.githubusercontent.com/konpyutaika/nifikop/master/config/crd/bases/nifi.konpyutaika.com_nifiusergroups.yaml +kubectl apply -f https://raw.githubusercontent.com/konpyutaika/nifikop/master/config/crd/bases/nifi.konpyutaika.com_nifidataflows.yaml +kubectl apply -f https://raw.githubusercontent.com/konpyutaika/nifikop/master/config/crd/bases/nifi.konpyutaika.com_nifiparametercontexts.yaml +kubectl apply -f https://raw.githubusercontent.com/konpyutaika/nifikop/master/config/crd/bases/nifi.konpyutaika.com_nifiregistryclients.yaml +``` + +Now deploy the helm chart : + +```bash +# You have to create the namespace before executing following command +helm install nifikop \ + oci://ghcr.io/konpyutaika/helm-charts/nifikop \ + --namespace=nifi \ + --version 0.14.1 \ + --set image.tag=v0.14.1-release \ + --set resources.requests.memory=256Mi \ + --set resources.requests.cpu=250m \ + --set resources.limits.memory=256Mi \ + --set resources.limits.cpu=250m \ + --set namespaces={"nifi"} +``` + +:::note +Add the following parameter if you are using this instance to only deploy unsecured clusters : `--set certManager.enabled=false` +::: + +## Create custom storage class + +We recommend to use a **custom StorageClass** to leverage the volume binding mode `WaitForFirstConsumer` + +```bash +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: exampleStorageclass +parameters: + type: pd-standard +provisioner: kubernetes.io/gce-pd +reclaimPolicy: Delete +volumeBindingMode: WaitForFirstConsumer +``` + +:::tip +Remember to set your NiFiCluster CR properly to use the newly created StorageClass. +::: + +## Deploy NiFi cluster + +And after you can deploy a simple NiFi cluster. + +```bash +# Add your zookeeper svc name to the configuration +kubectl create -n nifi -f config/samples/simplenificluster.yaml +``` diff --git a/site/website/versioned_docs/version-v0.14.1/2_setup/2_platform_setup/1_gke.md b/site/website/versioned_docs/version-v0.14.1/2_setup/2_platform_setup/1_gke.md new file mode 100644 index 0000000000..c798f74434 --- /dev/null +++ b/site/website/versioned_docs/version-v0.14.1/2_setup/2_platform_setup/1_gke.md @@ -0,0 +1,42 @@ +--- +id: 1_gke +title: Google Kubernetes Engine +sidebar_label: Google Kubernetes Engine +--- + +Follow these instructions to prepare a GKE cluster for NiFiKop + +1. Setup environment variables. + +```sh +export GCP_PROJECT= +export GCP_ZONE= +export CLUSTER_NAME= +``` + +2. Create a new cluster. + +```sh +gcloud container clusters create $CLUSTER_NAME \ + --cluster-version latest \ + --machine-type=n1-standard-1 \ + --num-nodes 4 \ + --zone $GCP_ZONE \ + --project $GCP_PROJECT +``` + +3. Retrieve your credentials for `kubectl`. + +```sh +cloud container clusters get-credentials $CLUSTER_NAME \ + --zone $GCP_ZONE \ + --project $GCP_PROJECT +``` + +4. Grant cluster administrator (admin) permissions to the current user. To create the necessary RBAC rules for NiFiKop, the current user requires admin permissions. + +```sh +kubectl create clusterrolebinding cluster-admin-binding \ + --clusterrole=cluster-admin \ + --user=$(gcloud config get-value core/account) +``` diff --git a/site/website/versioned_docs/version-v0.14.1/2_setup/2_platform_setup/2_k3d.md b/site/website/versioned_docs/version-v0.14.1/2_setup/2_platform_setup/2_k3d.md new file mode 100644 index 0000000000..d268a05e94 --- /dev/null +++ b/site/website/versioned_docs/version-v0.14.1/2_setup/2_platform_setup/2_k3d.md @@ -0,0 +1,26 @@ +--- +id: 2_k3d +title: K3D +sidebar_label: K3D +--- + +Follow these instructions to prepare k3d for NiFiKop installation with sufficient resources to run NiFiKop and some basic applications. + +## Prerequisites + +- Administrative privileges are required to run k3d. + +## Installation steps + +1. Install the latest version of [k3d](https://k3d.io/v5.3.0/#installation), version 5.3.0 or later. +2. Create your Kubernetes cluster. This example uses Kubernetes version 1.21.10. You can change the version to any Kubernetes version supported by NiFiKop by altering the --kubernetes-version value: + + ```sh + k3d cluster create --image rancher/k3s:v1.21.10-k3s1 --wait + ``` + +3. Expose your NiFi cluster: + + ```sh + k3d cluster edit k3s-default --port-add ":@loadbalancer" + ``` \ No newline at end of file diff --git a/site/website/versioned_docs/version-v0.14.1/2_setup/3_install/1_customizable_install_with_helm.md b/site/website/versioned_docs/version-v0.14.1/2_setup/3_install/1_customizable_install_with_helm.md new file mode 100644 index 0000000000..da804cf39b --- /dev/null +++ b/site/website/versioned_docs/version-v0.14.1/2_setup/3_install/1_customizable_install_with_helm.md @@ -0,0 +1,199 @@ +--- +id: 1_customizable_install_with_helm +title: Customizable install with Helm +sidebar_label: Customizable install with Helm +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +## Prerequisites + +- Perform any necessary [plateform-specific setup](../2_platform_setup/1_gke.md) +- [Install a Helm client](https://github.com/helm/helm#install) with a version higher than 3 + +## Introduction + +This Helm chart install NiFiKop the Nifi Kubernetes operator to create/configure/manage NiFi +clusters in a Kubernetes Namespace. + +It will use Custom Ressources Definition CRDs: + +- `nificlusters.nifi.konpyutaika.com`, +- `nifiusers.nifi.konpyutaika.com`, +- `nifiusergroups.nifi.konpyutaika.com`, +- `nifiregistryclients.nifi.konpyutaika.com`, +- `nifiparametercontexts.nifi.konpyutaika.com`, +- `nifidataflows.nifi.konpyutaika.com`, + +### Configuration + +The following tables lists the configurable parameters of the NiFi Operator Helm chart and their default values. +| Parameter | Description | Default | +|----------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------| +| `image.repository` | Image | `ghcr.io/konpyutaika/docker-images/nifikop` | +| `image.tag` | Image tag | `v0.14.1-release` | +| `image.pullPolicy` | Image pull policy | `Always` | +| `image.imagePullSecrets.enabled` | Enable tue use of secret for docker image | `false` | +| `image.imagePullSecrets.name` | Name of the secret to connect to docker registry | - | +| `certManager.enabled` | Enable cert-manager integration | `true` | +| `rbacEnable` | If true, create & use RBAC resources | `true` | +| `labels` | Labels to add to all deployed resources | `{}` | +| `annotations` | Annotations to add to all deployed resources | `{}` | +| `resources` | Pod resource requests & limits | `{}` | +| `metrics.enabled` | deploy service for metrics | `false` | +| `metrics.port` | Set port for operator metrics | `8081` | +| `logLevel` | Log level to output | `Info` | +| `logEncoding` | Log encoding to use. Either `json` or `console` | `json` | +| `certManager.clusterScoped` | If true setup cluster scoped resources | `false` | +| `namespaces` | List of namespaces where Operator watches for custom resources. Make sure the operator ServiceAccount is granted `get` permissions on this `Node` resource when using limited RBACs. | `""` i.e. all namespaces | +| `nodeSelector` | Node selector configuration for operator pod | `{}` | +| `affinity` | Node affinity configuration for operator pod | `{}` | +| `tolerations` | Toleration configuration for operator pod | `{}` | +| `serviceAccount.create` | Whether the SA creation is delegated to the chart or not | `true` | +| `serviceAccount.name` | Name of the SA used for NiFiKop deployment | release name | + +Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example, + +Alternatively, a YAML file that specifies the values for the above parameters can be provided while installing the chart. For example, + +```console +$ helm install nifikop \ + konpyutaika/nifikop \ + -f values.yaml +``` + +### Installing the Chart + +:::important Skip CRDs +In the case where you don't want to deploy the crds using helm (`--skip-crds`) you need to deploy manually the crds beforehand: + +```bash +kubectl apply -f https://raw.githubusercontent.com/konpyutaika/nifikop/master/config/crd/bases/nifi.konpyutaika.com_nificlusters.yaml +kubectl apply -f https://raw.githubusercontent.com/konpyutaika/nifikop/master/config/crd/bases/nifi.konpyutaika.com_nifiusers.yaml +kubectl apply -f https://raw.githubusercontent.com/konpyutaika/nifikop/master/config/crd/bases/nifi.konpyutaika.com_nifiusergroups.yaml +kubectl apply -f https://raw.githubusercontent.com/konpyutaika/nifikop/master/config/crd/bases/nifi.konpyutaika.com_nifidataflows.yaml +kubectl apply -f https://raw.githubusercontent.com/konpyutaika/nifikop/master/config/crd/bases/nifi.konpyutaika.com_nifiparametercontexts.yaml +kubectl apply -f https://raw.githubusercontent.com/konpyutaika/nifikop/master/config/crd/bases/nifi.konpyutaika.com_nifiregistryclients.yaml +``` + +::: + + + + +```bash +helm install nifikop konpyutaika/nifikop \ + --dry-run \ + --set logLevel=Debug \ + --set namespaces={"nifikop"} +``` + + + + +```bash +helm install konpyutaika/nifikop +``` + + + + + +```bash +helm install nifikop konpyutaika/nifikop --set namespaces={"nifikop"} +``` + + + + +> the `--replace` flag allow you to reuses a charts release name + +### Listing deployed charts + +```bash +helm list +``` + +### Get Status for the helm deployment + +```bash +helm status nifikop +``` + +## Uninstaling the Charts + +If you want to delete the operator from your Kubernetes cluster, the operator deployment +should be deleted. + +```bash +helm del nifikop +``` + +The command removes all the Kubernetes components associated with the chart and deletes the helm release. + +:::tip +The CRD created by the chart are not removed by default and should be manually cleaned up (if required) +::: + +Manually delete the CRD: + +```bash +kubectl delete crd nificlusters.nifi.konpyutaika.com +kubectl delete crd nifiusers.nifi.konpyutaika.com +kubectl delete crd nifiusergroups.nifi.konpyutaika.com +kubectl delete crd nifiregistryclients.nifi.konpyutaika.com +kubectl delete crd nifiparametercontexts.nifi.konpyutaika.com +kubectl delete crd nifidataflows.nifi.konpyutaika.com +``` + +:::warning +If you delete the CRD then +It will delete **ALL** Clusters that has been created using this CRD!!! +Please never delete a CRD without very good care +::: + +Helm always keeps records of what releases happened. Need to see the deleted releases ? + +```bash +helm list --deleted +``` + +Need to see all of the releases (deleted and currently deployed, as well as releases that +failed) ? + +```bash +helm list --all +``` + +Because Helm keeps records of deleted releases, a release name cannot be re-used. (If you really need to re-use a +release name, you can use the `--replace` flag, but it will simply re-use the existing release and replace its +resources.) + +Note that because releases are preserved in this way, you can rollback a deleted resource, and have it re-activate. + +To purge a release + +```bash +helm delete --purge nifikop +``` + +## Troubleshooting + +### Install of the CRD + +By default, the chart will install the CRDs, but this installation is global for the whole +cluster, and you may want to not modify the already deployed CRDs. + +In this case there is a parameter to say to not install the CRDs : + +``` +$ helm install --name nifikop ./helm/nifikop --set namespaces={"nifikop"} --skip-crds +``` diff --git a/site/website/versioned_docs/version-v0.14.1/3_tasks/1_nifi_cluster/1_nodes_configuration.md b/site/website/versioned_docs/version-v0.14.1/3_tasks/1_nifi_cluster/1_nodes_configuration.md new file mode 100644 index 0000000000..0260901b17 --- /dev/null +++ b/site/website/versioned_docs/version-v0.14.1/3_tasks/1_nifi_cluster/1_nodes_configuration.md @@ -0,0 +1,9 @@ +--- +id: 1_nodes_configuration +title: Nodes configuration +sidebar_label: Nodes configuration +--- + +:::warning +WIP +::: \ No newline at end of file diff --git a/site/website/versioned_docs/version-v0.14.1/3_tasks/1_nifi_cluster/2_cluster_scaling.md b/site/website/versioned_docs/version-v0.14.1/3_tasks/1_nifi_cluster/2_cluster_scaling.md new file mode 100644 index 0000000000..1c9f4cda61 --- /dev/null +++ b/site/website/versioned_docs/version-v0.14.1/3_tasks/1_nifi_cluster/2_cluster_scaling.md @@ -0,0 +1,237 @@ +--- +id: 2_cluster_scaling +title: Cluster Scaling +sidebar_label: Cluster Scaling +--- + +This tasks shows you how to perform a gracefull cluster scale up and scale down. + +## Before you begin + +- Setup NiFiKop by following the instructions in the [Installation guide](../../2_setup/1_getting_started.md). +- Deploy the [Simple NiFi](../../2_setup/1_getting_started.md#easy-way-installing-with-helm) sample cluster. +- Review the [Node](../../5_references/1_nifi_cluster/4_node.md) references doc. + +## About this task + +The [Simple NiFi](../../2_setup/1_getting_started.md#easy-way-installing-with-helm) example consists of a three nodes NiFi cluster. +A node decommission must follow a strict procedure, described in the [NiFi documentation](https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#decommission-nodes) : + +1. Disconnect the node +2. Once disconnect completes, offload the node. +3. Once offload completes, delete the node. +4. Once the delete request has finished, stop/remove the NiFi service on the host. + + +For the moment, we have implemented it as follows in the operator : + + 1. Disconnect the node + 2. Once disconnect completes, offload the node. + 3. Once offload completes, delete the pod. + 4. Once the pod deletion completes, delete the node. + 5. Once the delete request has finished, remove the node from the NifiCluster status. + +In addition, we have a regular check that ensure that all nodes have been removed. + +In this task, you will first perform a scale up, in adding an new node. Then, you will remove another node that the one created, and observe the decommission's steps. + +## Scale up : Add a new node + +For this task, we will simply add a node with the same configuration than the other ones, if you want to know more about how to add a node with an other configuration let's have a look to the [Node configuration](./1_nodes_configuration.md) documentation page. + +1. Add and run a dataflow as the example : + +![Scaling dataflow](/img/3_tasks/1_nifi_cluster/2_cluster_scaling/scaling_dataflow.png) + +2. Add a new node to the list of `NifiCluster.Spec.Nodes` field, by following the [Node object definition](../../5_references/1_nifi_cluster/4_node.md) documentation: + +```yaml +apiVersion: nifi.konpyutaika.com/v1alpha1 +kind: NifiCluster +metadata: + name: simplenifi +spec: + service: + headlessEnabled: true + zkAddress: "zookeepercluster-client.zookeeper:2181" + zkPath: "/simplenifi" + clusterImage: "apache/nifi:1.12.1" + oneNifiNodePerNode: false + nodeConfigGroups: + default_group: + isNode: true + storageConfigs: + - mountPath: "/opt/nifi/nifi-current/logs" + name: logs + pvcSpec: + accessModes: + - ReadWriteOnce + storageClassName: "standard" + resources: + requests: + storage: 10Gi + serviceAccountName: "default" + resourcesRequirements: + limits: + cpu: "2" + memory: 3Gi + requests: + cpu: "1" + memory: 1Gi + nodes: + - id: 0 + nodeConfigGroup: "default_group" + - id: 1 + nodeConfigGroup: "default_group" + - id: 2 + nodeConfigGroup: "default_group" +# >>>> START: The new node + - id: 25 + nodeConfigGroup: "default_group" +# <<<< END + propagateLabels: true + nifiClusterTaskSpec: + retryDurationMinutes: 10 + listenersConfig: + internalListeners: + - type: "http" + name: "http" + containerPort: 8080 + - type: "cluster" + name: "cluster" + containerPort: 6007 + - type: "s2s" + name: "s2s" + containerPort: 10000 +``` + +:::important +**Note :** The `Node.Id` field must be unique in the `NifiCluster.Spec.Nodes` list. +::: + +3. Apply the new `NifiCluster` configuration : + +```sh +kubectl -n nifi apply -f config/samples/simplenificluster.yaml +``` + +4. You should now have the following resources into kubernetes : + +```console +kubectl get pods,configmap,pvc -l nodeId=25 +NAME READY STATUS RESTARTS AGE +pod/simplenifi-25-nodem5jh4 1/1 Running 0 11m + +NAME DATA AGE +configmap/simplenifi-config-25 7 11m + +NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE +persistentvolumeclaim/simplenifi-25-storagehwn24 Bound pvc-7da86076-728e-11ea-846d-42010a8400f2 10Gi RWO standard 11m +``` + +And if you go on the NiFi UI, in the cluster administration page : + +![Scale up, cluster list](/img/3_tasks/1_nifi_cluster/2_cluster_scaling/scaleup_cluster_list.png) + +5. You now have data on the new node : + +![Scale up, cluster distribution](/img/3_tasks/1_nifi_cluster/2_cluster_scaling/scaleup_distribution.png) + +## Scaledown : Gracefully remove node + +For this task, we will simply remove a node and look at that the decommission's steps. + +1. Remove the node from the list of `NifiCluster.Spec.Nodes` field : + +```yaml +apiVersion: nifi.konpyutaika.com/v1alpha1 +kind: NifiCluster +metadata: + name: simplenifi +spec: + headlessServiceEnabled: true + zkAddresse: "zookeepercluster-client.zookeeper:2181" + zkPath: "/simplenifi" + clusterImage: "apache/nifi:1.11.3" + oneNifiNodePerNode: false + nodeConfigGroups: + default_group: + isNode: true + storageConfigs: + - mountPath: "/opt/nifi/nifi-current/logs" + name: logs + pvcSpec: + accessModes: + - ReadWriteOnce + storageClassName: "standard" + resources: + requests: + storage: 10Gi + serviceAccountName: "default" + resourcesRequirements: + limits: + cpu: "2" + memory: 3Gi + requests: + cpu: "1" + memory: 1Gi + nodes: + - id: 0 + nodeConfigGroup: "default_group" + - id: 1 + nodeConfigGroup: "default_group" +# >>>> START: node removed +# - id: 2 +# nodeConfigGroup: "default_group" +# <<<< END + - id: 25 + nodeConfigGroup: "default_group" + propagateLabels: true + nifiClusterTaskSpec: + retryDurationMinutes: 10 + listenersConfig: + internalListeners: + - type: "http" + name: "http" + containerPort: 8080 + - type: "cluster" + name: "cluster" + containerPort: 6007 + - type: "s2s" + name: "s2s" + containerPort: 10000 +``` + +2. Apply the new `NifiCluster` configuration : + +```sh +kubectl -n nifi apply -f config/samples/simplenificluster.yaml +``` + +3. You can follow the node's action step status in the `NifiCluster.Status` description : + +```console +kubectl describe nificluster simplenifi + +... +Status: + Nodes State: + ... + 2: + Configuration State: ConfigInSync + Graceful Action State: + Action State: GracefulDownscaleRequired + Error Message: + ... +... +``` + +:::tip +The list of decommision's step and their corresponding value for the `Nifi Cluster.Status.Node State.Graceful ActionState.ActionStep` field is described into the [Node State page](../../5_references/1_nifi_cluster/5_node_state.md#actionstep) +::: + +4. Once the scaledown successfully performed, you should have the data offloaded on the other nodes, and the node state removed from the `NifiCluster.Status.NodesState` list : + +:::warning +Keep in mind that the [`NifiCluster.Spec.nifiClusterTaskSpec.retryDurationMinutes`](../../5_references/1_nifi_cluster/1_nifi_cluster.md#nificlustertaskspec) should be long enough to perform the whole procedure, or you will have some rollback and retry loop. +::: \ No newline at end of file diff --git a/site/website/versioned_docs/version-v0.14.1/3_tasks/1_nifi_cluster/3_external_dns.md b/site/website/versioned_docs/version-v0.14.1/3_tasks/1_nifi_cluster/3_external_dns.md new file mode 100644 index 0000000000..27c33dc5ec --- /dev/null +++ b/site/website/versioned_docs/version-v0.14.1/3_tasks/1_nifi_cluster/3_external_dns.md @@ -0,0 +1,9 @@ +--- +id: 3_external_dns +title: External DNS +sidebar_label: External DNS +--- + +:::warning +WIP +::: \ No newline at end of file diff --git a/site/website/versioned_docs/version-v0.14.1/3_tasks/1_nifi_cluster/4_external_cluster.md b/site/website/versioned_docs/version-v0.14.1/3_tasks/1_nifi_cluster/4_external_cluster.md new file mode 100644 index 0000000000..00c9344864 --- /dev/null +++ b/site/website/versioned_docs/version-v0.14.1/3_tasks/1_nifi_cluster/4_external_cluster.md @@ -0,0 +1,93 @@ +--- +id: 4_external_cluster +title: External cluster +sidebar_label: External cluster +--- + +This task shows you how to configure an external cluster. + +## Common configuration + +The operator allows you to manage the Dataflow lifecycle for internal (i.e cluster managed by the operator) and external NiFi cluster. +A NiFi cluster is considered as external as soon as the `NifiCluster` resource used as reference in other NiFi resource explicitly detailed the way to comunicate with the cluster. + +This feature allows you : + +- To automate your Dataflow CI/CD using yaml +- To manage the same way your Dataflow management wherever your cluster is, on bare metal, VMs, k8s, on-premise or on cloud. + +To deploy different resources (`NifiRegistryClient`, `NifiUser`, `NifiUserGroup`, `NifiParameterContext`, `NifiDataflow`) you simply have to declare a `NifiCluster` resource explaining how to discuss with the external cluster, and refer to this resource as usual using the `Spec.ClusterRef` field. + +To declare an external cluster you have to follow this kind of configuration : + +```yaml +apiVersion: nifi.konpyutaika.com/v1alpha1 +kind: NifiCluster +metadata: + name: externalcluster +spec: + # rootProcessGroupId contains the uuid of the root process group for this cluster. + rootProcessGroupId: 'd37bee03-017a-1000-cff7-4eaaa82266b7' + # nodeURITemplate used to dynamically compute node uri. + nodeURITemplate: 'nifi0%d.integ.mapreduce.m0.p.fti.net:9090' + # all node requiresunique id + nodes: + - id: 1 + - id: 2 + - id: 3 + # type defines if the cluster is internal (i.e manager by the operator) or external. + # :Enum={"external","internal"} + type: 'external' + # clientType defines if the operator will use basic or tls authentication to query the NiFi cluster. + # Enum={"tls","basic"} + clientType: 'basic' + # secretRef reference the secret containing the informations required to authenticate to the cluster. + secretRef: + name: nifikop-credentials + namespace: nifikop-nifi +``` + +- The `Spec.RootProcessGroupId` field is required to give the ability to the operator of managing root level policy and default deployment and policy. +- The `Spec.NodeURITemplate` field, defines the hostname template of your NiFi cluster nodes, the operator will use this information and the list of id specified in `Spec.Nodes` field to generate the hostname of the nodes (in the configuration above you will have : `nifi01.integ.mapreduce.m0.p.fti.net:9090`, `nifi02.integ.mapreduce.m0.p.fti.net:9090`, `nifi03.integ.mapreduce.m0.p.fti.net:9090`). +- The `Spec.Type` field defines the type of cluster that this resource is refering to, by default it is `internal`, in our case here we just want to use this resource to reference an existing NiFi cluster, so we set this field to `external`. +- The `Spec.ClientType` field defines how we want to authenticate to the NiFi cluster API, for now we are supporting two modes : + - `tls` : using client TLS certificate. + - `basic` : using a username and a password to get an access token. +- The `Spec.SecretRef` defines a reference to a secret which contains the sensitive values that will be used by the operator to authenticate to the NiFi cluster API (ie in basic mode it will contain the password and username). + +:::warning +The id of node only support `int32` as type, so if the hostname of your nodes doesn't match with this, you can't use this feature. +::: + +## Secret configuration for Basic authentication + +When you are using the basic authentication, you have to pass some informations into the secret that is referenced into the `NifiCluster` resource: + +- `username` : the username associated to the user that will be used by the operator to request the REST API. +- `password` : the password associated to the user that will be used by the operator to request the REST API. +- `ca.crt (optional)`: the certificate authority to trust the server certificate if needed + +The following command shows how you can create this secret : + +```console +kubectl create secret generic nifikop-credentials \ + --from-file=username=./secrets/username\ + --from-file=password=./secrets/password\ + --from-file=ca.crt=./secrets/ca.crt\ + -n nifikop-nifi +``` + +:::info +When you use the basic authentication, the operator will create a secret `-basic-secret` containing for each node an access token that will be maintained by the operator. +::: + +## Secret configuration for TLS authentication + +When you are using the tls authentication, you have to pass some information into the secret that is referenced into the `NifiCluster` resource: + +- `tls.key` : The user private key. +- `tls.crt` : The user certificate. +- `password` : the password associated to the user that will be used by the operator to request the REST API. +- `ca.crt`: The CA certificate +- `truststore.jks`: +- `keystore.jks`: diff --git a/site/website/versioned_docs/version-v0.14.1/3_tasks/2_security/1_ssl.md b/site/website/versioned_docs/version-v0.14.1/3_tasks/2_security/1_ssl.md new file mode 100644 index 0000000000..0dc262f377 --- /dev/null +++ b/site/website/versioned_docs/version-v0.14.1/3_tasks/2_security/1_ssl.md @@ -0,0 +1,159 @@ +--- +id: 1_ssl +title: Securing NiFi with SSL +sidebar_label: SSL +--- + +The `NiFi operator` makes securing your NiFi cluster with SSL. You may provide your own certificates, or instruct the operator to create them for from your cluster configuration. + +Below this is an example configuration required to secure your cluster with SSL : + +```yaml +apiVersion: nifi.konpyutaika.com/v1alpha1 +kind: NifiCluster +... +spec: + ... + managedAdminUsers: + - identity : "alexandre.guitton@orange.com" + name: "aguitton" + ... + readOnlyConfig: + # NifiProperties configuration that will be applied to the node. + nifiProperties: + webProxyHosts: + - nifistandard2.trycatchlearn.fr:8443 + ... + ... + listenersConfig: + internalListeners: + - type: "https" + name: "https" + containerPort: 8443 + - type: "cluster" + name: "cluster" + containerPort: 6007 + - type: "s2s" + name: "s2s" + containerPort: 10000 + sslSecrets: + tlsSecretName: "test-nifikop" + create: true +``` + +- `managedAdminUsers` : list of users account which will be configured as admin into NiFi cluster, please check [](../4_nifi_user_group#managed-groups-for-simple-setup) for more information. +- `readOnlyConfig.nifiProperties.webProxyHosts` : A list of allowed HTTP Host header values to consider when NiFi is running securely and will be receiving requests to a different host[:port] than it is bound to. [web-properties](https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#web-properties) + +If `listenersConfig.sslSecrets.create` is set to `false`, the operator will look for the secret at `listenersConfig.sslSecrets.tlsSecretName` and expect these values : + +| key | value | +|-----|-------| +| caCert | The CA certificate | +| caKey | The CA private key | +| clientCert | A client certificate (this will be used by operator for NiFI operations) | +| clientKey | The private key for clientCert | + +## Using an existing Issuer + +As described in the [Reference section](../../5_references/1_nifi_cluster/6_listeners_config.md#sslsecrets), instead of using a self-signed certificate as CA, you can use an existing one. +In order to do so, you only have to refer it into your `Spec.ListenerConfig.SslSecrets.IssuerRef` field. + +### Example : Let's encrypt + +Let's say you have an existing DNS server, with [external dns](https://github.com/kubernetes-sigs/external-dns) deployed into your cluster's namespace. +You can easily use Let's encrypt as authority for your certificate. + +To do this, you have to : + +1. Create an issuer : + +```yaml +apiVersion: cert-manager.io/v1alpha2 +kind: Issuer +metadata: + name: letsencrypt-staging +spec: + acme: + # You must replace this email address with your own. + # Let's Encrypt will use this to contact you about expiring + # certificates, and issues related to your account. + email: + server: https://acme-staging-v02.api.letsencrypt.org/directory + privateKeySecretRef: + # Secret resource used to store the account's private key. + name: example-issuer-account-key + # Add a single challenge solver, HTTP01 using nginx + solvers: + - http01: + ingress: + ingressTemplate: + metadata: + annotations: + "external-dns.alpha.kubernetes.io/ttl": "5" +``` + +2. Setup External dns and correctly create your issuer into your cluster configuration : + +```yaml +apiVersion: nifi.konpyutaika.com/v1alpha1 +kind: NifiCluster +... +spec: + ... + clusterSecure: true + siteToSiteSecure: true + ... + listenersConfig: + clusterDomain: + useExternalDNS: true + ... + sslSecrets: + tlsSecretName: "test-nifikop" + create: true + issuerRef: + name: letsencrypt-staging + kind: Issuer +``` + +## Create SSL credentials + +You may use `NifiUser` resource to create new certificates for your applications, allowing them to query your Nifi cluster. + +To create a new client you will need to generate new certificates sign by the CA. The operator can automate this for you using the `NifiUser` CRD : + +```console +cat << EOF | kubectl apply -n nifi -f - +apiVersion: nifi.konpyutaika.com/v1alpha1 +kind: NifiUser +metadata: + name: example-client + namespace: nifi +spec: + clusterRef: + name: nifi + secretName: example-client-secret +EOF +``` + +This will create a user and store its credentials in the secret `example-client-secret`. The secret contains these fields : + +| key | value | +|-----|-------| +| ca.crt | The CA certificate | +| tls.crt | The user certificate | +| tls.key | The user private key | + +You can then mount these secret to your pod. Alternatively, you can write them to your local machine by running: + +```console +kubectl get secret example-client-secret -o jsonpath="{['data']['ca\.crt']}" | base64 -d > ca.crt +kubectl get secret example-client-secret -o jsonpath="{['data']['tls\.crt']}" | base64 -d > tls.crt +kubectl get secret example-client-secret -o jsonpath="{['data']['tls\.key']}" | base64 -d > tls.key +``` + +The operator can also include a Java keystore format (JKS) with your user secret if you'd like. Add `includeJKS`: `true` to the `spec` like shown above, and then the user-secret will gain these additional fields : + +| key | value | +|-----|-------| +| tls.jks | The java keystore containing both the user keys and the CA (use this for your keystore AND truststore) | +| pass.txt | The password to decrypt the JKS (this will be randomly generated) | \ No newline at end of file diff --git a/site/website/versioned_docs/version-v0.14.1/3_tasks/2_security/2_authentication/1_oidc.md b/site/website/versioned_docs/version-v0.14.1/3_tasks/2_security/2_authentication/1_oidc.md new file mode 100644 index 0000000000..06cf173abe --- /dev/null +++ b/site/website/versioned_docs/version-v0.14.1/3_tasks/2_security/2_authentication/1_oidc.md @@ -0,0 +1,42 @@ +--- +id: 1_oidc +title: OpenId Connect +sidebar_label: OpenId Connect +--- + +To enable authentication via OpenId Connect refering to [NiFi Administration guide](https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html) required some configuration into `nifi.properties`. + +In addition and to ensure multiple identity provider support, we recommended to add the following configuration to your `nifi.properties` : + +```sh +nifi.security.identity.mapping.pattern.dn=CN=([^,]*)(?:, (?:O|OU)=.*)? +nifi.security.identity.mapping.value.dn=$1 +nifi.security.identity.mapping.transform.dn=NONE +``` + +To perform this with `NiFiKop` you just have to configure the `Spec.NifiProperties.OverrideConfigs` field with your OIDC configuration, for example : + +```yaml +apiVersion: nifi.konpyutaika.com/v1alpha1 +kind: NifiCluster +... +spec: + ... + readOnlyConfig: + # NifiProperties configuration that will be applied to the node. + nifiProperties: + webProxyHosts: + - nifistandard2.trycatchlearn.fr:8443 + # Additionnals nifi.properties configuration that will override the one produced based + # on template and configurations. + overrideConfigs: | + nifi.security.user.oidc.discovery.url= + nifi.security.user.oidc.client.id= + nifi.security.user.oidc.client.secret= + nifi.security.identity.mapping.pattern.dn=CN=([^,]*)(?:, (?:O|OU)=.*)? + nifi.security.identity.mapping.value.dn=$1 + nifi.security.identity.mapping.transform.dn=NONE + ... + ... +... +``` \ No newline at end of file diff --git a/site/website/versioned_docs/version-v0.14.1/3_tasks/2_security/2_authorization/1_custom_authorizer.md b/site/website/versioned_docs/version-v0.14.1/3_tasks/2_security/2_authorization/1_custom_authorizer.md new file mode 100644 index 0000000000..515a8dd806 --- /dev/null +++ b/site/website/versioned_docs/version-v0.14.1/3_tasks/2_security/2_authorization/1_custom_authorizer.md @@ -0,0 +1,83 @@ +--- +id: 1_authorizer +title: Custom User Authorizers +sidebar_label: Custom Authorizers +--- + +:::info +This is an advanced configuration topic. In most cases, the default NiFi authorizer configuration is sufficient. +::: + +According to the NiFi Admin Guide, an [Authorizer](https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#authorizer-configuration) grants users the privileges to manage users and policies by creating preliminary authorizations at startup. By default, the `StandardManagedAuthorizer` leverages a `FileUserGroupProvider` and a `FileAccessPolicyProvider` which are file-based rules for each user you allow to interact with your NiFi cluster. + +In many cases, the default authorizer configuration is enough to control access to a NiFi cluster. However, there may be advanced cases where the default [`managed-authorizer`](https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#standardmanagedauthorizer) isn't sufficient to make every authorization decision you need. In this case, you can provide a custom authorizer extension and use that instead. + +Suppose a custom Authorizer is written and deployed with NiFi that reads the rules from a remote database rather than a local file. We'll call this `DatabaseAuthorizer`. Also suppose it is composed of a `DatabaseUserGroupProvider` and a `DatabaseAccessPolicyProvider`. In order to leverage these, they must end up on NiFi's classpath. + +In order to use this authorizer, you need to update NiFi's `authorizers.xml` configuration. This can be done through NiFiKOp by setting either the `Spec.readOnlyConfig.authorizerConfig.replaceTemplateConfigMap` or `Spec.readOnlyConfig.authorizerConfig.replaceTemplateSecretConfig`. The NiFiKOp deployment is dynamic in that node identities are determined at deploy time, so the authorizer configuration is templated to account for this. This means that the replacement ConfigMap or Secret must also be templated. + +Following the example, the below would be a sufficient authorizer template replacement: + +```yaml +{{- $nodeList := .NodeList }} +{{- $clusterName := .ClusterName }} +{{- $namespace := .Namespace }} + + + file-user-group-provider + org.apache.nifi.authorization.FileUserGroupProvider + ../data/users.xml + + {{ .ControllerUser }} +{{- range $i, $host := .NodeList }} + {{ $host }} +{{- end }} + + + database-user-group-provider + my.custom.DatabaseUserGroupProvider + +{{- range $i, $host := .NodeList }} + {{ $host }} +{{- end }} + + + file-access-policy-provider + org.apache.nifi.authorization.FileAccessPolicyProvider + file-user-group-provider + ../data/authorizations.xml + {{ .ControllerUser }} + +{{- range $i, $host := .NodeList }} + {{ $host }} +{{- end }} + + + + database-access-policy-provider + my.custom.DatabaseAccessPolicyProvider + +{{- range $i, $host := .NodeList }} + {{ $host }} +{{- end }} + + + + managed-authorizer + org.apache.nifi.authorization.StandardManagedAuthorizer + file-access-policy-provider + + + custom-database-authorizer + my.custom.DatabaseAuthorizer + database-access-policy-provider + + +``` + +And finally, the NiFi property `nifi.security.user.authorizer` indicates which of the configured authorizers in the authorizers.xml file to use. Following the example, we'd set the property to: + +```sh +nifi.security.user.authorizer=custom-database-authorizer +``` + diff --git a/site/website/versioned_docs/version-v0.14.1/3_tasks/3_nifi_dataflow.md b/site/website/versioned_docs/version-v0.14.1/3_tasks/3_nifi_dataflow.md new file mode 100644 index 0000000000..e944fd01aa --- /dev/null +++ b/site/website/versioned_docs/version-v0.14.1/3_tasks/3_nifi_dataflow.md @@ -0,0 +1,126 @@ +--- +id: 3_nifi_dataflow +title: Provisioning NiFi Dataflows +sidebar_label: NiFi Dataflows +--- + +You can create NiFi dataflows either : + +* directly against the cluster through its REST API (using UI or some home made scripts), or +* via the `NifiDataflow` CRD. + +If you want more details about the design, just have a look on the [design page](../1_concepts/2_design_principes.md#dataflow-lifecycle-management) + +To deploy a [NifiDataflow] you have to start by deploying a [NifiRegistryClient] because **NiFiKop** manages dataflow using the [NiFi Registry feature](https://nifi.apache.org/registry). + +Below is an example of [NifiRegistryClient] : + +```yaml +apiVersion: nifi.konpyutaika.com/v1alpha1 +kind: NifiRegistryClient +metadata: + name: registry-client-example + namespace: nifikop +spec: + clusterRef: + name: nc + namespace: nifikop + description: "Registry client managed by NiFiKop" + uri: "http://nifi.hostname.com:18080" +``` + +Once you have deployed your [NifiRegistryClient], you have the possibility of defining a configuration that you will apply to your [NifiDataflow]. + +This configuration is defined using the [NifiParameterContext] CRD, which NiFiKop will convert into a [Parameter context](https://nifi.apache.org/docs/nifi-docs/html/user-guide.html#parameter-contexts). + + +Below is an example of [NifiParameterContext] : + +```yaml +apiVersion: nifi.konpyutaika.com/v1alpha1 +kind: NifiParameterContext +metadata: + name: dataflow-lifecycle + namespace: demo +spec: + description: "It is a test" + clusterRef: + name: nc + namespace: nifikop + secretRefs: + - name: secret-params + namespace: nifikop + parameters: + - name: test + value: toto + description: tutu + - name: test2 + value: toto + description: toto +``` + +As you can see, in the [NifiParameterContext] you can refer to some secrets that will be converted into [sensitive parameter](https://nifi.apache.org/docs/nifi-docs/html/user-guide.html#using-parameters-with-sensitive-properties). + +Here is an example of secret that you can create that will be used by the configuration above : + +```console +kubectl create secret generic secret-params \ + --from-literal=secret1=yop \ + --from-literal=secret2=yep \ + -n nifikop +``` + +:::warning +As a sensitive value cannot be retrieved through the Rest API, to update the value of a sensitive parameter, you have to : + +- remove it from the secret +- wait for the next loop +- insert the parameter with the new value inside the secret + +or you can simply create a new [NifiParameterContext] and refer it into your [NifiDataflow]. +::: + +You can now deploy your [NifiDataflow] by referencing the previous objects : + +```yaml +apiVersion: nifi.konpyutaika.com/v1alpha1 +kind: NifiDataflow +metadata: + name: dataflow-lifecycle +spec: + parentProcessGroupID: "16cfd2ec-0174-1000-0000-00004b9b35cc" + bucketId: "01ced6cc-0378-4893-9403-f6c70d080d4f" + flowId: "9b2fb465-fb45-49e7-94fe-45b16b642ac9" + flowVersion: 2 + syncMode: always + skipInvalidControllerService: true + skipInvalidComponent: true + clusterRef: + name: nc + namespace: nifikop + registryClientRef: + name: registry-client-example + namespace: nifikop + parameterContextRef: + name: dataflow-lifecycle + namespace: demo + updateStrategy: drain +``` + +To find details about the versioned flow information required check the [official documentation](https://nifi.apache.org/docs/nifi-registry-docs/index.html) + +You have two modes of control from your dataflow by the operator : + +1 - `Spec.SyncMode == never` : The operator will deploy the dataflow as described in the resource, and never control it (unless you change the field to `always`). It is useful when you want to deploy your dataflow without starting it. + +2 - `Spec.SyncMode == once` : The operator will deploy the dataflow as described in the resource, run it once, and never control it again (unless you change the field to `always`). It is useful when you want to deploy your dataflow in a dev environment, and you want to update the dataflow. + +3 - `Spec.SyncMode == always` : The operator will deploy and ensure the dataflow lifecycle, it will avoid all manual modification directly from the Cluster (e.g remove the process group, remove the versioning, update the parent process group, make some local changes ...). If you want to perform update, rollback or stuff like this, you have to simply update the [NifiDataflow] resource. + +:::important +More information about `Spec.UpdateStrategy` [here](../5_references/5_nifi_dataflow.md#dataflowupdatestrategy) +::: + +[NifiDataflow]: ../5_references/5_nifi_dataflow.md +[NifiRegistryClient]: ../5_references/3_nifi_registry_client.md +[NifiParameterContext]: ../5_references/4_nifi_parameter_context.md \ No newline at end of file diff --git a/site/website/versioned_docs/version-v0.14.1/3_tasks/4_nifi_user_group.md b/site/website/versioned_docs/version-v0.14.1/3_tasks/4_nifi_user_group.md new file mode 100644 index 0000000000..f89c2102a2 --- /dev/null +++ b/site/website/versioned_docs/version-v0.14.1/3_tasks/4_nifi_user_group.md @@ -0,0 +1,168 @@ +--- +id: 4_nifi_user_group +title: Provisioning NiFi Users and Groups +sidebar_label: NiFi Users and Groups +--- + +## User management + +The `NifiUser` resource was already introduced for the [SSL credentials](./2_security/1_ssl.md#create-ssl-credentials) concerns. +What we are covering here is the NiFi user management part introduced in this resource. + +When you create a `NifiUser` resource the operator will : + +1. Try to check if a user already exists with the same name on the NiFi cluster, if it does, the operator will set [NifiUser.Status.Id](./2_security/1_ssl.md#create-ssl-credentials) to bind it with the kubernetes resource. +2. If no user is found, the operator will create and manage it (i.e it will ensure the synchronisation with the NiFi Cluster). + +```yaml +apiVersion: nifi.konpyutaika.com/v1alpha1 +kind: NifiUser +metadata: + name: aguitton +spec: + # identity field is use to define the user identity on NiFi cluster side, + # it use full when the user's name doesn't suite with Kubernetes resource name. + identity: alexandre.guitton@orange.com + # Contains the reference to the NifiCluster with the one the registry client is linked. + clusterRef: + name: nc + namespace: nifikop + # Whether or not the the operator also include a Java keystore format (JKS) with you secret + includeJKS: false + # Whether or not a certificate will be created for this user. + createCert: false + # defines the list of access policies that will be granted to the group. + accessPolicies: + # defines the kind of access policy, could be "global" or "component". + - type: component + # defines the kind of action that will be granted, could be "read" or "write" + action: read + # resource defines the kind of resource targeted by this access policies, please refer to the following page : + # https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#access-policies + resource: /data + # componentType is used if the type is "component", it's allow to define the kind of component on which is the + # access policy + componentType: "process-groups" + # componentId is used if the type is "component", it's allow to define the id of the component on which is the + # access policy + componentId: "" +``` + +By default the user name that will be used is the name of the resource. + +But as there are some constraints on this name (e.g [RFC 1123](https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-subdomain-names)) that doesn't match with those applied on NiFi, you can override it with the `NifiUser.Spec.Identity` field which is more permissive. +In the example above the kubernetes resource name will be `aguitton` but the NiFi use created on the cluster will be `alexandre.guitton@orange.com`. + +In the case the user will not authenticate himself using TLS authentication, the operator doesn't have to create a certificate, so just set `NifiUser.Spec.CreateCert` to false. + +For each user, you have the ability to define a list of [AccessPolicies](../5_references/2_nifi_user.md#accesspolicy) to give a list of access to your user. +In the example above we are giving to user `alexandre.guitton@orange.com` the right to view metadata et content for the root process group in flowfile queues in outbound connections and through provenance events. + +## UserGroup management + +To simplify the access management Apache NiFi allows to define groups containing a list of users, on which we apply a list of access policies. +This part is supported by the operator using the `NifiUserGroup` resource : + + +```yaml +apiVersion: nifi.konpyutaika.com/v1alpha1 +kind: NifiUserGroup +metadata: + name: group-test +spec: + # Contains the reference to the NifiCluster with the one the registry client is linked. + clusterRef: + name: nc + namespace: nifikop + # contains the list of reference to NifiUsers that are part to the group. + usersRef: + - name: nc-0-node.nc-headless.nifikop.svc.cluster.local +# namespace: nifikop + - name: nc-controller.nifikop.mgt.cluster.local + # defines the list of access policies that will be granted to the group. + accessPolicies: + # defines the kind of access policy, could be "global" or "component". + - type: global + # defines the kind of action that will be granted, could be "read" or "write" + action: read + # resource defines the kind of resource targeted by this access policies, please refer to the following page : + # https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#access-policies + resource: /counters +# # componentType is used if the type is "component", it's allow to define the kind of component on which is the +# # access policy +# componentType: "process-groups" +# # componentId is used if the type is "component", it's allow to define the id of the component on which is the +# # access policy +# componentId: "" +``` + +When you create a `NifiUserGroup` resource, the operator will create and manage a group named `${resource namespace}-${resource name}` in Nifi. +To declare the users that are part of this group, you just have to declare them in the [NifiUserGroup.UsersRef](../5_references/6_nifi_usergroup.md#userreference) field. + +:::important +The [NifiUserGroup.UsersRef](../5_references/6_nifi_usergroup.md#userreference) requires to declare the name and namespace of a `NifiUser` resource, so it is previously required to declare the resource. + +It's required to create the resource even if the user is already declared in NiFi Cluster (In that case the operator will just sync the kubernetes resource). +::: + +Like for `NifiUser` you can declare a list of [AccessPolicies](../5_references/2_nifi_user.md#accesspolicy) to give a list of access to your user. + +In the example above we are giving to users `nc-0-node.nc-headless.nifikop.svc.cluster.local` and `nc-controller.nifikop.mgt.cluster.local` the right to view the counters informations. + +## Managed groups for simple setup + +In some case these two features could be heavy to define, for example when you have 10 dataflows with one cluster for each of them, it will lead in a lot of `.yaml` files ... +To simplify this, we implement in the operator 2 `managed groups` : + +- **Admins :** a group giving access to everything on the NiFi Cluster, +- **Readers :** a group giving access as viewer on the NiFi Cluster. + +You can directly define the list of users who belong to each of them in the `NifiCluster.Spec` field : + +```yaml +apiVersion: nifi.konpyutaika.com/v1alpha1 +kind: NifiCluster +metadata: + name: mynifi +spec: + ... + oneNifiNodePerNode: false + # + propagateLabels: true + managedAdminUsers: + - identity : "alexandre.guitton@orange.com" + name: "aguitton" + - identity : "nifiuser@orange.com" + name: "nifiuser" + managedReaderUsers: + - identity : "toto@orange.com" + name: "toto" + ... +``` + +In this example the operator will create and manage 3 `NifiUsers` : + +- **aguitton**, with the identity : `alexandre.guitton@orange.com` +- **nifiuser**, with the identity : `nifiuser@orange.com` +- **toto**, with the identity : `toto@orange.com` + +And create and manage two groups : + +- **managed-admins :** that will contain 3 users (**aguitton**, **nifiuser**, **nc-controller.nifikop.mgt.cluster.local** which is the controller user). +- **managed-readers :** that will contain 1 user (**toto**) + +And the rest of the stuff will be reconciled and managed as described for `NifiUsers` and `NifiUserGroups`. + +:::note +There is one more group that is created and managed by the operator, this is the **managed-nodes** group, for each node a `NifiUser` is created, and we automatically add them to this group to give them the right list of accesses. + +To get the list of managed groups just check the list of `NifiUserGroup` : + +```console +kubectl get -n nifikop nifiusergroups.nifi.konpyutaika.com +NAME AGE +managed-admins 6d7h +managed-nodes 6d7h +managed-readers 6d7h +``` +::: \ No newline at end of file diff --git a/site/website/versioned_docs/version-v0.14.1/4_examples/1_simple_nifi_cluster.md b/site/website/versioned_docs/version-v0.14.1/4_examples/1_simple_nifi_cluster.md new file mode 100644 index 0000000000..224511cdbe --- /dev/null +++ b/site/website/versioned_docs/version-v0.14.1/4_examples/1_simple_nifi_cluster.md @@ -0,0 +1,5 @@ +--- +id: 1_simple_nifi_cluster +title: Simple NiFi cluster +sidebar_label: Simple NiFi cluster +--- \ No newline at end of file diff --git a/site/website/versioned_docs/version-v0.14.1/5_references/1_nifi_cluster/1_nifi_cluster.md b/site/website/versioned_docs/version-v0.14.1/5_references/1_nifi_cluster/1_nifi_cluster.md new file mode 100644 index 0000000000..4977ceeff6 --- /dev/null +++ b/site/website/versioned_docs/version-v0.14.1/5_references/1_nifi_cluster/1_nifi_cluster.md @@ -0,0 +1,202 @@ +--- +id: 1_nifi_cluster +title: NiFi cluster +sidebar_label: NiFi cluster +--- + +`NifiCluster` describes the desired state of the NiFi cluster we want to setup through the operator. + +```yaml +apiVersion: nifi.konpyutaika.com/v1alpha1 +kind: NifiCluster +metadata: + name: simplenifi +spec: + service: + headlessEnabled: true + annotations: + tyty: ytyt + labels: + tete: titi + pod: + annotations: + toto: tata + labels: + titi: tutu + zkAddress: 'zookeepercluster-client.zookeeper:2181' + zkPath: '/simplenifi' + clusterImage: 'apache/nifi:1.11.3' + oneNifiNodePerNode: false + nodeConfigGroups: + default_group: + isNode: true + podMetadata: + annotations: + node-annotation: "node-annotation-value" + labels: + node-label: "node-label-value" + externalVolumeConfigs: + - name: example-volume + mountPath: "/opt/nifi/example" + secret: + secretName: "raw-controller" + storageConfigs: + - mountPath: '/opt/nifi/nifi-current/logs' + name: logs + pvcSpec: + accessModes: + - ReadWriteOnce + storageClassName: 'standard' + resources: + requests: + storage: 10Gi + serviceAccountName: 'default' + resourcesRequirements: + limits: + cpu: '2' + memory: 3Gi + requests: + cpu: '1' + memory: 1Gi + nodes: + - id: 1 + nodeConfigGroup: 'default_group' + - id: 2 + nodeConfigGroup: 'default_group' + propagateLabels: true + nifiClusterTaskSpec: + retryDurationMinutes: 10 + listenersConfig: + internalListeners: + - type: 'http' + name: 'http' + containerPort: 8080 + - type: 'cluster' + name: 'cluster' + containerPort: 6007 + - type: 's2s' + name: 's2s' + containerPort: 10000 + externalServices: + - name: 'clusterip' + spec: + type: ClusterIP + portConfigs: + - port: 8080 + internalListenerName: 'http' + metadata: + annotations: + toto: tata + labels: + titi: tutu +``` + +## NifiCluster + +| Field | Type | Description | Required | Default | +| -------- | ----------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------- | -------- | ------- | +| metadata | [ObjectMetadata](https://godoc.org/k8s.io/apimachinery/pkg/apis/meta/v1#ObjectMeta) | is metadata that all persisted resources must have, which includes all objects users must create. | No | nil | +| spec | [NifiClusterSpec](#nificlusterspec) | defines the desired state of NifiCluster. | No | nil | +| status | [NifiClusterStatus](#nificlusterstatus) | defines the observed state of NifiCluster. | No | nil | + +## NifiClusterSpec + +| Field | Type | Description | Required | Default | +| ------------------ |----------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| ---------------- |--------------------------| +| clientType | Enum={"tls","basic"} | defines if the operator will use basic or tls authentication to query the NiFi cluster. | No | `tls` | +| type | Enum={"external","internal"} | defines if the cluster is internal (i.e manager by the operator) or external. | No | `internal` | +| nodeURITemplate | string | used to dynamically compute node uri. | if external type | - | +| nifiURI | stringused access through a LB uri. | if external type | - | +| rootProcessGroupId | string | contains the uuid of the root process group for this cluster. | if external type | - | +| secretRef | \[ \][SecretReference](../4_nifi_parameter_context#secretreference) | reference the secret containing the informations required to authentiticate to the cluster. | if external type | - | +| proxyUrl | string | defines the proxy required to query the NiFi cluster. | if external type | - | +|service| [ServicePolicy](#servicepolicy) | defines the policy for services owned by NiFiKop operator. |No| - | +|pod| [PodPolicy](#podpolicy) | defines the policy for pod owned by NiFiKop operator. |No| - | +|zkAddress| string | specifies the ZooKeeper connection string in the form hostname:port where host and port are those of a Zookeeper server. |No| "" | +|zkPath| string | specifies the Zookeeper chroot path as part of its Zookeeper connection string which puts its data under same path in the global ZooKeeper namespace. |Yes| "/" | +|initContainerImage| string | can override the default image used into the init container to check if ZoooKeeper server is reachable.. |Yes| "busybox" | +|initContainers| \[ \]string | defines additional initContainers configurations. |No| \[ \] | +|clusterImage| string | can specify the whole nificluster image in one place. |No| "" | +|oneNifiNodePerNode| boolean | if set to true every nifi node is started on a new node, if there is not enough node to do that it will stay in pending state. If set to false the operator also tries to schedule the nifi node to a unique node but if the node number is insufficient the nifi node will be scheduled to a node where a nifi node is already running. |No| nil | +|propagateLabels| boolean | - |Yes| false | +|managedAdminUsers| \[ \][ManagedUser](#managedusers) | contains the list of users that will be added to the managed admin group (with all rights). |No| [] | +|managedReaderUsers| \[ \][ManagedUser](#managedusers) | contains the list of users that will be added to the managed admin group (with all rights). |No| [] | +|readOnlyConfig| [ReadOnlyConfig](./2_read_only_config.md) | specifies the read-only type Nifi config cluster wide, all theses will be merged with node specified readOnly configurations, so it can be overwritten per node. |No| nil | +|nodeUserIdentityTemplate| string | specifies the template to be used when naming the node user identity (e.g. node-%d-mysuffix) |Yes| "node-%d-\" | +|nodeConfigGroups| map\[string\][NodeConfig](./3_node_config.md) | specifies multiple node configs with unique name |No| nil | +|nodes| \[ \][Node](./3_node_config.md) | specifies the list of cluster nodes, all node requires an image, unique id, and storageConfigs settings |Yes| nil +|disruptionBudget| [DisruptionBudget](#disruptionbudget) | defines the configuration for PodDisruptionBudget. |No| nil | +|ldapConfiguration| [LdapConfiguration](#ldapconfiguration) | specifies the configuration if you want to use LDAP. |No| nil | +|nifiClusterTaskSpec| [NifiClusterTaskSpec](#nificlustertaskspec) | specifies the configuration of the nifi cluster Tasks. |No| nil | +|listenersConfig| [ListenersConfig](./6_listeners_config.md) | specifies nifi's listener specifig configs. |No| - | +|sidecarConfigs| \[ \][Container](https://godoc.org/k8s.io/api/core/v1#Container) | Defines additional sidecar configurations. [Check documentation for more informations] | +|externalServices| \[ \][ExternalServiceConfigs](./7_external_service_config.md) | specifies settings required to access nifi externally. |No| - | +|topologySpreadConstraints| \[ \][TopologySpreadConstraint](https://godoc.org/k8s.io/api/core/v1#TopologySpreadConstraint) | specifies any TopologySpreadConstraint objects to be applied to all nodes. |No| nil | +|nifiControllerTemplate| string | NifiControllerTemplate specifies the template to be used when naming the node controller (e.g. %s-mysuffix) **Warning: once defined don't change this value either the operator will no longer be able to manage the cluster** |Yes| "%s-controller" | +|controllerUserIdentity| string | ControllerUserIdentity specifies what to call the static admin user's identity **Warning: once defined don't change this value either the operator will no longer be able to manage the cluster** |Yes| false | + + +## NifiClusterStatus + +| Field | Type | Description | Required | Default | +| ------------------ | ------------------------------------------- | ------------------------------------------------------------- | -------- | ------- | +| nodesState | map\[string\][NodeState](./5_node_state.md) | Store the state of each nifi node. | No | - | +| State | [ClusterState](#clusterstate) | Store the state of each nifi node. | Yes | - | +| rootProcessGroupId | string | contains the uuid of the root process group for this cluster. | No | - | + +## ServicePolicy + +| Field | Type | Description | Required | Default | +| --------------- |---------------------| --------------------------------------------------------------------------------------------------------------------------------------------------- |----------|-----------------------------------------------------------| +| headlessEnabled | boolean | specifies if the cluster should use headlessService for Nifi or individual services using service per nodes may come an handy case of service mesh. | Yes | false | +| serviceTemplate | string | specifies the template to be used when naming the service. | Yes | If headlessEnabled = true ? "%s-headless" = "%s-all-node" | +| annotations | map\[string\]string | Annotations specifies the annotations to attach to services the NiFiKop operator creates | No | - | +| labels | map\[string\]string | Labels specifies the labels to attach to services the NiFiKop operator creates | No | - | + + +## PodPolicy + +| Field | Type | Description | Required | Default | +| ----------- | ------------------- | ------------------------------------------------------------------------------------ | -------- | ------- | +| annotations | map\[string\]string | Annotations specifies the annotations to attach to pods the NiFiKop operator creates | No | - | +| labels | map\[string\]string | Labels specifies the Labels to attach to pods the NiFiKop operator creates | No | - | +| hostAliases | \[\][HostAlias](https://pkg.go.dev/k8s.io/api/core/v1#HostAlias) | A list of host aliases to include in every pod's /etc/hosts configuration in the scenario where DNS is not available. | No | \[\] | + +## ManagedUsers + +| Field | Type | Description | Required | Default | +| ------ | ------ | ----------------------------------------------------------------------------------------------------------------------------------- | -------- | ------- | +| identity | string | identity field is use to define the user identity on NiFi cluster side, it use full when the user's name doesn't suite with Kubernetes resource name. | No | - | +| name | string | name field is use to name the NifiUser resource, if not identity is provided it will be used to name the user on NiFi cluster side. | Yes | - | + +## DisruptionBudget + +| Field | Type | Description | Required | Default | +| -------- | ------ | ----------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------- | +| create | bool | if set to true, will create a podDisruptionBudget. | No | - | +| budget | string | the budget to set for the PDB, can either be static number or a percentage. | Yes | - | + +## LdapConfiguration + +| Field | Type | Description | Required | Default | +| ------------ | ------- | ----------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------- | +| enabled | boolean | if set to true, we will enable ldap usage into nifi.properties configuration. | No | false | +| url | string | space-separated list of URLs of the LDAP servers (i.e. ldap://${hostname}:${port}). | No | "" | +| searchBase | string | base DN for searching for users (i.e. CN=Users,DC=example,DC=com). | No | "" | +| searchFilter | string | Filter for searching for users against the 'User Search Base'. (i.e. sAMAccountName={0}). The user specified name is inserted into '{0}'. | No | "" | + +## NifiClusterTaskSpec + +| Field | Type | Description | Required | Default | +| -------------------- | ---- | ------------------------------------------------------------- | -------- | ------- | +| retryDurationMinutes | int | describes the amount of time the Operator waits for the task. | Yes | 5 | + +## ClusterState + +| Name | Value | Description | +| --------------------------- | ----------------------- | ------------------------------------------------------ | +| NifiClusterInitializing | ClusterInitializing | states that the cluster is still in initializing stage | +| NifiClusterInitialized | ClusterInitialized | states that the cluster is initialized | +| NifiClusterReconciling | ClusterReconciling | states that the cluster is still in reconciling stage | +| NifiClusterRollingUpgrading | ClusterRollingUpgrading | states that the cluster is rolling upgrading | +| NifiClusterRunning | ClusterRunning | states that the cluster is in running state | diff --git a/site/website/versioned_docs/version-v0.14.1/5_references/1_nifi_cluster/2_read_only_config.md b/site/website/versioned_docs/version-v0.14.1/5_references/1_nifi_cluster/2_read_only_config.md new file mode 100644 index 0000000000..883d420355 --- /dev/null +++ b/site/website/versioned_docs/version-v0.14.1/5_references/1_nifi_cluster/2_read_only_config.md @@ -0,0 +1,222 @@ +--- +id: 2_read_only_config +title: Read only configurations +sidebar_label: Read only configurations +--- + +ReadOnlyConfig object specifies the read-only type Nifi config cluster wide, all theses will be merged with node specified readOnly configurations, so it can be overwritten per node. + +```yaml +readOnlyConfig: + # MaximumTimerDrivenThreadCount define the maximum number of threads for timer driven processors available to the system. + maximumTimerDrivenThreadCount: 30 + # MaximumEventDrivenThreadCount define the maximum number of threads for event driven processors available to the system. + maximumEventDrivenThreadCount: 10 + # Logback configuration that will be applied to the node + logbackConfig: + # logback.xml configuration that will replace the one produced based on template + replaceConfigMap: + # The key of the value,in data content, that we want use. + data: logback.xml + # Name of the configmap that we want to refer. + name: raw + # Namespace where is located the secret that we want to refer. + namespace: nifikop + # logback.xml configuration that will replace the one produced based on template and overrideConfigMap + replaceSecretConfig: + # The key of the value,in data content, that we want use. + data: logback.xml + # Name of the configmap that we want to refer. + name: raw + # Namespace where is located the secret that we want to refer. + namespace: nifikop + # Authorizer configuration that will be applied to the node + authorizerConfig: + # An authorizers.xml configuration template that will replace the default template seen in authorizers.go + replaceTemplateConfigMap: + # The key of the value, in data content, that we want use. + data: authorizers.xml + # Name of the configmap that we want to refer. + name: raw + # Namespace where is located the secret that we want to refer. + namespace: nifikop + # An authorizers.xml configuration template that will replace the default template seen in authorizers.go and the replaceTemplateConfigMap + replaceTemplateSecretConfig: + # The key of the value,in data content, that we want use. + data: authorizers.xml + # Name of the configmap that we want to refer. + name: raw + # Namespace where is located the secret that we want to refer. + namespace: nifikop + # NifiProperties configuration that will be applied to the node. + nifiProperties: + # Additionnals nifi.properties configuration that will override the one produced based on template and + # configuration + overrideConfigMap: + # The key of the value,in data content, that we want use. + data: nifi.properties + # Name of the configmap that we want to refer. + name: raw + # Namespace where is located the secret that we want to refer. + namespace: nifikop. + # Additionnals nifi.properties configuration that will override the one produced based + # on template, configurations, overrideConfigMap and overrideConfigs. + overrideSecretConfig: + # The key of the value,in data content, that we want use. + data: nifi.properties + # Name of the configmap that we want to refer. + name: raw + # Namespace where is located the secret that we want to refer. + namespace: nifikop + # Additionnals nifi.properties configuration that will override the one produced based + # on template, configurations and overrideConfigMap + overrideConfigs: | + nifi.ui.banner.text=NiFiKop + # A comma separated list of allowed HTTP Host header values to consider when NiFi + # is running securely and will be receiving requests to a different host[:port] than it is bound to. + # https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#web-properties + # webProxyHosts: + # Nifi security client auth + needClientAuth: false + # Indicates which of the configured authorizers in the authorizers.xml file to use + # https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#authorizer-configuration + # authorizer: + # ZookeeperProperties configuration that will be applied to the node. + zookeeperProperties: + # # Additionnals zookeeeper.properties configuration that will override the one produced based on template and + # # configuration + # overrideConfigMap: + # # The key of the value,in data content, that we want use. + # data: zookeeeper.properties + # # Name of the configmap that we want to refer. + # name: raw + # # Namespace where is located the secret that we want to refer. + # namespace: nifikop. + # # Additionnals zookeeeper.properties configuration that will override the one produced based + # # on template, configurations, overrideConfigMap and overrideConfigs. + # overrideSecretConfig: + # # The key of the value,in data content, that we want use. + # data: zookeeeper.properties + # # Name of the configmap that we want to refer. + # name: raw + # # Namespace where is located the secret that we want to refer. + # namespace: nifikop + # Additionnals zookeeper.properties configuration that will override the one produced based + # on template and configurations. + overrideConfigs: | + initLimit=15 + autopurge.purgeInterval=24 + syncLimit=5 + tickTime=2000 + dataDir=./state/zookeeper + autopurge.snapRetainCount=30 + # BootstrapProperties configuration that will be applied to the node. + bootstrapProperties: + # # Additionnals bootstrap.properties configuration that will override the one produced based on template and + # # configuration + # overrideConfigMap: + # # The key of the value,in data content, that we want use. + # data: bootstrap.properties + # # Name of the configmap that we want to refer. + # name: raw + # # Namespace where is located the secret that we want to refer. + # namespace: nifikop. + # # Additionnals bootstrap.properties configuration that will override the one produced based + # # on template, configurations, overrideConfigMap and overrideConfigs. + # overrideSecretConfig: + # # The key of the value,in data content, that we want use. + # data: bootstrap.properties + # # Name of the configmap that we want to refer. + # name: raw + # # Namespace where is located the secret that we want to refer. + # namespace: nifikop + # JVM memory settings + nifiJvmMemory: "512m" + # Additionnals bootstrap.properties configuration that will override the one produced based + # on template and configurations. + # https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#bootstrap_properties + overrideConfigs: | + # java.arg.4=-Djava.net.preferIPv4Stack=true +``` + +## ReadOnlyConfig + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|maximumTimerDrivenThreadCount|int32|define the maximum number of threads for timer driven processors available to the system.|No|10| +|maximumEventDrivenThreadCount|int32|define the maximum number of threads for event driven processors available to the system.|No|1| +|additionalSharedEnvs|\[ \][corev1.EnvVar](https://pkg.go.dev/k8s.io/api/core/v1#EnvVar)|define a set of additional env variables that will shared between all init containers and ontainers in the pod..|No|\[ \]| +|nifiProperties|[NifiProperties](#nifiproperties)|nifi.properties configuration that will be applied to the node.|No|nil| +|zookeeperProperties|[ZookeeperProperties](#zookeeperproperties)|zookeeper.properties configuration that will be applied to the node.|No|nil| +|bootstrapProperties|[BootstrapProperties](#bootstrapproperties)|bootstrap.conf configuration that will be applied to the node.|No|nil| +|logbackConfig|[LogbackConfig](#logbackconfig)|logback.xml configuration that will be applied to the node.|No|nil| +|authorizerConfig|[AuthorizerConfig](#authorizerconfig)|authorizers.xml configuration template that will be applied to the node.|No|nil| +|bootstrapNotificationServicesConfig|[BootstrapNotificationServices](#bootstrapnotificationservices)|bootstrap_notification_services.xml configuration that will be applied to the node.|No|nil| + + + +## NifiProperties + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|overrideConfigMap|[ConfigmapReference](#configmapreference)|Additionnals nifi.properties configuration that will override the one produced based on template and configuration.|No|nil| +|overrideConfigs|string|Additionnals nifi.properties configuration that will override the one produced based on template, configurations and overrideConfigMap.|No|""| +|overrideSecretConfig|[SecretConfigReference](#secretconfigreference)|Additionnals nifi.properties configuration that will override the one produced based on template, configurations, overrideConfigMap and overrideConfigs.|No|nil| +|webProxyHosts|\[ \]string| A list of allowed HTTP Host header values to consider when NiFi is running securely and will be receiving requests to a different host[:port] than it is bound to. [web-properties](https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#web-properties)|No|""| +|needClientAuth|boolean|Nifi security client auth.|No|false| +|authorizer|string|Indicates which of the configured authorizers in the authorizers.xml file to use [authorizer-configuration](https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#authorizer-configuration)|No|"managed-authorizer"| + + +## ZookeeperProperties + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|overrideConfigMap|[ConfigmapReference](#configmapreference)|Additionnals zookeeper.properties configuration that will override the one produced based on template and configuration.|No|nil| +|overrideConfigs|string|Additionnals zookeeper.properties configuration that will override the one produced based on template, configurations and overrideConfigMap.|No|""| +|overrideSecretConfig|[SecretConfigReference](#secretconfigreference)|Additionnals zookeeper.properties configuration that will override the one produced based on template, configurations, overrideConfigMap and overrideConfigs.|No|nil| + +## BootstrapProperties + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|overrideConfigMap|[ConfigmapReference](#configmapreference)|Additionnals bootstrap.properties configuration that will override the one produced based on template and configuration.|No|nil| +|overrideConfigs|string|Additionnals bootstrap.properties configuration that will override the one produced based on template, configurations and overrideConfigMap.|No|""| +|overrideSecretConfig|[SecretConfigReference](#secretconfigreference)|Additionnals bootstrap.properties configuration that will override the one produced based on template, configurations, overrideConfigMap and overrideConfigs.|No|nil| +|NifiJvmMemory|string|JVM memory settings.|No|"512m"| + +## LogbackConfig + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|replaceConfigMap|[ConfigmapReference](#configmapreference)|logback.xml configuration that will replace the one produced based on template.|No|nil| +|replaceSecretConfig|[SecretConfigReference](#secretconfigreference)|logback.xml configuration that will replace the one produced based on template and overrideConfigMap.|No|nil| + +## AuthorizerConfig + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|replaceTemplateConfigMap|[ConfigmapReference](#configmapreference)|authorizers.xml configuration template that will replace the default template.|No|nil| +|replaceTemplateSecretConfig|[SecretConfigReference](#secretconfigreference)|authorizers.xml configuration that will replace the default template and the replaceTemplateConfigMap.|No|nil| + +## BootstrapNotificationServicesConfig + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|replaceConfigMap|[ConfigmapReference](#configmapreference)|bootstrap_notifications_services.xml configuration that will replace the one produced based on template.|No|nil| +|replaceSecretConfig|[SecretConfigReference](#secretconfigreference)|bootstrap_notifications_services.xml configuration that will replace the one produced based on template and overrideConfigMap.|No|nil| + +## ConfigmapReference + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|name|string|Name of the configmap that we want to refer.|Yes|""| +|namespace|string|Namespace where is located the configmap that we want to refer.|No|""| +|data|string|The key of the value,in data content, that we want use.|Yes|""| + +## SecretConfigReference + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|name|string|Name of the secret that we want to refer.|Yes|""| +|namespace|string|Namespace where is located the secret that we want to refer.|No|""| +|data|string|The key of the value,in data content, that we want use.|Yes|""| \ No newline at end of file diff --git a/site/website/versioned_docs/version-v0.14.1/5_references/1_nifi_cluster/3_node_config.md b/site/website/versioned_docs/version-v0.14.1/5_references/1_nifi_cluster/3_node_config.md new file mode 100644 index 0000000000..9675b31f0d --- /dev/null +++ b/site/website/versioned_docs/version-v0.14.1/5_references/1_nifi_cluster/3_node_config.md @@ -0,0 +1,108 @@ +--- +id: 3_node_config +title: Node configuration +sidebar_label: Node configuration +--- + +NodeConfig defines the node configuration + +```yaml + default_group: + # provenanceStorage allow to specify the maximum amount of data provenance information to store at a time + # https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#write-ahead-provenance-repository-properties + provenanceStorage: "10 GB" + #RunAsUser define the id of the user to run in the Nifi image + # +kubebuilder:validation:Minimum=1 + runAsUser: 1000 + # Set this to true if the instance is a node in a cluster. + # https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#basic-cluster-setup + isNode: true + # Additionnal metadata to merge to the pod associated + podMetadata: + annotations: + node-annotation: "node-annotation-value" + labels: + node-label: "node-label-value" + # Docker image used by the operator to create the node associated + # https://hub.docker.com/r/apache/nifi/ +# image: "apache/nifi:1.11.2" + # nodeAffinity can be specified, operator populates this value if new pvc added later to node + # https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#node-affinity +# nodeAffinity: + # imagePullPolicy define the pull policy for NiFi cluster docker image + imagePullPolicy: IfNotPresent + # priorityClassName define the name of the priority class to be applied to these nodes + priorityClassName: "example-priority-class-name" + # externalVolumeConfigs specifies a list of volume to mount into the main container. + externalVolumeConfigs: + - name: example-volume + mountPath: "/opt/nifi/example" + secret: + secretName: "raw-controller" + # storageConfigs specifies the node related configs + storageConfigs: + # Name of the storage config, used to name PV to reuse into sidecars for example. + - name: provenance-repository + # Path where the volume will be mount into the main nifi container inside the pod. + mountPath: "/opt/nifi/provenance_repository" + # Kubernetes PVC spec + # https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim + pvcSpec: + accessModes: + - ReadWriteOnce + storageClassName: "standard" + resources: + requests: + storage: 10Gi + - mountPath: "/opt/nifi/nifi-current/logs" + name: logs + pvcSpec: + accessModes: + - ReadWriteOnce + storageClassName: "standard" + resources: + requests: + storage: 10Gi +``` + +## NodeConfig + +| Field | Type |Description|Required|Default| +|-----------------------|----------------------------------------------------------------------------------------------|-----------|--------|--------| +| provenanceStorage | string |provenanceStorage allow to specify the maximum amount of data provenance information to store at a time: [write-ahead-provenance-repository-properties](https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#write-ahead-provenance-repository-properties)|No|"8 GB"| +| runAsUser | int64 |define the id of the user to run in the Nifi image|No|1000| +| fsGroup | int64 |define the id of the group for each volumes in Nifi image|No|1000| +| isNode | boolean |Set this to true if the instance is a node in a cluster: [basic-cluster-setup](https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#basic-cluster-setup)|No|true| +| image | string | Docker image used by the operator to create the node associated. [Nifi docker registry](https://hub.docker.com/r/apache/nifi/)|No|""| +| imagePullPolicy | [PullPolicy](https://godoc.org/k8s.io/api/core/v1#PullPolicy) | define the pull policy for NiFi cluster docker image.)|No|""| +| nodeAffinity | string | operator populates this value if new pvc added later to node [node-affinity](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#node-affinity)|No|nil| +| storageConfigs | \[ \][StorageConfig](#storageconfig) |specifies the node related configs.|No|nil| +| externalVolumeConfigs | \[ \][ExternalVolumeConfig](#externalvolumeconfig) |specifies a list of volume to mount into the main container.|No|nil| +| serviceAccountName | string |specifies the serviceAccount used for this specific node.|No|"default"| +| resourcesRequirements | [ResourceRequirements](https://godoc.org/k8s.io/api/core/v1#ResourceRequirements) | works exactly like Container resources, the user can specify the limit and the requests through this property [manage-compute-resources-container](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/).|No|nil| +| imagePullSecrets | \[ \][LocalObjectReference](https://godoc.org/k8s.io/api/core/v1#TypedLocalObjectReference) |specifies the secret to use when using private registry.|No|nil| +| nodeSelector | map\[string\]string |nodeSelector can be specified, which set the pod to fit on a node [nodeselector](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector)|No|nil| +| tolerations | \[ \][Toleration](https://godoc.org/k8s.io/api/core/v1#Toleration) |tolerations can be specified, which set the pod's tolerations [taint-and-toleration](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/#concepts).|No|nil| +| podMetadata | [Metadata](#metadata) |define additionnal metadata to merge to the pod associated.|No|nil| +| hostAliases | \[\][HostAlias](https://pkg.go.dev/k8s.io/api/core/v1#HostAlias) | A list of host aliases to include in each pod's /etc/hosts configuration in the scenario where DNS is not available. | No | \[\] | +| priorityClassName | string | Specify the name of the priority class to apply to pods created with this node config | No | nil| + +## StorageConfig + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|name|string|Name of the storage config, used to name PV to reuse into sidecars for example.|Yes| - | +|mountPath|string|Path where the volume will be mount into the main nifi container inside the pod.|Yes| - | +|pvcSpec|[PersistentVolumeClaimSpec](https://godoc.org/k8s.io/api/core/v1#PersistentVolumeClaimSpec)|Kubernetes PVC spec. [create-a-persistentvolumeclaim](https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim).|Yes| - | + +## ExternalVolumeConfig + +| Field |Type| Description |Required|Default| +|-------------------------------------------------------------------|----|-------------|--------|--------| +|| [VolueMount](https://pkg.go.dev/k8s.io/api/core/v1#VolumeMount) |describes a mounting of a Volume within a container.| Yes | - | +|| [VolumeSource](https://pkg.go.dev/k8s.io/api/core/v1#VolumeSource) | VolumeSource represents the location and type of the mounted volume. | Yes | - | + +## Metadata + +| annotations | map\[string\]string | Additionnal annotation to merge to the pod associated [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set). |No|nil| +| nodeLabels | map\[string\]string | Additionnal labels to merge to the pod associated [labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set). |No|nil| diff --git a/site/website/versioned_docs/version-v0.14.1/5_references/1_nifi_cluster/4_node.md b/site/website/versioned_docs/version-v0.14.1/5_references/1_nifi_cluster/4_node.md new file mode 100644 index 0000000000..63fab4f388 --- /dev/null +++ b/site/website/versioned_docs/version-v0.14.1/5_references/1_nifi_cluster/4_node.md @@ -0,0 +1,60 @@ +--- +id: 4_node +title: Node +sidebar_label: Node +--- + +Node defines the nifi node basic configuration + +```yaml + - id: 0 + # nodeConfigGroup can be used to ease the node configuration, if set only the id is required + nodeConfigGroup: "default_group" + # readOnlyConfig can be used to pass Nifi node config + # which has type read-only these config changes will trigger rolling upgrade + readOnlyConfig: + nifiProperties: + overrideConfigs: | + nifi.ui.banner.text=NiFiKop - Node 0 + # node configuration +# nodeConfig: + - id: 2 + # readOnlyConfig can be used to pass Nifi node config + # which has type read-only these config changes will trigger rolling upgrade + readOnlyConfig: + overrideConfigs: | + nifi.ui.banner.text=NiFiKop - Node 2 + # node configuration + nodeConfig: + resourcesRequirements: + limits: + cpu: "2" + memory: 3Gi + requests: + cpu: "1" + memory: 1Gi + storageConfigs: + # Name of the storage config, used to name PV to reuse into sidecars for example. + - name: provenance-repository + # Path where the volume will be mount into the main nifi container inside the pod. + mountPath: "/opt/nifi/provenance_repository" + # Kubernetes PVC spec + # https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim + pvcSpec: + accessModes: + - ReadWriteOnce + storageClassName: "standard" + resources: + requests: + storage: 8Gi +``` + +## Node + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|id|int32| unique Node id. |Yes| - | +|nodeConfigGroup|string| can be used to ease the node configuration, if set only the id is required |No| "" | +|readOnlyConfig|[ReadOnlyConfig](./2_read_only_config.md)| readOnlyConfig can be used to pass Nifi node config which has type read-only these config changes will trigger rolling upgrade.| No | nil | +|nodeConfig|[NodeConfig](./3_node_config.md)| node configuration. |No| nil | + diff --git a/site/website/versioned_docs/version-v0.14.1/5_references/1_nifi_cluster/5_node_state.md b/site/website/versioned_docs/version-v0.14.1/5_references/1_nifi_cluster/5_node_state.md new file mode 100644 index 0000000000..9b04fc0195 --- /dev/null +++ b/site/website/versioned_docs/version-v0.14.1/5_references/1_nifi_cluster/5_node_state.md @@ -0,0 +1,73 @@ +--- +id: 5_node_state +title: Node state +sidebar_label: Node state +--- + +Holds information about nifi state + +## NodeState + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|gracefulActionState|[GracefulActionState](#gracefulactionstate)| holds info about nifi cluster action status.| - | - | +|configurationState|[ConfigurationState](#configurationstate)| holds info about the config.| - | - | +|initClusterNode|[InitClusterNode](#initclusternode)| contains if this nodes was part of the initial cluster.| - | - | +|podIsReady|bool| True if the pod for this node is up and running. Otherwise false.| - | - | +|creationTime|[v1.Time](https://pkg.go.dev/k8s.io/apimachinery/pkg/apis/meta/v1#Time)| The time at which this node was created and added to the cluster| - | - | + + +## GracefulActionState + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|errorMessage|string| holds the information what happened with Nifi Cluster. | - | "" | +|actionStep|[ActionStep](#actionstep)| holds info about the action step ran.| No | nil | +|taskStarted|string| hold the time when the execution started.| No | "" | +|actionState|[State](#state)| holds the information about Action state.| No | nil | + +## ConfigurationState + +|Name|Value|Description| +|-----|----|------------| +|ConfigInSync|ConfigInSync|states that the generated nodeConfig is in sync with the Node| +|ConfigOutOfSync|ConfigOutOfSync|states that the generated nodeConfig is out of sync with the Node| + +## InitClusterNode + +|Name|Value|Description| +|-----|----|------------| +|IsInitClusterNode|true|states the node is part of initial cluster setup| +|NotInitClusterNode|false|states the node is not part of initial cluster setup| + +## State + +### Upscale + +|Name|Value|Description| +|-----|----|------------| +|GracefulUpscaleRequired|GracefulUpscaleRequired|states that a node upscale is required.| +|GracefulUpscaleRunning|GracefulUpscaleRunning|states that the node upscale task is still running.| +|GracefulUpscaleSucceeded|GracefulUpscaleSucceeded|states the node is updated gracefully.| + +### Downscale + +|Name|Value|Description| +|-----|----|------------| +|GracefulDownscaleRequired|GracefulDownscaleRequired|states that a node downscale is required| +|GracefulDownscaleRunning|GracefulDownscaleRunning|states that the node downscale is still running in| +|GracefulUpscaleSucceeded|GracefulUpscaleSucceeded|states the node is updated gracefully| + +## ActionStep +|Name|Value|Description| +|-----|----|------------| +|DisconnectNodeAction|DISCONNECTING|states that the NiFi node is disconnecting from NiFi Cluster.| +|DisconnectStatus|DISCONNECTED|states that the NiFi node is disconnected from NiFi Cluster.| +|OffloadNodeAction|OFFLOADING|states that the NiFi node is offloading data to NiFi Cluster.| +|OffloadStatus|OFFLOADED|states that the NiFi node offloaded data to NiFi Cluster.| +|RemovePodAction|POD_REMOVING|states that the NiFi node pod and object related are removing by operator.| +|RemovePodStatus|POD_REMOVED|states that the NiFi node pod and object related have been removed by operator.| +|RemoveNodeAction|REMOVING|states that the NiFi node is removing from NiFi Cluster.| +|RemoveStatus|REMOVED|states that the NiFi node is removed from NiFi Cluster.| +|ConnectNodeAction|CONNECTING|states that the NiFi node is connecting to the NiFi Cluster.| +|ConnectStatus|CONNECTED|states that the NiFi node is connected to the NiFi Cluster.| \ No newline at end of file diff --git a/site/website/versioned_docs/version-v0.14.1/5_references/1_nifi_cluster/6_listeners_config.md b/site/website/versioned_docs/version-v0.14.1/5_references/1_nifi_cluster/6_listeners_config.md new file mode 100644 index 0000000000..9fb1f10a04 --- /dev/null +++ b/site/website/versioned_docs/version-v0.14.1/5_references/1_nifi_cluster/6_listeners_config.md @@ -0,0 +1,59 @@ +--- +id: 6_listeners_config +title: Listeners Config +sidebar_label: Listeners Config +--- + +ListenersConfig defines the Nifi listener types : + +```yaml + listenersConfig: + internalListeners: + - type: "https" + name: "https" + containerPort: 8443 + - type: "cluster" + name: "cluster" + containerPort: 6007 + - type: "s2s" + name: "s2s" + containerPort: 10000 + - type: "prometheus" + name: "prometheus" + containerPort: 9090 + - type: "load-balance" + name: "load-balance" + containerPort: 6342 + sslSecrets: + tlsSecretName: "test-nifikop" + create: true +``` + +## ListenersConfig + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|internalListeners|\[ \][InternalListener](#internallistener)| specifies settings required to access nifi internally.| Yes | - | +|sslSecrets|[SSLSecrets](#sslsecrets)| contains information about ssl related kubernetes secrets if one of the listener setting type set to ssl these fields must be populated to.| Yes | nil | +|clusterDomain|string| allow to override the default cluster domain which is "cluster.local".| Yes | `cluster.local` | +|useExternalDNS|string| allow to manage externalDNS usage by limiting the DNS names associated to each nodes and load balancer: `-node-...`| Yes | false | + +## InternalListener + +Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|type|enum{ "cluster", "http", "https", "s2s", "prometheus", "load-balance"}| allow to specify if we are in a specific nifi listener it's allowing to define some required information such as Cluster Port, Http Port, Https Port, S2S, Load Balance port, or Prometheus port| Yes | - | +|name|string| an identifier for the port which will be configured. | Yes | - | +|containerPort|int32| the containerPort. | Yes | - | + + +## SSLSecrets + +Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|tlsSecretName|string| should contain all ssl certs required by nifi including: caCert, caKey, clientCert, clientKey serverCert, serverKey, peerCert, peerKey. | Yes | - | +|create|boolean| tells the installed cert manager to create the required certs keys. | Yes | - | +|clusterScoped|boolean| defines if the Issuer created is cluster or namespace scoped. | Yes | - | +|issuerRef|[ObjectReference](https://docs.cert-manager.io/en/release-0.9/reference/api-docs/index.html#objectreference-v1alpha1)| IssuerRef allow to use an existing issuer to act as CA: https://cert-manager.io/docs/concepts/issuer/ | No | - | +|pkiBackend|enum{"cert-manager"}| | Yes | - | + diff --git a/site/website/versioned_docs/version-v0.14.1/5_references/1_nifi_cluster/7_external_service_config.md b/site/website/versioned_docs/version-v0.14.1/5_references/1_nifi_cluster/7_external_service_config.md new file mode 100644 index 0000000000..1e62f13136 --- /dev/null +++ b/site/website/versioned_docs/version-v0.14.1/5_references/1_nifi_cluster/7_external_service_config.md @@ -0,0 +1,56 @@ +--- +id: 7_external_service_config +title: External Service Config +sidebar_label: External Service Config +--- + +ListenersConfig defines the Nifi listener types : + +```yaml + externalServices: + - name: "clusterip" + spec: + type: ClusterIP + portConfigs: + - port: 8080 + internalListenerName: "http" + metadata: + annotations: + toto: tata + labels: + titi: tutu +``` + +## ExternalServiceConfig + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|name|string| must be unique within a namespace. Name is primarily intended for creation idempotence and configuration.| Yes | - | +|metadata|[Metadata](#metadata)|define additionnal metadata to merge to the service associated.| No | - | +|spec|[ExternalServiceSpec](#externalservicespec)| defines the behavior of a service.| Yes | | + +## ExternalServiceSpec + +Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|portConfigs||\[ \][PortConfig](#portconfig)| Contains the list port for the service and the associated listener| Yes | - | +|clusterIP|string| More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies | No | - | +|type|[ServiceType](https://godoc.org/k8s.io/api/core/v1#ServiceType)| type determines how the Service is exposed. Defaults to ClusterIP. Valid options are ExternalName, ClusterIP, NodePort, and LoadBalancer. | No | - | +|externalIPs|\[ \]string| externalIPs is a list of IP addresses for which nodes in the cluster will also accept traffic for this service. These IPs are not managed by Kubernetes | No | - | +|loadBalancerIP|string| Only applies to Service Type: LoadBalancer. LoadBalancer will get created with the IP specified in this field. | No | - | +|loadBalancerSourceRanges|\[ \]string| If specified and supported by the platform, this will restrict traffic through the cloud-provider load-balancer will be restricted to the specified client IPs | No | - | +|externalName|string| externalName is the external reference that kubedns or equivalent will return as a CNAME record for this service. No proxying will be involved. | No | - | + +## PortConfig + +Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|port|int32| The port that will be exposed by this service. | Yes | - | +|internalListenerName| string| The name of the listener which will be used as target container. | Yes | - | + +## Metadata + +Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +| annotations | map\[string\]string | Additionnal annotation to merge to the service associated [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set). |No|nil| +| nodeLabels | map\[string\]string | Additionnal labels to merge to the service associated [labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set). |No|nil| diff --git a/site/website/versioned_docs/version-v0.14.1/5_references/2_nifi_user.md b/site/website/versioned_docs/version-v0.14.1/5_references/2_nifi_user.md new file mode 100644 index 0000000000..6030e318d6 --- /dev/null +++ b/site/website/versioned_docs/version-v0.14.1/5_references/2_nifi_user.md @@ -0,0 +1,101 @@ +--- +id: 2_nifi_user +title: NiFi User +sidebar_label: NiFi User +--- + +`NifiUser` is the Schema for the nifi users API. + +```yaml +apiVersion: nifi.konpyutaika.com/v1alpha1 +kind: NifiUser +metadata: + name: aguitton +spec: + identity: alexandre.guitton@orange.com + clusterRef: + name: nc + namespace: nifikop + createCert: false +``` + +## NifiUser +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|metadata|[ObjectMetadata](https://godoc.org/k8s.io/apimachinery/pkg/apis/meta/v1#ObjectMeta)|is metadata that all persisted resources must have, which includes all objects users must create.|No|nil| +|spec|[NifiUserSpec](#nifiuserspec)|defines the desired state of NifiUser.|No|nil| +|status|[NifiUserStatus](#nifiuserstatus)|defines the observed state of NifiUser.|No|nil| + +## NifiUserSpec + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|identity|string| used to define the user identity on NiFi cluster side, when the user's name doesn't suit with Kubernetes resource name. |No| - | +|secretName|string| name of the secret where all cert resources will be stored. |No| - | +|clusterRef|[ClusterReference](#clusterreference)| contains the reference to the NifiCluster with the one the user is linked. |Yes| - | +|DNSNames|\[ \]string| list of DNSNames that the user will used to request the NifiCluster (allowing to create the right certificates associated). |Yes| - | +|includeJKS|boolean| whether or not the the operator also include a Java keystore format (JKS) with you secret. |Yes| - | +|createCert|boolean| whether or not a certificate will be created for this user. |No| - | +|accessPolicies|\[ \][AccessPolicy](#accesspolicy)| defines the list of access policies that will be granted to the group. |No| [] | + + +## NifiUserStatus + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|id|string| the nifi user's node id.|Yes| - | +|version|string| the last nifi user's node revision version catched.|Yes| - | + +## ClusterReference + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|name|string| name of the NifiCluster. |Yes| - | +|namespace|string| the NifiCluster namespace location. |Yes| - | + +## AccessPolicy + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|type|[AccessPolicyType](#accesspolicytype)| defines the kind of access policy, could be "global" or "component". |Yes| - | +|action|[AccessPolicyAction](#accesspolicyaction)| defines the kind of action that will be granted, could be "read" or "write". |Yes| - | +|resource|[AccessPolicyResource](#accesspolicyresource)| defines the kind of resource targeted by this access policies, please refer to the following page : https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#access-policies |Yes| - | +|componentType|string| used if the type is "component", it allows to define the kind of component on which is the access policy. |No| - | +|componentId|string| used if the type is "component", it allows to define the id of the component on which is the access policy. |No| - | + +## AccessPolicyType + +|Name|Value|Description| +|-----|----|------------| +|GlobalAccessPolicyType|global|Global access policies govern the following system level authorizations| +|ComponentAccessPolicyType|component|Component level access policies govern the following component level authorizations| + +## AccessPolicyAction + +|Name|Value|Description| +|-----|----|------------| +|ReadAccessPolicyAction|read|Allows users to view| +|WriteAccessPolicyAction|write|Allows users to modify| + +## AccessPolicyResource + +|Name|Value|Description| +|-----|----|------------| +|FlowAccessPolicyResource|/flow|About the UI| +|ControllerAccessPolicyResource|/controller| about the controller including Reporting Tasks, Controller Services, Parameter Contexts and Nodes in the Cluster| +|ParameterContextAccessPolicyResource|/parameter-context|About the Parameter Contexts. Access to Parameter Contexts are inherited from the "access the controller" policies unless overridden.| +|ProvenanceAccessPolicyResource|/provenance|Allows users to submit a Provenance Search and request Event Lineage| +|RestrictedComponentsAccessPolicyResource|/restricted-components|About the restricted components assuming other permissions are sufficient. The restricted components may indicate which specific permissions are required. Permissions can be granted for specific restrictions or be granted regardless of restrictions. If permission is granted regardless of restrictions, the user can create/modify all restricted components.| +|PoliciesAccessPolicyResource|/policies|About the policies for all components| +|TenantsAccessPolicyResource|/tenants| About the users and user groups| +|SiteToSiteAccessPolicyResource|/site-to-site|Allows other NiFi instances to retrieve Site-To-Site details| +|SystemAccessPolicyResource|/system|Allows users to view System Diagnostics| +|ProxyAccessPolicyResource|/proxy|Allows proxy machines to send requests on the behalf of others| +|CountersAccessPolicyResource|/counters|About counters| +|ComponentsAccessPolicyResource|/| About the component configuration details| +|OperationAccessPolicyResource|/operation|to operate components by changing component run status (start/stop/enable/disable), remote port transmission status, or terminating processor threads| +|ProvenanceDataAccessPolicyResource|/provenance-data|to view provenance events generated by this component| +|DataAccessPolicyResource|/data|About metadata and content for this component in flowfile queues in outbound connections and through provenance events| +|PoliciesComponentAccessPolicyResource|/policies|-| +|DataTransferAccessPolicyResource|/data-transfer|Allows a port to receive data from NiFi instances| + diff --git a/site/website/versioned_docs/version-v0.14.1/5_references/3_nifi_registry_client.md b/site/website/versioned_docs/version-v0.14.1/5_references/3_nifi_registry_client.md new file mode 100644 index 0000000000..60e3365f99 --- /dev/null +++ b/site/website/versioned_docs/version-v0.14.1/5_references/3_nifi_registry_client.md @@ -0,0 +1,42 @@ +--- +id: 3_nifi_registry_client +title: NiFi Registry Client +sidebar_label: NiFi Registry Client +--- + +`NifiRegistryClient` is the Schema for the NiFi registry client API. + +```yaml +apiVersion: nifi.konpyutaika.com/v1alpha1 +kind: NifiRegistryClient +metadata: + name: squidflow +spec: + clusterRef: + name: nc + namespace: nifikop + description: "Squidflow demo" + uri: "http://nifi-registry:18080" +``` + +## NifiRegistryClient +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|metadata|[ObjectMetadata](https://godoc.org/k8s.io/apimachinery/pkg/apis/meta/v1#ObjectMeta)|is metadata that all persisted resources must have, which includes all objects registry clients must create.|No|nil| +|spec|[NifiRegistryClientSpec](#nifiregistryclientspec)|defines the desired state of NifiRegistryClient.|No|nil| +|status|[NifiRegistryClientStatus](#nifiregistryclientstatus)|defines the observed state of NifiRegistryClient.|No|nil| + +## NifiRegistryClientsSpec + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|description|string| describes the Registry client. |No| - | +|uri|string| URI of the NiFi registry that should be used for pulling the flow. |Yes| - | +|clusterRef|[ClusterReference](./2_nifi_user.md#clusterreference)| contains the reference to the NifiCluster with the one the user is linked. |Yes| - | + +## NifiRegistryClientStatus + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|id|string| nifi registry client's id. |Yes| - | +|version|int64| the last nifi registry client revision version catched. |Yes| - | \ No newline at end of file diff --git a/site/website/versioned_docs/version-v0.14.1/5_references/4_nifi_parameter_context.md b/site/website/versioned_docs/version-v0.14.1/5_references/4_nifi_parameter_context.md new file mode 100644 index 0000000000..253c418d9c --- /dev/null +++ b/site/website/versioned_docs/version-v0.14.1/5_references/4_nifi_parameter_context.md @@ -0,0 +1,86 @@ +--- +id: 4_nifi_parameter_context +title: NiFi Parameter Context +sidebar_label: NiFi Parameter Context +--- + +`NifiParameterContext` is the Schema for the NiFi parameter context API. + +```yaml +apiVersion: nifi.konpyutaika.com/v1alpha1 +kind: NifiParameterContext +metadata: + name: dataflow-lifecycle +spec: + description: "It is a test" + clusterRef: + name: nc + namespace: nifikop + secretRefs: + - name: secret-params + namespace: nifikop + parameters: + - name: test + value: toto + description: tutu + - name: test2 + description: toto + sensistive: true +``` + +## NifiParameterContext + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|metadata|[ObjectMetadata](https://godoc.org/k8s.io/apimachinery/pkg/apis/meta/v1#ObjectMeta)|is metadata that all persisted resources must have, which includes all objects parameter contexts must create.|No|nil| +|spec|[NifiParameterContextSpec](#NifiParameterContextspec)|defines the desired state of NifiParameterContext.|No|nil| +|status|[NifiParameterContextStatus](#NifiParameterContextstatus)|defines the observed state of NifiParameterContext.|No|nil| + +## NifiParameterContextsSpec + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|description|string| describes the Parameter Context. |No| - | +|parameters|\[ \][Parameter](#parameter)| a list of non-sensitive Parameters. |Yes| - | +|secretRefs|\[ \][SecretReference](#secretreference)| a list of secret containing sensitive parameters (the key will name of the parameter) |No| - | +|clusterRef|[ClusterReference](./2_nifi_user.md#clusterreference)| contains the reference to the NifiCluster with the one the user is linked. |Yes| - | +|disableTakeOver|bool| whether or not the operator should take over an existing parameter context if its name is the same. |No| - | + +## NifiParameterContextStatus + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|id|string| nifi parameter context's id. |Yes| - | +|version|int64| the last nifi parameter context revision version catched. |Yes| - | +|latestUpdateRequest|[ParameterContextUpdateRequest](#parametercontextupdaterequest)|the latest update request. |Yes| - | +|version|int64| the last nifi parameter context revision version catched. |Yes| - | + +## Parameter + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|name|string| the name of the Parameter. |Yes| - | +|value|string| the value of the Parameter. |No| - | +|description|string| the description of the Parameter. |No| - | +|sensitive|string| Whether the parameter is sensitive or not. |No| false | + +## SecretReference + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|name|string| name of the secret. |Yes| - | +|namespace|string| the secret namespace location. |Yes| - | + + +## ParameterContextUpdateRequest + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|id|string| the id of the update request. |Yes| - | +|uri|string| the uri for this request. |Yes| - | +|submissionTime|string| the timestamp of when the request was submitted This property is read only. |Yes| - | +|lastUpdated|string| the timestamp of when the request was submitted This property is read only. |Yes| - | +|complete|bool| whether or not this request has completed. |Yes| false | +|failureReason|string| an explication of why the request failed, or null if this request has not failed. |Yes| - | +|percentCompleted|int32| the percentage complete of the request, between 0 and 100. |Yes| - | +|state|string| the state of the request. |Yes| - | \ No newline at end of file diff --git a/site/website/versioned_docs/version-v0.14.1/5_references/5_nifi_dataflow.md b/site/website/versioned_docs/version-v0.14.1/5_references/5_nifi_dataflow.md new file mode 100644 index 0000000000..ee3b54b5d5 --- /dev/null +++ b/site/website/versioned_docs/version-v0.14.1/5_references/5_nifi_dataflow.md @@ -0,0 +1,136 @@ +--- +id: 5_nifi_dataflow +title: NiFi Dataflow +sidebar_label: NiFi Dataflow +--- + +`NifiDataflow` is the Schema for the NiFi dataflow API. + +```yaml +apiVersion: nifi.konpyutaika.com/v1alpha1 +kind: NifiDataflow +metadata: + name: dataflow-lifecycle +spec: + parentProcessGroupID: "16cfd2ec-0174-1000-0000-00004b9b35cc" + bucketId: "01ced6cc-0378-4893-9403-f6c70d080d4f" + flowId: "9b2fb465-fb45-49e7-94fe-45b16b642ac9" + flowVersion: 2 + flowPosition: + posX: 0 + posY: 0 + syncMode: always + skipInvalidControllerService: true + skipInvalidComponent: true + clusterRef: + name: nc + namespace: nifikop + registryClientRef: + name: squidflow + namespace: nifikop + parameterContextRef: + name: dataflow-lifecycle + namespace: nifikop + updateStrategy: drain +``` + +## NifiDataflow + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|metadata|[ObjectMetadata](https://godoc.org/k8s.io/apimachinery/pkg/apis/meta/v1#ObjectMeta)|is metadata that all persisted resources must have, which includes all objects dataflows must create.|No|nil| +|spec|[NifiDataflowSpec](#NifiDataflowspec)|defines the desired state of NifiDataflow.|No|nil| +|status|[NifiDataflowStatus](#NifiDataflowstatus)|defines the observed state of NifiDataflow.|No|nil| + + +## NifiDataflowsSpec + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|parentProcessGroupID|string|the UUID of the parent process group where you want to deploy your dataflow, if not set deploy at root level. |No| - | +|bucketId|string|the UUID of the Bucket containing the flow. |Yes| - | +|flowId|string|the UUID of the flow to run. |Yes| - | +|flowVersion|*int32|the version of the flow to run. |Yes| - | +|flowPosition|[FlowPosition](#flowposition)|the position of your dataflow in the canvas. |No| - | +|syncMode|Enum={"never","always","once"}|if the flow will be synchronized once, continuously or never. |No| always | +|skipInvalidControllerService|bool|whether the flow is considered as ran if some controller services are still invalid or not. |Yes| false | +|skipInvalidComponent|bool|whether the flow is considered as ran if some components are still invalid or not. |Yes| false | +|updateStrategy|[DataflowUpdateStrategy](#dataflowupdatestrategy)|describes the way the operator will deal with data when a dataflow will be updated : Drop or Drain |Yes| drain | +|clusterRef|[ClusterReference](./2_nifi_user.md#clusterreference)| contains the reference to the NifiCluster with the one the user is linked. |Yes| - | +|parameterContextRef|[ParameterContextReference](./4_nifi_parameter_context.md#parametercontextreference)| contains the reference to the ParameterContext with the one the dataflow is linked. |No| - | +|registryClientRef|[RegistryClientReference](./3_nifi_registry_client.md#registryclientreference)| contains the reference to the NifiRegistry with the one the dataflow is linked. |Yes| - | + +## NifiDataflowStatus + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|processGroupID|string| process Group ID. |Yes| - | +|state|[DataflowState](#dataflowstate)| the dataflow current state. |Yes| - | +|latestUpdateRequest|[UpdateRequest](#updaterequest)|the latest update request sent. |Yes| - | +|latestDropRequest|[DropRequest](#droprequest)|the latest queue drop request sent. |Yes| - | + +## DataflowUpdateStrategy + +|Name|Value|Description| +|-----|----|------------| +|DrainStrategy|drain|leads to shutting down only input components (Input processors, remote input process group) and dropping all flowfiles from the flow.| +|DropStrategy|drop|leads to shutting down all components and dropping all flowfiles from the flow.| + +## DataflowState + +|Name|Value|Description| +|-----|----|------------| +|DataflowStateCreated|Created|describes the status of a NifiDataflow as created.| +|DataflowStateStarting|Starting|describes the status of a NifiDataflow as starting.| +|DataflowStateRan|Ran|describes the status of a NifiDataflow as running.| +|DataflowStateOutOfSync|OutOfSync|describes the status of a NifiDataflow as out of sync.| +|DataflowStateInSync|InSync|describes the status of a NifiDataflow as in sync.| + +## UpdateRequest + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|type|[DataflowUpdateRequestType](#dataflowupdaterequesttype)|defines the type of versioned flow update request. |Yes| - | +|id|string|the id of the update request. |Yes| - | +|uri|string|the uri for this request. |Yes| - | +|lastUpdated|string|the last time this request was updated. |Yes| - | +|complete|bool| whether or not this request has completed. |Yes| false | +|failureReason|string| an explication of why the request failed, or null if this request has not failed. |Yes| - | +|percentCompleted|int32| the percentage complete of the request, between 0 and 100. |Yes| 0 | +|state|string| the state of the request. |Yes| - | + +## DropRequest + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|connectionId|string|the connection id. |Yes| - | +|id|string|the id for this drop request. |Yes| - | +|uri|string|the uri for this request. |Yes| - | +|lastUpdated|string|the last time this request was updated. |Yes| - | +|finished|bool|whether the request has finished. |Yes| false | +|failureReason|string|an explication of why the request failed, or null if this request has not failed. |Yes| - | +|percentCompleted|int32|the percentage complete of the request, between 0 and 100. |Yes| 0 | +|currentCount|int32|the number of flow files currently queued. |Yes| 0 | +|currentSize|int64| the size of flow files currently queued in bytes. |Yes| 0 | +|current|string|the count and size of flow files currently queued. |Yes| - | +|originalCount|int32|the number of flow files to be dropped as a result of this request. |Yes| 0 | +|originalSize|int64| the size of flow files to be dropped as a result of this request in bytes. |Yes| 0 | +|original|string|the count and size of flow files to be dropped as a result of this request. |Yes| - | +|droppedCount|int32|the number of flow files that have been dropped thus far. |Yes| 0 | +|droppedSize|int64| the size of flow files currently queued in bytes. |Yes| 0 | +|Dropped|string|the count and size of flow files that have been dropped thus far. |Yes| - | +|state|string|the state of the request. |Yes| - | + +## DataflowUpdateRequestType + +|Name|Value|Description| +|-----|----|------------| +|RevertRequestType|Revert|defines a revert changes request.| +|UpdateRequestType|Update|defines an update version request.| + +## FlowPosition + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|posX|int64|the x coordinate. |No| - | +|posY|int64|the y coordinate. |No| - | \ No newline at end of file diff --git a/site/website/versioned_docs/version-v0.14.1/5_references/6_nifi_usergroup.md b/site/website/versioned_docs/version-v0.14.1/5_references/6_nifi_usergroup.md new file mode 100644 index 0000000000..5ad51bbec5 --- /dev/null +++ b/site/website/versioned_docs/version-v0.14.1/5_references/6_nifi_usergroup.md @@ -0,0 +1,55 @@ +--- +id: 6_nifi_usergroup +title: NiFi UserGroup +sidebar_label: NiFi UserGroup +--- + +`NifiUserGroup` is the Schema for the nifi user groups API. + +```yaml +apiVersion: nifi.konpyutaika.com/v1alpha1 +kind: NifiUserGroup +metadata: + name: group-test +spec: + clusterRef: + name: nc + namespace: nifikop + usersRef: + - name: nc-0-node.nc-headless.nifikop.svc.cluster.local + - name: nc-controller.nifikop.mgt.cluster.local + accessPolicies: + - type: global + action: read + resource: /counters +``` + +## NifiUser +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|metadata|[ObjectMetadata](https://godoc.org/k8s.io/apimachinery/pkg/apis/meta/v1#ObjectMeta)|is metadata that all persisted resources must have, which includes all objects usergroups must create.|No|nil| +|spec|[NifiUserGroupSpec](#nifiusergroupspec)|defines the desired state of NifiUserGroup.|No|nil| +|status|[NifiUserGroupStatus](#nifiusergroupstatus)|defines the observed state of NifiUserGroup.|No|nil| + +## NifiUserGroupSpec + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|clusterRef|[ClusterReference](./2_nifi_user.md#clusterreference)| contains the reference to the NifiCluster with the one the user is linked. |Yes| - | +|usersRef|\[ \][UserReference](#userref)| contains the list of reference to NifiUsers that are part to the group. |No| [] | +|accessPolicies|\[ \][AccessPolicy](./2_nifi_user.md#accesspolicy)| defines the list of access policies that will be granted to the group. |No| [] | + +## NifiUserGroupStatus + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|id|string| the nifi usergroup's node id.|Yes| - | +|version|string| the last nifi usergroup's node revision version catched.|Yes| - | + +## UserReference + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|name|string| name of the NifiUser. |Yes| - | +|namespace|string| the NifiUser namespace location. |Yes| - | + diff --git a/site/website/versioned_docs/version-v0.14.1/5_references/7_nifi_nodegroup_autoscaler.md b/site/website/versioned_docs/version-v0.14.1/5_references/7_nifi_nodegroup_autoscaler.md new file mode 100644 index 0000000000..7faf3f5445 --- /dev/null +++ b/site/website/versioned_docs/version-v0.14.1/5_references/7_nifi_nodegroup_autoscaler.md @@ -0,0 +1,59 @@ +--- +id: 7_nifi_nodegroup_autoscaler +title: NiFi NodeGroup Autoscaler +sidebar_label: NiFi NodeGroup Autoscaler +--- + +`NifiNodeGroupAutoscaler` is the Schema through which you configure automatic scaling of `NifiCluster` deployments. + +```yaml +apiVersion: nifi.konpyutaika.com/v1alpha1 +kind: NifiNodeGroupAutoscaler +metadata: + name: nifinodegroupautoscaler-sample +spec: + # contains the reference to the NifiCluster with the one the node group autoscaler is linked. + clusterRef: + name: nificluster-name + namespace: nifikop + # defines the id of the NodeConfig contained in NifiCluster.Spec.NodeConfigGroups + nodeConfigGroupId: default-node-group + # The selector used to identify nodes in NifiCluster.Spec.Nodes this autoscaler will manage + # Use Node.Labels in combination with this selector to clearly define which nodes will be managed by this autoscaler + nodeLabelsSelector: + matchLabels: + nifi_cr: nificluster-name + nifi_node_group: default-node-group + # the strategy used to decide how to add nodes to a nifi cluster + upscaleStrategy: simple + # the strategy used to decide how to remove nodes from an existing cluster + downscaleStrategy: lifo +``` + +## NifiNodeGroupAutoscaler +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|metadata|[ObjectMetadata](https://godoc.org/k8s.io/apimachinery/pkg/apis/meta/v1#ObjectMeta)|is metadata that all persisted resources must have, which includes all objects nodegroupautoscalers must create.|No|nil| +|spec|[NifiNodeGroupAutoscalerSpec](#nifinodegroupautoscalerspec)|defines the desired state of NifiNodeGroupAutoscaler.|No|nil| +|status|[NifiNodeGroupAutoscalerStatus](#nifinodegroupautoscalerstatus)|defines the observed state of NifiNodeGroupAutoscaler.|No|nil| + +## NifiNodeGroupAutoscalerSpec + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|clusterRef|[ClusterReference](./2_nifi_user.md#clusterreference)| contains the reference to the NifiCluster containing the node group this autoscaler should manage. |Yes| - | +|nodeConfigGroupId| string | defines the id of the [NodeConfig](./1_nifi_cluster/3_node_config.md) contained in `NifiCluster.Spec.NodeConfigGroups`. |Yes| - | +|nodeLabelsSelector|[LabelSelector](https://pkg.go.dev/k8s.io/apimachinery/pkg/apis/meta/v1#LabelSelector)| defines the set of labels used to identify nodes in a `NifiCluster` node config group. Use `Node.Labels` in combination with this selector to clearly define which nodes will be managed by this autoscaler. Take care to avoid having mutliple autoscalers managing the same nodes. |Yes| - | +|readOnlyConfig| [ReadOnlyConfig](./1_nifi_cluster/2_read_only_config.md) | defines a readOnlyConfig to apply to each node in this node group. Any settings here will override those set in the configured `nodeConfigGroupId`. |Yes| - | +|nodeConfig| [NodeConfig](./1_nifi_cluster/3_node_config.md) | defines a nodeConfig to apply to each node in this node group. Any settings here will override those set in the configured `nodeConfigGroupId`. |Yes| - | +|upscaleStrategy| string | The strategy NiFiKop will use to scale up the nodes managed by this autoscaler. Must be one of {`simple`}. |Yes| - | +|downscaleStrategy| string | The strategy NiFiKop will use to scale down the nodes managed by this autoscaler. Must be one of {`lifo`}. |Yes| - | +|replicas| int | the initial number of replicas to configure the `HorizontalPodAutoscaler` with. After the initial configuration, this `replicas` configuration will be automatically updated by the Kubernetes `HorizontalPodAutoscaler` controller. |No| 1 | + +## NifiNodeGroupAutoscalerStatus + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|state|string| the state of the nodegroup autoscaler. This is set by the autoscaler. |No| - | +|replicas|int| the current number of replicas running in the node group this autoscaler is managing. This is set by the autoscaler.|No| - | +|selector|string| the [selector](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) used by the `HorizontalPodAutoscaler` controller to identify the replicas in this node group. This is set by the autoscaler.|No| - | \ No newline at end of file diff --git a/site/website/versioned_docs/version-v0.14.1/6_contributing/0_contribution_organization.md b/site/website/versioned_docs/version-v0.14.1/6_contributing/0_contribution_organization.md new file mode 100644 index 0000000000..59d6c9afa8 --- /dev/null +++ b/site/website/versioned_docs/version-v0.14.1/6_contributing/0_contribution_organization.md @@ -0,0 +1,66 @@ +--- +id: 0_contribution_organization +title: Contribution organization +sidebar_label: Contribution organization +--- + +## New ownership for more community oriented + +The NiFiKop operator was originally started by Orange in March 2020 as [Orange-OpenSource/nifikop](https://github.com/Orange-OpenSource/nifikop), +and then forked as `konpyutaika/nifikop` in March 20222 : but this is the same codebase and the same developers. + +We made this decision in concert with Orange team, because some legal restrictions would not have allowed to involve and serve external community around this operator efficiently. +Therefore,we have chosen to fork the source code into another organization and repository, which will allow a more open ownership and community-oriented development. + +It is important to notice that Orange will still continue to work and contribute to the operator, but as part of the community :) + +## Organizations + +With this ownership move, we decided to set up a new project management, with the aims to be more and more community-oriented + +### Slack channel + +One of the most important topics we want to improve is probably the communication around the operator's development. +To achieve this, we have created a new Slack open to anyone who wants [to join](https://join.slack.com/t/konpytika/shared_invite/zt-14md072lv-Jr8mqYoeUrqzfZF~YGUpXA), +with two main channels: + +- [#nifikop-news](https://konpytika.slack.com/archives/C035FHN1MNG): There we will announce each new release, and communicate about next objectives for the operator. +- [#nifikop-discussion](https://konpytika.slack.com/archives/C035X6KP684): Direct discussion between each member of the community to design new needs, fix issues and help each other. + +### Tech scoping + +As we want to involve as much as possible the people on the operator, we will introduce a new support for brainstorming and designing new major features. + +This is the Tech Scoping, whose main objective is to describe the problem statement that we are trying to solve, +the different approaches that could solve it, and together discuss and challenge them to define the solution to be implemented. + +You can find all the tech scoping in this [Google Drive repository](https://drive.google.com/drive/folders/1-A__UxEdRBZrwEUJu4lMF4LJtIstrnT0?usp=sharing) + +### Teams + +#### NiFiKop Leads + +This group is currently composed of : + +- [Alexandre Guitton](https://github.com/erdrix) as original owner and developer of the operator +- [Julien Guitton](https://github.com/juldrixx) as representative of Orange contribution + +The mains objectives of this group are to : + +- Define the global roadmap of the operator, +- Ensure the reviews and validations of the PRs, +- Review and validate the Tech Scoping. + +This group aims to be more representative of the community, so if the operator community grows or if there is a needs, we would be happy to have more people in this group :) + +#### NiFiKop Contributors + +This group is currently composed of : + +The mains objectives of this group are to : + +- Manage issues to help people, +- Review PRs (not validation), +- Create and edit Tech Scoping for new features. + +This is an open group, so feel free to contact a NiFiKop Leader on Slack to join :) \ No newline at end of file diff --git a/site/website/versioned_docs/version-v0.14.1/6_contributing/1_developer_guide.md b/site/website/versioned_docs/version-v0.14.1/6_contributing/1_developer_guide.md new file mode 100644 index 0000000000..e494a8fb47 --- /dev/null +++ b/site/website/versioned_docs/version-v0.14.1/6_contributing/1_developer_guide.md @@ -0,0 +1,144 @@ +--- +id: 1_developer_guide +title: Developer guide +sidebar_label: Developer guide +--- + +## Operator SDK + +### Prerequisites + +NiFiKop has been validated with : + +- [go](https://golang.org/doc/install) version v1.17+. +- [docker](https://docs.docker.com/get-docker/) version 18.09+ +- [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) version v1.16+ +- [Helm](https://helm.sh/) version v3.4.2 +- [Operator sdk](https://github.com/operator-framework/operator-sdk) version v1.18.1 + +### Initial setup + +Checkout the project. + +```bash +git clone https://github.com/konpyutaika/nifikop.git +cd nifikop +``` + +### Operator sdk + +The full list of command is available here : https://sdk.operatorframework.io/docs/upgrading-sdk-version/v1.0.0/#cli-changes + +### Build NiFiKop + +#### Local environment + +If you prefer working directly with your local go environment you can simply uses : + +```bash +make build +``` + +### Run NiFiKop + +We can quickly run NiFiKop in development mode (on your local host), then it will use your kubectl configuration file to connect to your kubernetes cluster. + +There are several ways to execute your operator : + +- Using your IDE directly +- Executing directly the Go binary +- deploying using the Helm charts + +If you want to configure your development IDE, you need to give it environment variables so that it will uses to connect to kubernetes. + +```bash +KUBECONFIG={path/to/your/kubeconfig} +WATCH_NAMESPACE={namespace_to_watch} +POD_NAME={name for operator pod} +LOG_LEVEL=Debug +OPERATOR_NAME=ide +``` + +#### Run the Operator Locally with the Go Binary + +This method can be used to run the operator locally outside of the cluster. This method may be preferred during development as it facilitates faster deployment and testing. + +Set the name of the operator in an environment variable + +```bash +export OPERATOR_NAME=nifi-operator +``` + +Deploy the CRDs. + +```bash +kubectl apply -f config/crd/bases/nifi.konpyutaika.com_nificlusters.yaml +kubectl apply -f config/crd/bases/nifi.konpyutaika.com_nifidataflows.yaml +kubectl apply -f config/crd/bases/nifi.konpyutaika.com_nifiparametercontexts.yaml +kubectl apply -f config/crd/bases/nifi.konpyutaika.com_nifiregistryclients.yaml +kubectl apply -f config/crd/bases/nifi.konpyutaika.com_nifiusergroups.yaml +kubectl apply -f config/crd/bases/nifi.konpyutaika.com_nifiusers.yaml +``` + +And deploy the operator. + +```bash +make run +``` + +This will run the operator in the `default` namespace using the default Kubernetes config file at `$HOME/.kube/config`. + +#### Deploy using the Helm Charts + +This section provides an instructions for running the operator Helm charts with an image that is built from the local branch. + +Build the image from the current branch. + +```bash +export DOCKER_REPO_BASE={your-docker-repo} +make docker-build +``` + +Push the image to docker hub (or to whichever repo you want to use) + +```bash +$ make docker-push +``` + +:::info +The image tag is a combination of the version as defined in `verion/version.go` and the branch name. +::: + +Install the Helm chart. + +```bash +helm install skeleton ./helm/nifikop \ + --set image.tag=v0.5.1-release \ + --namespace-{"nifikop"} +``` + +:::important +The `image.repository` and `image.tag` template variables have to match the names from the image that we pushed in the previous step. +::: + +:::info +We set the chart name to the branch, but it can be anything. +::: + +Lastly, verify that the operator is running. + +```console +$ kubectl get pods -n nifikop +NAME READY STATUS RESTARTS AGE +skeleton-nifikop-8946b89dc-4cfs9 1/1 Running 0 7m45s +``` + +## Helm + +The NiFiKop operator is released in the `konpyutaika-incubator` helm repository. + +In order to package the chart you need to run the following command. + +```bash +make helm-package +``` diff --git a/site/website/versioned_docs/version-v0.14.1/6_contributing/2_reporting_bugs.md b/site/website/versioned_docs/version-v0.14.1/6_contributing/2_reporting_bugs.md new file mode 100644 index 0000000000..376f0f76fc --- /dev/null +++ b/site/website/versioned_docs/version-v0.14.1/6_contributing/2_reporting_bugs.md @@ -0,0 +1,25 @@ +--- +id: 2_reporting_bugs +title: Reporting bugs +sidebar_label: Reporting bugs +--- + +If any part of the NiFiKop project has bugs or documentation mistakes, please let us know by [opening an issue](https://github.com/konpyutaika/nifikop/issues/new). We treat bugs and mistakes very seriously and believe no issue is too small. Before creating a bug report, please check that an issue reporting the same problem does not already exist. + +To make the bug report accurate and easy to understand, please try to create bug reports that are: + +- Specific. Include as much details as possible: which version, what environment, what configuration, etc. + +- Reproducible. Include the steps to reproduce the problem. We understand some issues might be hard to reproduce, please include the steps that might lead to the problem. + +- Isolated. Please try to isolate and reproduce the bug with minimum dependencies. It would significantly slow down the speed to fix a bug if too many dependencies are involved in a bug report. Debugging external systems that rely on operator-sdk is out of scope, but we are happy to provide guidance in the right direction or help with using operator-sdk itself. + +- Unique. Do not duplicate existing bug report. + +- Scoped. One bug per report. Do not follow up with another bug inside one report. + +It may be worthwhile to read [Elika Etemad’s article on filing good bug reports][filing-good-bugs] before creating a bug report. + +We might ask for further information to locate a bug. A duplicated bug report will be closed. + +[filing-good-bugs]: http://fantasai.inkedblade.net/style/talks/filing-good-bugs/ \ No newline at end of file diff --git a/site/website/versioned_docs/version-v0.14.1/6_contributing/3_credits.md b/site/website/versioned_docs/version-v0.14.1/6_contributing/3_credits.md new file mode 100644 index 0000000000..e04adf7f4b --- /dev/null +++ b/site/website/versioned_docs/version-v0.14.1/6_contributing/3_credits.md @@ -0,0 +1,11 @@ +--- +id: 3_credits +title: Credits +sidebar_label: Credits +--- + +This implementation is based on other Open-Source project, and lot of the community ideas. Particular thanks to : + +- Operator implementation based on [banzaicloud/kafka-operator](https://github.com/banzaicloud/kafka-operator) +- NiFi kubernetes setup configuration inspired from [cetic/helm-nifi](https://github.com/cetic/helm-nifi) +- Implementation is based on [Operator SDK](https://github.com/operator-framework/operator-sdk) \ No newline at end of file diff --git a/site/website/versioned_docs/version-v0.14.1/7_upgrade/1_v0.7.x_to_v0.8.0.md b/site/website/versioned_docs/version-v0.14.1/7_upgrade/1_v0.7.x_to_v0.8.0.md new file mode 100644 index 0000000000..4cf1e08601 --- /dev/null +++ b/site/website/versioned_docs/version-v0.14.1/7_upgrade/1_v0.7.x_to_v0.8.0.md @@ -0,0 +1,165 @@ +--- +id: 1_v0.7.x_to_v0.8.0 +title: v0.7.x to v0.8.0 +sidebar_label: v0.7.x to v0.8.0 +--- + +Guide to migrate operator resources built using `nifi.orange.com/v1alpha1` to `nifi.konpyutaika/v1alpha1`. + +## Getting started + +The goal is to migrate your NiFiKop resources from the old CRDs to the new ones without any service interruption. + +To do this, it is necessary to have both versions of CRDs available on Kubernetes and to have the old operator stopped (to prevent any manipulation on the resources). +Then launch the script developed in nodejs presented in the following. The script will copy the resources in the old CRDs to the new CRDs keeping only the relevant fields (labels, annotations, name and spec) and then copy the status. + +## Prerequisites + +- [nodejs](https://nodejs.org/en/download/) version 15.3.0+ +- [npm](https://docs.npmjs.com/cli/v7/configuring-npm/install) version 7.0.14+ + +## Initial setup + +Create a nodejs project and download the required dependencies: + +```bash +npm init -y +npm install @kubernetes/client-node@0.16.3 minimist@1.2.6 +``` + +In `package.json` add the following script: + +```json +"start": "node --no-warnings index.js" +``` + +Your `package.json` should look like that: + +```json +{ + "name": "nifikop_crd_migration", + "version": "1.0.0", + "description": "Script to migrate from the old CRDs to the new CRDs.", + "main": "index.js", + "scripts": { + "start": "node --no-warnings index.js", + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [ + "K8S", + "NiFiKop", + "CRDs" + ], + "license": "ISC", + "dependencies": { + "@kubernetes/client-node": "^0.16.3", + "minimist": "^1.2.6" + } +} +``` + +## Script setup + +Create the file `index.js` with the following content: + +```js +process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = 0; +const k8s = require('@kubernetes/client-node'); + +const kc = new k8s.KubeConfig(); +kc.loadFromDefault(); + +const k8sApi = kc.makeApiClient(k8s.CustomObjectsApi); + +const KONPYUTAIKA_GROUP = 'nifi.konpyutaika.com'; +const KONPYUTAIKA_GROUP_VERSION = 'v1alpha1'; +const ORANGE_GROUP = 'nifi.orange.com'; +const ORANGE_GROUP_VERSION = 'v1alpha1'; + +const call = async (SRC_GRP, SRC_GRP_VER, DST_GRP, DST_GRP_VER, KIND_PLURAL, NAMESPACE) => { + console.log(`Listing ${KIND_PLURAL} of ${SRC_GRP}/${SRC_GRP_VER} in ${NAMESPACE}...`); + const listResources = (await k8sApi.listNamespacedCustomObject(SRC_GRP, SRC_GRP_VER, NAMESPACE, KIND_PLURAL)).body.items; + return Promise.all(listResources.map(async (resource) => { + try { + console.log(`Found ${resource.kind} "${resource.metadata.name}" of ${resource.apiVersion} in ${NAMESPACE}`); + + if (resource.metadata.ownerReferences) { + console.log(`${resource.kind} ${resource.metadata.name} mananged by something else (ownerRefereces is set).`); + return; + } + + const bodyResource = { + apiVersion: `${DST_GRP}/${DST_GRP_VER}`, + kind: resource.kind, + metadata: { + name: resource.metadata.name, + annotations: resource.metadata.annotations, + labels: resource.metadata.labels + }, + spec: resource.spec + }; + + console.log(`Creating ${bodyResource.kind} "${bodyResource.metadata.name}" of ${bodyResource.apiVersion} in ${NAMESPACE}...`); + const newResource = (await k8sApi.createNamespacedCustomObject(DST_GRP, DST_GRP_VER, NAMESPACE, KIND_PLURAL, bodyResource)).body; + console.log('...done creating.'); + + const bodyStatus = { + apiVersion: newResource.apiVersion, + kind: newResource.kind, + metadata: { + name: newResource.metadata.name, + resourceVersion: newResource.metadata.resourceVersion + }, + status: resource.status + }; + + console.log(`Copying status from ${resource.kind} "${resource.metadata.name}" of ${newResource.apiVersion} to ${newResource.kind} "${newResource.metadata.name}" of ${newResource.apiVersion} in ${NAMESPACE}...`); + const newResourceWithStatus = (await k8sApi.replaceNamespacedCustomObjectStatus(DST_GRP, DST_GRP_VER, NAMESPACE, KIND_PLURAL, bodyStatus.metadata.name, bodyStatus)).body; + console.log('...done copying.'); + return newResourceWithStatus; + } + catch (e) { + console.error(e.body ? e.body.message ? e.body.message : e.body : e); + } + })); +}; + +const argv = require('minimist')(process.argv.slice(2)); + +let NAMESPACE = argv.namespace ? argv.namespace.length > 0 ? argv.namespace : 'default' : 'default'; +let KIND_PLURAL = { + cluster: 'nificlusters', + dataflow: 'nifidataflows', + parametercontext: 'nifiparametercontexts', + registryclient: 'nifiregistryclients', + user: 'nifiusers', + usergroup: 'nifiusergroups', +}; + +if (!argv.type) { + console.error('Type not provided'); + process.exit(1); +} + +if (!KIND_PLURAL[argv.type]) { + console.error(`Type ${argv.type} is not one of the following types: ${Object.keys(KIND_PLURAL)}`); + process.exit(1); +} + +console.log(`########### START: ${KIND_PLURAL[argv.type]} ###########`); +call( ORANGE_GROUP, ORANGE_GROUP_VERSION, KONPYUTAIKA_GROUP, KONPYUTAIKA_GROUP_VERSION, KIND_PLURAL[argv.type], NAMESPACE) + .then(r => console.log('############ END ############')) + .catch(e => console.error(e)); +``` + +## Run script + +To migrate the resources, run the following command: + +```bash +npm start -- --type= --namespace= +``` + +with +- ``: NiFiKop resource type (cluster, dataflow, user, usergroup, parametercontext or registryclient) +- `:` Kubernetes namespace where the resources will be migrated \ No newline at end of file diff --git a/site/website/versioned_sidebars/version-v0.14.1-sidebars.json b/site/website/versioned_sidebars/version-v0.14.1-sidebars.json new file mode 100644 index 0000000000..11ff81fdb1 --- /dev/null +++ b/site/website/versioned_sidebars/version-v0.14.1-sidebars.json @@ -0,0 +1,91 @@ +{ + "docs": { + "Concepts": [ + "1_concepts/1_introduction", + "1_concepts/2_design_principes", + "1_concepts/3_features", + "1_concepts/4_roadmap" + ], + "Setup": [ + "2_setup/1_getting_started", + { + "type": "category", + "label": "Platform Setup", + "items": [ + "2_setup/2_platform_setup/1_gke", + "2_setup/2_platform_setup/2_k3d" + ] + }, + { + "type": "category", + "label": "Install", + "items": [ + "2_setup/3_install/1_customizable_install_with_helm" + ] + } + ], + "Tasks": [ + { + "type": "category", + "label": "NiFi Cluster", + "items": [ + "3_tasks/1_nifi_cluster/2_cluster_scaling", + "3_tasks/1_nifi_cluster/4_external_cluster" + ] + }, + { + "type": "category", + "label": "Security", + "items": [ + "3_tasks/2_security/1_ssl", + { + "type": "category", + "label": "Authentication", + "items": [ + "3_tasks/2_security/2_authentication/1_oidc" + ] + }, + { + "type": "category", + "label": "Authorization", + "items": [ + "3_tasks/2_security/2_authorization/1_authorizer" + ] + } + ] + }, + "3_tasks/3_nifi_dataflow", + "3_tasks/4_nifi_user_group" + ], + "Reference": [ + { + "type": "category", + "label": "NiFi Cluster", + "items": [ + "5_references/1_nifi_cluster/1_nifi_cluster", + "5_references/1_nifi_cluster/2_read_only_config", + "5_references/1_nifi_cluster/3_node_config", + "5_references/1_nifi_cluster/4_node", + "5_references/1_nifi_cluster/5_node_state", + "5_references/1_nifi_cluster/6_listeners_config", + "5_references/1_nifi_cluster/7_external_service_config" + ] + }, + "5_references/2_nifi_user", + "5_references/3_nifi_registry_client", + "5_references/4_nifi_parameter_context", + "5_references/5_nifi_dataflow", + "5_references/6_nifi_usergroup", + "5_references/7_nifi_nodegroup_autoscaler" + ], + "Contributing": [ + "6_contributing/0_contribution_organization", + "6_contributing/1_developer_guide", + "6_contributing/2_reporting_bugs", + "6_contributing/3_credits" + ], + "Upgrade Guides": [ + "7_upgrade/1_v0.7.x_to_v0.8.0" + ] + } +} diff --git a/site/website/versions.json b/site/website/versions.json index b1d980f36c..3c841e9107 100644 --- a/site/website/versions.json +++ b/site/website/versions.json @@ -1,4 +1,5 @@ [ + "v0.14.1", "v0.14.0", "v0.13.0", "v0.12.0", diff --git a/version/version.go b/version/version.go index af997dbf08..de1a2236c4 100644 --- a/version/version.go +++ b/version/version.go @@ -1,5 +1,5 @@ package version var ( - Version = "0.14.0" + Version = "0.14.1" )