From bf5159e320f4ad5d3dfa472f8f098afc494447a8 Mon Sep 17 00:00:00 2001 From: Juldrixx <31806759+juldrixx@users.noreply.github.com> Date: Fri, 15 Nov 2024 12:03:02 +0100 Subject: [PATCH] Bump version 1.11.4 (#487) --- .github/ISSUE_TEMPLATE/bug-report.yml | 2 +- .github/ISSUE_TEMPLATE/support-question.yml | 2 +- CHANGELOG.md | 8 +- config/manager/kustomization.yaml | 2 +- config/manager/manager.yaml | 2 +- .../keycloak-example/step-1/operator.yaml | 2 +- helm/nifi-cluster/Chart.yaml | 2 +- helm/nifi-cluster/README.md | 2 +- helm/nifikop/Chart.yaml | 4 +- helm/nifikop/README.md | 2 +- helm/nifikop/values.yaml | 2 +- site/docs/2_deploy_nifikop/1_quick_start.md | 4 +- .../2_customizable_install_with_helm.md | 2 +- .../1_concepts/1_start_here.md | 34 ++ .../1_concepts/2_design_principles.md | 53 +++ .../version-v1.11.4/1_concepts/3_features.md | 67 +++ .../version-v1.11.4/1_concepts/4_roadmap.md | 93 ++++ .../2_deploy_nifikop/1_quick_start.md | 181 +++++++ .../2_customizable_install_with_helm.md | 237 ++++++++++ .../2_deploy_nifikop/3_kubectl_plugin.md | 34 ++ .../1_manage_clusters/0_design_principles.md | 35 ++ .../1_deploy_cluster/1_quick_start.md | 155 ++++++ .../1_deploy_cluster/2_nodes_configuration.md | 402 ++++++++++++++++ .../3_expose_cluster/1_kubernetes_service.md | 156 ++++++ .../3_expose_cluster/2_istio_service_mesh.md | 137 ++++++ .../1_deploy_cluster/4_ssl_configuration.md | 161 +++++++ .../5_users_authentication/1_oidc.md | 42 ++ .../1_custom_user_authorizer.md | 83 ++++ .../2_cluster_scaling/1_scaling_mechanism.md | 248 ++++++++++ .../2_auto_scaling/0_design_principles.md | 44 ++ .../2_auto_scaling/1_using_keda.md | 443 ++++++++++++++++++ .../1_manage_clusters/3_external_cluster.md | 91 ++++ .../1_users_management.md | 57 +++ .../2_groups_management.md | 54 +++ .../3_managed_groups.md | 60 +++ .../3_manage_dataflows/0_design_principles.md | 29 ++ .../3_manage_dataflows/1_deploy_dataflow.md | 126 +++++ .../1_deploy_connection.md | 127 +++++ .../4_compatibility_versions.md | 235 ++++++++++ .../1_nifi_cluster/1_nifi_cluster.md | 252 ++++++++++ .../1_nifi_cluster/2_read_only_config.md | 223 +++++++++ .../1_nifi_cluster/3_node_config.md | 123 +++++ .../5_references/1_nifi_cluster/4_node.md | 66 +++ .../1_nifi_cluster/5_node_state.md | 73 +++ .../1_nifi_cluster/6_listeners_config.md | 63 +++ .../7_external_service_config.md | 85 ++++ .../5_references/2_nifi_user.md | 101 ++++ .../5_references/3_nifi_registry_client.md | 42 ++ .../5_references/4_nifi_parameter_context.md | 124 +++++ .../5_references/5_nifi_dataflow.md | 140 ++++++ .../5_references/6_nifi_usergroup.md | 57 +++ .../7_nifi_nodegroup_autoscaler.md | 59 +++ .../5_references/8_nifi_connection.md | 151 ++++++ .../0_contribution_organization.md | 66 +++ .../6_contributing/1_developer_guide.md | 144 ++++++ .../6_contributing/2_reporting_bugs.md | 25 + .../6_contributing/3_credits.md | 11 + .../7_upgrade_guides/1_v0.7.x_to_v0.8.0.md | 165 +++++++ .../7_upgrade_guides/2_v0.14.1_to_v0.15.0.md | 35 ++ .../7_upgrade_guides/3_v0.16.0_to_v1.0.0.md | 47 ++ .../7_upgrade_guides/4_v1.3.1_to_v1.4.0.md | 38 ++ .../version-v1.11.4-sidebars.json | 154 ++++++ site/website/versions.json | 1 + version/version.go | 2 +- 64 files changed, 5650 insertions(+), 17 deletions(-) create mode 100644 site/website/versioned_docs/version-v1.11.4/1_concepts/1_start_here.md create mode 100644 site/website/versioned_docs/version-v1.11.4/1_concepts/2_design_principles.md create mode 100644 site/website/versioned_docs/version-v1.11.4/1_concepts/3_features.md create mode 100644 site/website/versioned_docs/version-v1.11.4/1_concepts/4_roadmap.md create mode 100644 site/website/versioned_docs/version-v1.11.4/2_deploy_nifikop/1_quick_start.md create mode 100644 site/website/versioned_docs/version-v1.11.4/2_deploy_nifikop/2_customizable_install_with_helm.md create mode 100644 site/website/versioned_docs/version-v1.11.4/2_deploy_nifikop/3_kubectl_plugin.md create mode 100644 site/website/versioned_docs/version-v1.11.4/3_manage_nifi/1_manage_clusters/0_design_principles.md create mode 100644 site/website/versioned_docs/version-v1.11.4/3_manage_nifi/1_manage_clusters/1_deploy_cluster/1_quick_start.md create mode 100644 site/website/versioned_docs/version-v1.11.4/3_manage_nifi/1_manage_clusters/1_deploy_cluster/2_nodes_configuration.md create mode 100644 site/website/versioned_docs/version-v1.11.4/3_manage_nifi/1_manage_clusters/1_deploy_cluster/3_expose_cluster/1_kubernetes_service.md create mode 100644 site/website/versioned_docs/version-v1.11.4/3_manage_nifi/1_manage_clusters/1_deploy_cluster/3_expose_cluster/2_istio_service_mesh.md create mode 100644 site/website/versioned_docs/version-v1.11.4/3_manage_nifi/1_manage_clusters/1_deploy_cluster/4_ssl_configuration.md create mode 100644 site/website/versioned_docs/version-v1.11.4/3_manage_nifi/1_manage_clusters/1_deploy_cluster/5_users_authentication/1_oidc.md create mode 100644 site/website/versioned_docs/version-v1.11.4/3_manage_nifi/1_manage_clusters/1_deploy_cluster/6_users_authorization/1_custom_user_authorizer.md create mode 100644 site/website/versioned_docs/version-v1.11.4/3_manage_nifi/1_manage_clusters/2_cluster_scaling/1_scaling_mechanism.md create mode 100644 site/website/versioned_docs/version-v1.11.4/3_manage_nifi/1_manage_clusters/2_cluster_scaling/2_auto_scaling/0_design_principles.md create mode 100644 site/website/versioned_docs/version-v1.11.4/3_manage_nifi/1_manage_clusters/2_cluster_scaling/2_auto_scaling/1_using_keda.md create mode 100644 site/website/versioned_docs/version-v1.11.4/3_manage_nifi/1_manage_clusters/3_external_cluster.md create mode 100644 site/website/versioned_docs/version-v1.11.4/3_manage_nifi/2_manage_users_and_accesses/1_users_management.md create mode 100644 site/website/versioned_docs/version-v1.11.4/3_manage_nifi/2_manage_users_and_accesses/2_groups_management.md create mode 100644 site/website/versioned_docs/version-v1.11.4/3_manage_nifi/2_manage_users_and_accesses/3_managed_groups.md create mode 100644 site/website/versioned_docs/version-v1.11.4/3_manage_nifi/3_manage_dataflows/0_design_principles.md create mode 100644 site/website/versioned_docs/version-v1.11.4/3_manage_nifi/3_manage_dataflows/1_deploy_dataflow.md create mode 100644 site/website/versioned_docs/version-v1.11.4/3_manage_nifi/4_manage_connections/1_deploy_connection.md create mode 100644 site/website/versioned_docs/version-v1.11.4/4_compatibility_versions.md create mode 100644 site/website/versioned_docs/version-v1.11.4/5_references/1_nifi_cluster/1_nifi_cluster.md create mode 100644 site/website/versioned_docs/version-v1.11.4/5_references/1_nifi_cluster/2_read_only_config.md create mode 100644 site/website/versioned_docs/version-v1.11.4/5_references/1_nifi_cluster/3_node_config.md create mode 100644 site/website/versioned_docs/version-v1.11.4/5_references/1_nifi_cluster/4_node.md create mode 100644 site/website/versioned_docs/version-v1.11.4/5_references/1_nifi_cluster/5_node_state.md create mode 100644 site/website/versioned_docs/version-v1.11.4/5_references/1_nifi_cluster/6_listeners_config.md create mode 100644 site/website/versioned_docs/version-v1.11.4/5_references/1_nifi_cluster/7_external_service_config.md create mode 100644 site/website/versioned_docs/version-v1.11.4/5_references/2_nifi_user.md create mode 100644 site/website/versioned_docs/version-v1.11.4/5_references/3_nifi_registry_client.md create mode 100644 site/website/versioned_docs/version-v1.11.4/5_references/4_nifi_parameter_context.md create mode 100644 site/website/versioned_docs/version-v1.11.4/5_references/5_nifi_dataflow.md create mode 100644 site/website/versioned_docs/version-v1.11.4/5_references/6_nifi_usergroup.md create mode 100644 site/website/versioned_docs/version-v1.11.4/5_references/7_nifi_nodegroup_autoscaler.md create mode 100644 site/website/versioned_docs/version-v1.11.4/5_references/8_nifi_connection.md create mode 100644 site/website/versioned_docs/version-v1.11.4/6_contributing/0_contribution_organization.md create mode 100644 site/website/versioned_docs/version-v1.11.4/6_contributing/1_developer_guide.md create mode 100644 site/website/versioned_docs/version-v1.11.4/6_contributing/2_reporting_bugs.md create mode 100644 site/website/versioned_docs/version-v1.11.4/6_contributing/3_credits.md create mode 100644 site/website/versioned_docs/version-v1.11.4/7_upgrade_guides/1_v0.7.x_to_v0.8.0.md create mode 100644 site/website/versioned_docs/version-v1.11.4/7_upgrade_guides/2_v0.14.1_to_v0.15.0.md create mode 100644 site/website/versioned_docs/version-v1.11.4/7_upgrade_guides/3_v0.16.0_to_v1.0.0.md create mode 100644 site/website/versioned_docs/version-v1.11.4/7_upgrade_guides/4_v1.3.1_to_v1.4.0.md create mode 100644 site/website/versioned_sidebars/version-v1.11.4-sidebars.json diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml index 36419f7244..5f4f32f882 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.yml +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -45,7 +45,7 @@ body: attributes: label: NiFiKop version description: NiFiKop release or git SHA - placeholder: v1.11.3-release + placeholder: v1.11.4-release validations: required: true diff --git a/.github/ISSUE_TEMPLATE/support-question.yml b/.github/ISSUE_TEMPLATE/support-question.yml index 063f87388a..7005e1446e 100644 --- a/.github/ISSUE_TEMPLATE/support-question.yml +++ b/.github/ISSUE_TEMPLATE/support-question.yml @@ -33,7 +33,7 @@ body: attributes: label: NiFiKop version description: NiFiKop release or git SHA - placeholder: v1.11.3-release + placeholder: v1.11.4-release - type: input attributes: diff --git a/CHANGELOG.md b/CHANGELOG.md index ed5b8642d2..eb81b1b562 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,12 +6,16 @@ ### Fixed Bugs -- [PR #486](https://github.com/konpyutaika/nifikop/pull/486) - **[Operator/NifiUserGroup]** Fixed policy creation/update on Root PG. - ### Deprecated ### Removed +## v1.11.4 + +### Fixed Bugs + +- [PR #486](https://github.com/konpyutaika/nifikop/pull/486) - **[Operator/NifiUserGroup]** Fixed policy creation/update on Root PG. + ## v1.11.3 ### Changed diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 1b855a0c80..ba7e7ad42a 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -13,4 +13,4 @@ kind: Kustomization images: - name: controller newName: ghcr.io/konpyutaika/docker-images/nifikop - newTag: 1.11.3-master + newTag: 1.11.4-master diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index c85f98bc8f..bf9ae6900b 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -70,7 +70,7 @@ spec: - /manager args: - --leader-elect - image: ghcr.io/konpyutaika/docker-images/nifikop:v1.11.3-release + image: ghcr.io/konpyutaika/docker-images/nifikop:v1.11.4-release name: nifikop securityContext: allowPrivilegeEscalation: false diff --git a/config/samples/keycloak-example/step-1/operator.yaml b/config/samples/keycloak-example/step-1/operator.yaml index 4ea92d3bc7..ac39f7d915 100644 --- a/config/samples/keycloak-example/step-1/operator.yaml +++ b/config/samples/keycloak-example/step-1/operator.yaml @@ -1,4 +1,4 @@ -# nifikop 1.11.3 +# nifikop 1.11.4 rbacEnable: true namespaces: - nifi diff --git a/helm/nifi-cluster/Chart.yaml b/helm/nifi-cluster/Chart.yaml index f4753e917c..e794514b8a 100644 --- a/helm/nifi-cluster/Chart.yaml +++ b/helm/nifi-cluster/Chart.yaml @@ -30,7 +30,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 1.11.3 +version: 1.11.4 # This is the NiFi version to be deployed appVersion: "1.28.0" diff --git a/helm/nifi-cluster/README.md b/helm/nifi-cluster/README.md index 32cbf24226..632111fe0c 100644 --- a/helm/nifi-cluster/README.md +++ b/helm/nifi-cluster/README.md @@ -1,6 +1,6 @@ # nifi-cluster -![Version: 1.11.3](https://img.shields.io/badge/Version-1.11.3-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.28.0](https://img.shields.io/badge/AppVersion-1.28.0-informational?style=flat-square) +![Version: 1.11.4](https://img.shields.io/badge/Version-1.11.4-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.28.0](https://img.shields.io/badge/AppVersion-1.28.0-informational?style=flat-square) A Helm chart for deploying NiFi clusters in Kubernetes diff --git a/helm/nifikop/Chart.yaml b/helm/nifikop/Chart.yaml index 0cce48f51f..e93d4d060d 100644 --- a/helm/nifikop/Chart.yaml +++ b/helm/nifikop/Chart.yaml @@ -4,8 +4,8 @@ name: nifikop home: https://github.com/konpyutaika/nifikop sources: - https://github.com/konpyutaika/nifikop -version: 1.11.3 -appVersion: 1.11.3-release +version: 1.11.4 +appVersion: 1.11.4-release icon: https://konpyutaika.github.io/nifikop/img/nifikop.png maintainers: - name: erdrix diff --git a/helm/nifikop/README.md b/helm/nifikop/README.md index 09c69ef59c..6fe3a0f44b 100644 --- a/helm/nifikop/README.md +++ b/helm/nifikop/README.md @@ -23,7 +23,7 @@ The following tables lists the configurable parameters of the NiFi Operator Helm | Parameter | Description | Default | | -------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |--------------------------| | `image.repository` | Image | `konpyutaika/nifikop` | -| `image.tag` | Image tag | `v1.11.3-release` | +| `image.tag` | Image tag | `v1.11.4-release` | | `image.pullPolicy` | Image pull policy | `Always` | | `image.imagePullSecrets.enabled` | Enable the use of secret for docker image | `false` | | `image.imagePullSecrets.name` | Name of the secret to connect to docker registry | - | diff --git a/helm/nifikop/values.yaml b/helm/nifikop/values.yaml index 5703431f8f..5df21e3528 100644 --- a/helm/nifikop/values.yaml +++ b/helm/nifikop/values.yaml @@ -2,7 +2,7 @@ ## image: repository: ghcr.io/konpyutaika/docker-images/nifikop - tag: v1.11.3-release + tag: v1.11.4-release pullPolicy: Always imagePullSecrets: enabled: false diff --git a/site/docs/2_deploy_nifikop/1_quick_start.md b/site/docs/2_deploy_nifikop/1_quick_start.md index d656d7a5a3..18617e579e 100644 --- a/site/docs/2_deploy_nifikop/1_quick_start.md +++ b/site/docs/2_deploy_nifikop/1_quick_start.md @@ -144,8 +144,8 @@ Now deploy the helm chart: helm install nifikop \ oci://ghcr.io/konpyutaika/helm-charts/nifikop \ --namespace=nifi \ - --version 1.11.3 \ - --set image.tag=v1.11.3-release \ + --version 1.11.4 \ + --set image.tag=v1.11.4-release \ --set resources.requests.memory=256Mi \ --set resources.requests.cpu=250m \ --set resources.limits.memory=256Mi \ diff --git a/site/docs/2_deploy_nifikop/2_customizable_install_with_helm.md b/site/docs/2_deploy_nifikop/2_customizable_install_with_helm.md index 2775984e31..6ea53dfc80 100644 --- a/site/docs/2_deploy_nifikop/2_customizable_install_with_helm.md +++ b/site/docs/2_deploy_nifikop/2_customizable_install_with_helm.md @@ -35,7 +35,7 @@ The following tables lists the configurable parameters of the NiFi Operator Helm | Parameter | Description | Default | |----------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------| | `image.repository` | Image | `ghcr.io/konpyutaika/docker-images/nifikop` | -| `image.tag` | Image tag | `v1.11.3-release` | +| `image.tag` | Image tag | `v1.11.4-release` | | `image.pullPolicy` | Image pull policy | `Always` | | `image.imagePullSecrets.enabled` | Enable tue use of secret for docker image | `false` | | `image.imagePullSecrets.name` | Name of the secret to connect to docker registry | - | diff --git a/site/website/versioned_docs/version-v1.11.4/1_concepts/1_start_here.md b/site/website/versioned_docs/version-v1.11.4/1_concepts/1_start_here.md new file mode 100644 index 0000000000..1121eeb65f --- /dev/null +++ b/site/website/versioned_docs/version-v1.11.4/1_concepts/1_start_here.md @@ -0,0 +1,34 @@ +--- +id: 1_start_here +title: Start here +sidebar_label: Start here +--- + +The Konpyūtāika NiFi operator is a Kubernetes operator to automate provisioning, management, autoscaling and operations of [Apache NiFi](https://nifi.apache.org/) clusters deployed to K8s. + +## Overview + +Apache NiFi is an open-source solution that support powerful and scalable directed graphs of data routing, transformation, and system mediation logic. +Some of the high-level capabilities and objectives of Apache NiFi include, and some of the main features of the **NiFiKop** are: + +- **Fine grained** node configuration support +- Graceful rolling upgrade +- graceful NiFi cluster **scaling** +- NiFi cluster **auto-scaling** +- Encrypted communication using SSL +- the provisioning of secure NiFi clusters +- Advanced Dataflow and user management via CRD + +Some of the roadmap features: + +- Automatic reaction and self healing based on alerts (plugin system, with meaningful default alert plugins) +- graceful NiFi cluster **rebalancing** + +## Motivation + +There are already some approaches to operating NiFi on Kubernetes, however, we did not find them appropriate for use in a highly dynamic environment, nor capable of meeting our needs. + +- [Helm chart](https://github.com/cetic/helm-nifi) +- [Cloudera Nifi Operator](https://blog.cloudera.com/cloudera-flow-management-goes-cloud-native-with-apache-nifi-on-red-hat-openshift-kubernetes-platform/) + +Finally, our motivation is to build an open source solution and a community which drives the innovation and features of this operator. \ No newline at end of file diff --git a/site/website/versioned_docs/version-v1.11.4/1_concepts/2_design_principles.md b/site/website/versioned_docs/version-v1.11.4/1_concepts/2_design_principles.md new file mode 100644 index 0000000000..ed7e36b801 --- /dev/null +++ b/site/website/versioned_docs/version-v1.11.4/1_concepts/2_design_principles.md @@ -0,0 +1,53 @@ +--- +id: 2_design_principles +title: Design Principles +sidebar_label: Design Principles +--- + +This operator is built following the logic implied by the [operator sdk framework](https://sdk.operatorframework.io/). +What we want to offer with NiFiKop is that we provide as much automation as possible to manage NiFi at scale and in the most automated way possible. + +## Separation of concerns + +Kubernetes is designed for automation. Right out of the box, the Kubernetes core has a lot of automation features built in. You can use Kubernetes to automate the deployment and execution of workloads, and you can automate how Kubernetes does it. + +The Kubernetes operator model concept allows you to extend cluster behavior without changing the Kubernetes code itself by binding controllers to one or more custom resources. Operators are clients of the Kubernetes API that act as controllers for a custom resource. + +There are two things we can think of when we talk about operators in Kubernetes: + +- Automate the deployment of my entire stack. +- Automate the actions required by the deployment. + +For NiFiKop, we focus primarily on NiFi for the stack concept, what does that mean? + +- We do not manage other components that can be integrated with NiFi Cluster like Prometheus, Zookeeper, NiFi registry etc. +- We want to provide as many tools as possible to automate the work on NiFi (cluster deployment, data flow and user management, etc.). + +We consider that for NiFiKop to be a low-level operator, focused on NiFi and only NiFi, and if we want to manage a complex stack with e.g. Zookeeper, NiFi Registry, Prometheus etc. we need something else working at a higher level, like Helm charts or another operator controlling NiFiKop and other resources. + +## One controller per resource + +The operator should reflect as much as possible the behavior of the solution we want to automate. If we take our case with NiFi, what we can say is that: + +- You can have one or more NiFi clusters +- On your cluster you can define a NiFi registry client, but it is not mandatory. +- You can also define users and groups and deploy a DataFlow if you want. + +This means that your cluster is not defined by what is deployed on it, and what you deploy on it does not depend on it. +To be more explicit, just because I deploy a NiFi cluster doesn't mean the DataFlow deployed on it will stay there, we can move the DataFlow from one cluster to another. + +To manage this, we need to create different kinds of resources ([NifiCluster], [NifiDataflow], [NifiParameterContext], [NifiUser], [NifiUserGroup], [NifiRegistryClient], [NifiNodeGroupAutoscaler], [NifiConnection]) with one controller per resource that will manage its own resource. +In this way, we can say: + +- I deploy a NiFiCluster +- I define a NiFiDataflow that will be deployed on this cluster, and if I want to change cluster, I just have to change the `ClusterRef` to change cluster + + +[NifiCluster]: ../5_references/1_nifi_cluster +[NifiDataflow]: ../5_references/5_nifi_dataflow +[NifiParameterContext]: ../5_references/4_nifi_parameter_context +[NifiUser]: ../5_references/2_nifi_user +[NifiUserGroup]: ../5_references/6_nifi_usergroup +[NifiRegistryClient]: ../5_references/3_nifi_registry_client +[NifiNodeGroupAutoscaler]: ../5_references/7_nifi_nodegroup_autoscaler +[NifiConnection]: ../5_references/8_nifi_connection \ No newline at end of file diff --git a/site/website/versioned_docs/version-v1.11.4/1_concepts/3_features.md b/site/website/versioned_docs/version-v1.11.4/1_concepts/3_features.md new file mode 100644 index 0000000000..612c36afa5 --- /dev/null +++ b/site/website/versioned_docs/version-v1.11.4/1_concepts/3_features.md @@ -0,0 +1,67 @@ +--- +id: 3_features +title: Features +sidebar_label: Features +--- + +To highligt some of the features we needed and were not possible with the operators available, please keep reading + +## Fine Grained Node Config Support + +We needed to be able to react to events in a fine-grained way for each Node - and not in the limited way StatefulSet does (which, for example, removes the most recently created Nodes). Some of the available solutions try to overcome these deficits by placing scripts inside the container to generate configs at runtime (a good example is our [Cassandra Operator](https://github.com/Orange-OpenSource/casskop)), whereas the Orange NiFi operator's configurations are deterministically placed in specific Configmaps. + +## Graceful NiFi Cluster Scaling + +Apache NiFi is a good candidate to create an operator, because everything is made to orchestrate it through REST Api calls. With this comes automation of actions such as scaling, following all required steps: https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#decommission-nodes. + +## Graceful Rolling Upgrade + +Operator supports graceful rolling upgrade. It means the operator will check if the cluster is healthy. + +## Dynamic Configuration Support + +NiFi operates with two type of configs: + +- Read-only +- PerNode + +Read only config requires node restart to update all the others may be updated dynamically. +Operator CRD distinguishes these fields, and proceed with the right action. It can be a rolling upgrade, or +a dynamic reconfiguration. + +## Dataflow lifecycle management via CRD + +In a cloud native approach, we are looking for important management features, which we have applied to NiFi Dataflow: + +- **Automated deployment:** Based on the NiFi registry, you can describe your `NiFiDataflow` resource that will be deployed and run on the targeted NiFi cluster. +- **Portability:** On kubernetes everything is a yaml file, so with NiFiKop we give you the ability to describe your clusters but also the `registry clients`, `parameter contexts` and `dataflows` of your NiFi application, so that you can redeploy the same thing in a different namespace or cluster. +- **State management:** With NiFiKop resources, you can describe what you want, and the operator deals with the NiFi Rest API to make sure the resource stays in sync (even if someone manually makes changes directly on NiFi cluster). +- **Configurations:** Based on the `Parameter Contexts`, NiFiKop allows you to associate to your `Dataflow` (= your applications) with a different configuration depending on the environment ! + +## Users and access policies management + +Without the management of users and access policies associated, it was not possible to have a fully automated NiFi cluster setup due to: + +- **Node scaling:** when a new node joins the cluster it needs to have some roles like `proxy user request`, `view data` etc., by managing users and access policies we can easily create a user for this node with the right accesses. +- **Operator admin rights:** For the operator to manage efficiently the cluster it needs a lot of rights as `deploying process groups`, `empty the queues` etc., these rights are not available by default when you set a user as [InitialAdmin](https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#initial-admin-identity). Once again by giving the ability to define users and access policies we go through this. +- **User's access:** as seen just below we need to define the operator as `InitialAdmin`, in this situation there is no more users that can access to the web UI to manually give access to other users. That's why we extend the `InitialAdmin` concept into the operator, giving the ability to define a list of users as admins. + +In addition to these requirements to have a fully automated and managed cluster, we introduced some useful features: + +- **User management:** using `NifiUser` resource, you are able to create (or bind an existing) user in NiFi cluster and apply some access policies that will be managed and continuously synced by the operator. +- **Group management:** using `NifiUserGroup` resource, you can create groups in NiFi cluster and apply access policies and a list of `NifiUser` that will be managed and continuously synced by the operator. +- **Default group:** As the definition of `NifiUser` and `NifiUserGroup` resources could be heavy for some simple use cases, we also decided to define two default groups that you can feed with a list of users that will be created and managed by the operator (no kubernetes resources to create): + - **Admins:** a group giving access to everything on the NiFi Cluster, + - **Readers:** a group giving access as viewer on the NiFi Cluster. + +By introducing this feature we are giving you the ability to fully automate your deployment, from the NiFi Cluster to your managed NiFi Dataflow. + +## Automatic horizontal NiFi cluster scaling via CRD + +NiFiKop supports automatically horizontally scaling `NifiCluster` node groups with a `NifiNodeGroupAutoscaler` custom resource. + +- **Kubernetes native:** The `NifiNodeGroupAutoscaler` controller implements the [Kubernetes scale subresource](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#scale-subresource) and creates a Kubernetes [`HorizontalPodAutoscaler`](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) to automatically scale the pods that NiFiKop creates for `NifiCluster` deployments. +- **Metrics-driven autoscaling:** The `HorizontalPodAutoscaler` can be driven by pod usage metrics (e.g. RAM, CPU) or through NiFi application metrics scraped by Prometheus. +- **Flexible NifiCluster node group autoscaling:** The `NifiNodeGroupAutoscaler` scales specific node groups in your `NifiCluster` and you may have as many autoscalers as you like per `NifiCluster` deployment. For example, a `NifiNodeGroupAutoscaler` may manage high-memory or high-cpu sets of nodes for volume burst scenarios or it may manage every node in your cluster. + +Through this set of features, you may elect to have NiFiKop configure automatic horizontal autoscaling for any subset of nodes in your `NifiCluster` deployment. \ No newline at end of file diff --git a/site/website/versioned_docs/version-v1.11.4/1_concepts/4_roadmap.md b/site/website/versioned_docs/version-v1.11.4/1_concepts/4_roadmap.md new file mode 100644 index 0000000000..637a0c6cff --- /dev/null +++ b/site/website/versioned_docs/version-v1.11.4/1_concepts/4_roadmap.md @@ -0,0 +1,93 @@ +--- +id: 4_roadmap +title: Roadmap +sidebar_label: Roadmap +--- + +## Available + +### NiFi cluster installation + +| | | +| --------------------- | --------- | +| Status | Done | +| Priority | High | +| Targeted Start date | Jan 2020 | + +### Graceful NiFi Cluster Scaling + +| | | +| --------------------- | --------- | +| Status | Done | +| Priority | High | +| Targeted Start date | Jan 2020 | + +Apache NiFi is a good candidate to create an operator, because everything is made to orchestrate it through REST Api calls. With this comes automation of actions such as scaling, following all required steps: https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#decommission-nodes. + +### Communication via SSL + +| | | +| --------------------- | -------- | +| Status | Done | +| Priority | High | +| Targeted Start date | May 2020 | + + +The operator fully automates NiFi's SSL support. +The operator can provision the required secrets and certificates for you, or you can provide your own. + +### Dataflow lifecycle management via CRD + +| | | +| --------------------- | --------- | +| Status | Done | +| Priority | High | +| Targeted Start date | Aug 2020 | + +### Users & access policies management + +| | | +| --------------------- | ----- | +| Status | Done| +| Priority | High | +| Targeted Start date | November 2020 | + +The operator fully automates NiFi's user and access policies management. + +## Backlog + +### Monitoring via Prometheus + +| | | +| --------------------- | -------- | +| Status | To Do | +| Priority | High | +| Targeted Start date | Oct 2020 | + +The NiFi operator exposes NiFi JMX metrics to Prometheus. + +### Auto scaling + +| | | +| --------------------- |-----------| +| Status | To Do | +| Priority | High | +| Targeted Start date | Sept 2022 | + +Enable the NiFi cluster to be scaled by HPA kubernetes feature: + +- upscale cluster (add a new Node) +- downscale cluster (remove a Node) + +### Seamless Istio mesh support + +| | | +| --------------------- | ----- | +| Status | To Do | +| Priority | Low | +| Targeted Start date | - | + +- Operator allows to use ClusterIP services instead of Headless, which still works better in case of Service meshes. +- To avoid too early nifi initialization, which might lead to unready sidecar container. The operator will use a small script to +mitigate this behaviour. All NiFi image can be used the only one requirement is an available **wget** command. +- To access a NiFi cluster which runs inside the mesh. Operator will supports creating Istio ingress gateways. \ No newline at end of file diff --git a/site/website/versioned_docs/version-v1.11.4/2_deploy_nifikop/1_quick_start.md b/site/website/versioned_docs/version-v1.11.4/2_deploy_nifikop/1_quick_start.md new file mode 100644 index 0000000000..18617e579e --- /dev/null +++ b/site/website/versioned_docs/version-v1.11.4/2_deploy_nifikop/1_quick_start.md @@ -0,0 +1,181 @@ +--- +id: 1_quick_start +title: Quick start +sidebar_label: Quick start +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +For information about versions compatibilty of the operator features with Kubernetes and Apache NiFi, let's have look of the [version compatibility page](../4_compatibility_versions) + +## Getting Started + +### Cluster Setup + +For local testing we recommend following one of the following setup guides: + +- [Docker Desktop (Mac)](https://docs.docker.com/desktop/kubernetes) +- [Minikube](https://minikube.sigs.k8s.io/docs/start) + :::note + Start Minikube with at least 4gb RAM with `minikube start --memory=4000` + ::: +- [Kind](https://kind.sigs.k8s.io/docs/user/quick-start/) +- For testing on GKE you can [create a cluster with the command line or the Cloud Console UI](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-zonal-cluster). +- For testing on EKS you can [install eksctl](https://eksctl.io/introduction/) and run `eksctl create cluster` to create an EKS cluster/VPC/subnets/etc. This process should take 10-15 minutes. + +### Install kubectl + +If you do not already have the CLI tool `kubectl` installed, please follow [these instructions to install](https://kubernetes.io/docs/tasks/tools/). + +### Configure kubectl + +Configure `kubectl` to connect to your cluster by using `kubectl config use-context my-cluster-name`. + +- For GKE + - Configure `gcloud` with `gcloud auth login`. + - On the Google Cloud Console, the cluster page will have a `Connect` button, which will give a command to run locally that looks like + ```console + gcloud container clusters get-credentials CLUSTER_NAME --zone ZONE_NAME --project PROJECT_NAME. + ``` + - Use `kubectl config get-contexts` to show the contexts available. + - Run `kubectl config use-context ${gke context}` to access the cluster from `kubectl`. +- For EKS + - [Configure your AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html) to connect to your project. + - Install [eksctl](https://eksctl.io/introduction/) + - Run `eksctl utils write-kubeconfig --cluster=${CLUSTER NAME}` to make the context available to `kubectl` + - Use `kubectl config get-context`s to show the contexts available. + - Run `kubectl config use-context ${eks context}` to access the cluster with `kubectl`. + +## Install cert-manager + +The NiFiKop operator uses `cert-manager` for issuing certificates to users and and nodes, so you'll need to have it setup in case you want to deploy a secured cluster with authentication enabled. The minimum supported cert-manager version is v1.0. + + + + +```bash +# Install the CustomResourceDefinitions and cert-manager itself +kubectl apply -f \ + https://github.com/jetstack/cert-manager/releases/download/v1.7.2/cert-manager.yaml +``` + + + + +```bash +# Install CustomResourceDefinitions first +kubectl apply --validate=false -f \ + https://github.com/jetstack/cert-manager/releases/download/v1.7.2/cert-manager.crds.yaml + +# Add the jetstack helm repo +helm repo add jetstack https://charts.jetstack.io +helm repo update + +# You have to create the namespace before executing following command +helm install cert-manager \ + --namespace cert-manager \ + --version v1.7.2 jetstack/cert-manager +``` + + + + +## Deploy NiFiKop + +You can deploy the operator using a Helm chart [Helm chart](https://github.com/konpyutaika/nifikop/tree/master/helm): + +> To install an other version of the operator use `helm install --name=nifikop --namespace=nifi --set operator.image.tag=x.y.z konpyutaika-incubator/nifikop` + +In the case where you don't want to deploy the crds using helm (`--skip-crds`), you have to deploy manually the crds: + +```bash +kubectl apply -f https://raw.githubusercontent.com/konpyutaika/nifikop/master/config/crd/bases/nifi.konpyutaika.com_nificlusters.yaml +kubectl apply -f https://raw.githubusercontent.com/konpyutaika/nifikop/master/config/crd/bases/nifi.konpyutaika.com_nifiusers.yaml +kubectl apply -f https://raw.githubusercontent.com/konpyutaika/nifikop/master/config/crd/bases/nifi.konpyutaika.com_nifiusergroups.yaml +kubectl apply -f https://raw.githubusercontent.com/konpyutaika/nifikop/master/config/crd/bases/nifi.konpyutaika.com_nifidataflows.yaml +kubectl apply -f https://raw.githubusercontent.com/konpyutaika/nifikop/master/config/crd/bases/nifi.konpyutaika.com_nifiparametercontexts.yaml +kubectl apply -f https://raw.githubusercontent.com/konpyutaika/nifikop/master/config/crd/bases/nifi.konpyutaika.com_nifiregistryclients.yaml +kubectl apply -f https://raw.githubusercontent.com/konpyutaika/nifikop/master/config/crd/bases/nifi.konpyutaika.com_nifinodegroupautoscalers.yaml +kubectl apply -f https://raw.githubusercontent.com/konpyutaika/nifikop/master/config/crd/bases/nifi.konpyutaika.com_nificonnections.yaml +``` + +:::important Conversion webhook +In case you keep the conversion webhook enabled (to handle the conversion of resources from `v1alpha1` to `v1`) +you will need to add the following settings to your yaml definition of CRDs: + +```yaml +... +annotations: + cert-manager.io/inject-ca-from: ${namespace}/${certificate_name} +... +spec: + ... + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: ${namespace} + name: ${webhook_service_name} + path: /convert + conversionReviewVersions: + - v1 + - v1alpha1 + ... +``` + +Where: +- `namespace`: is the namespace in which you will deploy your helm chart. +- `certificate_name`: is `${helm release name}-webhook-cert` +- `webhook_service_name`: is `${helm release name}-webhook-cert` +::: + +Now deploy the helm chart: + +```bash +# You have to create the namespace before executing following command +helm install nifikop \ + oci://ghcr.io/konpyutaika/helm-charts/nifikop \ + --namespace=nifi \ + --version 1.11.4 \ + --set image.tag=v1.11.4-release \ + --set resources.requests.memory=256Mi \ + --set resources.requests.cpu=250m \ + --set resources.limits.memory=256Mi \ + --set resources.limits.cpu=250m \ + --set namespaces={"nifi"} +``` + + +:::note +Add the following parameter if you are using this instance to only deploy unsecured clusters: `--set certManager.enabled=false` +::: + +### On OpenShift +The restricted SCC according to the DOC from OpenShift: "Denies access to all host features and requires pods to be run with a UID, and SELinux context that are allocated to the namespace." So in order to deploy NiFiKop on OpenShift we need to get the openshift.io/sa.scc.uid-range annotation of the namespace that we will deploy NiFiKop into. + +Get the uid for the nifi namespace: +```bash +uid=$(kubectl get namespace nifi -o=jsonpath='{.metadata.annotations.openshift\.io/sa\.scc\.supplemental-groups}' | sed 's/\/10000$//' | tr -d '[:space:]') +``` +Set RunAsUser on install with helm: +```bash +helm install nifikop \ + nifikop \ + --namespace=nifi \ + --version 1.1.1 \ + --set image.tag=v1.1.1-release \ + --set resources.requests.memory=256Mi \ + --set resources.requests.cpu=250m \ + --set resources.limits.memory=256Mi \ + --set resources.limits.cpu=250m \ + --set namespaces={"nifi"} \ + --set runAsUser=$uid +``` diff --git a/site/website/versioned_docs/version-v1.11.4/2_deploy_nifikop/2_customizable_install_with_helm.md b/site/website/versioned_docs/version-v1.11.4/2_deploy_nifikop/2_customizable_install_with_helm.md new file mode 100644 index 0000000000..6ea53dfc80 --- /dev/null +++ b/site/website/versioned_docs/version-v1.11.4/2_deploy_nifikop/2_customizable_install_with_helm.md @@ -0,0 +1,237 @@ +--- +id: 2_customizable_install_with_helm +title: Customizable install with Helm +sidebar_label: Customizable install with Helm +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +## Prerequisites + +- Perform any necessary [plateform-specific setup](./1_quick_start) +- [Install a Helm client](https://github.com/helm/helm#install) with a version higher than 3 + +## Introduction + +This Helm chart install NiFiKop the Nifi Kubernetes operator to create/configure/manage NiFi +clusters in a Kubernetes Namespace. + +It will use Custom Ressources Definition CRDs: + +- `nificlusters.nifi.konpyutaika.com`, +- `nifiusers.nifi.konpyutaika.com`, +- `nifiusergroups.nifi.konpyutaika.com`, +- `nifiregistryclients.nifi.konpyutaika.com`, +- `nifiparametercontexts.nifi.konpyutaika.com`, +- `nifidataflows.nifi.konpyutaika.com`, +- `nifinodegroupautoscalers.nifi.konpyutaika.com`, +- `nificonnections.nifi.konpyutaika.com`, + +### Configuration + +The following tables lists the configurable parameters of the NiFi Operator Helm chart and their default values. + +| Parameter | Description | Default | +|----------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------| +| `image.repository` | Image | `ghcr.io/konpyutaika/docker-images/nifikop` | +| `image.tag` | Image tag | `v1.11.4-release` | +| `image.pullPolicy` | Image pull policy | `Always` | +| `image.imagePullSecrets.enabled` | Enable tue use of secret for docker image | `false` | +| `image.imagePullSecrets.name` | Name of the secret to connect to docker registry | - | +| `certManager.enabled` | Enable cert-manager integration | `true` | +| `rbacEnable` | If true, create & use RBAC resources | `true` | +| `labels` | Labels to add to all deployed resources | `{}` | +| `annotations` | Annotations to add to all deployed resources | `{}` | +| `resources` | Pod resource requests & limits | `{}` | +| `metrics.enabled` | deploy service for metrics | `false` | +| `metrics.port` | Set port for operator metrics | `8081` | +| `logLevel` | Log level to output | `Info` | +| `logEncoding` | Log encoding to use. Either `json` or `console` | `json` | +| `certManager.clusterScoped` | If true setup cluster scoped resources | `false` | +| `namespaces` | List of namespaces where Operator watches for custom resources. Make sure the operator ServiceAccount is granted `get` permissions on this `Node` resource when using limited RBACs. | `""` i.e. all namespaces | +| `nodeSelector` | Node selector configuration for operator pod | `{}` | +| `affinity` | Node affinity configuration for operator pod | `{}` | +| `tolerations` | Toleration configuration for operator pod | `{}` | +| `serviceAccount.create` | Whether the SA creation is delegated to the chart or not | `true` | +| `serviceAccount.name` | Name of the SA used for NiFiKop deployment | release name | +| `webhook.enabled` | Enable webhook migration | `true` | +| `runAsUser` | Specify RunAsUser uid for NiFiKop operator pod | `1000` | + +Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example, + +Alternatively, a YAML file that specifies the values for the above parameters can be provided while installing the chart. For example, + +```console +$ helm install nifikop \ + konpyutaika/nifikop \ + -f values.yaml +``` + +### Installing the Chart + +:::important Skip CRDs +In the case where you don't want to deploy the crds using helm (`--skip-crds`) you need to deploy manually the crds beforehand: + +```bash +kubectl apply -f https://raw.githubusercontent.com/konpyutaika/nifikop/master/config/crd/bases/nifi.konpyutaika.com_nificlusters.yaml +kubectl apply -f https://raw.githubusercontent.com/konpyutaika/nifikop/master/config/crd/bases/nifi.konpyutaika.com_nifiusers.yaml +kubectl apply -f https://raw.githubusercontent.com/konpyutaika/nifikop/master/config/crd/bases/nifi.konpyutaika.com_nifiusergroups.yaml +kubectl apply -f https://raw.githubusercontent.com/konpyutaika/nifikop/master/config/crd/bases/nifi.konpyutaika.com_nifidataflows.yaml +kubectl apply -f https://raw.githubusercontent.com/konpyutaika/nifikop/master/config/crd/bases/nifi.konpyutaika.com_nifiparametercontexts.yaml +kubectl apply -f https://raw.githubusercontent.com/konpyutaika/nifikop/master/config/crd/bases/nifi.konpyutaika.com_nifiregistryclients.yaml +kubectl apply -f https://raw.githubusercontent.com/konpyutaika/nifikop/master/config/crd/bases/nifi.konpyutaika.com_nifinodegroupautoscalers.yaml +kubectl apply -f https://raw.githubusercontent.com/konpyutaika/nifikop/master/config/crd/bases/nifi.konpyutaika.com_nificonnections.yaml +``` + +::: + +:::important Conversion webhook +In case you keep the conversions webhook enabled (to handle the conversion of resources from `v1alpha1` to `v1`) +you will need to add the following settings to your yaml definition of CRDs: + +```yaml +... +annotations: + cert-manager.io/inject-ca-from: ${namespace}/${certificate_name} +... +spec: + ... + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: ${namespace} + name: ${webhook_service_name} + path: /convert + conversionReviewVersions: + - v1 + - v1alpha1 + ... +``` + +Where: +- `namespace`: is the namespace in which you will deploy your helm chart. +- `certificate_name`: is `${helm release name}-webhook-cert` +- `webhook_service_name`: is `${helm release name}-webhook-cert` +::: + + + + +```bash +helm install nifikop konpyutaika/nifikop \ + --dry-run \ + --set logLevel=Debug \ + --set namespaces={"nifikop"} +``` + + + + +```bash +helm install konpyutaika/nifikop +``` + + + + + +```bash +helm install nifikop konpyutaika/nifikop --set namespaces={"nifikop"} +``` + + + + +> the `--replace` flag allow you to reuses a charts release name + +### Listing deployed charts + +```bash +helm list +``` + +### Get Status for the helm deployment + +```bash +helm status nifikop +``` + +## Uninstaling the Charts + +If you want to delete the operator from your Kubernetes cluster, the operator deployment +should be deleted. + +```bash +helm del nifikop +``` + +The command removes all the Kubernetes components associated with the chart and deletes the helm release. + +:::tip +The CRD created by the chart are not removed by default and should be manually cleaned up (if required) +::: + +Manually delete the CRD: + +```bash +kubectl delete crd nificlusters.nifi.konpyutaika.com +kubectl delete crd nifiusers.nifi.konpyutaika.com +kubectl delete crd nifiusergroups.nifi.konpyutaika.com +kubectl delete crd nifiregistryclients.nifi.konpyutaika.com +kubectl delete crd nifiparametercontexts.nifi.konpyutaika.com +kubectl delete crd nifidataflows.nifi.konpyutaika.com +``` + +:::warning +If you delete the CRD then +It will delete **ALL** Clusters that has been created using this CRD!!! +Please never delete a CRD without very good care +::: + +Helm always keeps records of what releases happened. Need to see the deleted releases ? + +```bash +helm list --deleted +``` + +Need to see all of the releases (deleted and currently deployed, as well as releases that +failed) ? + +```bash +helm list --all +``` + +Because Helm keeps records of deleted releases, a release name cannot be re-used. (If you really need to re-use a +release name, you can use the `--replace` flag, but it will simply re-use the existing release and replace its +resources.) + +Note that because releases are preserved in this way, you can rollback a deleted resource, and have it re-activate. + +To purge a release + +```bash +helm delete --purge nifikop +``` + +## Troubleshooting + +### Install of the CRD + +By default, the chart will install the CRDs, but this installation is global for the whole +cluster, and you may want to not modify the already deployed CRDs. + +In this case there is a parameter to say to not install the CRDs: + +``` +$ helm install --name nifikop ./helm/nifikop --set namespaces={"nifikop"} --skip-crds +``` diff --git a/site/website/versioned_docs/version-v1.11.4/2_deploy_nifikop/3_kubectl_plugin.md b/site/website/versioned_docs/version-v1.11.4/2_deploy_nifikop/3_kubectl_plugin.md new file mode 100644 index 0000000000..ee5ac8286f --- /dev/null +++ b/site/website/versioned_docs/version-v1.11.4/2_deploy_nifikop/3_kubectl_plugin.md @@ -0,0 +1,34 @@ +--- +id: 3_kubectl_plugin +title: Kubectl Plugin +sidebar_label: Kubectl Plugin +--- + +You can build the plugin and copy the exectuable into your PATH. + +For example on a UNIX machine: + +```console +make kubectl-nifikop && sudo cp ./bin/kubectl-nifikop /usr/local/bin +``` + +Then you can test the plugin: + +```console +$ kubectl nifikop +Usage: + nifikop [command] + +Available Commands: + completion Generate the autocompletion script for the specified shell + help Help about any command + nificluster + nificonnection + nifidataflow + nifigroupautoscaler + nifiregistryclient + nifiuser + nifiusergroup +``` + +Your NiFiKop plugin is now installed! \ No newline at end of file diff --git a/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/1_manage_clusters/0_design_principles.md b/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/1_manage_clusters/0_design_principles.md new file mode 100644 index 0000000000..5c58a0ef93 --- /dev/null +++ b/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/1_manage_clusters/0_design_principles.md @@ -0,0 +1,35 @@ +--- +id: 0_design_principles +title: Design Principles +sidebar_label: Design Principles +--- + +## Pod level management + +NiFi is a stateful application. The first piece of the puzzle is the Node, which is a simple server capable of createing/forming a cluster with other Nodes. Every Node has his own **unique** configuration which differs slightly from all others. + +All NiFi on Kubernetes setup use [StatefulSet](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/) to create a NiFi Cluster. Just to quickly recap from the K8s docs: + +>StatefulSet manages the deployment and scaling of a set of Pods, and provide guarantees about their ordering and uniqueness. Like a Deployment, a StatefulSet manages Pods that are based on an identical container spec. Unlike a Deployment, a StatefulSet maintains sticky identities for each of its Pods. These pods are created from the same spec, but are not interchangeable: each has a persistent identifier that is maintained across any rescheduling. + +How does this looks from the perspective of Apache NiFi ? + +With StatefulSet we get: +- unique Node IDs generated during Pod startup +- networking between Nodes with headless services +- unique Persistent Volumes for Nodes + +Using StatefulSet we **lose** the ability to: + +- modify the configuration of unique Nodes +- remove a specific Node from a cluster (StatefulSet always removes the most recently created Node) +- use multiple, different Persistent Volumes for each Node + +The NiFi Operator uses `simple` Pods, ConfigMaps, and PersistentVolumeClaims, instead of StatefulSet (based on the design used by [Banzai Cloud Kafka Operator](https://github.com/banzaicloud/kafka-operator)). +Using these resources allows us to build an Operator which is better suited to NiFi. + +With the NiFi operator we can: + +- modify the configuration of unique Nodes +- remove specific Nodes from clusters +- use multiple Persistent Volumes for each Node \ No newline at end of file diff --git a/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/1_manage_clusters/1_deploy_cluster/1_quick_start.md b/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/1_manage_clusters/1_deploy_cluster/1_quick_start.md new file mode 100644 index 0000000000..d0e702a968 --- /dev/null +++ b/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/1_manage_clusters/1_deploy_cluster/1_quick_start.md @@ -0,0 +1,155 @@ +--- +id: 1_quick_start +title: Quick start +sidebar_label: Quick start +--- + + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +## Create custom storage class + +We recommend to use a **custom StorageClass** to leverage the volume binding mode `WaitForFirstConsumer` + +```bash +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: exampleStorageclass +parameters: + type: pd-standard +provisioner: kubernetes.io/gce-pd +reclaimPolicy: Delete +volumeBindingMode: WaitForFirstConsumer +``` + +:::tip +Remember to set your NiFiCluster CR properly to use the newly created StorageClass. +::: + +## State management + +To manage its cluster and states, NiFi needs Zookeeper or rights on the Kubernetes cluster to manage `Leases` and `ConfigMaps` resources in the namespace where it is deployed. + +In the case of Zookeeper, you must first have a Zookeeper cluster if you don't already have one. +Otherwise, you need to provide the corresponding role to the NiFi cluster's `ServiceAccount`. + +> We believe in the `separation of concerns` principle, thus the NiFi operator does not install nor manage Zookeeper. + +### Installing Zookeeper + +To install Zookeeper we recommend using the [Bitnami's Zookeeper chart](https://github.com/bitnami/charts/tree/master/bitnami/zookeeper). + +```bash +helm install zookeeper oci://registry-1.docker.io/bitnamicharts/zookeeper \ + --namespace=zookeeper \ + --set resources.requests.memory=256Mi \ + --set resources.requests.cpu=250m \ + --set resources.limits.memory=256Mi \ + --set resources.limits.cpu=250m \ + --set global.storageClass=standard \ + --set networkPolicy.enabled=true \ + --set replicaCount=3 \ + --create-namespace +``` + +:::warning +Replace the `storageClass` parameter value with your own. +::: + +#### On OpenShift + +We need to get the uid/gid for the RunAsUser and the fsGroup for the namespace we deploy zookeeper in. + +Get the zookeeper allowed uid/gid. + +```bash +zookeper_uid=$(kubectl get namespace zookeeper -o=jsonpath='{.metadata.annotations.openshift\.io/sa\.scc\.supplemental-groups}' | sed 's/\/10000$//' | tr -d '[:space:]') +``` +Specify the runAsUser and fsGroup Parameter on install of zookeeper. + +```bash +helm install zookeeper oci://registry-1.docker.io/bitnamicharts/zookeeper \ + --set resources.requests.memory=256Mi \ + --set resources.requests.cpu=250m \ + --set resources.limits.memory=256Mi \ + --set resources.limits.cpu=250m \ + --set global.storageClass=standard \ + --set networkPolicy.enabled=true \ + --set replicaCount=3 \ + --set containerSecurityContext.runAsUser=$zookeper_uid \ + --set podSecurityContext.fsGroup=$zookeper_uid +``` + +### Enabling Kubernetes State Management + +When using native Kubernetes State Management from NiFi, you need to make sure that the `ServiceAccount` used by NiFi has the correct rights to manage the needed Kubernetes resources. + +```yaml +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: simplenifi + namespace: nifi +rules: +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["*"] +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["*"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: simplenifi + namespace: nifi +subjects: + - kind: ServiceAccount + name: default + namespace: nifi +roleRef: + kind: Role + name: simplenifi + apiGroup: rbac.authorization.k8s.io +``` + +:::info +In this case, you need to set `clusterManager` in `NiFiCluster`'s specification to `kubernetes`. +You can also use the Helm chart to create your cluster and it will take care of it for you. +::: + +## Deploy NiFi cluster + +And after you can deploy a simple NiFi cluster. + +```bash +# Add your zookeeper svc name to the configuration +kubectl create -n nifi -f config/samples/simplenificluster.yaml +``` + +### On OpenShift + +```bash +# Add your zookeeper svc name to the configuration +kubectl create -n nifi -f config/samples/simplenificluster.yaml +### On OpenShift + +We need to get the uid/gid for the RunAsUser and the fsGroup for the namespace we deploy our nificluster in. + +```bash +uid=$(kubectl get namespace nifi -o=jsonpath='{.metadata.annotations.openshift\.io/sa\.scc\.supplemental-groups}' | sed 's/\/10000$//' | tr -d '[:space:]') +``` + +Then update the config/samples/openshift file with our uid value. + +```bash +sed -i "s/1000690000/$uid/g" config/samples/openshift.yaml +``` + +And after you can deploy a simple NiFi cluster. + +```bash +kubectl create -n nifi -f config/samples/openshift.yaml +``` \ No newline at end of file diff --git a/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/1_manage_clusters/1_deploy_cluster/2_nodes_configuration.md b/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/1_manage_clusters/1_deploy_cluster/2_nodes_configuration.md new file mode 100644 index 0000000000..b0935d3844 --- /dev/null +++ b/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/1_manage_clusters/1_deploy_cluster/2_nodes_configuration.md @@ -0,0 +1,402 @@ +--- +id: 2_nodes_configuration +title: Nodes configuration +sidebar_label: Nodes configuration +--- + +In the quick start section, you deployed a simple `NifiCluster` resource, which deploys a NiFi cluster. But in many cases, you may need to tune the cluster nodes to match your needs. +In this section, we'll try to cover the various things that can be specified for cluster node configuration. + +## ReadOnlyConfig and NodeConfigGroups + +To set up your `NiFi` cluster with `NiFiKop`, the first thing to understand is the difference between `readOnlyConfig` and `nodeConfigGroups`. + +### Configure multiple node groups + +In NiFiKop you can define different types of nodes using the `Spec.NodeConfigGroups` field. This field allows you to define as many node configurations as you want. +Once a `NodeConfigGroup` has been defined, you can define it with your node declaration to say "I want to add a new node with this type of configuration". + +The main purpose of a [NodeConfigGroup] is to define the purely technical requirements for the pod that will be deployed (storage configurations, resource requirements, docker image, pod location, etc). + +For example, you can have this node group configuration: + +```yaml + # nodeConfigGroups specifies multiple node configs with unique name + nodeConfigGroups: + default_group: + # provenanceStorage allow to specify the maximum amount of data provenance information to store at a time + # https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#write-ahead-provenance-repository-properties + provenanceStorage: "10 GB" + #RunAsUser define the id of the user to run in the Nifi image + # +kubebuilder:validation:Minimum=1 + runAsUser: 1000 + serviceAccountName: "default" + # resourceRequirements works exactly like Container resources, the user can specify the limit and the requests + # through this property + # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ + resourcesRequirements: + limits: + cpu: "2" + memory: 3Gi + requests: + cpu: "1" + memory: 3Gi + high_mem_group: + # provenanceStorage allow to specify the maximum amount of data provenance information to store at a time + # https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#write-ahead-provenance-repository-properties + provenanceStorage: "10 GB" + #RunAsUser define the id of the user to run in the Nifi image + # +kubebuilder:validation:Minimum=1 + runAsUser: 1000 + serviceAccountName: "default" + # resourceRequirements works exactly like Container resources, the user can specify the limit and the requests + # through this property + # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ + resourcesRequirements: + limits: + cpu: "2" + memory: 30Gi + requests: + cpu: "1" + memory: 30Gi +``` + +In this example, we have defined two different node configurations: +- `default_group`: Saying that we want **3Gi** of RAM for a node using this configuration. +- `high_mem_group`: Saying that we want **30Gi** of RAM for a node using this configuration. + +- Now we can declare the nodes of our cluster using this configuration: + +```yaml +- id: 0 + nodeConfigGroup: "default_group" +- id: 2 + nodeConfigGroup: "high_mem_group" +- id: 3 + nodeConfigGroup: "high_mem_group" +- id: 5 + nodeConfig: + resourcesRequirements: + limits: + cpu: "2" + memory: 3Gi + requests: + cpu: "1" + memory: 1Gi +``` + +In this example, you can see that we have defined one node using the node configuration `default_group` (id = 0), 2 nodes using `high_mem_group` (id = 2 and 3) and you also have the possibility to define the node configuration group directly at the node level (not reusable by another node) like for node 5. + +#### Storage management + +One of the most important configurations for a node in the case of a NiFi cluster is data persistence. As we run on kubernetes, whenever a pod is deleted, all data that is not stored on persistent storage will be lost, which is really bad for a statefull application like NiFi. +To avoid this, you can define how and what you want to persist in NiFi in the [NodeConfigGroup]. + +##### Data persistence + +The first way to define data persistence is to use the [Spec.NodeConfigGroup.StorageConfig](../../../5_references/1_nifi_cluster/3_node_config#storageconfig) field. + +This field allows you to define a storage set giving: +- `name`: a unique name to identify the storage config +- `metadata`: labels and annotations to attach to the PVC getting created. +- `pvcSpec`: a Kubernetes PVC spec definition +- `mountPath`: the path where the volume will be mounted into the main nifi container inside the pod (i.e the path were you want the data to be persisted). + +:::note +If you don't replace them in the `nifi.properties` file using [NiFi-configurations](#nifi-configurations), here is a list of paths that should be associated with a storage configuration: +- `/opt/nifi/data`: contains + - `/opt/nifi/data/flow.xml.gz`: flow configuration files + - `/opt/nifi/data/archive`: NiFi archive + - `/opt/nifi/data/templates`: templates directory + - `/opt/nifi/data/database_repository`: Database persistence +- `/opt/nifi/nifi-current/logs`: NiFi logs files +- `/opt/nifi/flowfile_repository`: flowfiles repository +- `/opt/nifi/content_repository`: NiFi content repository +- `/opt/nifi/provenance_repository`: NiFi provenance repository +- `/opt/nifi/nifi-current/conf`: NiFi configurations +::: + +Here is an example we use in production for to persist data: + +```yaml +... +storageConfigs: + - mountPath: /opt/nifi/nifi-current/logs + name: logs + reclaimPolicy: Delete + metadata: + labels: + my-label: my-value + annotations: + my-annotation: my-value + pvcSpec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 100Gi + storageClassName: ssd-wait + - mountPath: /opt/nifi/data + name: data + reclaimPolicy: Delete + metadata: + labels: + my-label: my-value + annotations: + my-annotation: my-value + pvcSpec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Gi + storageClassName: ssd-wait + - mountPath: /opt/nifi/extensions + name: extensions-repository + reclaimPolicy: Delete + metadata: + labels: + my-label: my-value + annotations: + my-annotation: my-value + pvcSpec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi + storageClassName: ssd-wait + - mountPath: /opt/nifi/flowfile_repository + name: flowfile-repository + reclaimPolicy: Delete + metadata: + labels: + my-label: my-value + annotations: + my-annotation: my-value + pvcSpec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 100Gi + storageClassName: ssd-wait + - mountPath: /opt/nifi/nifi-current/conf + name: conf + reclaimPolicy: Delete + metadata: + labels: + my-label: my-value + annotations: + my-annotation: my-value + pvcSpec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi + storageClassName: ssd-wait + - mountPath: /opt/nifi/content_repository + name: content-repository + reclaimPolicy: Delete + metadata: + labels: + my-label: my-value + annotations: + my-annotation: my-value + pvcSpec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 500Gi + storageClassName: ssd-wait + - mountPath: /opt/nifi/provenance_repository + name: provenance-repository + reclaimPolicy: Delete + metadata: + labels: + my-label: my-value + annotations: + my-annotation: my-value + pvcSpec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 500Gi +``` + +##### External volumes + +In some cases, you may want to mount a volume that is not managed by the operator to add a configuration or persist data using an existing volume. To do this, use the [Spec.NodeConfigGroup.StorageConfig](../../../5_references/1_nifi_cluster/3_node_config#externalvolumeconfig) field. + +:::info +For a complete overview of node configuration possibilities, please refer to the reference page [NodeConfigGroup] +::: + +## NiFi configurations + +Once you have correctly defined the pods that will be deployed for your NiFi cluster, you may still have some configuration to do but at the NiFi level this time! +For this, the field to configure is [ReadOnlyConfig] which can be used at the global level `Spec.ReadOnlyConfig` or at the node level like for `NodeConfigGroup`. + +There is some configuration that can be passed directly into this field like: +- **maximumTimerDrivenThreadCount**: define the maximum number of threads for timer driven processors available to the system. +- **maximumEventDrivenThreadCount**: define the maximum number of threads for event driven processors available to the system (@DEPRECATED. This has no effect from NiFiKOp v1.9.0 or later). + +And other configurations (e.g. configuration files) that can be defined using `kubernetes Secret`, `ConfigMap` or directly using the `override` field. + +### ConfigMap, Secret, Inline + +For most of the configuration files that can be overwritten (see section below), there are 4 ways to define the configuration: +- `default`: if nothing is specified, a default configuration is defined by the operator and will be used as is. +- `kubernetes secret`: you reference a data key in a secret that will contain your configuration (used to define sensitive configurations like client secret, password, etc.) +- `kubernetes configMap`: you reference a data key in a configMap that will contain your configuration. +- `override field`: you define the configuration directly in the `NiFiCluster` object using a string. + +When more than one configuration type is defined, the following priority is applied when the same configuration field is defined more than once: `Secret` > `ConfigMap` > `Override` > `Default`, which follow the security priority. + +Let's take an example: + +```yaml + nifiProperties: + # Additionnal nifi.properties configuration that will override the one produced based on template and + # configuration + overrideConfigMap: + # The key of the value,in data content, that we want use. + data: nifi.properties + # Name of the configmap that we want to refer. + name: raw + # Namespace where is located the secret that we want to refer. + namespace: nifikop + # Additionnal nifi.properties configuration that will override the one produced based + # on template, configurations, overrideConfigMap and overrideConfigs. + overrideSecretConfig: + # The key of the value,in data content, that we want use. + data: nifi.properties + # Name of the configmap that we want to refer. + name: raw + # Namespace where is located the secret that we want to refer. + namespace: nifikop + # Additionnal nifi.properties configuration that will override the one produced based + # on template, configurations and overrideConfigMap + overrideConfigs: | + nifi.ui.banner.text=NiFiKop + nifi.sensitive.props.key=thisIsABadSensitiveKeyPassword +``` + +In this example if we have the `nifi.sensitive.props.key` key defined in the secret `raw`, it will override the one defined in `overrideConfigs` field. + +### Overridable configurations + +Here is the list of configuration that you can override for NiFi: +- [nifi.properties](https://github.com/konpyutaika/nifikop/blob/master/pkg/resources/templates/config/nifi_properties.go) +- [zookeeper.properties](https://github.com/konpyutaika/nifikop/blob/master/pkg/resources/templates/config/zookeeper_properties.go) +- [bootstrap.conf](https://github.com/konpyutaika/nifikop/blob/master/pkg/resources/templates/config/bootstrap_conf.go) +- [logback.xml](https://github.com/konpyutaika/nifikop/blob/master/pkg/resources/templates/config/logback.xml.go) +- [authorizers.xml](https://github.com/konpyutaika/nifikop/blob/master/pkg/resources/templates/config/authorizers.go) +- [bootstrap_notification_services.xml](https://github.com/konpyutaika/nifikop/blob/master/pkg/resources/templates/config/bootstrap_notifications_services.go) + +:::warning +Keep in mind that some changes to the default configuration may cause the operator's behavior to break, so keep that in mind! +Just because it's allowed doesn't mean it works 😄. +::: + +## Advanced configuration + +In some cases, using the default content or provenance configuration for storage may not be sufficient, for example you may need to create multiple directories for your content or provenance repository in order to [set up a high performance installation](https://community.cloudera.com/t5/Community-Articles/HDF-CFM-NIFI-Best-practices-for-setting-up-a-high/ta-p/244999). +As described in the NiFi Administration Guide, you can do this by using the [nifi.content.repository.directory.default*](https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#file-system-content-repository-properties) and [nifi.provenance.repository.directory.default*](https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#write-ahead-provenance-repository-properties) properties. + +Here is an example of how to do this in the `NiFiCluster` configuration: + +```yaml +... + readOnlyConfig: + nifiProperties: + overrideConfigs: | + nifi.content.repository.directory.dir1=../content-additional/dir1 + nifi.content.repository.directory.dir2=../content-additional/dir2 + nifi.content.repository.directory.dir3=../content-additional/dir3 + nifi.provenance.repository.directory.dir1=../provenance-additional/dir1 + nifi.provenance.repository.directory.dir2=../provenance-additional/dir2 +... + nodeConfigGroups: + default_group: + ... + storageConfigs: + - mountPath: "/opt/nifi/content-additional/dir1" + name: content-repository-dir1 + metadata: + labels: + my-label: my-value + annotations: + my-annotation: my-value + pvcSpec: + accessModes: + - ReadWriteOnce + storageClassName: {{ storageClassName }} + resources: + requests: + storage: 100G + - mountPath: "/opt/nifi/content-additional/dir2" + name: content-repository-dir2 + metadata: + labels: + my-label: my-value + annotations: + my-annotation: my-value + pvcSpec: + accessModes: + - ReadWriteOnce + storageClassName: {{ storageClassName }} + resources: + requests: + storage: 100G + - mountPath: "/opt/nifi/content-additional/dir3" + name: content-repository-dir3 + metadata: + labels: + my-label: my-value + annotations: + my-annotation: my-value + pvcSpec: + accessModes: + - ReadWriteOnce + storageClassName: {{ storageClassName }} + resources: + requests: + storage: 100G + - mountPath: "/opt/nifi/provenance-additional/dir1" + name: provenance-repository-dir1 + metadata: + labels: + my-label: my-value + annotations: + my-annotation: my-value + pvcSpec: + accessModes: + - ReadWriteOnce + storageClassName: {{ storageClassName }} + resources: + requests: + storage: 100G + - mountPath: "/opt/nifi/provenance-additional/dir2" + name: provenance-repository-dir2 + metadata: + labels: + my-label: my-value + annotations: + my-annotation: my-value + pvcSpec: + accessModes: + - ReadWriteOnce + storageClassName: {{ storageClassName }} + resources: + requests: + storage: 100G + ... +``` + + +[NodeConfigGroup]: ../../../5_references/1_nifi_cluster/3_node_config +[ReadOnlyConfig]: ../../../5_references/1_nifi_cluster/2_read_only_config \ No newline at end of file diff --git a/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/1_manage_clusters/1_deploy_cluster/3_expose_cluster/1_kubernetes_service.md b/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/1_manage_clusters/1_deploy_cluster/3_expose_cluster/1_kubernetes_service.md new file mode 100644 index 0000000000..c732638ff8 --- /dev/null +++ b/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/1_manage_clusters/1_deploy_cluster/3_expose_cluster/1_kubernetes_service.md @@ -0,0 +1,156 @@ +--- +id: 1_kubernetes_service +title: Kubernetes service +sidebar_label: Kubernetes service +--- + +The purpose of this section is to explain how expose your NiFi cluster and access it in and out of kubernetes. + +## Listener configuration + +The first part to manage when you are configuring your cluster is the ports that will be used for the internal need of NiFi, we will call them `internal listeners` +There is 6 types of `internal listeners`: +- **cluster**: Define the node’s protocol port (used by cluster nodes to discuss together). +- **http/https**: The HTTP or HTTPS port used to expose NiFi cluster UI (**Note**: use only one of them !) +- **s2s**: The remote input socket port for Site-to-Site communication +- **load-balance**: Cluster node load balancing port +- **prometheus**: Port that will be used to expose the promotheus reporting task + +To configure these listeners you must use the [Spec.ListernersConfig.InternalListeners](../../../../5_references/1_nifi_cluster/6_listeners_config#internallistener) field: + +```yaml +listenersConfig: + internalListeners: + - type: "https" + name: "https" + containerPort: 8443 + - type: "cluster" + name: "cluster" + containerPort: 6007 + - type: "s2s" + name: "s2s" + containerPort: 10000 + - type: "prometheus" + name: "prometheus" + containerPort: 9090 + - type: "load-balance" + name: "load-balance" + containerPort: 6342 +``` + +Here we defined a listener by specifying: +- `type`: one of the six described above (e.g `https`) +- `name`: name of the port that will be attached to pod (e.g `https`) +- `containerPort`: port that will be used by the container inside the pod and configured in NiFi configuration for the listener (e.g `8443`) + +If you look at the yaml description of the deployed pod, you should have something like this: + +```yaml + ports: + - containerPort: 8443 + name: https + protocol: TCP + - containerPort: 6007 + name: cluster + protocol: TCP + - containerPort: 6342 + name: load-balance + protocol: TCP + - containerPort: 10000 + name: s2s + protocol: TCP + - containerPort: 9090 + name: prometheus + protocol: TCP +``` + +:::important +Really important thing, you can add additional `internal listeners` without type, it means that these listeners are not related to internal NiFi behaviour. +It might be really useful, if you are exposing a NiFi processor through a port (e.g http endpoint to receive HTTP request inside of NiFi): + +```yaml +listenersConfig: + internalListeners: + ... + - name: "http-tracking" + containerPort: 8081 +``` +::: + +## Headless vs All service mode + +To internally expose the NiFi cluster nodes, there is one major constraint which is that each node must be accessible one by one by the controller and the other nodes! + +To do this, a single traditional Kubernetes service will not suffice, as it will balance the traffic between all nodes, which is not what we want. + +There are two ways to solve this problem: +- **Use a [headless service](https://kubernetes.io/docs/concepts/services-networking/service/#headless-services)**: this is the most appropriate way to expose your nodes internally, using this method you will simply deploy a service that will allow you to access each pod one by one via DNS resolution. +- **Use one service per node**: this way we create one service per node, giving you one cluster IP per node to access a single node directly! + +To configure how you want to expose your NiFi node internally, you simply set the `Spec.HeadlessEnabled` field, if true you will be in the first mode, if not in the second. + +:::note +In most cases, the `headless mode` should be used. An example where you need the other mode would be integration with Istio service mesh, which does not support headless service integration. +::: + + +## External service configuration + +Once you have considered how to expose your service internally in the k8s cluster, you may want to expose your cluster externally, for example to give access to your cluster to your users, or to expose your prometheus endpoint. +To configure the external exposure of your cluster pods, you should use the [Spec.ExternalServices](../../../../5_references/1_nifi_cluster/7_external_service_config) field. + +It takes as configuration: +- `name`: which will be used to name the kubernetes service. +- `spec`: + - `type`: how the service is exposed (following the definition of [ServiceType](https://godoc.org/k8s.io/api/core/v1#ServiceType)) + - `portConfigs`: a list of port configurations: + - `port`: the port that will be used by the service to expose the associated `internal listener`. + - `internalListernerName`: name of the `internal listener` to expose + +If we take a concrete example: + +```yaml +listenersConfig: + internalListeners: + - type: "https" + name: "https" + containerPort: 8443 + - type: "cluster" + name: "cluster" + containerPort: 6007 + - type: "s2s" + name: "s2s" + containerPort: 10000 + - type: "prometheus" + name: "prometheus" + containerPort: 9090 + - type: "load-balance" + name: "load-balance" + containerPort: 6342 + - name: "http-tracking" + containerPort: 8081 +externalServices: + - name: cluster-access + spec: + portConfigs: + - internalListenerName: https + port: 443 + - internalListenerName: http-tracking + port: 80 + type: LoadBalancer +``` + +Here, we expose our `internal listeners`: `https` and `http-tracking` through a Kubernetes service: `cluster-access`, with two different ports: `443` and `80`. +If you look at the services, you should see something like this. + +```console +kubectl get services +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +cluster-access LoadBalancer 10.88.21.98 35.180.241.132 443:30421/TCP,80:30521/TCP 20d +``` + +If you add the `external ip` in your `Spec.ReadOnlyConfig.NifiProperties.WebProxyHosts` field, you will be able to access your cluster by: `https://` and your NiFi processor http endpoint by: `http://`. + +:::note +For any additional configuration please refer to [this page](../../../../5_references/1_nifi_cluster/7_external_service_config). +::: diff --git a/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/1_manage_clusters/1_deploy_cluster/3_expose_cluster/2_istio_service_mesh.md b/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/1_manage_clusters/1_deploy_cluster/3_expose_cluster/2_istio_service_mesh.md new file mode 100644 index 0000000000..c881b311f9 --- /dev/null +++ b/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/1_manage_clusters/1_deploy_cluster/3_expose_cluster/2_istio_service_mesh.md @@ -0,0 +1,137 @@ +--- +id: 2_istio_service_mesh +title: Istio service mesh +sidebar_label: Istio service mesh +--- + +The purpose of this section is to explain how to expose your NiFi cluster using Istio on Kubernetes. + +## Istio configuration for HTTP + +To access to the NiFi cluster from the external world, it is needed to configure a `Gateway` and a `VirtualService` on Istio. + +The following example show how to define a `Gateway` that will be able to intercept all requests for a specific domain host on HTTP port 80. + +```yaml +apiVersion: networking.istio.io/v1beta1 +kind: Gateway +metadata: + name: nifi-gateway +spec: + selector: + istio: ingressgateway + servers: + - port: + number: 80 + name: http + protocol: HTTP + hosts: + - nifi.my-domain.com +``` + +In combination, we need to define also a `VirtualService` that redirect all requests itercepted by the `Gateway` to a specific service. + +```yaml +apiVersion: networking.istio.io/v1beta1 +kind: VirtualService +metadata: + name: nifi-vs +spec: + gateways: + - nifi-gateway + hosts: + - nifi.my-domain.com + http: + - match: + - uri: + prefix: / + route: + - destination: + host: nifi + port: + number: 8080 +``` + +## Istio configuration for HTTPS + +In case you are deploying a cluster and you want to enable the HTTPS protocol to manage user authentication, the configuration is more complex. To understand the reason of this, it is important to explain that in this scenario the HTTPS protocol with all certificates is managed directly by NiFi. This means that all requests passes through all Istio services in an encrypted way, so Istio can't manage a sticky session. +To solve this issue, the tricky is to limit the HTTPS session till the `Gateway`, then decrypt all requests in HTTP, manage the sticky session and then encrypt again in HTTPS before reach the NiFi node. +Istio allows to do this using a destination rule combined with the `VirtualService`. In the next example, we define a `Gateway` that accepts HTTPS traffic and transform it in HTTP. + +```yaml +apiVersion: networking.istio.io/v1beta1 +kind: Gateway +metadata: + name: nifi-gateway +spec: + selector: + istio: ingressgateway + servers: + - port: + number: 443 + name: https + protocol: HTTPS + tls: + mode: SIMPLE + credentialName: my-secret + hosts: + - nifi.my-domain.com +``` + +In combination, we need to define also a `VirtualService` that redirect all HTTP traffic to a specific the `ClusterIP` Service. + +```yaml +apiVersion: networking.istio.io/v1beta1 +kind: VirtualService +metadata: + name: nifi-vs +spec: + gateways: + - nifi-gateway + hosts: + - nifi.my-domain.com + http: + - match: + - uri: + prefix: / + route: + - destination: + host: ..svc.cluster.local + port: + number: 8443 +``` + +Please note that the service name configured as destination of the `VirtualService` is the name of the Service created with the following section in the cluster Deployment YAML + +```yaml +spec: + externalServices: + - name: "nifi-cluster" + spec: + type: ClusterIP + portConfigs: + - port: 8443 + internalListenerName: "https" +``` + +Finally the destination rule that redirect all HTTP traffic destinated to the `ClusterIP` Service to HTTPS encrypting it. + +```yaml +apiVersion: networking.istio.io/v1beta1 +kind: DestinationRule +metadata: + name: nifi-dr +spec: + host: ..svc.cluster.local + trafficPolicy: + tls: + mode: SIMPLE + loadBalancer: + consistentHash: + httpCookie: + name: __Secure-Authorization-Bearer + ttl: 0s +``` + +As you can see in the previous example, the destination rule also define how manage the sticky session based on httpCookie property called __Secure-Authorization-Bearer. + diff --git a/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/1_manage_clusters/1_deploy_cluster/4_ssl_configuration.md b/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/1_manage_clusters/1_deploy_cluster/4_ssl_configuration.md new file mode 100644 index 0000000000..ae767f0dc8 --- /dev/null +++ b/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/1_manage_clusters/1_deploy_cluster/4_ssl_configuration.md @@ -0,0 +1,161 @@ +--- +id: 4_ssl_configuration +title: SSL configuration +sidebar_label: SSL configuration +--- + +The `NiFi operator` makes securing your NiFi cluster with SSL easy. You may provide your own certificates, or instruct the operator to create them for you from your cluster configuration. + +Below this is an example configuration required to secure your cluster with SSL: + +```yaml +apiVersion: nifi.konpyutaika.com/v1 +kind: NifiCluster +... +spec: + ... + readOnlyConfig: + # NifiProperties configuration that will be applied to the node. + nifiProperties: + webProxyHosts: + - nifistandard2.trycatchlearn.fr:8443 + ... + ... + listenersConfig: + internalListeners: + - type: "https" + name: "https" + containerPort: 8443 + - type: "cluster" + name: "cluster" + containerPort: 6007 + - type: "s2s" + name: "s2s" + containerPort: 10000 + sslSecrets: + tlsSecretName: "test-nifikop" + create: true +``` + +- `readOnlyConfig.nifiProperties.webProxyHosts`: A list of allowed HTTP Host header values to consider when NiFi is running securely and will be receiving requests to a different host[:port] than it is bound to. [web-properties](https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#web-properties) + +If `listenersConfig.sslSecrets.create` is set to `false`, the operator will look for the secret at `listenersConfig.sslSecrets.tlsSecretName` and expect these values: + +| key | value | +|-----|-------| +| caCert | The CA certificate | +| caKey | The CA private key | +| clientCert | A client certificate (this will be used by operator for NiFI operations) | +| clientKey | The private key for clientCert | + +## Using an existing Issuer + +As described in the [Reference section](../../../5_references/1_nifi_cluster/6_listeners_config.md#sslsecrets), instead of using a self-signed certificate as CA, you can use an existing one. +In order to do so, you only have to refer it into your `Spec.ListenerConfig.SslSecrets.IssuerRef` field. + +:::warning +When using [cert-manager Issuer](https://cert-manager.io/docs/concepts/issuer/), please make sure that the hostname +(default `clusterName-id-node.namespace.svc.cluster.local`) for the nodes in cluster is 64 bytes or less, otherwise the webhook +of cert-manager will fail. You can try to use shorter name for NiFiCluster or modify `nodeUserIdentityTemplate` to keep +the name length under 64 bytes. +::: + +### Example: Let's Encrypt + +Let's say you have an existing DNS server, with [external dns](https://github.com/kubernetes-sigs/external-dns) deployed into your cluster's namespace. +You can easily use [Let's Encrypt](https://letsencrypt.org/) as an authority for your certificate. + +To do this, you have to: + +1. Create an issuer: + +```yaml +apiVersion: cert-manager.io/v1alpha2 +kind: Issuer +metadata: + name: letsencrypt-staging +spec: + acme: + # You must replace this email address with your own. + # Let's Encrypt will use this to contact you about expiring + # certificates, and issues related to your account. + email: + server: https://acme-staging-v02.api.letsencrypt.org/directory + privateKeySecretRef: + # Secret resource used to store the account's private key. + name: example-issuer-account-key + # Add a single challenge solver, HTTP01 using nginx + solvers: + - http01: + ingress: + ingressTemplate: + metadata: + annotations: + "external-dns.alpha.kubernetes.io/ttl": "5" +``` + +2. Setup External dns and correctly create your issuer into your cluster configuration: + +```yaml +apiVersion: nifi.konpyutaika.com/v1 +kind: NifiCluster +... +spec: + ... + clusterSecure: true + siteToSiteSecure: true + ... + listenersConfig: + clusterDomain: + useExternalDNS: true + ... + sslSecrets: + tlsSecretName: "test-nifikop" + create: true + issuerRef: + name: letsencrypt-staging + kind: Issuer +``` + +## Create SSL credentials + +You may use `NifiUser` resource to create new user certificates for your applications, allowing them to authenticate and query your Nifi cluster. + +To create a new client you will need to generate new certificates sign by the CA. The operator can automate this for you using the `NifiUser` CRD: + +```console +cat << EOF | kubectl apply -n nifi -f - +apiVersion: nifi.konpyutaika.com/v1 +kind: NifiUser +metadata: + name: example-client + namespace: nifi +spec: + clusterRef: + name: nifi + secretName: example-client-secret +EOF +``` + +This will create a user and store its credentials in the secret `example-client-secret`. The secret contains these fields: + +| key | value | +|-----|-------| +| ca.crt | The CA certificate | +| tls.crt | The user certificate | +| tls.key | The user private key | + +You can then mount these secret to your pod. Alternatively, you can write them to your local machine by running: + +```console +kubectl get secret example-client-secret -o jsonpath="{['data']['ca\.crt']}" | base64 -d > ca.crt +kubectl get secret example-client-secret -o jsonpath="{['data']['tls\.crt']}" | base64 -d > tls.crt +kubectl get secret example-client-secret -o jsonpath="{['data']['tls\.key']}" | base64 -d > tls.key +``` + +The operator can also include a Java keystore format (JKS) with your user secret if you'd like. Add `includeJKS`: `true` to the `spec` like shown above, and then the user-secret will gain these additional fields: + +| key | value | +|-----|-------| +| tls.jks | The java keystore containing both the user keys and the CA (use this for your keystore AND truststore) | +| pass.txt | The password to decrypt the JKS file (this will be randomly generated) | diff --git a/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/1_manage_clusters/1_deploy_cluster/5_users_authentication/1_oidc.md b/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/1_manage_clusters/1_deploy_cluster/5_users_authentication/1_oidc.md new file mode 100644 index 0000000000..c7b296d0e6 --- /dev/null +++ b/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/1_manage_clusters/1_deploy_cluster/5_users_authentication/1_oidc.md @@ -0,0 +1,42 @@ +--- +id: 1_oidc +title: OpenId Connect +sidebar_label: OpenId Connect +--- + +To enable authentication via OpenId Connect refering to [NiFi Administration guide](https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html) required some configuration into `nifi.properties`. + +In addition and to ensure multiple identity provider support, we recommended to add the following configuration to your `nifi.properties`: + +```sh +nifi.security.identity.mapping.pattern.dn=CN=([^,]*)(?:, (?:O|OU)=.*)? +nifi.security.identity.mapping.value.dn=$1 +nifi.security.identity.mapping.transform.dn=NONE +``` + +To perform this with `NiFiKop` you just have to configure the `Spec.NifiProperties.OverrideConfigs` field with your OIDC configuration, for example: + +```yaml +apiVersion: nifi.konpyutaika.com/v1 +kind: NifiCluster +... +spec: + ... + readOnlyConfig: + # NifiProperties configuration that will be applied to the node. + nifiProperties: + webProxyHosts: + - nifistandard2.trycatchlearn.fr:8443 + # Additionnal nifi.properties configuration that will override the one produced based + # on template and configurations. + overrideConfigs: | + nifi.security.user.oidc.discovery.url= + nifi.security.user.oidc.client.id= + nifi.security.user.oidc.client.secret= + nifi.security.identity.mapping.pattern.dn=CN=([^,]*)(?:, (?:O|OU)=.*)? + nifi.security.identity.mapping.value.dn=$1 + nifi.security.identity.mapping.transform.dn=NONE + ... + ... +... +``` \ No newline at end of file diff --git a/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/1_manage_clusters/1_deploy_cluster/6_users_authorization/1_custom_user_authorizer.md b/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/1_manage_clusters/1_deploy_cluster/6_users_authorization/1_custom_user_authorizer.md new file mode 100644 index 0000000000..f905789d69 --- /dev/null +++ b/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/1_manage_clusters/1_deploy_cluster/6_users_authorization/1_custom_user_authorizer.md @@ -0,0 +1,83 @@ +--- +id: 1_custom_user_authorizer +title: Custom User Authorizers +sidebar_label: Custom User Authorizers +--- + +:::info +This is an advanced configuration topic. In most cases, the default NiFi authorizer configuration is sufficient. +::: + +According to the NiFi Admin Guide, an [Authorizer](https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#authorizer-configuration) grants users the privileges to manage users and policies by creating preliminary authorizations at startup. By default, the `StandardManagedAuthorizer` leverages a `FileUserGroupProvider` and a `FileAccessPolicyProvider` which are file-based rules for each user you allow to interact with your NiFi cluster. + +In many cases, the default authorizer configuration is enough to control access to a NiFi cluster. However, there may be advanced cases where the default [`managed-authorizer`](https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#standardmanagedauthorizer) isn't sufficient to make every authorization decision you need. In this case, you can provide a custom authorizer extension and use that instead. + +Suppose a custom Authorizer is written and deployed with NiFi that reads the rules from a remote database rather than a local file. We'll call this `DatabaseAuthorizer`. Also suppose it is composed of a `DatabaseUserGroupProvider` and a `DatabaseAccessPolicyProvider`. In order to leverage these, they must end up on NiFi's classpath. + +In order to use this authorizer, you need to update NiFi's `authorizers.xml` configuration. This can be done through NiFiKOp by setting either the `Spec.readOnlyConfig.authorizerConfig.replaceTemplateConfigMap` or `Spec.readOnlyConfig.authorizerConfig.replaceTemplateSecretConfig`. The NiFiKOp deployment is dynamic in that node identities are determined at deploy time, so the authorizer configuration is templated to account for this. This means that the replacement ConfigMap or Secret must also be templated. + +Following the example, the below would be a sufficient authorizer template replacement: + +```yaml +{{- $nodeList := .NodeList }} +{{- $clusterName := .ClusterName }} +{{- $namespace := .Namespace }} + + + file-user-group-provider + org.apache.nifi.authorization.FileUserGroupProvider + ../data/users.xml + + {{ .ControllerUser }} +{{- range $i, $host := .NodeList }} + {{ $host }} +{{- end }} + + + database-user-group-provider + my.custom.DatabaseUserGroupProvider + +{{- range $i, $host := .NodeList }} + {{ $host }} +{{- end }} + + + file-access-policy-provider + org.apache.nifi.authorization.FileAccessPolicyProvider + file-user-group-provider + ../data/authorizations.xml + {{ .ControllerUser }} + +{{- range $i, $host := .NodeList }} + {{ $host }} +{{- end }} + + + + database-access-policy-provider + my.custom.DatabaseAccessPolicyProvider + +{{- range $i, $host := .NodeList }} + {{ $host }} +{{- end }} + + + + managed-authorizer + org.apache.nifi.authorization.StandardManagedAuthorizer + file-access-policy-provider + + + custom-database-authorizer + my.custom.DatabaseAuthorizer + database-access-policy-provider + + +``` + +And finally, the NiFi property `nifi.security.user.authorizer` indicates which of the configured authorizers in the authorizers.xml file to use. Following the example, we'd set the property to: + +```sh +nifi.security.user.authorizer=custom-database-authorizer +``` + diff --git a/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/1_manage_clusters/2_cluster_scaling/1_scaling_mechanism.md b/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/1_manage_clusters/2_cluster_scaling/1_scaling_mechanism.md new file mode 100644 index 0000000000..6642a005a7 --- /dev/null +++ b/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/1_manage_clusters/2_cluster_scaling/1_scaling_mechanism.md @@ -0,0 +1,248 @@ +--- +id: 1_scaling_mechanism +title: Scaling mechanism +sidebar_label: Scaling mechanism +--- + +This tasks shows you how to perform a gracefull cluster scale up and scale down. + +## Before you begin + +- Setup NiFiKop by following the instructions in the [Installation guide](../../../2_deploy_nifikop/1_quick_start). +- Deploy the [Simple NiFi](../1_deploy_cluster/1_quick_start) sample cluster. +- Review the [Node](../../../5_references/1_nifi_cluster/4_node) references doc. + +## About this task + +The [Simple NiFi](../1_deploy_cluster/1_quick_start) example consists of a three nodes NiFi cluster. +A node decommission must follow a strict procedure, described in the [NiFi documentation](https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#decommission-nodes): + +1. Disconnect the node +2. Once disconnect completes, offload the node. +3. Once offload completes, delete the node. +4. Once the delete request has finished, stop/remove the NiFi service on the host. + + +For the moment, we have implemented it as follows in the operator: + +1. Disconnect the node +2. Once disconnect completes, offload the node. +3. Once offload completes, delete the pod. +4. Once the pod deletion completes, delete the node. +5. Once the delete request has finished, remove the node from the NifiCluster status. + +In addition, we have a regular check that ensure that all nodes have been removed. + +In this task, you will first perform a scale up, in adding an new node. Then, you will remove another node that the one created, and observe the decommission's steps. + +## Scale up: Add a new node + +For this task, we will simply add a node with the same configuration than the other ones, if you want to know more about how to add a node with an other configuration let's have a look to the [Node configuration](../1_deploy_cluster/2_nodes_configuration) documentation page. + +1. Add and run a dataflow as the example: + +![Scaling dataflow](/img/3_tasks/1_nifi_cluster/2_cluster_scaling/scaling_dataflow.png) + +2. Add a new node to the list of `NifiCluster.Spec.Nodes` field, by following the [Node object definition](../../../5_references/1_nifi_cluster/4_node) documentation: + +```yaml +apiVersion: nifi.konpyutaika.com/v1 +kind: NifiCluster +metadata: + name: simplenifi +spec: + service: + headlessEnabled: true + clusterManager: zookeeper + zkAddress: "zookeepercluster-client.zookeeper:2181" + zkPath: "/simplenifi" + clusterImage: "apache/nifi:1.12.1" + oneNifiNodePerNode: false + nodeConfigGroups: + default_group: + isNode: true + storageConfigs: + - mountPath: "/opt/nifi/nifi-current/logs" + name: logs + metadata: + labels: + my-label: my-value + annotations: + my-annotation: my-value + pvcSpec: + accessModes: + - ReadWriteOnce + storageClassName: "standard" + resources: + requests: + storage: 10Gi + serviceAccountName: "default" + resourcesRequirements: + limits: + cpu: "2" + memory: 3Gi + requests: + cpu: "1" + memory: 1Gi + nodes: + - id: 0 + nodeConfigGroup: "default_group" + - id: 1 + nodeConfigGroup: "default_group" + - id: 2 + nodeConfigGroup: "default_group" +# >>>> START: The new node + - id: 25 + nodeConfigGroup: "default_group" +# <<<< END + propagateLabels: true + nifiClusterTaskSpec: + retryDurationMinutes: 10 + listenersConfig: + internalListeners: + - type: "http" + name: "http" + containerPort: 8080 + - type: "cluster" + name: "cluster" + containerPort: 6007 + - type: "s2s" + name: "s2s" + containerPort: 10000 +``` + +:::important +**Note:** The `Node.Id` field must be unique in the `NifiCluster.Spec.Nodes` list. +::: + +3. Apply the new `NifiCluster` configuration: + +```sh +kubectl -n nifi apply -f config/samples/simplenificluster.yaml +``` + +4. You should now have the following resources into kubernetes: + +```console +kubectl get pods,configmap,pvc -l nodeId=25 +NAME READY STATUS RESTARTS AGE +pod/simplenifi-25-nodem5jh4 1/1 Running 0 11m + +NAME DATA AGE +configmap/simplenifi-config-25 7 11m + +NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE +persistentvolumeclaim/simplenifi-25-storagehwn24 Bound pvc-7da86076-728e-11ea-846d-42010a8400f2 10Gi RWO standard 11m +``` + +And if you go on the NiFi UI, in the cluster administration page: + +![Scale up, cluster list](/img/3_tasks/1_nifi_cluster/2_cluster_scaling/scaleup_cluster_list.png) + +5. You now have data on the new node: + +![Scale up, cluster distribution](/img/3_tasks/1_nifi_cluster/2_cluster_scaling/scaleup_distribution.png) + +## Scaledown: Gracefully remove node + +For this task, we will simply remove a node and look at that the decommissions steps. + +1. Remove the node from the list of `NifiCluster.Spec.Nodes` field: + +```yaml +apiVersion: nifi.konpyutaika.com/v1 +kind: NifiCluster +metadata: + name: simplenifi +spec: + headlessServiceEnabled: true + zkAddress: "zookeepercluster-client.zookeeper:2181" + zkPath: "/simplenifi" + clusterImage: "apache/nifi:1.11.3" + oneNifiNodePerNode: false + nodeConfigGroups: + default_group: + isNode: true + storageConfigs: + - mountPath: "/opt/nifi/nifi-current/logs" + name: logs + metadata: + labels: + my-label: my-value + annotations: + my-annotation: my-value + pvcSpec: + accessModes: + - ReadWriteOnce + storageClassName: "standard" + resources: + requests: + storage: 10Gi + serviceAccountName: "default" + resourcesRequirements: + limits: + cpu: "2" + memory: 3Gi + requests: + cpu: "1" + memory: 1Gi + nodes: + - id: 0 + nodeConfigGroup: "default_group" + - id: 1 + nodeConfigGroup: "default_group" +# >>>> START: node removed +# - id: 2 +# nodeConfigGroup: "default_group" +# <<<< END + - id: 25 + nodeConfigGroup: "default_group" + propagateLabels: true + nifiClusterTaskSpec: + retryDurationMinutes: 10 + listenersConfig: + internalListeners: + - type: "http" + name: "http" + containerPort: 8080 + - type: "cluster" + name: "cluster" + containerPort: 6007 + - type: "s2s" + name: "s2s" + containerPort: 10000 +``` + +2. Apply the new `NifiCluster` configuration: + +```sh +kubectl -n nifi apply -f config/samples/simplenificluster.yaml +``` + +3. You can follow the node's action step status in the `NifiCluster.Status` description: + +```console +kubectl describe nificluster simplenifi + +... +Status: + Nodes State: + ... + 2: + Configuration State: ConfigInSync + Graceful Action State: + Action State: GracefulDownscaleRequired + Error Message: + ... +... +``` + +:::tip +The list of decommisions step and their corresponding value for the `Nifi Cluster.Status.Node State.Graceful ActionState.ActionStep` field is described into the [Node State page](../../../5_references/1_nifi_cluster/5_node_state#actionstep) +::: + +4. Once the scaledown successfully performed, you should have the data offloaded on the other nodes, and the node state removed from the `NifiCluster.Status.NodesState` list: + +:::warning +Keep in mind that the [`NifiCluster.Spec.nifiClusterTaskSpec.retryDurationMinutes`](../../../5_references/1_nifi_cluster/1_nifi_cluster.md#nificlustertaskspec) should be long enough to perform the whole procedure, or you will have some rollback and retry loop. +::: \ No newline at end of file diff --git a/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/1_manage_clusters/2_cluster_scaling/2_auto_scaling/0_design_principles.md b/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/1_manage_clusters/2_cluster_scaling/2_auto_scaling/0_design_principles.md new file mode 100644 index 0000000000..2575452336 --- /dev/null +++ b/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/1_manage_clusters/2_cluster_scaling/2_auto_scaling/0_design_principles.md @@ -0,0 +1,44 @@ +--- +id: 0_design_principles +title: Design Principles +sidebar_label: Design Principles +--- + +:::info +These feature have been scpoed by the community, please find the discussion and technical scoping [here] (https://docs.google.com/document/d/1QNGSNNjWx4CGt5-NvX9ZArQMfyrwjw-B95f54GUNdB0/edit#heading=h.t9xh94v7viuj). +::: + +## Design reflexion + +If you read the technical scoping above, we explored many options for enabling automatic scaling of NiFi clusters. +After much discussion, it turned out that we wanted to mimic the approach and design behind auto-scaling a deployment with [HPA] (https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/). + +If we look at how this works, you define a `Deployment`, which will manage a `ReplicaSet` which will manage `Pods`. And you define your `HPA` which will manage the scale field of the `Deployment`. +For our `NiFiCluster` we considered the same kind of separation of concerns: we define a new resource `NifiNodeGroupAutoScaler` that manages the `NifiCluster` that will manage the `Pods`. And you define your `HPA` which will manage the scale field of the `Deployment`. + +This is the basis of the thinking. There was another inspiration for designing the functionality, which is that we wanted to keep the possibility of different types of node groups and manage them separately, so we pushed by thinking about similar existing models, and we thought about how in the Kubernetes Cloud Cluster (EKS, GKE etc.) nodes can be managed. +You can define fixed groups of nodes, you can auto-scale others. + +And finally, we wanted to separate the `NifiCluster` itself from the `autoscaling management` and allow mixing the two, allowing you to have a cluster initially with no scaling at all, add scaling from a subset of nodes with a given configuration, and finally disable autoscaling without any impact. + +## Implementation + +Referring to the official guideline, the recommended approach is to implement [the sub resource scale in the CRD](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#scale-subresource). + +This approach requires to define: +- `specReplicasPath` defines the JSONPath inside of a custom resource that corresponds to `scale.spec.replicas` +- `statusReplicasPath` defines the JSONPath inside of a custom resource that corresponds to `scale.status.replicas` +- `labelSelectorPath` defines the JSONPath inside of a custom resource that corresponds to `scale.Status.Selector` + +we add a new resource: [NifiNodeGroupAutoScaler](../../../../5_references/7_nifi_nodegroup_autoscaler), with the following fields: +- `spec.nifiClusterRef`: reference to the NiFi cluster resource that will be autoscaled +- `spec.nodeConfigGroupId`: reference to the nodeConfigGroup that will be used for nodes managed by the auto scaling. +- `spec.readOnlyConfig`: defines a readOnlyConfig to apply to each node in this node group. Any settings here will override those set in the configured `NifiCluster`. +- `spec.nodeConfig`: defines a nodeConfig to apply to each node in this node group. Any settings here will override those set in the configured `nodeConfigGroupId`. +- `spec.replicas`: current number of replicas expected for the node config group +- `spec.upscaleStrategy`: strategy used to upscale (simple) +- `spec.downscaleStrategy`: strategy used to downscale (lifo) + +Here is a representation of dependencies: + +![auto scaling schema](/img/auto_scaling.jpg) \ No newline at end of file diff --git a/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/1_manage_clusters/2_cluster_scaling/2_auto_scaling/1_using_keda.md b/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/1_manage_clusters/2_cluster_scaling/2_auto_scaling/1_using_keda.md new file mode 100644 index 0000000000..1597a82220 --- /dev/null +++ b/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/1_manage_clusters/2_cluster_scaling/2_auto_scaling/1_using_keda.md @@ -0,0 +1,443 @@ +--- +id: 1_using_keda +title: Using KEDA +sidebar_label: Using KEDA +--- + +## Deploy KDEA + +### What is KEDA ? + +[KEDA] is a Kubernetes-based Event Driven Autoscaler. With [KEDA], you can drive the scaling of any container in Kubernetes based on the number of events needing to be processed. + +[KEDA] is a single-purpose and lightweight component that can be added into any Kubernetes cluster. [KEDA] works alongside standard Kubernetes components like the Horizontal Pod Autoscaler and can extend functionality without overwriting or duplication. With [KEDA] you can explicitly map the apps you want to use event-driven scale, with other apps continuing to function. This makes [KEDA] a flexible and safe option to run alongside any number of any other Kubernetes applications or frameworks. + +[KEDA] can be a very powerful tool for integration with NiFi because we can auto-scale based on a service that your DataFlow will consume (e.g. PubSub, etc.) or with NiFi metrics exposed using Prometheus. + +### Deployment + +Following the [documentation](https://keda.sh/docs/2.8/deploy/) here are the steps to deploy KEDA. + +Deploying KEDA with Helm is very simple: + +- Add Helm repo + +````console +helm repo add kedacore https://kedacore.github.io/charts +```` + +- Update Helm repo + +````console +helm repo update +```` + +- Install keda Helm chart + +```console +kubectl create namespace keda +helm install keda kedacore/keda --namespace keda +``` + +[KEDA]: https://keda.sh/ + +### Deploy NiFI cluster + +Use your own NiFi cluster deployment, for this example we will add a specific `NodeConfigGroup` which will be used for auto-scaling nodes, and add the configuration for Prometheus: + +```yaml +... +spec: + ... + listenersConfig: + internalListeners: + ... + - containerPort: 9090 + name: prometheus + type: prometheus + ... + ... + nodeConfigGroups: + auto_scaling: + isNode: true + resourcesRequirements: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: "1" + memory: 1Gi + serviceAccountName: external-dns + storageConfigs: + - mountPath: /opt/nifi/nifi-current/logs + name: logs + metadata: + labels: + my-label: my-value + annotations: + my-annotation: my-value + pvcSpec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: ssd-wait + - mountPath: /opt/nifi/data + name: data + metadata: + labels: + my-label: my-value + annotations: + my-annotation: my-value + pvcSpec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: ssd-wait + - mountPath: /opt/nifi/extensions + name: extensions-repository + metadata: + labels: + my-label: my-value + annotations: + my-annotation: my-value + pvcSpec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: ssd-wait + - mountPath: /opt/nifi/flowfile_repository + name: flowfile-repository + metadata: + labels: + my-label: my-value + annotations: + my-annotation: my-value + pvcSpec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: ssd-wait + - mountPath: /opt/nifi/nifi-current/conf + name: conf + metadata: + labels: + my-label: my-value + annotations: + my-annotation: my-value + pvcSpec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: ssd-wait + - mountPath: /opt/nifi/content_repository + name: content-repository + metadata: + labels: + my-label: my-value + annotations: + my-annotation: my-value + pvcSpec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: ssd-wait + - mountPath: /opt/nifi/provenance_repository + name: provenance-repository + metadata: + labels: + my-label: my-value + annotations: + my-annotation: my-value + pvcSpec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: ssd-wait + ... +``` + +### Deploy NiFi cluster autoscaling group + +Now we will deploy a `NifiNodeGroupAutoscaler` to define how and what we want to autoscale: + +```yaml +apiVersion: nifi.konpyutaika.com/v1alpha1 +kind: NifiNodeGroupAutoscaler +metadata: + name: nifinodegroupautoscaler-sample + namespace: clusters +spec: + # contains the reference to the NifiCluster with the one the node group autoscaler is linked. + clusterRef: + name: cluster + namespace: clusters + # defines the id of the NodeConfig contained in NifiCluster.Spec.NodeConfigGroups + nodeConfigGroupId: auto_scaling + # readOnlyConfig can be used to pass Nifi node config + # which has type read-only these config changes will trigger rolling upgrade + readOnlyConfig: + nifiProperties: + overrideConfigs: | + nifi.ui.banner.text=NiFiKop - Scale Group + # This is an example of a node config you can apply to each replica in this node group. + # Any settings here will override those in the configured nodeConfigGroupId +# nodeConfig: +# nodeSelector: +# node_type: high-mem + # The selector used to identify nodes in NifiCluster.Spec.Nodes this autoscaler will manage + # Use Node.Labels in combination with this selector to clearly define which nodes will be managed by this autoscaler + nodeLabelsSelector: + matchLabels: + nifi_cr: cluster + nifi_node_group: auto-scaling + # the strategy used to decide how to add nodes to a nifi cluster + upscaleStrategy: simple + # the strategy used to decide how to remove nodes from an existing cluster + downscaleStrategy: lifo +``` + +Here we will autoscale using the `NodeConfigGroup`: auto_scaling. + +### Deploy Prometheus + +For this example, we will base the autoscaling on some metrics of NiFi cluster, to deploy Prometheus we use [prometheus operator](https://github.com/prometheus-operator/prometheus-operator). + +- Create dedicated namespace: + +```console +kubectl create namespace monitoring-system +``` + +- Add Helm repo + +````console +helm repo add prometheus https://prometheus-community.github.io/helm-charts +```` + +- Update Helm repo + +````console +helm repo update +```` + +- Deploy prometheus operator + +```console +helm install prometheus prometheus/kube-prometheus-stack --namespace monitoring-system \ + --set prometheusOperator.createCustomResource=false \ + --set prometheusOperator.logLevel=debug \ + --set prometheusOperator.alertmanagerInstanceNamespaces=monitoring-system \ + --set prometheusOperator.namespaces.additional[0]=monitoring-system \ + --set prometheusOperator.prometheusInstanceNamespaces=monitoring-system \ + --set prometheusOperator.thanosRulerInstanceNamespaces=monitoring-system \ + --set defaultRules.enable=false \ + --set alertmanager.enabled=false \ + --set grafana.enabled=false \ + --set kubeApiServer.enabled=false \ + --set kubelet.enabled=false \ + --set kubeControllerManager.enabled=false \ + --set coreDNS.enabled=false \ + --set kubeEtcd.enabled=false \ + --set kubeScheduler.enabled=false \ + --set kubeProxy.enabled=false \ + --set kubeStateMetrics.enabled=false \ + --set prometheus.enabled=false +``` + +- Deploy the `ServiceAccount`, `ClusterRole` and `ClusterRoleBinding` resources: + +```yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: prometheus + namespace: monitoring-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: prometheus +rules: +- apiGroups: [""] + resources: + - nodes + - nodes/metrics + - services + - endpoints + - pods + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: + - configmaps + verbs: ["get"] +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: ["get", "list", "watch"] +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: prometheus +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus +subjects: +- kind: ServiceAccount + name: prometheus + namespace: monitoring-system +``` + +- Deploy the `Prometheus` resource: + +```yaml +apiVersion: monitoring.coreos.com/v1 +kind: Prometheus +metadata: + name: prometheus + namespace: monitoring-system +spec: + enableAdminAPI: false + evaluationInterval: 30s + logLevel: debug + podMonitorSelector: + matchExpressions: + - key: release + operator: In + values: + - prometheus + resources: + requests: + memory: 400Mi + scrapeInterval: 30s + serviceAccountName: prometheus + serviceMonitorSelector: + matchExpressions: + - key: app + operator: In + values: + - nifi-cluster +``` + +- Deploy the `ServiceMonitor` resource: + +```yaml +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: cluster + namespace: monitoring-system + labels: + app: nifi-cluster + nifi_cr: cluster +spec: + selector: + matchLabels: + app: nifi + nifi_cr: cluster + namespaceSelector: + matchNames: + - clusters + endpoints: + - interval: 10s + port: prometheus + path: /metrics + honorLabels: true + relabelings: + - sourceLabels: [__meta_kubernetes_pod_ip] + separator: ; + regex: (.*) + targetLabel: pod_ip + replacement: $1 + action: replace + - sourceLabels: [__meta_kubernetes_pod_label_nodeId] + separator: ; + regex: (.*) + targetLabel: nodeId + replacement: $1 + action: replace + - sourceLabels: [__meta_kubernetes_pod_label_nifi_cr] + separator: ; + regex: (.*) + targetLabel: nifi_cr + replacement: $1 + action: replace +``` + +You should now have a `prometheus-prometheus-0` pod and a `prometheus-operated` service, you can access your prometheus using port forwarding: + +```console +kubectl port-forward -n monitoring-system service/prometheus-operated 9090:9090 +``` + +You should be able to connect to your prometheus instance on `http://localhost:9090`, check that you can query your NiFi metrics correctly. + +### Deploy Scale object + +The last step is to deploy your [ScaledObject](https://keda.sh/docs/2.10/concepts/scaling-deployments/#scaledobject-spec) to define how to scale your NiFi node: + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: cluster + namespace: clusters +spec: + scaleTargetRef: + apiVersion: nifi.konpyutaika.com/v1alpha1 # Optional. Default: apps/v1 + kind: NifiNodeGroupAutoscaler # Optional. Default: Deployment + name: nifinodegroupautoscaler-sample # Mandatory. Must be in the same namespace as the ScaledObject + envSourceContainerName: nifi # Optional. Default: .spec.template.spec.containers[0] + pollingInterval: 30 # Optional. Default: 30 seconds + cooldownPeriod: 300 # Optional. Default: 300 seconds + idleReplicaCount: 0 # Optional. Default: ignored, must be less than minReplicaCount + minReplicaCount: 1 # Optional. Default: 0 + maxReplicaCount: 3 # Optional. Default: 100 + fallback: # Optional. Section to specify fallback options + failureThreshold: 5 # Mandatory if fallback section is included + replicas: 1 # Mandatory if fallback section is included + # advanced: # Optional. Section to specify advanced options + # restoreToOriginalReplicaCount: true # Optional. Whether the target resource should be scaled back to original replicas count, after the ScaledObject is deleted + # horizontalPodAutoscalerConfig: # Optional. Section to specify HPA related options + # name: {name-of-hpa-resource} # Optional. Default: keda-hpa-{scaled-object-name} + # behavior: # Optional. Use to modify HPA's scaling behavior + # scaleDown: + # stabilizationWindowSeconds: 300 <--- + # policies: + # - type: Percent + # value: 100 + # periodSeconds: 15 + triggers: + - type: prometheus + metadata: + serverAddress: http://prometheus-operated.monitoring-system.svc:9090 + metricName: http_requests_total + threshold: + query: +``` + +Now everything is ready, you must have an `HPA` deployed that manage your `NifiNodeGroupAutoscaler` + +```console +kubectl get -n clusters hpa +NAME REFERENCE TARGETS MINPODS MAXPODS REPLICAS AGE +keda-hpa-cluster NifiNodeGroupAutoscaler/nifinodegroupautoscaler-sample 1833m/2 (avg) 1 3 2 25d +``` \ No newline at end of file diff --git a/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/1_manage_clusters/3_external_cluster.md b/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/1_manage_clusters/3_external_cluster.md new file mode 100644 index 0000000000..b45b6da0f8 --- /dev/null +++ b/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/1_manage_clusters/3_external_cluster.md @@ -0,0 +1,91 @@ +--- +id: 3_external_cluster +title: External cluster +sidebar_label: External cluster +--- + +## Common configuration + +The operator allows you to manage the Dataflow lifecycle for internal (i.e cluster managed by the operator) and external NiFi cluster. +A NiFi cluster is considered as external as soon as the `NifiCluster` resource used as reference in other NiFi resource explicitly detailed the way to communicate with the cluster. + +This feature allows you: + +- To automate your Dataflow CI/CD using yaml +- To manage the same way your Dataflow management wherever your cluster is, on bare metal, VMs, k8s, on-premise or on cloud. + +To deploy different resources (`NifiRegistryClient`, `NifiUser`, `NifiUserGroup`, `NifiParameterContext`, `NifiDataflow`) you simply have to declare a `NifiCluster` resource explaining how to discuss with the external cluster, and refer to this resource as usual using the `Spec.ClusterRef` field. + +To declare an external cluster you have to follow this kind of configuration: + +```yaml +apiVersion: nifi.konpyutaika.com/v1 +kind: NifiCluster +metadata: + name: externalcluster +spec: + # rootProcessGroupId contains the uuid of the root process group for this cluster. + rootProcessGroupId: 'd37bee03-017a-1000-cff7-4eaaa82266b7' + # nodeURITemplate used to dynamically compute node uri. + nodeURITemplate: 'nifi0%d.integ.mapreduce.m0.p.fti.net:9090' + # all node requiresunique id + nodes: + - id: 1 + - id: 2 + - id: 3 + # type defines if the cluster is internal (i.e manager by the operator) or external. + # :Enum={"external","internal"} + type: 'external' + # clientType defines if the operator will use basic or tls authentication to query the NiFi cluster. + # Enum={"tls","basic"} + clientType: 'basic' + # secretRef reference the secret containing the informations required to authenticate to the cluster. + secretRef: + name: nifikop-credentials + namespace: nifikop-nifi +``` + +- The `Spec.RootProcessGroupId` field is required to give the ability to the operator of managing root level policy and default deployment and policy. +- The `Spec.NodeURITemplate` field, defines the hostname template of your NiFi cluster nodes, the operator will use this information and the list of id specified in `Spec.Nodes` field to generate the hostname of the nodes (in the configuration above you will have: `nifi01.integ.mapreduce.m0.p.fti.net:9090`, `nifi02.integ.mapreduce.m0.p.fti.net:9090`, `nifi03.integ.mapreduce.m0.p.fti.net:9090`). +- The `Spec.Type` field defines the type of cluster that this resource is refering to, by default it is `internal`, in our case here we just want to use this resource to reference an existing NiFi cluster, so we set this field to `external`. +- The `Spec.ClientType` field defines how we want to authenticate to the NiFi cluster API, for now we are supporting two modes: + - `tls`: using client TLS certificate. + - `basic`: using a username and a password to get an access token. +- The `Spec.SecretRef` defines a reference to a secret which contains the sensitive values that will be used by the operator to authenticate to the NiFi cluster API (ie in basic mode it will contain the password and username). + +:::warning +The id of node only support `int32` as type, so if the hostname of your nodes doesn't match with this, you can't use this feature. +::: + +## Secret configuration for Basic authentication + +When you are using the basic authentication, you have to pass some informations into the secret that is referenced into the `NifiCluster` resource: + +- `username`: the username associated to the user that will be used by the operator to request the REST API. +- `password`: the password associated to the user that will be used by the operator to request the REST API. +- `ca.crt (optional)`: the certificate authority to trust the server certificate if needed + +The following command shows how you can create this secret: + +```console +kubectl create secret generic nifikop-credentials \ + --from-file=username=./secrets/username\ + --from-file=password=./secrets/password\ + --from-file=ca.crt=./secrets/ca.crt\ + -n nifikop-nifi +``` + +:::info +When you use the basic authentication, the operator will create a secret `-basic-secret` containing for each node an access token that will be maintained by the operator. +::: + +## Secret configuration for TLS authentication + +When you are using the tls authentication, you have to pass some information into the secret that is referenced into the `NifiCluster` resource: + +- `tls.key`: The user private key. +- `tls.crt`: The user certificate. +- `password`: the password associated to the user that will be used by the operator to request the REST API. +- `ca.crt`: The CA certificate +- `truststore.jks`: +- `keystore.jks`: \ No newline at end of file diff --git a/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/2_manage_users_and_accesses/1_users_management.md b/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/2_manage_users_and_accesses/1_users_management.md new file mode 100644 index 0000000000..fa047b0279 --- /dev/null +++ b/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/2_manage_users_and_accesses/1_users_management.md @@ -0,0 +1,57 @@ +--- +id: 1_users_management +title: Users management +sidebar_label: Users management +--- + +The `NifiUser` resource was already introduced for the [SSL credentials](../1_manage_clusters/1_deploy_cluster/4_ssl_configuration#create-ssl-credentials) concerns. +What we are covering here is the NiFi user management part introduced in this resource. + +When you create a `NifiUser` resource the operator will: + +1. Try to check if a user already exists with the same name on the NiFi cluster, if it does, the operator will set [NifiUser.Status.Id](../1_manage_clusters/1_deploy_cluster/4_ssl_configuration#create-ssl-credentials) to bind it with the kubernetes resource. +2. If no user is found, the operator will create and manage it (i.e it will ensure the synchronisation with the NiFi Cluster). + +```yaml +apiVersion: nifi.konpyutaika.com/v1 +kind: NifiUser +metadata: + name: aguitton +spec: + # identity field is use to define the user identity on NiFi cluster side, + # it use full when the user's name doesn't suite with Kubernetes resource name. + identity: alexandre.guitton@konpyutaika.com + # Contains the reference to the NifiCluster with the one the registry client is linked. + clusterRef: + name: nc + namespace: nifikop + # Whether or not the the operator also include a Java keystore format (JKS) with you secret + includeJKS: false + # Whether or not a certificate will be created for this user. + createCert: false + # defines the list of access policies that will be granted to the group. + accessPolicies: + # defines the kind of access policy, could be "global" or "component". + - type: component + # defines the kind of action that will be granted, could be "read" or "write" + action: read + # resource defines the kind of resource targeted by this access policies, please refer to the following page: + # https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#access-policies + resource: /data + # componentType is used if the type is "component", it's allow to define the kind of component on which is the + # access policy + componentType: "process-groups" + # componentId is used if the type is "component", it's allow to define the id of the component on which is the + # access policy + componentId: "" +``` + +By default the user name that will be used is the name of the resource. + +But as there are some constraints on this name (e.g [RFC 1123](https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-subdomain-names)) that doesn't match with those applied on NiFi, you can override it with the `NifiUser.Spec.Identity` field which is more permissive. +In the example above the kubernetes resource name will be `aguitton` but the NiFi use created on the cluster will be `alexandre.guitton@konpyutaika.com`. + +In the case the user will not authenticate himself using TLS authentication, the operator doesn't have to create a certificate, so just set `NifiUser.Spec.CreateCert` to false. + +For each user, you have the ability to define a list of [AccessPolicies](../../5_references/2_nifi_user#accesspolicy) to give a list of access to your user. +In the example above we are giving to user `alexandre.guitton@konpyutaika.com` the right to view metadata et content for the root process group in flowfile queues in outbound connections and through provenance events. \ No newline at end of file diff --git a/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/2_manage_users_and_accesses/2_groups_management.md b/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/2_manage_users_and_accesses/2_groups_management.md new file mode 100644 index 0000000000..8e5ea70236 --- /dev/null +++ b/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/2_manage_users_and_accesses/2_groups_management.md @@ -0,0 +1,54 @@ +--- +id: 2_groups_management +title: Groups management +sidebar_label: Groups management +--- + +To simplify the access management Apache NiFi allows to define groups containing a list of users, on which we apply a list of access policies. +This part is supported by the operator using the `NifiUserGroup` resource: + + +```yaml +apiVersion: nifi.konpyutaika.com/v1 +kind: NifiUserGroup +metadata: + name: group-test +spec: + # Contains the reference to the NifiCluster with the one the registry client is linked. + clusterRef: + name: nc + namespace: nifikop + # contains the list of reference to NifiUsers that are part to the group. + usersRef: + - name: nc-0-node.nc-headless.nifikop.svc.cluster.local +# namespace: nifikop + - name: nc-controller.nifikop.mgt.cluster.local + # defines the list of access policies that will be granted to the group. + accessPolicies: + # defines the kind of access policy, could be "global" or "component". + - type: global + # defines the kind of action that will be granted, could be "read" or "write" + action: read + # resource defines the kind of resource targeted by this access policies, please refer to the following page: + # https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#access-policies + resource: /counters +# # componentType is used if the type is "component", it's allow to define the kind of component on which is the +# # access policy +# componentType: "process-groups" +# # componentId is used if the type is "component", it's allow to define the id of the component on which is the +# # access policy +# componentId: "" +``` + +When you create a `NifiUserGroup` resource, the operator will create and manage a group named `${resource namespace}-${resource name}` in Nifi. +To declare the users that are part of this group, you just have to declare them in the [NifiUserGroup.UsersRef](../../5_references/6_nifi_usergroup#userreference) field. + +:::important +The [NifiUserGroup.UsersRef](../../5_references/6_nifi_usergroup#userreference) requires to declare the name and namespace of a `NifiUser` resource, so it is previously required to declare the resource. + +It's required to create the resource even if the user is already declared in NiFi Cluster (In that case the operator will just sync the kubernetes resource). +::: + +Like for `NifiUser` you can declare a list of [AccessPolicies](../../5_references/2_nifi_user#accesspolicy) to give a list of access to your user. + +In the example above we are giving to users `nc-0-node.nc-headless.nifikop.svc.cluster.local` and `nc-controller.nifikop.mgt.cluster.local` the right to view the counters information. diff --git a/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/2_manage_users_and_accesses/3_managed_groups.md b/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/2_manage_users_and_accesses/3_managed_groups.md new file mode 100644 index 0000000000..322f842698 --- /dev/null +++ b/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/2_manage_users_and_accesses/3_managed_groups.md @@ -0,0 +1,60 @@ +--- +id: 3_managed_groups +title: Managed groups +sidebar_label: Managed groups +--- + +In some case these two features could be heavy to define, for example when you have 10 dataflows with one cluster for each of them, it will lead in a lot of `.yaml` files. +To simplify this, we implement in the operator 2 `managed groups`: + +- **Admins:** a group giving access to everything on the NiFi Cluster, +- **Readers:** a group giving access as viewer on the NiFi Cluster. + +You can directly define the list of users who belong to each of them in the `NifiCluster.Spec` field: + +```yaml +apiVersion: nifi.konpyutaika.com/v1 +kind: NifiCluster +metadata: + name: mynifi +spec: + ... + oneNifiNodePerNode: false + propagateLabels: true + managedAdminUsers: + - identity: "alexandre.guitton@konpyutaika.com" + name: "aguitton" + - identity: "nifiuser@konpyutaika.com" + name: "nifiuser" + managedReaderUsers: + - identity: "toto@konpyutaika.com" + name: "toto" + ... +``` + +In this example the operator will create and manage 3 `NifiUsers`: + +- **aguitton**, with the identity: `alexandre.guitton@konpyutaika.com` +- **nifiuser**, with the identity: `nifiuser@konpyutaika.com` +- **toto**, with the identity: `toto@konpyutaika.com` + +And create and manage two groups: + +- **managed-admins:** that will contain 3 users (**aguitton**, **nifiuser**, **nc-controller.nifikop.mgt.cluster.local** which is the controller user). +- **managed-readers:** that will contain 1 user (**toto**) + +And the rest of the stuff will be reconciled and managed as described for `NifiUsers` and `NifiUserGroups`. + +:::note +There is one more group that is created and managed by the operator, this is the **managed-nodes** group, for each node a `NifiUser` is created, and we automatically add them to this group to give them the right list of accesses. + +To get the list of managed groups just check the list of `NifiUserGroup`: + +```console +kubectl get -n nifikop nifiusergroups.nifi.konpyutaika.com +NAME AGE +managed-admins 6d7h +managed-nodes 6d7h +managed-readers 6d7h +``` +::: \ No newline at end of file diff --git a/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/3_manage_dataflows/0_design_principles.md b/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/3_manage_dataflows/0_design_principles.md new file mode 100644 index 0000000000..6b02d7d57c --- /dev/null +++ b/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/3_manage_dataflows/0_design_principles.md @@ -0,0 +1,29 @@ +--- +id: 0_design_principles +title: Design Principles +sidebar_label: Design Principles +--- + +The [Dataflow Lifecycle management feature](../../1_concepts/3_features#dataflow-lifecycle-management-via-crd) introduces 3 new CRDs: + +- **NiFiRegistryClient:** Allowing you to declare a [NiFi registry client](https://nifi.apache.org/docs/nifi-registry-docs/html/getting-started.html#connect-nifi-to-the-registry). +- **NiFiParameterContext:** Allowing you to create parameter context, with two kinds of parameters, a simple `map[string]string` for non-sensitive parameters and a `list of secrets` which contains sensitive parameters. +- **NiFiDataflow:** Allowing you to declare a Dataflow based on a `NiFiRegistryClient` and optionally a `ParameterContext`, which will be deployed and managed by the operator on the `targeted NiFi cluster`. + +The following diagram shows the interactions between all the components: + +![dataflow lifecycle management schema](/img/1_concepts/2_design_principes/dataflow_lifecycle_management_schema.jpg) + +With each CRD comes a new controller, with a reconcile loop: + +- **NiFiRegistryClient's controller:** + +![NiFi registry client's reconcile loop](/img/1_concepts/2_design_principes/registry_client_reconcile_loop.jpeg) + +- **NiFiParameterContext's controller:** + +![NiFi parameter context's reconcile loop](/img/1_concepts/2_design_principes/parameter_context_reconcile_loop.jpeg) + +- **NiFiDataflow's controller:** + +![NiFi dataflow's reconcile loop](/img/1_concepts/2_design_principes/dataflow_reconcile_loop.jpeg) \ No newline at end of file diff --git a/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/3_manage_dataflows/1_deploy_dataflow.md b/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/3_manage_dataflows/1_deploy_dataflow.md new file mode 100644 index 0000000000..31d6a2ddc3 --- /dev/null +++ b/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/3_manage_dataflows/1_deploy_dataflow.md @@ -0,0 +1,126 @@ +--- +id: 1_deploy_dataflow +title: Deploy dataflow +sidebar_label: Deploy dataflow +--- + +You can create NiFi dataflows either: + +* directly against the cluster through its REST API (using UI or some home made scripts), or +* via the `NifiDataflow` CRD. + +If you want more details about the design, just have a look on the [design page](./0_design_principles#dataflow-lifecycle-management) + +To deploy a [NifiDataflow] you have to start by deploying a [NifiRegistryClient] because **NiFiKop** manages dataflow using the [NiFi Registry feature](https://nifi.apache.org/registry). + +Below is an example of [NifiRegistryClient]: + +```yaml +apiVersion: nifi.konpyutaika.com/v1 +kind: NifiRegistryClient +metadata: + name: registry-client-example + namespace: nifikop +spec: + clusterRef: + name: nc + namespace: nifikop + description: "Registry client managed by NiFiKop" + uri: "http://nifi.hostname.com:18080" +``` + +Once you have deployed your [NifiRegistryClient], you have the possibility of defining a configuration that you will apply to your [NifiDataflow]. + +This configuration is defined using the [NifiParameterContext] CRD, which NiFiKop will convert into a [Parameter context](https://nifi.apache.org/docs/nifi-docs/html/user-guide.html#parameter-contexts). + + +Below is an example of [NifiParameterContext]: + +```yaml +apiVersion: nifi.konpyutaika.com/v1 +kind: NifiParameterContext +metadata: + name: dataflow-lifecycle + namespace: demo +spec: + description: "It is a test" + clusterRef: + name: nc + namespace: nifikop + secretRefs: + - name: secret-params + namespace: nifikop + parameters: + - name: test + value: toto + description: tutu + - name: test2 + value: toto + description: toto +``` + +As you can see, in the [NifiParameterContext] you can refer to some secrets that will be converted into [sensitive parameter](https://nifi.apache.org/docs/nifi-docs/html/user-guide.html#using-parameters-with-sensitive-properties). + +Here is an example of secret that you can create that will be used by the configuration above: + +```console +kubectl create secret generic secret-params \ + --from-literal=secret1=yop \ + --from-literal=secret2=yep \ + -n nifikop +``` + +:::warning +As a sensitive value cannot be retrieved through the Rest API, to update the value of a sensitive parameter, you have to: + +- remove it from the secret +- wait for the next loop +- insert the parameter with the new value inside the secret + +or you can simply create a new [NifiParameterContext] and refer it into your [NifiDataflow]. +::: + +You can now deploy your [NifiDataflow] by referencing the previous objects: + +```yaml +apiVersion: nifi.konpyutaika.com/v1 +kind: NifiDataflow +metadata: + name: dataflow-lifecycle +spec: + parentProcessGroupID: "16cfd2ec-0174-1000-0000-00004b9b35cc" + bucketId: "01ced6cc-0378-4893-9403-f6c70d080d4f" + flowId: "9b2fb465-fb45-49e7-94fe-45b16b642ac9" + flowVersion: 2 + syncMode: always + skipInvalidControllerService: true + skipInvalidComponent: true + clusterRef: + name: nc + namespace: nifikop + registryClientRef: + name: registry-client-example + namespace: nifikop + parameterContextRef: + name: dataflow-lifecycle + namespace: demo + updateStrategy: drain +``` + +To find details about the versioned flow information required check the [official documentation](https://nifi.apache.org/docs/nifi-registry-docs/index.html) + +You have two modes of control from your dataflow by the operator: + +1 - `Spec.SyncMode == never`: The operator will deploy the dataflow as described in the resource, and never control it (unless you change the field to `always`). It is useful when you want to deploy your dataflow without starting it. + +2 - `Spec.SyncMode == once`: The operator will deploy the dataflow as described in the resource, run it once, and never control it again (unless you change the field to `always`). It is useful when you want to deploy your dataflow in a dev environment, and you want to update the dataflow. + +3 - `Spec.SyncMode == always`: The operator will deploy and ensure the dataflow lifecycle, it will avoid all manual modification directly from the Cluster (e.g remove the process group, remove the versioning, update the parent process group, make some local changes ...). If you want to perform update, rollback or stuff like this, you have to simply update the [NifiDataflow] resource. + +:::important +More information about `Spec.UpdateStrategy` [here](../../5_references/5_nifi_dataflow#componentupdatestrategy) +::: + +[NifiDataflow]: ../../5_references/5_nifi_dataflow +[NifiRegistryClient]: ../../5_references/3_nifi_registry_client +[NifiParameterContext]: ../../5_references/4_nifi_parameter_context \ No newline at end of file diff --git a/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/4_manage_connections/1_deploy_connection.md b/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/4_manage_connections/1_deploy_connection.md new file mode 100644 index 0000000000..1180c156d0 --- /dev/null +++ b/site/website/versioned_docs/version-v1.11.4/3_manage_nifi/4_manage_connections/1_deploy_connection.md @@ -0,0 +1,127 @@ +--- +id: 1_deploy_connection +title: Deploy connection +sidebar_label: Deploy connection +--- + +You can create NiFi connections either: + +* directly against the cluster through its REST API (using UI or some home made scripts), or +* via the `NifiConnection` CRD. + +To deploy a [NifiConnection] you have to start by deploying at least 2 [NifiDataflows] because **NiFiKop** manages connection between 2 [NifiDataflows]. + +If you want more details about how to deploy [NifiDataflow], just have a look on the [how to deploy dataflow page](../3_manage_dataflows/1_deploy_dataflow). + +Below is an example of 2 [NifiDataflows] named respectively `input` and `output`: + +```yaml +apiVersion: nifi.konpyutaika.com/v1alpha1 +kind: NifiDataflow +metadata: + name: input + namespace: nifikop +spec: + clusterRef: + name: nc + namespace: nifikop + bucketId: deedb9f6-65a4-44e9-a1c9-722008fcd0ba + flowId: ab95431d-980d-41bd-904a-fac4bd864ba6 + flowVersion: 1 + registryClientRef: + name: registry-client-example + namespace: nifikop + skipInvalidComponent: true + skipInvalidControllerService: true + syncMode: always + updateStrategy: drain + flowPosition: + posX: 0 + posY: 0 +--- +apiVersion: nifi.konpyutaika.com/v1alpha1 +kind: NifiDataflow +metadata: + name: output + namespace: nifikop +spec: + clusterRef: + name: nc + namespace: nifikop + bucketId: deedb9f6-65a4-44e9-a1c9-722008fcd0ba + flowId: fc5363eb-801e-432f-aa94-488838674d07 + flowVersion: 2 + registryClientRef: + name: registry-client-example + namespace: nifikop + skipInvalidComponent: true + skipInvalidControllerService: true + syncMode: always + updateStrategy: drain + flowPosition: + posX: 750 + posY: 0 +``` + +We will obtain the following initial setup: +![Initial setup](/img/3_tasks/4_manage_connections/1_deploy_connections/initial_setup.jpg) + +:::important +The `input` dataflow must have an `output port` and the `output` dataflow must have an `input port`. +::: + +Now that we have 2 [NifiDataflows], we can connect them with a [NifiConnection]. + +Below is an example of a [NifiConnection] named `connection` between the 2 previously deployed dataflows: + +```yaml +apiVersion: nifi.konpyutaika.com/v1alpha1 +kind: NifiConnection +metadata: + name: connection + namespace: nifikop +spec: + source: + name: input + namespace: nifikop + subName: output + type: dataflow + destination: + name: output + namespace: nifikop + subName: input + type: dataflow + configuration: + backPressureDataSizeThreshold: 100 GB + backPressureObjectThreshold: 10000 + flowFileExpiration: 1 hour + labelIndex: 0 + bends: + - posX: 550 + posY: 550 + - posX: 550 + posY: 440 + - posX: 550 + posY: 88 + updateStrategy: drain +``` + +You will obtain the following setup: +![Connection setup](/img/3_tasks/4_manage_connections/1_deploy_connections/connection_setup.jpg) + +The `prioritizers` field takes a list of prioritizers, and the order of the list matters in NiFi so it matters in the resource. + +- `prioriters=[NewestFlowFileFirstPrioritizer, FirstInFirstOutPrioritizer, OldestFlowFileFirstPrioritizer]` ![Connection prioritizers 0](/img/3_tasks/4_manage_connections/1_deploy_connections/connection_prioritizers_0.jpg) +- `prioriters=[FirstInFirstOutPrioritizer, NewestFlowFileFirstPrioritizer, OldestFlowFileFirstPrioritizer]` ![Connection prioritizers 1](/img/3_tasks/4_manage_connections/1_deploy_connections/connection_prioritizers_0.jpg) +- `prioriters=[PriorityAttributePrioritizer]` ![Connection prioritizers 2](/img/3_tasks/4_manage_connections/1_deploy_connections/connection_prioritizers_0.jpg) + +The `labelIndex` field will place the label of the connection according to the bends. +If we take the previous bending configuration, you will get this setup for these labelIndex: + +- `labelIndex=0`: ![Connection labelIndex 0](/img/3_tasks/4_manage_connections/1_deploy_connections/connection_labelindex_0.jpg) +- `labelIndex=1`: ![Connection labelIndex 1](/img/3_tasks/4_manage_connections/1_deploy_connections/connection_labelindex_1.jpg) +- `labelIndex=2`: ![Connection labelIndex 2](/img/3_tasks/4_manage_connections/1_deploy_connections/connection_labelindex_2.jpg) + +[NifiDataflow]: ../../5_references/5_nifi_dataflow +[NifiDataflows]: ../../5_references/5_nifi_dataflow +[NifiConnection]: ../../5_references/8_nifi_connection \ No newline at end of file diff --git a/site/website/versioned_docs/version-v1.11.4/4_compatibility_versions.md b/site/website/versioned_docs/version-v1.11.4/4_compatibility_versions.md new file mode 100644 index 0000000000..e38d3207cc --- /dev/null +++ b/site/website/versioned_docs/version-v1.11.4/4_compatibility_versions.md @@ -0,0 +1,235 @@ +--- +id: 4_compatibility_versions +title: Compatibility versions +sidebar_label: Compatibility versions +--- + +**Official supported Kubernetes version**: `1.20+` + + +### NiFi cluster + +Nifikop supports the following NiFi cluster features: + +| NiFi Version | Cluster deployment | Standalone deployment | Cluster nodes configuration | Cluster rolling upgrade | Cluster scaling | Cluster auto-scaling | Prometheus Reporting | Kubernetes Cluster & State management | +| ------------- | ------------------ | --------------------- | --------------------------- | ----------------------- | --------------- | -------------------- | -------------------- | ------------------------------------- | +| NiFi 1.16 | Yes | No | Yes | Yes | Yes | Yes | Yes | No | +| NiFi 1.17 | Yes | No | Yes | Yes | Yes | Yes | Yes | No | +| NiFi 1.18 | Yes | No | Yes | Yes | Yes | Yes | Yes | No | +| NiFi 1.19 | Yes | No | Yes | Yes | Yes | Yes | Yes | No | +| NiFi 1.20 | Yes | No | Yes | Yes | Yes | Yes | Yes | No | +| NiFi 1.21 | Yes | No | Yes | Yes | Yes | Yes | Yes | No | +| NiFi 1.22 | Yes | No | Yes | Yes | Yes | Yes | Yes | No | +| NiFi 1.23 | Yes | No | Yes | Yes | Yes | Yes | Yes | No | +| NiFi 1.24 | Yes | No | Yes | Yes | Yes | Yes | Yes | No | +| NiFi 1.25 | Yes | No | Yes | Yes | Yes | Yes | Yes | No | +| NiFi 1.26 | Yes | No | Yes | Yes | Yes | Yes | Yes | No | +| NiFi 1.27 | Yes | No | Yes | Yes | Yes | Yes | Yes | No | +| NiFi 1.28 | Yes | No | Yes | Yes | Yes | Yes | Yes | No | +| NiFi 2.0.0-M1 | Yes | No | Yes | Yes | Yes | Yes | Yes | No | +| NiFi 2.0.0-M2 | Yes | No | Yes | Yes | Yes | Yes | Yes | Yes | +| NiFi 2.0.0-M3 | Yes | No | Yes | Yes | Yes | Yes | Yes | Yes | +| NiFi 2.0.0-M4 | Yes | No | Yes | Yes | Yes | Yes | Yes | Yes | +| NiFi 2.0.0 | Yes | No | Yes | Yes | Yes | Yes | Yes | Yes | + +### NiFi external cluster + +Nifikop supports the following features for externally deployed clusters: + +| NiFi Version | Basic authentication | TLS authentication | +|---------------|----------------------|--------------------| +| NiFi 1.16 | Yes | Yes | +| NiFi 1.17 | Yes | Yes | +| NiFi 1.18 | Yes | Yes | +| NiFi 1.19 | Yes | Yes | +| NiFi 1.20 | Yes | Yes | +| NiFi 1.21 | Yes | Yes | +| NiFi 1.22 | Yes | Yes | +| NiFi 1.23 | Yes | Yes | +| NiFi 1.24 | Yes | Yes | +| NiFi 1.25 | Yes | Yes | +| NiFi 1.26 | Yes | Yes | +| NiFi 1.27 | Yes | Yes | +| NiFi 1.28 | Yes | Yes | +| NiFi 2.0.0-M1 | Yes | Yes | +| NiFi 2.0.0-M2 | Yes | Yes | +| NiFi 2.0.0-M3 | Yes | Yes | +| NiFi 2.0.0-M4 | Yes | Yes | +| NiFi 2.0.0 | Yes | Yes | + +### NiFi users + +Nifikop supports the following features for configuring users and user policies: + +| NiFi Version | User deployment | User policies | +|---------------|-----------------|---------------| +| NiFi 1.16 | Yes | Yes | +| NiFi 1.17 | Yes | Yes | +| NiFi 1.18 | Yes | Yes | +| NiFi 1.19 | Yes | Yes | +| NiFi 1.20 | Yes | Yes | +| NiFi 1.21 | Yes | Yes | +| NiFi 1.22 | Yes | Yes | +| NiFi 1.23 | Yes | Yes | +| NiFi 1.24 | Yes | Yes | +| NiFi 1.25 | Yes | Yes | +| NiFi 1.26 | Yes | Yes | +| NiFi 1.27 | Yes | Yes | +| NiFi 1.28 | Yes | Yes | +| NiFi 2.0.0-M1 | Yes | Yes | +| NiFi 2.0.0-M2 | Yes | Yes | +| NiFi 2.0.0-M3 | Yes | Yes | +| NiFi 2.0.0-M4 | Yes | Yes | +| NiFi 2.0.0 | Yes | Yes | + +### NiFi user groups + +Nifikop supports the following features for configuring user groups: + +| NiFi Version | Group deployment | Group policies | +|---------------|------------------|----------------| +| NiFi 1.16 | Yes | Yes | +| NiFi 1.17 | Yes | Yes | +| NiFi 1.18 | Yes | Yes | +| NiFi 1.19 | Yes | Yes | +| NiFi 1.20 | Yes | Yes | +| NiFi 1.21 | Yes | Yes | +| NiFi 1.22 | Yes | Yes | +| NiFi 1.23 | Yes | Yes | +| NiFi 1.24 | Yes | Yes | +| NiFi 1.25 | Yes | Yes | +| NiFi 1.26 | Yes | Yes | +| NiFi 1.27 | Yes | Yes | +| NiFi 1.28 | Yes | Yes | +| NiFi 2.0.0-M1 | Yes | Yes | +| NiFi 2.0.0-M2 | Yes | Yes | +| NiFi 2.0.0-M3 | Yes | Yes | +| NiFi 2.0.0-M4 | Yes | Yes | +| NiFi 2.0.0 | Yes | Yes | + +### NiFi dataflow + +Nifikop supports the following features for managing dataflows: + +| NiFi Version | Dataflow deployment | Dataflow rollback | Dataflow version upgrade | Dataflow cluster migration | +|---------------|---------------------|-------------------|--------------------------|----------------------------| +| NiFi 1.16 | No | Yes | Yes | Yes | +| NiFi 1.17 | No | Yes | Yes | Yes | +| NiFi 1.18 | Yes | Yes | Yes | Yes | +| NiFi 1.19 | Yes | Yes | Yes | Yes | +| NiFi 1.20 | Yes | Yes | Yes | Yes | +| NiFi 1.21 | Yes | Yes | Yes | Yes | +| NiFi 1.22 | Yes | Yes | Yes | Yes | +| NiFi 1.23 | Yes | Yes | Yes | Yes | +| NiFi 1.24 | Yes | Yes | Yes | Yes | +| NiFi 1.25 | Yes | Yes | Yes | Yes | +| NiFi 1.26 | Yes | Yes | Yes | Yes | +| NiFi 1.27 | Yes | Yes | Yes | Yes | +| NiFi 1.28 | Yes | Yes | Yes | Yes | +| NiFi 2.0.0-M1 | Yes | Yes | Yes | Yes | +| NiFi 2.0.0-M2 | Yes | Yes | Yes | Yes | +| NiFi 2.0.0-M3 | Yes | Yes | Yes | Yes | +| NiFi 2.0.0-M4 | Yes | Yes | Yes | Yes | +| NiFi 2.0.0 | Yes | Yes | Yes | Yes | + +### NiFi registry + +Nifikop supports the following features for managing registries: + +| NiFi Version | Registry deployment | +|---------------|---------------------| +| NiFi 1.16 | No | +| NiFi 1.17 | No | +| NiFi 1.18 | Yes | +| NiFi 1.19 | Yes | +| NiFi 1.20 | Yes | +| NiFi 1.21 | Yes | +| NiFi 1.22 | Yes | +| NiFi 1.23 | Yes | +| NiFi 1.24 | Yes | +| NiFi 1.25 | Yes | +| NiFi 1.26 | Yes | +| NiFi 1.27 | Yes | +| NiFi 1.28 | Yes | +| NiFi 2.0.0-M1 | Yes | +| NiFi 2.0.0-M2 | Yes | +| NiFi 2.0.0-M3 | Yes | +| NiFi 2.0.0-M4 | Yes | +| NiFi 2.0.0 | Yes | + + +### NiFi parameter context + +Nifikop supports the following features for managing parameter contexts: + +| NiFi Version | Parameter context deployment | Parameter context inheritance | Parameter context cluster migration | +|---------------|------------------------------|-------------------------------|-------------------------------------| +| NiFi 1.16 | Yes | Yes | No | +| NiFi 1.17 | Yes | Yes | No | +| NiFi 1.18 | Yes | Yes | No | +| NiFi 1.19 | Yes | Yes | No | +| NiFi 1.20 | Yes | Yes | No | +| NiFi 1.21 | Yes | Yes | No | +| NiFi 1.22 | Yes | Yes | No | +| NiFi 1.23 | Yes | Yes | No | +| NiFi 1.24 | Yes | Yes | No | +| NiFi 1.25 | Yes | Yes | No | +| NiFi 1.26 | Yes | Yes | No | +| NiFi 1.27 | Yes | Yes | No | +| NiFi 1.28 | Yes | Yes | No | +| NiFi 2.0.0-M1 | Yes | Yes | No | +| NiFi 2.0.0-M2 | Yes | Yes | No | +| NiFi 2.0.0-M3 | Yes | Yes | No | +| NiFi 2.0.0-M4 | Yes | Yes | No | +| NiFi 2.0.0 | Yes | Yes | No | + + +### NiFi auto scaling + +Nifikop supports the following features for cluster auto-scaling + +| NiFi Version | Auto scaling group deployment | Auto scaling group FIFO | +|---------------|-------------------------------|-------------------------| +| NiFi 1.16 | Yes | Yes | +| NiFi 1.17 | Yes | Yes | +| NiFi 1.18 | Yes | Yes | +| NiFi 1.19 | Yes | Yes | +| NiFi 1.20 | Yes | Yes | +| NiFi 1.21 | Yes | Yes | +| NiFi 1.22 | Yes | Yes | +| NiFi 1.23 | Yes | Yes | +| NiFi 1.24 | Yes | Yes | +| NiFi 1.25 | Yes | Yes | +| NiFi 1.26 | Yes | Yes | +| NiFi 1.27 | Yes | Yes | +| NiFi 1.28 | Yes | Yes | +| NiFi 2.0.0-M1 | Yes | Yes | +| NiFi 2.0.0-M2 | Yes | Yes | +| NiFi 2.0.0-M3 | Yes | Yes | +| NiFi 2.0.0-M4 | Yes | Yes | +| NiFi 2.0.0 | Yes | Yes | + +### NiFi connection + +Nifikop supports for the following features for connecting two dataflows together: + +| NiFi Version | Connection deployment | Connection cluster migration | Connection multi cluster | +|---------------|-----------------------|------------------------------|--------------------------| +| NiFi 1.16 | Yes | Yes | No | +| NiFi 1.17 | Yes | Yes | No | +| NiFi 1.18 | Yes | Yes | No | +| NiFi 1.19 | Yes | Yes | No | +| NiFi 1.20 | Yes | Yes | No | +| NiFi 1.21 | Yes | Yes | No | +| NiFi 1.22 | Yes | Yes | No | +| NiFi 1.23 | Yes | Yes | No | +| NiFi 1.24 | Yes | Yes | No | +| NiFi 1.25 | Yes | Yes | No | +| NiFi 1.26 | Yes | Yes | No | +| NiFi 1.27 | Yes | Yes | No | +| NiFi 1.28 | Yes | Yes | No | +| NiFi 2.0.0-M1 | Yes | Yes | No | +| NiFi 2.0.0-M2 | Yes | Yes | No | +| NiFi 2.0.0-M3 | Yes | Yes | No | +| NiFi 2.0.0-M4 | Yes | Yes | No | +| NiFi 2.0.0 | Yes | Yes | No | diff --git a/site/website/versioned_docs/version-v1.11.4/5_references/1_nifi_cluster/1_nifi_cluster.md b/site/website/versioned_docs/version-v1.11.4/5_references/1_nifi_cluster/1_nifi_cluster.md new file mode 100644 index 0000000000..f71047372a --- /dev/null +++ b/site/website/versioned_docs/version-v1.11.4/5_references/1_nifi_cluster/1_nifi_cluster.md @@ -0,0 +1,252 @@ +--- +id: 1_nifi_cluster +title: NiFi cluster +sidebar_label: NiFi cluster +--- + +`NifiCluster` describes the desired state of the NiFi cluster we want to setup through the operator. + +```yaml +apiVersion: nifi.konpyutaika.com/v1 +kind: NifiCluster +metadata: + name: simplenifi +spec: + service: + headlessEnabled: true + annotations: + tyty: ytyt + labels: + cluster-name: simplenifi + tete: titi + clusterManager: zookeeper + zkAddress: "zookeeper.zookeeper:2181" + zkPath: /simplenifi + externalServices: + - metadata: + annotations: + toto: tata + labels: + cluster-name: driver-simplenifi + titi: tutu + name: driver-ip + spec: + portConfigs: + - internalListenerName: http + port: 8080 + type: ClusterIP + clusterImage: "apache/nifi:1.28.0" + initContainerImage: "bash:5.2.2" + oneNifiNodePerNode: true + readOnlyConfig: + nifiProperties: + overrideConfigs: | + nifi.sensitive.props.key=thisIsABadSensitiveKeyPassword + pod: + annotations: + toto: tata + labels: + cluster-name: simplenifi + titi: tutu + nodeConfigGroups: + default_group: + imagePullPolicy: IfNotPresent + isNode: true + serviceAccountName: default + storageConfigs: + - mountPath: "/opt/nifi/nifi-current/logs" + name: logs + reclaimPolicy: Delete + pvcSpec: + accessModes: + - ReadWriteOnce + storageClassName: "standard" + resources: + requests: + storage: 10Gi + resourcesRequirements: + limits: + cpu: "1" + memory: 2Gi + requests: + cpu: "1" + memory: 2Gi + nodes: + - id: 1 + nodeConfigGroup: "default_group" + - id: 2 + nodeConfigGroup: "default_group" + propagateLabels: true + nifiClusterTaskSpec: + retryDurationMinutes: 10 + listenersConfig: + internalListeners: + - containerPort: 8080 + type: http + name: http + - containerPort: 6007 + type: cluster + name: cluster + - containerPort: 10000 + type: s2s + name: s2s + - containerPort: 9090 + type: prometheus + name: prometheus + - containerPort: 6342 + type: load-balance + name: load-balance + +``` + +## NifiCluster + +| Field | Type | Description | Required | Default | +| -------- | ----------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------- | -------- | ------- | +| metadata | [ObjectMetadata](https://godoc.org/k8s.io/apimachinery/pkg/apis/meta/v1#ObjectMeta) | is metadata that all persisted resources must have, which includes all objects users must create. | No | nil | +| spec | [NifiClusterSpec](#nificlusterspec) | defines the desired state of NifiCluster. | No | nil | +| status | [NifiClusterStatus](#nificlusterstatus) | defines the observed state of NifiCluster. | No | nil | + +## NifiClusterSpec + +| Field | Type | Description | Required | Default | +| ------------------------- | ---------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------- | -------------------------- | +| clientType | Enum={"tls","basic"} | defines if the operator will use basic or tls authentication to query the NiFi cluster. | No | `tls` | +| type | Enum={"external","internal"} | defines if the cluster is internal (i.e manager by the operator) or external. | No | `internal` | +| nodeURITemplate | string | used to dynamically compute node uri. | if external type | - | +| nifiURI | stringused access through a LB uri. | if external type | - | +| rootProcessGroupId | string | contains the uuid of the root process group for this cluster. | if external type | - | +| secretRef | \[ \][SecretReference](../4_nifi_parameter_context#secretreference) | reference the secret containing the informations required to authentiticate to the cluster. | if external type | - | +| proxyUrl | string | defines the proxy required to query the NiFi cluster. | if external type | - | +| service | [ServicePolicy](#servicepolicy) | defines the policy for services owned by NiFiKop operator. | No | - | +| pod | [PodPolicy](#podpolicy) | defines the policy for pod owned by NiFiKop operator. | No | - | +| clusterManager | [ClusterManagerType](#clustermanagertype) | specifies which manager will handle the cluster leader election and state management. | No | zookeeper | +| zkAddress | string | specifies the ZooKeeper connection string in the form hostname:port where host and port are those of a Zookeeper server. | No | "" | +| zkPath | string | specifies the Zookeeper chroot path as part of its Zookeeper connection string which puts its data under same path in the global ZooKeeper namespace. | Yes | "/" | +| initContainerImage | string | can override the default image used into the init container to check if ZoooKeeper server is reachable. | Yes | "bash" | +| initContainers | \[ \]string | defines additional initContainers configurations. | No | \[ \] | +| clusterImage | string | can specify the whole nificluster image in one place. | No | "" | +| oneNifiNodePerNode | boolean | if set to true every nifi node is started on a new node, if there is not enough node to do that it will stay in pending state. If set to false the operator also tries to schedule the nifi node to a unique node but if the node number is insufficient the nifi node will be scheduled to a node where a nifi node is already running. | No | nil | +| propagateLabels | boolean | whether the labels defined on the `NifiCluster` metadata will be propagated to resources created by the operator or not. | Yes | false | +| managedAdminUsers | \[ \][ManagedUser](#managedusers) | contains the list of users that will be added to the managed admin group (with all rights). | No | [] | +| managedReaderUsers | \[ \][ManagedUser](#managedusers) | contains the list of users that will be added to the managed admin group (with all rights). | No | [] | +| readOnlyConfig | [ReadOnlyConfig](./2_read_only_config) | specifies the read-only type Nifi config cluster wide, all theses will be merged with node specified readOnly configurations, so it can be overwritten per node. | No | nil | +| nodeUserIdentityTemplate | string | specifies the template to be used when naming the node user identity (e.g. node-%d-mysuffix) | Yes | "node-%d-\" | +| nodeConfigGroups | map\[string\][NodeConfig](./3_node_config) | specifies multiple node configs with unique name | No | nil | +| nodes | \[ \][Node](./3_node_config) | specifies the list of cluster nodes, all node requires an image, unique id, and storageConfigs settings | Yes | nil | +| disruptionBudget | [DisruptionBudget](#disruptionbudget) | defines the configuration for PodDisruptionBudget. | No | nil | +| ldapConfiguration | [LdapConfiguration](#ldapconfiguration) | specifies the configuration if you want to use LDAP. | No | nil | +| singleUserConfiguration | [SingleUserConfiguration](#singleuserconfiguration) | specifies the configuration if you want to use SingleUser. | No | nil | +| nifiClusterTaskSpec | [NifiClusterTaskSpec](#nificlustertaskspec) | specifies the configuration of the nifi cluster Tasks. | No | nil | +| listenersConfig | [ListenersConfig](./6_listeners_config) | specifies nifi's listener specifig configs. | No | - | +| sidecarConfigs | \[ \][Container](https://godoc.org/k8s.io/api/core/v1#Container) | Defines additional sidecar configurations. [Check documentation for more informations] | +| externalServices | \[ \][ExternalServiceConfigs](./7_external_service_config) | specifies settings required to access nifi externally. | No | - | +| topologySpreadConstraints | \[ \][TopologySpreadConstraint](https://godoc.org/k8s.io/api/core/v1#TopologySpreadConstraint) | specifies any TopologySpreadConstraint objects to be applied to all nodes. | No | nil | +| nifiControllerTemplate | string | NifiControllerTemplate specifies the template to be used when naming the node controller (e.g. %s-mysuffix) **Warning: once defined don't change this value either the operator will no longer be able to manage the cluster** | Yes | "%s-controller" | +| controllerUserIdentity | string | ControllerUserIdentity specifies what to call the static admin user's identity **Warning: once defined don't change this value either the operator will no longer be able to manage the cluster** | Yes | false | + + +## NifiClusterStatus + +| Field | Type | Description | Required | Default | +| ------------------ | ---------------------------------------- | ------------------------------------------------------------- | -------- | ------- | +| nodesState | map\[string\][NodeState](./5_node_state) | Store the state of each nifi node. | No | - | +| State | [ClusterState](#clusterstate) | Store the state of each nifi node. | Yes | - | +| rootProcessGroupId | string | contains the uuid of the root process group for this cluster. | No | - | + +## ServicePolicy + +| Field | Type | Description | Required | Default | +| --------------- | ------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | --------------------------------------------------------- | +| headlessEnabled | boolean | specifies if the cluster should use headlessService for Nifi or individual services using service per nodes may come an handy case of service mesh. | Yes | false | +| serviceTemplate | string | specifies the template to be used when naming the service. | Yes | If headlessEnabled = true ? "%s-headless" = "%s-all-node" | +| annotations | map\[string\]string | Annotations specifies the annotations to attach to services the NiFiKop operator creates | No | - | +| labels | map\[string\]string | Labels specifies the labels to attach to services the NiFiKop operator creates | No | - | + + +## PodPolicy + +| Field | Type | Description | Required | Default | +| -------------- | ---------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------- | -------- | ------- | +| annotations | map\[string\]string | Annotations specifies the annotations to attach to pods the NiFiKop operator creates | No | - | +| labels | map\[string\]string | Labels specifies the Labels to attach to pods the NiFiKop operator creates | No | - | +| hostAliases | \[ \][HostAlias](https://pkg.go.dev/k8s.io/api/core/v1#HostAlias) | A list of host aliases to include in every pod's /etc/hosts configuration in the scenario where DNS is not available. | No | \[ \] | +| readinessProbe | [Probe](https://pkg.go.dev/k8s.io/api/core/v1#Probe) | The readiness probe that the `Pod` is configured with. If not provided, a default will be used. | No | nil | +| livenessProbe | [Probe](https://pkg.go.dev/k8s.io/api/core/v1#Probe) | The liveness probe that the `Pod` is configured with. If not provided, a default will be used. | No | nil | + +## ManagedUsers + +| Field | Type | Description | Required | Default | +| -------- | ------ | ----------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------- | +| identity | string | identity field is use to define the user identity on NiFi cluster side, it use full when the user's name doesn't suite with Kubernetes resource name. | No | - | +| name | string | name field is use to name the NifiUser resource, if not identity is provided it will be used to name the user on NiFi cluster side. | Yes | - | + +## DisruptionBudget + +| Field | Type | Description | Required | Default | +| ------ | ------ | --------------------------------------------------------------------------- | -------- | ------- | +| create | bool | if set to true, will create a podDisruptionBudget. | No | - | +| budget | string | the budget to set for the PDB, can either be static number or a percentage. | Yes | - | + +## LdapConfiguration + +| Field | Type | Description | Required | Default | +| ----------------------- | ------- | ----------------------------------------------------------------------------------------------------------------------------------------- | -------- | ----------- | +| enabled | boolean | if set to true, we will enable ldap usage into nifi.properties configuration. | No | false | +| url | string | space-separated list of URLs of the LDAP servers (i.e. ldap://$\{hostname}:$\{port}). | No | "" | +| searchBase | string | base DN for searching for users (i.e. CN=Users,DC=example,DC=com). | No | "" | +| searchFilter | string | Filter for searching for users against the 'User Search Base'. (i.e. sAMAccountName={0}). The user specified name is inserted into '{0}'. | No | "" | +| authenticationStrategy | string | How the connection to the LDAP server is authenticated. Possible values are ANONYMOUS, SIMPLE, LDAPS, or START_TLS. | No | START_TLS | +| managerDn | string | The DN of the manager that is used to bind to the LDAP server to search for users. | No | "" | +| managerPassword | string | The password of the manager that is used to bind to the LDAP server to search for users. | No | "" | +| tlsKeystore | string | Path to the Keystore that is used when connecting to LDAP using LDAPS or START_TLS. Not required for LDAPS. Only used for mutual TLS | No | "" | +| tlsKeystorePassword | string | Password for the Keystore that is used when connecting to LDAP using LDAPS or START_TLS. | No | "" | +| tlsKeystoreType | string | Type of the Keystore that is used when connecting to LDAP using LDAPS or START_TLS (i.e. JKS or PKCS12). | No | "" | +| tlsTruststore | string | Path to the Truststore that is used when connecting to LDAP using LDAPS or START_TLS. Required for LDAPS | No | "" | +| tlsTruststorePassword | string | Password for the Truststore that is used when connecting to LDAP using LDAPS or START_TLS. | No | "" | +| tlsTruststoreType | string | Type of the Truststore that is used when connecting to LDAP using LDAPS or START_TLS (i.e. JKS or PKCS12). | No | "" | +| clientAuth | string | Client authentication policy when connecting to LDAP using LDAPS or START_TLS. Possible values are REQUIRED, WANT, NONE. | No | "" | +| protocol | string | Protocol to use when connecting to LDAP using LDAPS or START_TLS. (i.e. TLS, TLSv1.1, TLSv1.2, etc). | No | "" | +| shutdownGracefully | string | Specifies whether the TLS should be shut down gracefully before the target context is closed. Defaults to false. | No | "" | +| referralStrategy | string | Strategy for handling referrals. Possible values are FOLLOW, IGNORE, THROW. | No | FOLLOW | +| identityStrategy | string | Strategy to identify users. Possible values are USE_DN and USE_USERNAME. | No | USE_DN | + +## SingleUserConfiguration + +| Field | Type | Description | Required | Default | +| ----------------- | -------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------------------------------------------ | +| enabled | boolean | specifies whether or not the cluster should use single user authentication for Nifi | No | false | +| authorizerEnabled | boolean | specifies if the cluster should use use the single-user-authorizer instead of the managed-authorizer (if enabled, the creation of users and user groups will not work in NiFi, and the single user will have no rights by default.) | No | true | +| secretRef | [SecretReference](../4_nifi_parameter_context#secretreference) | references the secret containing the informations required to authentiticate to the cluster | No | nil | +| secretKeys | [UserSecretKeys](#usersecretkeys) | references the keys from the secret containing the user name and password. | No | \{username:"username", password:"password"} | + +## NifiClusterTaskSpec + +| Field | Type | Description | Required | Default | +| -------------------- | ---- | ------------------------------------------------------------------------------------------------------------------------------------------ | -------- | ------- | +| retryDurationMinutes | int | describes the time the operator waits before going back and retrying a cluster task, which can be: scale up, scale down, rolling upgrade..| Yes | 5 | + +## ClusterState + +| Name | Value | Description | +| --------------------------- | ----------------------- | ------------------------------------------------------ | +| NifiClusterInitializing | ClusterInitializing | states that the cluster is still in initializing stage | +| NifiClusterInitialized | ClusterInitialized | states that the cluster is initialized | +| NifiClusterReconciling | ClusterReconciling | states that the cluster is still in reconciling stage | +| NifiClusterRollingUpgrading | ClusterRollingUpgrading | states that the cluster is rolling upgrading | +| NifiClusterRunning | ClusterRunning | states that the cluster is in running state | +| NifiClusterNoNodes | NifiClusterNoNodes | states that the cluster has no nodes | + +## UserSecretKeys + +| Field | Type | Description | Required | Default | +| -------- | ------ | ------------------------------------------------------------------ | -------- | -------- | +| username | string | specifies he name of the secret key to retrieve the user name. | No | username | +| password | string | specifies he name of the secret key to retrieve the user password. | No | password | + +## ClusterManagerType + +| Name | Value | Description | +| ------------------------ | ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- | +| ZookeeperClusterManager | zookeeper | indicates that the cluster leader election and state management will be managed with ZooKeeper. When Zookeeper is configured, you must also configure `NifiCluster.spec.zkPath` and `NifiCluster.spec.zkAddress`. | +| KubernetesClusterManager | kubernetes | indicates that the cluster leader election and state management will be managed with Kubernetes resources, with `Leases` and `ConfigMaps` respectively. | \ No newline at end of file diff --git a/site/website/versioned_docs/version-v1.11.4/5_references/1_nifi_cluster/2_read_only_config.md b/site/website/versioned_docs/version-v1.11.4/5_references/1_nifi_cluster/2_read_only_config.md new file mode 100644 index 0000000000..7f9e1fc4b1 --- /dev/null +++ b/site/website/versioned_docs/version-v1.11.4/5_references/1_nifi_cluster/2_read_only_config.md @@ -0,0 +1,223 @@ +--- +id: 2_read_only_config +title: Read only configurations +sidebar_label: Read only configurations +--- + +ReadOnlyConfig object specifies the read-only type Nifi config cluster wide, all theses will be merged with node specified readOnly configurations, so it can be overwritten per node. + +```yaml +readOnlyConfig: + # MaximumTimerDrivenThreadCount define the maximum number of threads for timer driven processors available to the system. + maximumTimerDrivenThreadCount: 30 + # MaximumEventDrivenThreadCount define the maximum number of threads for event driven processors available to the system (@DEPRECATED. This has no effect from NiFiKOp v1.9.0 or later). + maximumEventDrivenThreadCount: 10 + # Logback configuration that will be applied to the node + logbackConfig: + # logback.xml configuration that will replace the one produced based on template + replaceConfigMap: + # The key of the value,in data content, that we want use. + data: logback.xml + # Name of the configmap that we want to refer. + name: raw + # Namespace where is located the secret that we want to refer. + namespace: nifikop + # logback.xml configuration that will replace the one produced based on template and overrideConfigMap + replaceSecretConfig: + # The key of the value,in data content, that we want use. + data: logback.xml + # Name of the configmap that we want to refer. + name: raw + # Namespace where is located the secret that we want to refer. + namespace: nifikop + # Authorizer configuration that will be applied to the node + authorizerConfig: + # An authorizers.xml configuration template that will replace the default template seen in authorizers.go + replaceTemplateConfigMap: + # The key of the value, in data content, that we want use. + data: authorizers.xml + # Name of the configmap that we want to refer. + name: raw + # Namespace where is located the secret that we want to refer. + namespace: nifikop + # An authorizers.xml configuration template that will replace the default template seen in authorizers.go and the replaceTemplateConfigMap + replaceTemplateSecretConfig: + # The key of the value,in data content, that we want use. + data: authorizers.xml + # Name of the configmap that we want to refer. + name: raw + # Namespace where is located the secret that we want to refer. + namespace: nifikop + # NifiProperties configuration that will be applied to the node. + nifiProperties: + # Additionnal nifi.properties configuration that will override the one produced based on template and + # configuration + overrideConfigMap: + # The key of the value,in data content, that we want use. + data: nifi.properties + # Name of the configmap that we want to refer. + name: raw + # Namespace where is located the secret that we want to refer. + namespace: nifikop. + # Additionnal nifi.properties configuration that will override the one produced based + # on template, configurations, overrideConfigMap and overrideConfigs. + overrideSecretConfig: + # The key of the value,in data content, that we want use. + data: nifi.properties + # Name of the configmap that we want to refer. + name: raw + # Namespace where is located the secret that we want to refer. + namespace: nifikop + # Additionnal nifi.properties configuration that will override the one produced based + # on template, configurations and overrideConfigMap + overrideConfigs: | + nifi.ui.banner.text=NiFiKop + # A comma separated list of allowed HTTP Host header values to consider when NiFi + # is running securely and will be receiving requests to a different host[:port] than it is bound to. + # https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#web-properties + # webProxyHosts: + # Nifi security client auth + needClientAuth: false + # Indicates which of the configured authorizers in the authorizers.xml file to use + # https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#authorizer-configuration + # authorizer: + # ZookeeperProperties configuration that will be applied to the node. + zookeeperProperties: + # # Additionnal zookeeeper.properties configuration that will override the one produced based on template and + # # configuration + # overrideConfigMap: + # # The key of the value,in data content, that we want use. + # data: zookeeeper.properties + # # Name of the configmap that we want to refer. + # name: raw + # # Namespace where is located the secret that we want to refer. + # namespace: nifikop. + # # Additionnal zookeeeper.properties configuration that will override the one produced based + # # on template, configurations, overrideConfigMap and overrideConfigs. + # overrideSecretConfig: + # # The key of the value,in data content, that we want use. + # data: zookeeeper.properties + # # Name of the configmap that we want to refer. + # name: raw + # # Namespace where is located the secret that we want to refer. + # namespace: nifikop + # Additionnal zookeeper.properties configuration that will override the one produced based + # on template and configurations. + overrideConfigs: | + initLimit=15 + autopurge.purgeInterval=24 + syncLimit=5 + tickTime=2000 + dataDir=./state/zookeeper + autopurge.snapRetainCount=30 + # BootstrapProperties configuration that will be applied to the node. + bootstrapProperties: + # # Additionnal bootstrap.conf configuration that will override the one produced based on template and + # # configuration + # overrideConfigMap: + # # The key of the value,in data content, that we want use. + # data: bootstrap.conf + # # Name of the configmap that we want to refer. + # name: raw + # # Namespace where is located the secret that we want to refer. + # namespace: nifikop. + # # Additionnal bootstrap.conf configuration that will override the one produced based + # # on template, configurations, overrideConfigMap and overrideConfigs. + # overrideSecretConfig: + # # The key of the value,in data content, that we want use. + # data: bootstrap.conf + # # Name of the configmap that we want to refer. + # name: raw + # # Namespace where is located the secret that we want to refer. + # namespace: nifikop + # JVM memory settings + nifiJvmMemory: "512m" + # Additionnal bootstrap.conf configuration that will override the one produced based + # on template and configurations. + # https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#bootstrap_properties + overrideConfigs: | + # java.arg.4=-Djava.net.preferIPv4Stack=true +``` + +## ReadOnlyConfig + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|maximumTimerDrivenThreadCount|int32|define the maximum number of threads for timer driven processors available to the system.|No|10| +|maximumEventDrivenThreadCount|int32|define the maximum number of threads for event driven processors available to the system (@DEPRECATED. This has no effect from NiFiKOp v1.9.0 or later).|No|1| +|additionalSharedEnvs|\[ \][corev1.EnvVar](https://pkg.go.dev/k8s.io/api/core/v1#EnvVar)|define a set of additional env variables that will shared between all init containers and containers in the pod.|No|\[ \]| +|additionalNifiEnvs|\[ \][corev1.EnvVar](https://pkg.go.dev/k8s.io/api/core/v1#EnvVar)|define a set of additional env variables that will only be embed in the nifi container.|No|\[ \]| +|nifiProperties|[NifiProperties](#nifiproperties)|nifi.properties configuration that will be applied to the node.|No|nil| +|zookeeperProperties|[ZookeeperProperties](#zookeeperproperties)|zookeeper.properties configuration that will be applied to the node.|No|nil| +|bootstrapProperties|[BootstrapProperties](#bootstrapproperties)|bootstrap.conf configuration that will be applied to the node.|No|nil| +|logbackConfig|[LogbackConfig](#logbackconfig)|logback.xml configuration that will be applied to the node.|No|nil| +|authorizerConfig|[AuthorizerConfig](#authorizerconfig)|authorizers.xml configuration template that will be applied to the node.|No|nil| +|bootstrapNotificationServicesConfig|[BootstrapNotificationServices](#bootstrapnotificationservices)|bootstrap_notification_services.xml configuration that will be applied to the node.|No|nil| + + + +## NifiProperties + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|overrideConfigMap|[ConfigmapReference](#configmapreference)|Additionnal nifi.properties configuration that will override the one produced based on template and configuration.|No|nil| +|overrideConfigs|string|Additionnal nifi.properties configuration that will override the one produced based on template, configurations and overrideConfigMap.|No|""| +|overrideSecretConfig|[SecretConfigReference](#secretconfigreference)|Additionnal nifi.properties configuration that will override the one produced based on template, configurations, overrideConfigMap and overrideConfigs.|No|nil| +|webProxyHosts|\[ \]string| A list of allowed HTTP Host header values to consider when NiFi is running securely and will be receiving requests to a different host[:port] than it is bound to. [web-properties](https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#web-properties)|No|""| +|needClientAuth|boolean|Nifi security client auth.|No|false| +|authorizer|string|Indicates which of the configured authorizers in the authorizers.xml file to use [authorizer-configuration](https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#authorizer-configuration)|No|"managed-authorizer"| + + +## ZookeeperProperties + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|overrideConfigMap|[ConfigmapReference](#configmapreference)|Additionnal zookeeper.properties configuration that will override the one produced based on template and configuration.|No|nil| +|overrideConfigs|string|Additionnal zookeeper.properties configuration that will override the one produced based on template, configurations and overrideConfigMap.|No|""| +|overrideSecretConfig|[SecretConfigReference](#secretconfigreference)|Additionnal zookeeper.properties configuration that will override the one produced based on template, configurations, overrideConfigMap and overrideConfigs.|No|nil| + +## BootstrapProperties + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|overrideConfigMap|[ConfigmapReference](#configmapreference)|Additionnal bootstrap.conf configuration that will override the one produced based on template and configuration.|No|nil| +|overrideConfigs|string|Additionnal bootstrap.conf configuration that will override the one produced based on template, configurations and overrideConfigMap.|No|""| +|overrideSecretConfig|[SecretConfigReference](#secretconfigreference)|Additionnal bootstrap.conf configuration that will override the one produced based on template, configurations, overrideConfigMap and overrideConfigs.|No|nil| +|NifiJvmMemory|string|JVM memory settings.|No|"512m"| + +## LogbackConfig + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|replaceConfigMap|[ConfigmapReference](#configmapreference)|logback.xml configuration that will replace the one produced based on template.|No|nil| +|replaceSecretConfig|[SecretConfigReference](#secretconfigreference)|logback.xml configuration that will replace the one produced based on template and overrideConfigMap.|No|nil| + +## AuthorizerConfig + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|replaceTemplateConfigMap|[ConfigmapReference](#configmapreference)|authorizers.xml configuration template that will replace the default template.|No|nil| +|replaceTemplateSecretConfig|[SecretConfigReference](#secretconfigreference)|authorizers.xml configuration that will replace the default template and the replaceTemplateConfigMap.|No|nil| + +## BootstrapNotificationServicesConfig + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|replaceConfigMap|[ConfigmapReference](#configmapreference)|bootstrap_notifications_services.xml configuration that will replace the one produced based on template.|No|nil| +|replaceSecretConfig|[SecretConfigReference](#secretconfigreference)|bootstrap_notifications_services.xml configuration that will replace the one produced based on template and overrideConfigMap.|No|nil| + +## ConfigmapReference + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|name|string|Name of the configmap that we want to refer.|Yes|""| +|namespace|string|Namespace where is located the configmap that we want to refer.|No|""| +|data|string|The key of the value,in data content, that we want use.|Yes|""| + +## SecretConfigReference + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|name|string|Name of the secret that we want to refer.|Yes|""| +|namespace|string|Namespace where is located the secret that we want to refer.|No|""| +|data|string|The key of the value,in data content, that we want use.|Yes|""| \ No newline at end of file diff --git a/site/website/versioned_docs/version-v1.11.4/5_references/1_nifi_cluster/3_node_config.md b/site/website/versioned_docs/version-v1.11.4/5_references/1_nifi_cluster/3_node_config.md new file mode 100644 index 0000000000..7c8b703171 --- /dev/null +++ b/site/website/versioned_docs/version-v1.11.4/5_references/1_nifi_cluster/3_node_config.md @@ -0,0 +1,123 @@ +--- +id: 3_node_config +title: Node configuration +sidebar_label: Node configuration +--- + +NodeConfig defines the node configuration + +```yaml + default_group: + # provenanceStorage allow to specify the maximum amount of data provenance information to store at a time + # https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#write-ahead-provenance-repository-properties + provenanceStorage: "10 GB" + #RunAsUser define the id of the user to run in the Nifi image + # +kubebuilder:validation:Minimum=1 + runAsUser: 1000 + # Set this to true if the instance is a node in a cluster. + # https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#basic-cluster-setup + isNode: true + # Additionnal metadata to merge to the pod associated + podMetadata: + annotations: + node-annotation: "node-annotation-value" + labels: + node-label: "node-label-value" + # Docker image used by the operator to create the node associated + # https://hub.docker.com/r/apache/nifi/ +# image: "apache/nifi:1.11.2" + # nodeAffinity can be specified, operator populates this value if new pvc added later to node + # https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#node-affinity +# nodeAffinity: + # imagePullPolicy define the pull policy for NiFi cluster docker image + imagePullPolicy: IfNotPresent + # priorityClassName define the name of the priority class to be applied to these nodes + priorityClassName: "example-priority-class-name" + # externalVolumeConfigs specifies a list of volume to mount into the main container. + externalVolumeConfigs: + - name: example-volume + mountPath: "/opt/nifi/example" + secret: + secretName: "raw-controller" + # storageConfigs specifies the node related configs + storageConfigs: + # Name of the storage config, used to name PV to reuse into sidecars for example. + - name: provenance-repository + # Retain this PVC throughout NifiCluster deletions. + reclaimPolicy: Retain + # Path where the volume will be mount into the main nifi container inside the pod. + mountPath: "/opt/nifi/provenance_repository" + # Metadata to attach to the PVC that gets created + metadata: + labels: + my-label: my-value + annotations: + my-annotation: my-value + # Kubernetes PVC spec + # https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim + pvcSpec: + accessModes: + - ReadWriteOnce + storageClassName: "standard" + resources: + requests: + storage: 10Gi + - mountPath: "/opt/nifi/nifi-current/logs" + name: logs + reclaimPolicy: Delete + pvcSpec: + accessModes: + - ReadWriteOnce + storageClassName: "standard" + resources: + requests: + storage: 10Gi +``` + +## NodeConfig + +| Field | Type |Description|Required|Default| +|-----------------------|----------------------------------------------------------------------------------------------|-----------|--------|--------| +| provenanceStorage | string |provenanceStorage allow to specify the maximum amount of data provenance information to store at a time: [write-ahead-provenance-repository-properties](https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#write-ahead-provenance-repository-properties)|No|"8 GB"| +| runAsUser | int64 |define the id of the user to run in the Nifi image|No|1000| +| fsGroup | int64 |define the id of the group for each volumes in Nifi image|No|1000| +| isNode | boolean |Set this to true if the instance is a node in a cluster: [basic-cluster-setup](https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#basic-cluster-setup)|No|true| +| image | string | Docker image used by the operator to create the node associated. [Nifi docker registry](https://hub.docker.com/r/apache/nifi/)|No|""| +| imagePullPolicy | [PullPolicy](https://godoc.org/k8s.io/api/core/v1#PullPolicy) | define the pull policy for NiFi cluster docker image.|No|""| +| nodeAffinity | string | operator populates this value if new pvc added later to node [node-affinity](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#node-affinity)|No|nil| +| seccompProfile | [SeccompProfile](https://godoc.org/k8s.io/api/core/v1#SeccompProfile) | overrides the default seccompProfile of the nodes pod | No | \{type: "RuntimeDefault"\} | +| securityContext | [SecurityContext](https://godoc.org/k8s.io/api/core/v1#SecurityContext) | overrides the default container security context for all containers in the pod | No | \{allowPrivilegeEscalation: false, capabilities: \{drop: \["all"\]\}\} | +| storageConfigs | \[ \][StorageConfig](#storageconfig) |specifies the node related configs.|No|nil| +| externalVolumeConfigs | \[ \][ExternalVolumeConfig](#externalvolumeconfig) |specifies a list of volume to mount into the main container.|No|nil| +| serviceAccountName | string |specifies the serviceAccount used for this specific node.|No|"default"| +| resourcesRequirements | [ResourceRequirements](https://godoc.org/k8s.io/api/core/v1#ResourceRequirements) | works exactly like Container resources, the user can specify the limit and the requests through this property [manage-compute-resources-container](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/).|No|nil| +| imagePullSecrets | \[ \][LocalObjectReference](https://godoc.org/k8s.io/api/core/v1#TypedLocalObjectReference) |specifies the secret to use when using private registry.|No|nil| +| nodeSelector | map\[string\]string |nodeSelector can be specified, which set the pod to fit on a node [nodeselector](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector)|No|nil| +| tolerations | \[ \][Toleration](https://godoc.org/k8s.io/api/core/v1#Toleration) |tolerations can be specified, which set the pod's tolerations [taint-and-toleration](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/#concepts).|No|nil| +| podMetadata | [Metadata](#metadata) |define additionnal metadata to merge to the Pod associated.|No|nil| +| hostAliases | \[ \][HostAlias](https://pkg.go.dev/k8s.io/api/core/v1#HostAlias) | A list of host aliases to include in each pod's /etc/hosts configuration in the scenario where DNS is not available. | No | \[ \] | +| priorityClassName | string | Specify the name of the priority class to apply to pods created with this node config | No | nil| + +## StorageConfig + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|name|string|Name of the storage config, used to name PV to reuse into sidecars for example.|Yes| - | +|mountPath|string|Path where the volume will be mount into the main nifi container inside the pod.|Yes| - | +|reclaimPolicy|[PersistentVolumeReclaimPolicy](https://pkg.go.dev/k8s.io/api/core/v1#PersistentVolumeReclaimPolicy)|The PVC reclaim policy. Must be one of \{Delete, Retain\}. Recycle is not supported. If Retain, the PVC is not deleted when the `NifiCluster` is deleted and it will be re-attached to only the node it was previously attached to when the `NifiCluster` is recreated. |No| Delete | +|metadata|[Metadata](#metadata)|Define additional metadata to merge to the PVC associated.|No| - | +|pvcSpec|[PersistentVolumeClaimSpec](https://godoc.org/k8s.io/api/core/v1#PersistentVolumeClaimSpec)|Kubernetes PVC spec. [create-a-persistentvolumeclaim](https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim).|Yes| - | + +## ExternalVolumeConfig + +| Field |Type| Description |Required|Default| +|-------------------------------------------------------------------|----|-------------|--------|--------| +|| [VolumeMount](https://pkg.go.dev/k8s.io/api/core/v1#VolumeMount) |describes a mounting of a Volume within a container.| Yes | - | +|| [VolumeSource](https://pkg.go.dev/k8s.io/api/core/v1#VolumeSource) | VolumeSource represents the location and type of the mounted volume. | Yes | - | + +## Metadata + +| Field |Type| Description |Required|Default| +|-------------------------------------------------------------------|----|-------------|--------|--------| +| annotations | map\[string\]string | Additionnal annotation to merge to the resource associated [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set). |No|nil| +| labels | map\[string\]string | Additionnal labels to merge to the resource associated [labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set). |No|nil| diff --git a/site/website/versioned_docs/version-v1.11.4/5_references/1_nifi_cluster/4_node.md b/site/website/versioned_docs/version-v1.11.4/5_references/1_nifi_cluster/4_node.md new file mode 100644 index 0000000000..a5282b0076 --- /dev/null +++ b/site/website/versioned_docs/version-v1.11.4/5_references/1_nifi_cluster/4_node.md @@ -0,0 +1,66 @@ +--- +id: 4_node +title: Node +sidebar_label: Node +--- + +Node defines the nifi node basic configuration + +```yaml + - id: 0 + # nodeConfigGroup can be used to ease the node configuration, if set only the id is required + nodeConfigGroup: "default_group" + # readOnlyConfig can be used to pass Nifi node config + # which has type read-only these config changes will trigger rolling upgrade + readOnlyConfig: + nifiProperties: + overrideConfigs: | + nifi.ui.banner.text=NiFiKop - Node 0 + # node configuration +# nodeConfig: + - id: 2 + # readOnlyConfig can be used to pass Nifi node config + # which has type read-only these config changes will trigger rolling upgrade + readOnlyConfig: + overrideConfigs: | + nifi.ui.banner.text=NiFiKop - Node 2 + # node configuration + nodeConfig: + resourcesRequirements: + limits: + cpu: "2" + memory: 3Gi + requests: + cpu: "1" + memory: 1Gi + storageConfigs: + # Name of the storage config, used to name PV to reuse into sidecars for example. + - name: provenance-repository + # Path where the volume will be mount into the main nifi container inside the pod. + mountPath: "/opt/nifi/provenance_repository" + # Metadata to attach to the PVC that gets created + metadata: + labels: + my-label: my-value + annotations: + my-annotation: my-value + # Kubernetes PVC spec + # https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim + pvcSpec: + accessModes: + - ReadWriteOnce + storageClassName: "standard" + resources: + requests: + storage: 8Gi +``` + +## Node + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|id|int32| unique Node id. |Yes| - | +|nodeConfigGroup|string| can be used to ease the node configuration, if set only the id is required |No| "" | +|readOnlyConfig|[ReadOnlyConfig](./2_read_only_config)| readOnlyConfig can be used to pass Nifi node config which has type read-only these config changes will trigger rolling upgrade.| No | nil | +|nodeConfig|[NodeConfig](./3_node_config)| node configuration. |No| nil | + diff --git a/site/website/versioned_docs/version-v1.11.4/5_references/1_nifi_cluster/5_node_state.md b/site/website/versioned_docs/version-v1.11.4/5_references/1_nifi_cluster/5_node_state.md new file mode 100644 index 0000000000..9b04fc0195 --- /dev/null +++ b/site/website/versioned_docs/version-v1.11.4/5_references/1_nifi_cluster/5_node_state.md @@ -0,0 +1,73 @@ +--- +id: 5_node_state +title: Node state +sidebar_label: Node state +--- + +Holds information about nifi state + +## NodeState + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|gracefulActionState|[GracefulActionState](#gracefulactionstate)| holds info about nifi cluster action status.| - | - | +|configurationState|[ConfigurationState](#configurationstate)| holds info about the config.| - | - | +|initClusterNode|[InitClusterNode](#initclusternode)| contains if this nodes was part of the initial cluster.| - | - | +|podIsReady|bool| True if the pod for this node is up and running. Otherwise false.| - | - | +|creationTime|[v1.Time](https://pkg.go.dev/k8s.io/apimachinery/pkg/apis/meta/v1#Time)| The time at which this node was created and added to the cluster| - | - | + + +## GracefulActionState + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|errorMessage|string| holds the information what happened with Nifi Cluster. | - | "" | +|actionStep|[ActionStep](#actionstep)| holds info about the action step ran.| No | nil | +|taskStarted|string| hold the time when the execution started.| No | "" | +|actionState|[State](#state)| holds the information about Action state.| No | nil | + +## ConfigurationState + +|Name|Value|Description| +|-----|----|------------| +|ConfigInSync|ConfigInSync|states that the generated nodeConfig is in sync with the Node| +|ConfigOutOfSync|ConfigOutOfSync|states that the generated nodeConfig is out of sync with the Node| + +## InitClusterNode + +|Name|Value|Description| +|-----|----|------------| +|IsInitClusterNode|true|states the node is part of initial cluster setup| +|NotInitClusterNode|false|states the node is not part of initial cluster setup| + +## State + +### Upscale + +|Name|Value|Description| +|-----|----|------------| +|GracefulUpscaleRequired|GracefulUpscaleRequired|states that a node upscale is required.| +|GracefulUpscaleRunning|GracefulUpscaleRunning|states that the node upscale task is still running.| +|GracefulUpscaleSucceeded|GracefulUpscaleSucceeded|states the node is updated gracefully.| + +### Downscale + +|Name|Value|Description| +|-----|----|------------| +|GracefulDownscaleRequired|GracefulDownscaleRequired|states that a node downscale is required| +|GracefulDownscaleRunning|GracefulDownscaleRunning|states that the node downscale is still running in| +|GracefulUpscaleSucceeded|GracefulUpscaleSucceeded|states the node is updated gracefully| + +## ActionStep +|Name|Value|Description| +|-----|----|------------| +|DisconnectNodeAction|DISCONNECTING|states that the NiFi node is disconnecting from NiFi Cluster.| +|DisconnectStatus|DISCONNECTED|states that the NiFi node is disconnected from NiFi Cluster.| +|OffloadNodeAction|OFFLOADING|states that the NiFi node is offloading data to NiFi Cluster.| +|OffloadStatus|OFFLOADED|states that the NiFi node offloaded data to NiFi Cluster.| +|RemovePodAction|POD_REMOVING|states that the NiFi node pod and object related are removing by operator.| +|RemovePodStatus|POD_REMOVED|states that the NiFi node pod and object related have been removed by operator.| +|RemoveNodeAction|REMOVING|states that the NiFi node is removing from NiFi Cluster.| +|RemoveStatus|REMOVED|states that the NiFi node is removed from NiFi Cluster.| +|ConnectNodeAction|CONNECTING|states that the NiFi node is connecting to the NiFi Cluster.| +|ConnectStatus|CONNECTED|states that the NiFi node is connected to the NiFi Cluster.| \ No newline at end of file diff --git a/site/website/versioned_docs/version-v1.11.4/5_references/1_nifi_cluster/6_listeners_config.md b/site/website/versioned_docs/version-v1.11.4/5_references/1_nifi_cluster/6_listeners_config.md new file mode 100644 index 0000000000..204f992e79 --- /dev/null +++ b/site/website/versioned_docs/version-v1.11.4/5_references/1_nifi_cluster/6_listeners_config.md @@ -0,0 +1,63 @@ +--- +id: 6_listeners_config +title: Listeners Config +sidebar_label: Listeners Config +--- + +ListenersConfig defines the Nifi listener types: + +```yaml + listenersConfig: + internalListeners: + - type: "https" + name: "https" + containerPort: 8443 + - type: "cluster" + name: "cluster" + containerPort: 6007 + - type: "s2s" + name: "s2s" + containerPort: 10000 + - type: "prometheus" + name: "prometheus" + containerPort: 9090 + - type: "load-balance" + name: "load-balance" + containerPort: 6342 + - name: "my-custom-listener-port" + containerPort: 1234 + protocol: "TCP" + sslSecrets: + tlsSecretName: "test-nifikop" + create: true +``` + +## ListenersConfig + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|internalListeners|\[ \][InternalListener](#internallistener)| specifies settings required to access nifi internally.| Yes | - | +|sslSecrets|[SSLSecrets](#sslsecrets)| contains information about ssl related kubernetes secrets if one of the listener setting type set to ssl these fields must be populated to.| Yes | nil | +|clusterDomain|string| allow to override the default cluster domain which is "cluster.local".| Yes | `cluster.local` | +|useExternalDNS|string| allow to manage externalDNS usage by limiting the DNS names associated to each nodes and load balancer: `-node-...`| Yes | false | + +## InternalListener + +Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|type|enum{ "cluster", "http", "https", "s2s", "prometheus", "load-balance"}| allow to specify if we are in a specific nifi listener it's allowing to define some required information such as Cluster Port, Http Port, Https Port, S2S, Load Balance port, or Prometheus port| Yes | - | +|name|string| an identifier for the port which will be configured. | Yes | - | +|containerPort|int32| the containerPort. | Yes | - | +|protocol|[Protocol](https://pkg.go.dev/k8s.io/api/core/v1#Protocol)| the network protocol for this listener. Must be one of the protocol enum values (i.e. TCP, UDP, SCTP). | No | `TCP` | + + +## SSLSecrets + +Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|tlsSecretName|string| should contain all ssl certs required by nifi including: caCert, caKey, clientCert, clientKey serverCert, serverKey, peerCert, peerKey. | Yes | - | +|create|boolean| tells the installed cert manager to create the required certs keys. | Yes | - | +|clusterScoped|boolean| defines if the Issuer created is cluster or namespace scoped. | Yes | - | +|issuerRef|[ObjectReference](https://docs.cert-manager.io/en/release-0.9/reference/api-docs/index.html#objectreference-v1alpha1)| IssuerRef allow to use an existing issuer to act as CA: https://cert-manager.io/docs/concepts/issuer/ | No | - | +|pkiBackend|enum{"cert-manager"}| | Yes | - | + diff --git a/site/website/versioned_docs/version-v1.11.4/5_references/1_nifi_cluster/7_external_service_config.md b/site/website/versioned_docs/version-v1.11.4/5_references/1_nifi_cluster/7_external_service_config.md new file mode 100644 index 0000000000..ac2e0ed512 --- /dev/null +++ b/site/website/versioned_docs/version-v1.11.4/5_references/1_nifi_cluster/7_external_service_config.md @@ -0,0 +1,85 @@ +--- +id: 7_external_service_config +title: External Service Config +sidebar_label: External Service Config +--- + +ListenersConfig defines the Nifi listener types: + +```yaml + externalServices: + - name: "clusterip" + spec: + type: ClusterIP + portConfigs: + - port: 8080 + internalListenerName: "http" + - port: 7182 + internalListenerName: "my-custom-listener" + protocol: TCP + metadata: + annotations: + toto: tata + labels: + titi: tutu +``` + +Load balancer example: + +```yaml +externalServices: + - name: "nlb" + spec: + type: LoadBalancer + loadBalancerClass: "service.k8s.aws/nlb" + portConfigs: + - port: 8080 + internalListenerName: "http" + - port: 7890 + internalListenerName: "my-custom-udp-listener" + protocol: UDP + metadata: + annotations: + toto: tata + labels: + titi: tutu +``` + +## ExternalServiceConfig + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|name|string| Must be unique within a namespace. Name is primarily intended for creation idempotence and configuration.| Yes | - | +|metadata|[Metadata](#metadata)| Defines additional metadata to merge with the associated service.| No | - | +|spec|[ExternalServiceSpec](#externalservicespec)| defines the behavior of a service.| Yes | | + +## ExternalServiceSpec + +Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|portConfigs||\[ \][PortConfig](#portconfig)| Contains the list port for the service and the associated listener| Yes | - | +|clusterIP|string| More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies | No | - | +|type|[ServiceType](https://godoc.org/k8s.io/api/core/v1#ServiceType)| type determines how the Service is exposed. Defaults to ClusterIP. Valid options are ExternalName, ClusterIP, NodePort, and LoadBalancer. | No | - | +|externalIPs|\[ \]string| externalIPs is a list of IP addresses for which nodes in the cluster will also accept traffic for this service. These IPs are not managed by Kubernetes | No | - | +|loadBalancerIP|string| Only applies to Service Type: LoadBalancer. LoadBalancer will get created with the IP specified in this field. | No | - | +|loadBalancerSourceRanges|\[ \]string| If specified and supported by the platform, this will restrict traffic through the cloud-provider load-balancer will be restricted to the specified client IPs | No | - | +|externalName|string| externalName is the external reference that kubedns or equivalent will return as a CNAME record for this service. No proxying will be involved. | No | - | +|loadBalancerClass|string| loadBalancerClass is the class of the load balancer implementation this Service belongs to. | No | - | +|externalTrafficPolicy|string| See the Kubernetes [traffic policies](https://kubernetes.io/docs/reference/networking/virtual-ips/#traffic-policies) documentation. | No | Depends on the `Service` type. | +|internalTrafficPolicy|string| See the Kubernetes [traffic policies](https://kubernetes.io/docs/reference/networking/virtual-ips/#traffic-policies) documentation. | No | Depends on the `Service` type. | + +## PortConfig + +Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|port|int32| The port that will be exposed by this service. | Yes | - | +|internalListenerName|string| The name of the listener which will be used as target container. | Yes | - | +|nodePort|int32| The port that will expose this service externally. (Only if the service is of type NodePort) | No | - | +|protocol|[Protocol](https://pkg.go.dev/k8s.io/api/core/v1#Protocol)| The network protocol for this service port. Must be one of the protocol enum values (i.e. TCP, UDP, SCTP). | No | `TCP` | + +## Metadata + +Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +| annotations | map\[string\]string | Additional annotations to merge with the associated service [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set). | No | `nil` | +| labels | map\[string\]string | Additional labels to merge with the associated service [labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set). | No | `nil` | diff --git a/site/website/versioned_docs/version-v1.11.4/5_references/2_nifi_user.md b/site/website/versioned_docs/version-v1.11.4/5_references/2_nifi_user.md new file mode 100644 index 0000000000..a6616e24d3 --- /dev/null +++ b/site/website/versioned_docs/version-v1.11.4/5_references/2_nifi_user.md @@ -0,0 +1,101 @@ +--- +id: 2_nifi_user +title: NiFi User +sidebar_label: NiFi User +--- + +`NifiUser` is the Schema for the nifi users API. + +```yaml +apiVersion: nifi.konpyutaika.com/v1 +kind: NifiUser +metadata: + name: aguitton +spec: + identity: alexandre.guitton@konpyutaika.com + clusterRef: + name: nc + namespace: nifikop + createCert: false +``` + +## NifiUser +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|metadata|[ObjectMetadata](https://godoc.org/k8s.io/apimachinery/pkg/apis/meta/v1#ObjectMeta)|is metadata that all persisted resources must have, which includes all objects users must create.|No|nil| +|spec|[NifiUserSpec](#nifiuserspec)|defines the desired state of NifiUser.|No|nil| +|status|[NifiUserStatus](#nifiuserstatus)|defines the observed state of NifiUser.|No|nil| + +## NifiUserSpec + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|identity|string| used to define the user identity on NiFi cluster side, when the user's name doesn't suit with Kubernetes resource name. |No| - | +|secretName|string| name of the secret where all cert resources will be stored. |No| - | +|clusterRef|[ClusterReference](#clusterreference)| contains the reference to the NifiCluster with the one the user is linked. |Yes| - | +|DNSNames|\[ \]string| list of DNSNames that the user will used to request the NifiCluster (allowing to create the right certificates associated). |Yes| - | +|includeJKS|boolean| whether or not the the operator also include a Java keystore format (JKS) with you secret. |Yes| - | +|createCert|boolean| whether or not a certificate will be created for this user. |No| - | +|accessPolicies|\[ \][AccessPolicy](#accesspolicy)| defines the list of access policies that will be granted to the group. |No| [] | + + +## NifiUserStatus + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|id|string| the nifi user's node id.|Yes| - | +|version|string| the last nifi user's node revision version catched.|Yes| - | + +## ClusterReference + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|name|string| name of the NifiCluster. |Yes| - | +|namespace|string| the NifiCluster namespace location. |Yes| - | + +## AccessPolicy + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|type|[AccessPolicyType](#accesspolicytype)| defines the kind of access policy, could be "global" or "component". |Yes| - | +|action|[AccessPolicyAction](#accesspolicyaction)| defines the kind of action that will be granted, could be "read" or "write". |Yes| - | +|resource|[AccessPolicyResource](#accesspolicyresource)| defines the kind of resource targeted by this access policies, please refer to the following page: https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#access-policies |Yes| - | +|componentType|string| used if the type is "component", it allows to define the kind of component on which is the access policy. |No| - | +|componentId|string| used if the type is "component", it allows to define the id of the component on which is the access policy. |No| - | + +## AccessPolicyType + +|Name|Value|Description| +|-----|----|------------| +|GlobalAccessPolicyType|global|Global access policies govern the following system level authorizations| +|ComponentAccessPolicyType|component|Component level access policies govern the following component level authorizations| + +## AccessPolicyAction + +|Name|Value|Description| +|-----|----|------------| +|ReadAccessPolicyAction|read|Allows users to view| +|WriteAccessPolicyAction|write|Allows users to modify| + +## AccessPolicyResource + +|Name|Value|Description| +|-----|----|------------| +|FlowAccessPolicyResource|/flow|About the UI| +|ControllerAccessPolicyResource|/controller| about the controller including Reporting Tasks, Controller Services, Parameter Contexts and Nodes in the Cluster| +|ParameterContextAccessPolicyResource|/parameter-contexts|About the Parameter Contexts. Access to Parameter Contexts are inherited from the "access the controller" policies unless overridden.| +|ProvenanceAccessPolicyResource|/provenance|Allows users to submit a Provenance Search and request Event Lineage| +|RestrictedComponentsAccessPolicyResource|/restricted-components|About the restricted components assuming other permissions are sufficient. The restricted components may indicate which specific permissions are required. Permissions can be granted for specific restrictions or be granted regardless of restrictions. If permission is granted regardless of restrictions, the user can create/modify all restricted components.| +|PoliciesAccessPolicyResource|/policies|About the policies for all components| +|TenantsAccessPolicyResource|/tenants| About the users and user groups| +|SiteToSiteAccessPolicyResource|/site-to-site|Allows other NiFi instances to retrieve Site-To-Site details| +|SystemAccessPolicyResource|/system|Allows users to view System Diagnostics| +|ProxyAccessPolicyResource|/proxy|Allows proxy machines to send requests on the behalf of others| +|CountersAccessPolicyResource|/counters|About counters| +|ComponentsAccessPolicyResource|/| About the component configuration details| +|OperationAccessPolicyResource|/operation|to operate components by changing component run status (start/stop/enable/disable), remote port transmission status, or terminating processor threads| +|ProvenanceDataAccessPolicyResource|/provenance-data|to view provenance events generated by this component| +|DataAccessPolicyResource|/data|About metadata and content for this component in flowfile queues in outbound connections and through provenance events| +|PoliciesComponentAccessPolicyResource|/policies|-| +|DataTransferAccessPolicyResource|/data-transfer|Allows a port to receive data from NiFi instances| + diff --git a/site/website/versioned_docs/version-v1.11.4/5_references/3_nifi_registry_client.md b/site/website/versioned_docs/version-v1.11.4/5_references/3_nifi_registry_client.md new file mode 100644 index 0000000000..000bb409d1 --- /dev/null +++ b/site/website/versioned_docs/version-v1.11.4/5_references/3_nifi_registry_client.md @@ -0,0 +1,42 @@ +--- +id: 3_nifi_registry_client +title: NiFi Registry Client +sidebar_label: NiFi Registry Client +--- + +`NifiRegistryClient` is the Schema for the NiFi registry client API. + +```yaml +apiVersion: nifi.konpyutaika.com/v1alpha1 +kind: NifiRegistryClient +metadata: + name: squidflow +spec: + clusterRef: + name: nc + namespace: nifikop + description: "Squidflow demo" + uri: "http://nifi-registry:18080" +``` + +## NifiRegistryClient +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|metadata|[ObjectMetadata](https://godoc.org/k8s.io/apimachinery/pkg/apis/meta/v1#ObjectMeta)|is metadata that all persisted resources must have, which includes all objects registry clients must create.|No|nil| +|spec|[NifiRegistryClientSpec](#nifiregistryclientspec)|defines the desired state of NifiRegistryClient.|No|nil| +|status|[NifiRegistryClientStatus](#nifiregistryclientstatus)|defines the observed state of NifiRegistryClient.|No|nil| + +## NifiRegistryClientsSpec + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|description|string| describes the Registry client. |No| - | +|uri|string| URI of the NiFi registry that should be used for pulling the flow. |Yes| - | +|clusterRef|[ClusterReference](./2_nifi_user#clusterreference)| contains the reference to the NifiCluster with the one the user is linked. |Yes| - | + +## NifiRegistryClientStatus + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|id|string| nifi registry client's id. |Yes| - | +|version|int64| the last nifi registry client revision version catched. |Yes| - | \ No newline at end of file diff --git a/site/website/versioned_docs/version-v1.11.4/5_references/4_nifi_parameter_context.md b/site/website/versioned_docs/version-v1.11.4/5_references/4_nifi_parameter_context.md new file mode 100644 index 0000000000..805acb6b38 --- /dev/null +++ b/site/website/versioned_docs/version-v1.11.4/5_references/4_nifi_parameter_context.md @@ -0,0 +1,124 @@ +--- +id: 4_nifi_parameter_context +title: NiFi Parameter Context +sidebar_label: NiFi Parameter Context +--- + +`NifiParameterContext` is the Schema for the NiFi parameter context API. + +```yaml +apiVersion: nifi.konpyutaika.com/v1 +kind: NifiParameterContext +metadata: + name: dataflow-lifecycle +spec: + description: "It is a test" + clusterRef: + name: nc + namespace: nifikop + secretRefs: + - name: secret-params + namespace: nifikop + parameters: + - name: test + value: toto + description: tutu + - name: test2 + description: toto + sensistive: true +--- +apiVersion: nifi.konpyutaika.com/v1 +kind: NifiParameterContext +metadata: + name: dataflow-lifecycle-child +spec: + description: "It is a child test" + clusterRef: + name: nc + namespace: nifikop + secretRefs: + - name: secret-params + namespace: nifikop + inheritedParameterContexts: + - name: dataflow-lifecycle + parameters: + - name: test + value: toto-child + description: tutu (child) +``` + +## NifiParameterContext + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|metadata|[ObjectMetadata](https://godoc.org/k8s.io/apimachinery/pkg/apis/meta/v1#ObjectMeta)|is metadata that all persisted resources must have, which includes all objects parameter contexts must create.|No|nil| +|spec|[NifiParameterContextSpec](#nifiparametercontextspec)|defines the desired state of NifiParameterContext.|No|nil| +|status|[NifiParameterContextStatus](#nifiparametercontextstatus)|defines the observed state of NifiParameterContext.|No|nil| + +## NifiParameterContextsSpec + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|description|string| describes the Parameter Context. |No| - | +|parameters|\[ \][Parameter](#parameter)| a list of non-sensitive Parameters. |Yes| - | +|secretRefs|\[ \][SecretReference](#secretreference)| a list of secret containing sensitive parameters (the key will name of the parameter) |No| - | +|clusterRef|[ClusterReference](./2_nifi_user#clusterreference)| contains the reference to the NifiCluster with the one the user is linked. |Yes| - | +|inheritedParameterContext|[ParameterContextReference](#parametercontextreference)| contains the reference(s) to the NiFiParameterContext it should inherit from. |No| - | +|disableTakeOver|bool| whether or not the operator should take over an existing parameter context if its name is the same. |No| - | + +## NifiParameterContextStatus + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|id|string| nifi parameter context's id. |Yes| - | +|version|int64| the last nifi parameter context revision version catched. |Yes| - | +|latestUpdateRequest|[ParameterContextUpdateRequest](#parametercontextupdaterequest)|the latest update request. |Yes| - | +|version|int64| the last nifi parameter context revision version catched. |Yes| - | +|LatestSecretsResourceVersion|\[ \][SecretResourceVersion](#secretResourceVersion)|the latest `resourceVersion` of the secrets. |No| - | + +## Parameter + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|name|string| the name of the Parameter. |Yes| - | +|value|string| the value of the Parameter. |No| - | +|description|string| the description of the Parameter. |No| - | +|sensitive|string| Whether the parameter is sensitive or not. |No| false | + +## SecretReference + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|name|string| name of the secret. |Yes| - | +|namespace|string| the secret namespace location. |Yes| - | + + +## ParameterContextUpdateRequest + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|id|string| the id of the update request. |Yes| - | +|uri|string| the uri for this request. |Yes| - | +|submissionTime|string| the timestamp of when the request was submitted This property is read only. |Yes| - | +|lastUpdated|string| the timestamp of when the request was submitted This property is read only. |Yes| - | +|complete|bool| whether or not this request has completed. |Yes| false | +|failureReason|string| an explication of why the request failed, or null if this request has not failed. |Yes| - | +|percentCompleted|int32| the percentage complete of the request, between 0 and 100. |Yes| - | +|state|string| the state of the request. |Yes| - | +|notFound|bool| whether or not this request was found. |Yes| false | +|notFoundRetryCount|int32| the number of consecutive retries made in case of a NotFound error (limit: 3). |Yes| 0 | + +## ParameterContextReference + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|name|string| name of the NifiParameterContext. |Yes| - | +|namespace|string| the NifiParameterContext namespace location. |No| - | + +## SecretResourceVersion + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|name|string| name of the secret. |Yes| - | +|namespace|string| namespace where is located the secret. |Yes| - | +|resourceVersion|string| resource version of the secret. |Yes| - | \ No newline at end of file diff --git a/site/website/versioned_docs/version-v1.11.4/5_references/5_nifi_dataflow.md b/site/website/versioned_docs/version-v1.11.4/5_references/5_nifi_dataflow.md new file mode 100644 index 0000000000..65a2c7eed2 --- /dev/null +++ b/site/website/versioned_docs/version-v1.11.4/5_references/5_nifi_dataflow.md @@ -0,0 +1,140 @@ +--- +id: 5_nifi_dataflow +title: NiFi Dataflow +sidebar_label: NiFi Dataflow +--- + +`NifiDataflow` is the Schema for the NiFi dataflow API. + +```yaml +apiVersion: nifi.konpyutaika.com/v1 +kind: NifiDataflow +metadata: + name: dataflow-lifecycle +spec: + parentProcessGroupID: "16cfd2ec-0174-1000-0000-00004b9b35cc" + bucketId: "01ced6cc-0378-4893-9403-f6c70d080d4f" + flowId: "9b2fb465-fb45-49e7-94fe-45b16b642ac9" + flowVersion: 2 + flowPosition: + posX: 0 + posY: 0 + syncMode: always + skipInvalidControllerService: true + skipInvalidComponent: true + clusterRef: + name: nc + namespace: nifikop + registryClientRef: + name: squidflow + namespace: nifikop + parameterContextRef: + name: dataflow-lifecycle + namespace: nifikop + updateStrategy: drain +``` + +## NifiDataflow + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|metadata|[ObjectMetadata](https://godoc.org/k8s.io/apimachinery/pkg/apis/meta/v1#ObjectMeta)|is metadata that all persisted resources must have, which includes all objects dataflows must create.|No|nil| +|spec|[NifiDataflowSpec](#nifidataflowspec)|defines the desired state of NifiDataflow.|No|nil| +|status|[NifiDataflowStatus](#nifidataflowstatus)|defines the observed state of NifiDataflow.|No|nil| + + +## NifiDataflowsSpec + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|parentProcessGroupID|string|the UUID of the parent process group where you want to deploy your dataflow, if not set deploy at root level. |No| - | +|bucketId|string|the UUID of the Bucket containing the flow. |Yes| - | +|flowId|string|the UUID of the flow to run. |Yes| - | +|flowVersion|*int32|the version of the flow to run. |Yes| - | +|flowPosition|[FlowPosition](#flowposition)|the position of your dataflow in the canvas. |No| - | +|syncMode|Enum={"never","always","once"}|if the flow will be synchronized once, continuously or never. |No| always | +|skipInvalidControllerService|bool|whether the flow is considered as ran if some controller services are still invalid or not. |Yes| false | +|skipInvalidComponent|bool|whether the flow is considered as ran if some components are still invalid or not. |Yes| false | +|updateStrategy|[ComponentUpdateStrategy](#componentupdatestrategy)|describes the way the operator will deal with data when a dataflow will be updated: Drop or Drain |Yes| drain | +|clusterRef|[ClusterReference](./2_nifi_user#clusterreference)| contains the reference to the NifiCluster with the one the user is linked. |Yes| - | +|parameterContextRef|[ParameterContextReference](./4_nifi_parameter_context#parametercontextreference)| contains the reference to the ParameterContext with the one the dataflow is linked. |No| - | +|registryClientRef|[RegistryClientReference](./3_nifi_registry_client#registryclientreference)| contains the reference to the NifiRegistry with the one the dataflow is linked. |Yes| - | + +## NifiDataflowStatus + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|processGroupID|string| process Group ID. |Yes| - | +|state|[DataflowState](#dataflowstate)| the dataflow current state. |Yes| - | +|latestUpdateRequest|[UpdateRequest](#updaterequest)|the latest update request sent. |Yes| - | +|latestDropRequest|[DropRequest](#droprequest)|the latest queue drop request sent. |Yes| - | + +## ComponentUpdateStrategy + +|Name|Value|Description| +|-----|----|------------| +|DrainStrategy|drain|leads to shutting down only input components (Input processors, remote input process group) and waiting the dataflow to be drained.| +|DropStrategy|drop|leads to shutting down all components and dropping all flowfiles from the flow.| + +## DataflowState + +|Name|Value|Description| +|-----|----|------------| +|DataflowStateCreated|Created|describes the status of a NifiDataflow as created.| +|DataflowStateStarting|Starting|describes the status of a NifiDataflow as starting.| +|DataflowStateRan|Ran|describes the status of a NifiDataflow as running.| +|DataflowStateOutOfSync|OutOfSync|describes the status of a NifiDataflow as out of sync.| +|DataflowStateInSync|InSync|describes the status of a NifiDataflow as in sync.| + +## UpdateRequest + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|type|[DataflowUpdateRequestType](#dataflowupdaterequesttype)|defines the type of versioned flow update request. |Yes| - | +|id|string|the id of the update request. |Yes| - | +|uri|string|the uri for this request. |Yes| - | +|lastUpdated|string|the last time this request was updated. |Yes| - | +|complete|bool| whether or not this request has completed. |Yes| false | +|failureReason|string| an explication of why the request failed, or null if this request has not failed. |Yes| - | +|percentCompleted|int32| the percentage complete of the request, between 0 and 100. |Yes| 0 | +|state|string| the state of the request. |Yes| - | +|notFound|bool| whether or not this request was found. |Yes| false | +|notFoundRetryCount|int32| the number of consecutive retries made in case of a NotFound error (limit: 3). |Yes| 0 | + +## DropRequest + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|connectionId|string|the connection id. |Yes| - | +|id|string|the id for this drop request. |Yes| - | +|uri|string|the uri for this request. |Yes| - | +|lastUpdated|string|the last time this request was updated. |Yes| - | +|finished|bool|whether the request has finished. |Yes| false | +|failureReason|string|an explication of why the request failed, or null if this request has not failed. |Yes| - | +|percentCompleted|int32|the percentage complete of the request, between 0 and 100. |Yes| 0 | +|currentCount|int32|the number of flow files currently queued. |Yes| 0 | +|currentSize|int64| the size of flow files currently queued in bytes. |Yes| 0 | +|current|string|the count and size of flow files currently queued. |Yes| - | +|originalCount|int32|the number of flow files to be dropped as a result of this request. |Yes| 0 | +|originalSize|int64| the size of flow files to be dropped as a result of this request in bytes. |Yes| 0 | +|original|string|the count and size of flow files to be dropped as a result of this request. |Yes| - | +|droppedCount|int32|the number of flow files that have been dropped thus far. |Yes| 0 | +|droppedSize|int64| the size of flow files currently queued in bytes. |Yes| 0 | +|Dropped|string|the count and size of flow files that have been dropped thus far. |Yes| - | +|state|string|the state of the request. |Yes| - | +|notFound|bool|whether or not this request was found. |Yes| false | +|notFoundRetryCount|int32| the number of consecutive retries made in case of a NotFound error (limit: 3). |Yes| 0 | + +## DataflowUpdateRequestType + +|Name|Value|Description| +|-----|----|------------| +|RevertRequestType|Revert|defines a revert changes request.| +|UpdateRequestType|Update|defines an update version request.| + +## FlowPosition + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|posX|int64|the x coordinate. |No| - | +|posY|int64|the y coordinate. |No| - | \ No newline at end of file diff --git a/site/website/versioned_docs/version-v1.11.4/5_references/6_nifi_usergroup.md b/site/website/versioned_docs/version-v1.11.4/5_references/6_nifi_usergroup.md new file mode 100644 index 0000000000..cbbdeacd06 --- /dev/null +++ b/site/website/versioned_docs/version-v1.11.4/5_references/6_nifi_usergroup.md @@ -0,0 +1,57 @@ +--- +id: 6_nifi_usergroup +title: NiFi UserGroup +sidebar_label: NiFi UserGroup +--- + +`NifiUserGroup` is the Schema for the nifi user groups API. + +```yaml +apiVersion: nifi.konpyutaika.com/v1 +kind: NifiUserGroup +metadata: + name: group-test +spec: + identity: "My Special Group" + clusterRef: + name: nc + namespace: nifikop + usersRef: + - name: nc-0-node.nc-headless.nifikop.svc.cluster.local + - name: nc-controller.nifikop.mgt.cluster.local + accessPolicies: + - type: global + action: read + resource: /counters +``` + +## NifiUserGroup +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|metadata|[ObjectMetadata](https://godoc.org/k8s.io/apimachinery/pkg/apis/meta/v1#ObjectMeta)|is metadata that all persisted resources must have, which includes all objects usergroups must create.|No|nil| +|spec|[NifiUserGroupSpec](#nifiusergroupspec)|defines the desired state of NifiUserGroup.|No|nil| +|status|[NifiUserGroupStatus](#nifiusergroupstatus)|defines the observed state of NifiUserGroup.|No|nil| + +## NifiUserGroupSpec + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|identity|string| Used to define the group's identity on NiFi cluster side, when the group's name doesn't suit Kubernetes resource name requirements. |No| - | +|clusterRef|[ClusterReference](./2_nifi_user#clusterreference)| contains the reference to the NifiCluster with the one the user is linked. |Yes| - | +|usersRef|\[ \][UserReference](#userref)| contains the list of reference to NifiUsers that are part to the group. |No| [] | +|accessPolicies|\[ \][AccessPolicy](./2_nifi_user#accesspolicy)| defines the list of access policies that will be granted to the group. |No| [] | + +## NifiUserGroupStatus + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|id|string| the nifi usergroup's node id.|Yes| - | +|version|string| the last nifi usergroup's node revision version catched.|Yes| - | + +## UserReference + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|name|string| name of the NifiUser. |Yes| - | +|namespace|string| the NifiUser namespace location. |Yes| - | + diff --git a/site/website/versioned_docs/version-v1.11.4/5_references/7_nifi_nodegroup_autoscaler.md b/site/website/versioned_docs/version-v1.11.4/5_references/7_nifi_nodegroup_autoscaler.md new file mode 100644 index 0000000000..28229a1cf1 --- /dev/null +++ b/site/website/versioned_docs/version-v1.11.4/5_references/7_nifi_nodegroup_autoscaler.md @@ -0,0 +1,59 @@ +--- +id: 7_nifi_nodegroup_autoscaler +title: NiFi NodeGroup Autoscaler +sidebar_label: NiFi NodeGroup Autoscaler +--- + +`NifiNodeGroupAutoscaler` is the Schema through which you configure automatic scaling of `NifiCluster` deployments. + +```yaml +apiVersion: nifi.konpyutaika.com/v1alpha1 +kind: NifiNodeGroupAutoscaler +metadata: + name: nifinodegroupautoscaler-sample +spec: + # contains the reference to the NifiCluster with the one the node group autoscaler is linked. + clusterRef: + name: nificluster-name + namespace: nifikop + # defines the id of the NodeConfig contained in NifiCluster.Spec.NodeConfigGroups + nodeConfigGroupId: default-node-group + # The selector used to identify nodes in NifiCluster.Spec.Nodes this autoscaler will manage + # Use Node.Labels in combination with this selector to clearly define which nodes will be managed by this autoscaler + nodeLabelsSelector: + matchLabels: + nifi_cr: nificluster-name + nifi_node_group: default-node-group + # the strategy used to decide how to add nodes to a nifi cluster + upscaleStrategy: simple + # the strategy used to decide how to remove nodes from an existing cluster + downscaleStrategy: lifo +``` + +## NifiNodeGroupAutoscaler +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|metadata|[ObjectMetadata](https://godoc.org/k8s.io/apimachinery/pkg/apis/meta/v1#ObjectMeta)|is metadata that all persisted resources must have, which includes all objects nodegroupautoscalers must create.|No|nil| +|spec|[NifiNodeGroupAutoscalerSpec](#nifinodegroupautoscalerspec)|defines the desired state of NifiNodeGroupAutoscaler.|No|nil| +|status|[NifiNodeGroupAutoscalerStatus](#nifinodegroupautoscalerstatus)|defines the observed state of NifiNodeGroupAutoscaler.|No|nil| + +## NifiNodeGroupAutoscalerSpec + +|Field| Type |Description|Required|Default| +|-----|-------------------------------------------------------------------------------------|-----------|--------|--------| +|clusterRef| [ClusterReference](./2_nifi_user#clusterreference) | contains the reference to the NifiCluster containing the node group this autoscaler should manage. |Yes| - | +|nodeConfigGroupId| string | defines the id of the [NodeConfig](./1_nifi_cluster/3_node_config) contained in `NifiCluster.Spec.NodeConfigGroups`. |Yes| - | +|nodeLabelsSelector| [LabelSelector](https://pkg.go.dev/k8s.io/apimachinery/pkg/apis/meta/v1#LabelSelector) | defines the set of labels used to identify nodes in a `NifiCluster` node config group. Use `Node.Labels` in combination with this selector to clearly define which nodes will be managed by this autoscaler. Take care to avoid having mutliple autoscalers managing the same nodes. |Yes| - | +|readOnlyConfig| [ReadOnlyConfig](./1_nifi_cluster/2_read_only_config) | defines a readOnlyConfig to apply to each node in this node group. Any settings here will override those set in the configured `nodeConfigGroupId`. |Yes| - | +|nodeConfig| [NodeConfig](./1_nifi_cluster/3_node_config) | defines a nodeConfig to apply to each node in this node group. Any settings here will override those set in the configured `nodeConfigGroupId`. |Yes| - | +|upscaleStrategy| string | The strategy NiFiKop will use to scale up the nodes managed by this autoscaler. Must be one of {`simple`}. |Yes| - | +|downscaleStrategy| string | The strategy NiFiKop will use to scale down the nodes managed by this autoscaler. Must be one of {`lifo`}. |Yes| - | +|replicas| int | the initial number of replicas to configure the `HorizontalPodAutoscaler` with. After the initial configuration, this `replicas` configuration will be automatically updated by the Kubernetes `HorizontalPodAutoscaler` controller. |No| 0 | + +## NifiNodeGroupAutoscalerStatus + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|--------| +|state|string| the state of the nodegroup autoscaler. This is set by the autoscaler. |No| - | +|replicas|int| the current number of replicas running in the node group this autoscaler is managing. This is set by the autoscaler.|No| - | +|selector|string| the [selector](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) used by the `HorizontalPodAutoscaler` controller to identify the replicas in this node group. This is set by the autoscaler.|No| - | \ No newline at end of file diff --git a/site/website/versioned_docs/version-v1.11.4/5_references/8_nifi_connection.md b/site/website/versioned_docs/version-v1.11.4/5_references/8_nifi_connection.md new file mode 100644 index 0000000000..418c40839e --- /dev/null +++ b/site/website/versioned_docs/version-v1.11.4/5_references/8_nifi_connection.md @@ -0,0 +1,151 @@ +--- +id: 8_nifi_connection +title: NiFi Connection +sidebar_label: NiFi Connection +--- + +`NifiConnection` is the Schema for the NiFi connection API. + +```yaml +apiVersion: nifi.konpyutaika.com/v1alpha1 +kind: NifiConnection +metadata: + name: connection + namespace: instances +spec: + source: + name: input + namespace: instances + subName: output_1 + type: dataflow + destination: + name: output + namespace: instances + subName: input_1 + type: dataflow + configuration: + flowFileExpiration: 1 hour + backPressureDataSizeThreshold: 100 GB + backPressureObjectThreshold: 10000 + loadBalanceStrategy: PARTITION_BY_ATTRIBUTE + loadBalancePartitionAttribute: partition_attribute + loadBalanceCompression: DO_NOT_COMPRESS + prioritizers: + - NewestFlowFileFirstPrioritizer + - FirstInFirstOutPrioritizer + labelIndex: 0 + bends: + - posX: 550 + posY: 550 + - posX: 550 + posY: 440 + - posX: 550 + posY: 88 + updateStrategy: drain +``` + +## NifiDataflow + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|-------| +|metadata|[ObjectMetadata](https://godoc.org/k8s.io/apimachinery/pkg/apis/meta/v1#ObjectMeta)|is metadata that all persisted resources must have, which includes all objects dataflows must create.|No|nil| +|spec|[NifiConnectionSpec](#nificonnectionspec)|defines the desired state of NifiDataflow.|No|nil| +|status|[NifiConnectionStatus](#nificonnectionstatus)|defines the observed state of NifiDataflow.|No|nil| + +## NifiConnectionSpec + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|-------| +|source|[ComponentReference](#componentreference)|the Source component of the connection. |Yes| - | +|destination|[ComponentReference](#componentreference)|the Destination component of the connection. |Yes| - | +|configuration|[ConnectionConfiguration](#connectionconfiguration)|the version of the flow to run. |Yes| - | +|updateStrategy|[ComponentUpdateStrategy](#componentupdatestrategy)|describes the way the operator will deal with data when a connection will be deleted: Drop or Drain |Yes| drain | + +## NifiConnectionStatus + +|Field|Type|Description|Required|Default| +|-----|----|-----------|--------|-------| +|connectionID|string| connection ID. |Yes| - | +|state|[ConnectionState](#connectionstate)| the connection current state. |Yes| - | + +## ComponentUpdateStrategy + +|Name|Value|Description| +|----|-----|-----------| +|DrainStrategy|drain|leads to block stopping of input/output component until they are empty.| +|DropStrategy|drop|leads to dropping all flowfiles from the connection.| + +## ConnectionState + +|Name|Value|Description| +|----|-----|-----------| +|ConnectionStateCreated|Created|describes the status of a NifiConnection as created.| +|ConnectionStateOutOfSync|OutOfSync|describes the status of a NifiConnection as out of sync.| +|ConnectionStateInSync|InSync|describes the status of a NifiConnection as in sync.| + +## ComponentReference + +|Name|Value|Description|Required|Default| +|----|-----|-----------|--------|-------| +|name|string|the name of the component.|Yes| - | +|namespace|string|the namespace of the component.|Yes| - | +|type|[ComponentType](#componenttype)|the type of the component (e.g. nifidataflow).|Yes| - | +|subName|string|the name of the sub component (e.g. queue or port name).|No| - | + +## ComponentType + +|Name|Value|Description| +|----|-----|-----------| +|ComponentDataflow|dataflow|indicates that the component is a NifiDataflow.| +|ComponentInputPort|input-port|indicates that the component is a NifiInputPort. **(not implemented)**| +|ComponentOutputPort|output-port|indicates that the component is a NifiOutputPort. **(not implemented)**| +|ComponentProcessor|processor|indicates that the component is a NifiProcessor. **(not implemented)**| +|ComponentFunnel|funnel|indicates that the component is a NifiFunnel. **(not implemented)**| +|ComponentProcessGroup|process-group|indicates that the component is a NifiProcessGroup. **(not implemented)**| + +## ConnectionConfiguration + +|Name|Value|Description|Required|Default| +|----|-----|-----------|--------|-------| +|flowFileExpiration|string|the maximum amount of time an object may be in the flow before it will be automatically aged out of the flow.|No| - | +|backPressureDataSizeThreshold|string|the maximum data size of objects that can be queued before back pressure is applied.|No| 1 GB | +|backPressureObjectThreshold|*int64|the maximum number of objects that can be queued before back pressure is applied.|No| 10000 | +|loadBalanceStrategy|[ConnectionLoadBalanceStrategy](#connectionloadbalancestrategy)|how to load balance the data in this Connection across the nodes in the cluster.|No| DO_NOT_LOAD_BALANCE | +|loadBalancePartitionAttribute|string|the FlowFile Attribute to use for determining which node a FlowFile will go to.|No| - | +|loadBalanceCompression|[ConnectionLoadBalanceCompression](#connectionloadbalancecompression)|whether or not data should be compressed when being transferred between nodes in the cluster.|No| DO_NOT_COMPRESS | +|prioritizers|\[ \][ConnectionPrioritizer](#connectionprioritizer)|the comparators used to prioritize the queue.|No| - | +|labelIndex|*int32|the index of the bend point where to place the connection label.|No| - | +|bends|\[ \][ConnectionBend](#connectionbend)|the bend points on the connection.|No| - | + +## ConnectionLoadBalanceStrategy + +|Name|Value|Description| +|----|-----|-----------| +|StrategyDoNotLoadBalance|DO_NOT_LOAD_BALANCE|do not load balance FlowFiles between nodes in the cluster.| +|StrategyPartitionByAttribute|PARTITION_BY_ATTRIBUTE|determine which node to send a given FlowFile to based on the value of a user-specified FlowFile Attribute. All FlowFiles that have the same value for said Attribute will be sent to the same node in the cluster.| +|StrategyRoundRobin|ROUND_ROBIN|flowFiles will be distributed to nodes in the cluster in a Round-Robin fashion. However, if a node in the cluster is not able to receive data as fast as other nodes, that node may be skipped in one or more iterations in order to maximize throughput of data distribution across the cluster.| +|StrategySingle|SINGLE|all FlowFiles will be sent to the same node. Which node they are sent to is not defined.| + +## ConnectionLoadBalanceCompression + +|Name|Value|Description| +|----|-----|-----------| +|CompressionDoNotCompress|DO_NOT_COMPRESS|flowFiles will not be compressed.| +|CompressionCompressAttributesOnly|COMPRESS_ATTRIBUTES_ONLY|flowFiles' attributes will be compressed, but the flowFiles' contents will not be.| +|CompressionCompressAttributesAndContent|COMPRESS_ATTRIBUTES_AND_CONTENT|flowFiles' attributes and content will be compressed.| + +## ConnectionPrioritizer + +|Name|Value|Description| +|----|-----|-----------| +|PrioritizerFirstInFirstOutPrioritizer|FirstInFirstOutPrioritizer|given two FlowFiles, the one that reached the connection first will be processed first.| +|PrioritizerNewestFlowFileFirstPrioritizer|NewestFlowFileFirstPrioritizer|given two FlowFiles, the one that is newest in the dataflow will be processed first.| +|PrioritizerOldestFlowFileFirstPrioritizer|OldestFlowFileFirstPrioritizer|given two FlowFiles, the one that is oldest in the dataflow will be processed first. 'This is the default scheme that is used if no prioritizers are selected'.| +|PrioritizerPriorityAttributePrioritizer|PriorityAttributePrioritizer|given two FlowFiles, an attribute called “priority” will be extracted. The one that has the lowest priority value will be processed first.| + +## ConnectionBend + +|Name|Value|Description|Required|Default| +|----|-----|-----------|--------|-------| +|posX|*int64|the x coordinate.|No| - | +|posY|*int64|the y coordinate.|No| - | diff --git a/site/website/versioned_docs/version-v1.11.4/6_contributing/0_contribution_organization.md b/site/website/versioned_docs/version-v1.11.4/6_contributing/0_contribution_organization.md new file mode 100644 index 0000000000..d046d0ce44 --- /dev/null +++ b/site/website/versioned_docs/version-v1.11.4/6_contributing/0_contribution_organization.md @@ -0,0 +1,66 @@ +--- +id: 0_contribution_organization +title: Contribution organization +sidebar_label: Contribution organization +--- + +## New ownership for more community oriented + +The NiFiKop operator was originally started by Orange in March 2020 as [Orange-OpenSource/nifikop](https://github.com/Orange-OpenSource/nifikop), +and then forked as `konpyutaika/nifikop` in March 2022: but this is the same codebase and the same developers. + +We made this decision in concert with Orange team, because some legal restrictions would not have allowed to involve and serve external community around this operator efficiently. +Therefore,we have chosen to fork the source code into another organization and repository, which will allow a more open ownership and community-oriented development. + +It is important to notice that Orange will still continue to work and contribute to the operator, but as part of the community 😄. + +## Organizations + +With this ownership move, we decided to set up a new project management, with the aims to be more and more community-oriented + +### Slack channel + +One of the most important topics we want to improve is probably the communication around the operator's development. +To achieve this, we have created a new Slack open to anyone who wants [to join](https://join.slack.com/t/konpytika/shared_invite/zt-14md072lv-Jr8mqYoeUrqzfZF~YGUpXA), +with two main channels: + +- [#nifikop-news](https://konpytika.slack.com/archives/C035FHN1MNG): There we will announce each new release, and communicate about next objectives for the operator. +- [#nifikop-discussion](https://konpytika.slack.com/archives/C035X6KP684): Direct discussion between each member of the community to design new needs, fix issues and help each other. + +### Tech scoping + +As we want to involve as much as possible the people on the operator, we will introduce a new support for brainstorming and designing new major features. + +This is the Tech Scoping, whose main objective is to describe the problem statement that we are trying to solve, +the different approaches that could solve it, and together discuss and challenge them to define the solution to be implemented. + +You can find all the tech scoping in this [Google Drive repository](https://drive.google.com/drive/folders/1-A__UxEdRBZrwEUJu4lMF4LJtIstrnT0?usp=sharing) + +### Teams + +#### NiFiKop Leads + +This group is currently composed of: + +- [Alexandre Guitton](https://github.com/erdrix) as original owner and developer of the operator +- [Julien Guitton](https://github.com/juldrixx) as representative of Orange contribution + +The mains objectives of this group are to: + +- Define the global roadmap of the operator, +- Ensure the reviews and validations of the PRs, +- Review and validate the Tech Scoping. + +This group aims to be more representative of the community, so if the operator community grows or if there is a needs, we would be happy to have more people in this group 😄. + +#### NiFiKop Contributors + +This group is currently composed of: + +The mains objectives of this group are to: + +- Manage issues to help people, +- Review PRs (not validation), +- Create and edit Tech Scoping for new features. + +This is an open group, so feel free to contact a NiFiKop Leader on Slack to join 😄. \ No newline at end of file diff --git a/site/website/versioned_docs/version-v1.11.4/6_contributing/1_developer_guide.md b/site/website/versioned_docs/version-v1.11.4/6_contributing/1_developer_guide.md new file mode 100644 index 0000000000..fc272453a7 --- /dev/null +++ b/site/website/versioned_docs/version-v1.11.4/6_contributing/1_developer_guide.md @@ -0,0 +1,144 @@ +--- +id: 1_developer_guide +title: Developer guide +sidebar_label: Developer guide +--- + +## Operator SDK + +### Prerequisites + +NiFiKop has been validated with: + +- [go](https://golang.org/doc/install) version v1.22+. +- [docker](https://docs.docker.com/get-docker/) version 18.09+ +- [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) version v1.16+ +- [Helm](https://helm.sh/) version v3.4.2 +- [Operator sdk](https://github.com/operator-framework/operator-sdk) version v1.33.0 + +### Initial setup + +Checkout the project. + +```bash +git clone https://github.com/konpyutaika/nifikop.git +cd nifikop +``` + +### Operator sdk + +The full list of commands is available here: https://sdk.operatorframework.io/docs/cli/ + +### Build NiFiKop + +#### Local environment + +If you prefer working directly with your local go environment you can simply uses: + +```bash +make build +``` + +### Run NiFiKop + +We can quickly run NiFiKop in development mode (on your local host), then it will use your kubectl configuration file to connect to your kubernetes cluster. + +There are several ways to execute your operator: + +- Using your IDE directly +- Executing directly the Go binary +- deploying using the Helm charts + +If you want to configure your development IDE, you need to give it environment variables so that it will uses to connect to kubernetes. + +```bash +KUBECONFIG={path/to/your/kubeconfig} +WATCH_NAMESPACE={namespace_to_watch} +POD_NAME={name for operator pod} +LOG_LEVEL=Debug +OPERATOR_NAME=ide +``` + +#### Run the Operator Locally with the Go Binary + +This method can be used to run the operator locally outside of the cluster. This method may be preferred during development as it facilitates faster deployment and testing. + +Set the name of the operator in an environment variable + +```bash +export OPERATOR_NAME=nifi-operator +``` + +Deploy the CRDs. + +```bash +kubectl apply -f config/crd/bases/nifi.konpyutaika.com_nificlusters.yaml +kubectl apply -f config/crd/bases/nifi.konpyutaika.com_nifidataflows.yaml +kubectl apply -f config/crd/bases/nifi.konpyutaika.com_nifiparametercontexts.yaml +kubectl apply -f config/crd/bases/nifi.konpyutaika.com_nifiregistryclients.yaml +kubectl apply -f config/crd/bases/nifi.konpyutaika.com_nifiusergroups.yaml +kubectl apply -f config/crd/bases/nifi.konpyutaika.com_nifiusers.yaml +``` + +And deploy the operator. + +```bash +make run +``` + +This will run the operator in the `default` namespace using the default Kubernetes config file at `$HOME/.kube/config`. + +#### Deploy using the Helm Charts + +This section provides an instructions for running the operator Helm charts with an image that is built from the local branch. + +Build the image from the current branch. + +```bash +export DOCKER_REGISTRY_BASE={your-docker-repo} +make docker-build +``` + +Push the image to docker hub (or to whichever repo you want to use) + +```bash +$ make docker-push +``` + +:::info +The image tag is a combination of the version as defined in `verion/version.go` and the branch name. +::: + +Install the Helm chart. + +```bash +helm install skeleton ./helm/nifikop \ + --set image.tag=v0.5.1-release \ + --namespace-{"nifikop"} +``` + +:::important +The `image.repository` and `image.tag` template variables have to match the names from the image that we pushed in the previous step. +::: + +:::info +We set the chart name to the branch, but it can be anything. +::: + +Lastly, verify that the operator is running. + +```console +$ kubectl get pods -n nifikop +NAME READY STATUS RESTARTS AGE +skeleton-nifikop-8946b89dc-4cfs9 1/1 Running 0 7m45s +``` + +## Helm + +The NiFiKop operator is released in the `konpyutaika-incubator` helm repository. + +In order to package the chart you need to run the following command. + +```bash +make helm-package +``` diff --git a/site/website/versioned_docs/version-v1.11.4/6_contributing/2_reporting_bugs.md b/site/website/versioned_docs/version-v1.11.4/6_contributing/2_reporting_bugs.md new file mode 100644 index 0000000000..376f0f76fc --- /dev/null +++ b/site/website/versioned_docs/version-v1.11.4/6_contributing/2_reporting_bugs.md @@ -0,0 +1,25 @@ +--- +id: 2_reporting_bugs +title: Reporting bugs +sidebar_label: Reporting bugs +--- + +If any part of the NiFiKop project has bugs or documentation mistakes, please let us know by [opening an issue](https://github.com/konpyutaika/nifikop/issues/new). We treat bugs and mistakes very seriously and believe no issue is too small. Before creating a bug report, please check that an issue reporting the same problem does not already exist. + +To make the bug report accurate and easy to understand, please try to create bug reports that are: + +- Specific. Include as much details as possible: which version, what environment, what configuration, etc. + +- Reproducible. Include the steps to reproduce the problem. We understand some issues might be hard to reproduce, please include the steps that might lead to the problem. + +- Isolated. Please try to isolate and reproduce the bug with minimum dependencies. It would significantly slow down the speed to fix a bug if too many dependencies are involved in a bug report. Debugging external systems that rely on operator-sdk is out of scope, but we are happy to provide guidance in the right direction or help with using operator-sdk itself. + +- Unique. Do not duplicate existing bug report. + +- Scoped. One bug per report. Do not follow up with another bug inside one report. + +It may be worthwhile to read [Elika Etemad’s article on filing good bug reports][filing-good-bugs] before creating a bug report. + +We might ask for further information to locate a bug. A duplicated bug report will be closed. + +[filing-good-bugs]: http://fantasai.inkedblade.net/style/talks/filing-good-bugs/ \ No newline at end of file diff --git a/site/website/versioned_docs/version-v1.11.4/6_contributing/3_credits.md b/site/website/versioned_docs/version-v1.11.4/6_contributing/3_credits.md new file mode 100644 index 0000000000..aaa134221c --- /dev/null +++ b/site/website/versioned_docs/version-v1.11.4/6_contributing/3_credits.md @@ -0,0 +1,11 @@ +--- +id: 3_credits +title: Credits +sidebar_label: Credits +--- + +This implementation is based on other Open-Source project, and lot of the community ideas. Particular thanks to: + +- Operator implementation based on [banzaicloud/kafka-operator](https://github.com/banzaicloud/kafka-operator) +- NiFi kubernetes setup configuration inspired from [cetic/helm-nifi](https://github.com/cetic/helm-nifi) +- Implementation is based on [Operator SDK](https://github.com/operator-framework/operator-sdk) \ No newline at end of file diff --git a/site/website/versioned_docs/version-v1.11.4/7_upgrade_guides/1_v0.7.x_to_v0.8.0.md b/site/website/versioned_docs/version-v1.11.4/7_upgrade_guides/1_v0.7.x_to_v0.8.0.md new file mode 100644 index 0000000000..4cf1e08601 --- /dev/null +++ b/site/website/versioned_docs/version-v1.11.4/7_upgrade_guides/1_v0.7.x_to_v0.8.0.md @@ -0,0 +1,165 @@ +--- +id: 1_v0.7.x_to_v0.8.0 +title: v0.7.x to v0.8.0 +sidebar_label: v0.7.x to v0.8.0 +--- + +Guide to migrate operator resources built using `nifi.orange.com/v1alpha1` to `nifi.konpyutaika/v1alpha1`. + +## Getting started + +The goal is to migrate your NiFiKop resources from the old CRDs to the new ones without any service interruption. + +To do this, it is necessary to have both versions of CRDs available on Kubernetes and to have the old operator stopped (to prevent any manipulation on the resources). +Then launch the script developed in nodejs presented in the following. The script will copy the resources in the old CRDs to the new CRDs keeping only the relevant fields (labels, annotations, name and spec) and then copy the status. + +## Prerequisites + +- [nodejs](https://nodejs.org/en/download/) version 15.3.0+ +- [npm](https://docs.npmjs.com/cli/v7/configuring-npm/install) version 7.0.14+ + +## Initial setup + +Create a nodejs project and download the required dependencies: + +```bash +npm init -y +npm install @kubernetes/client-node@0.16.3 minimist@1.2.6 +``` + +In `package.json` add the following script: + +```json +"start": "node --no-warnings index.js" +``` + +Your `package.json` should look like that: + +```json +{ + "name": "nifikop_crd_migration", + "version": "1.0.0", + "description": "Script to migrate from the old CRDs to the new CRDs.", + "main": "index.js", + "scripts": { + "start": "node --no-warnings index.js", + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [ + "K8S", + "NiFiKop", + "CRDs" + ], + "license": "ISC", + "dependencies": { + "@kubernetes/client-node": "^0.16.3", + "minimist": "^1.2.6" + } +} +``` + +## Script setup + +Create the file `index.js` with the following content: + +```js +process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = 0; +const k8s = require('@kubernetes/client-node'); + +const kc = new k8s.KubeConfig(); +kc.loadFromDefault(); + +const k8sApi = kc.makeApiClient(k8s.CustomObjectsApi); + +const KONPYUTAIKA_GROUP = 'nifi.konpyutaika.com'; +const KONPYUTAIKA_GROUP_VERSION = 'v1alpha1'; +const ORANGE_GROUP = 'nifi.orange.com'; +const ORANGE_GROUP_VERSION = 'v1alpha1'; + +const call = async (SRC_GRP, SRC_GRP_VER, DST_GRP, DST_GRP_VER, KIND_PLURAL, NAMESPACE) => { + console.log(`Listing ${KIND_PLURAL} of ${SRC_GRP}/${SRC_GRP_VER} in ${NAMESPACE}...`); + const listResources = (await k8sApi.listNamespacedCustomObject(SRC_GRP, SRC_GRP_VER, NAMESPACE, KIND_PLURAL)).body.items; + return Promise.all(listResources.map(async (resource) => { + try { + console.log(`Found ${resource.kind} "${resource.metadata.name}" of ${resource.apiVersion} in ${NAMESPACE}`); + + if (resource.metadata.ownerReferences) { + console.log(`${resource.kind} ${resource.metadata.name} mananged by something else (ownerRefereces is set).`); + return; + } + + const bodyResource = { + apiVersion: `${DST_GRP}/${DST_GRP_VER}`, + kind: resource.kind, + metadata: { + name: resource.metadata.name, + annotations: resource.metadata.annotations, + labels: resource.metadata.labels + }, + spec: resource.spec + }; + + console.log(`Creating ${bodyResource.kind} "${bodyResource.metadata.name}" of ${bodyResource.apiVersion} in ${NAMESPACE}...`); + const newResource = (await k8sApi.createNamespacedCustomObject(DST_GRP, DST_GRP_VER, NAMESPACE, KIND_PLURAL, bodyResource)).body; + console.log('...done creating.'); + + const bodyStatus = { + apiVersion: newResource.apiVersion, + kind: newResource.kind, + metadata: { + name: newResource.metadata.name, + resourceVersion: newResource.metadata.resourceVersion + }, + status: resource.status + }; + + console.log(`Copying status from ${resource.kind} "${resource.metadata.name}" of ${newResource.apiVersion} to ${newResource.kind} "${newResource.metadata.name}" of ${newResource.apiVersion} in ${NAMESPACE}...`); + const newResourceWithStatus = (await k8sApi.replaceNamespacedCustomObjectStatus(DST_GRP, DST_GRP_VER, NAMESPACE, KIND_PLURAL, bodyStatus.metadata.name, bodyStatus)).body; + console.log('...done copying.'); + return newResourceWithStatus; + } + catch (e) { + console.error(e.body ? e.body.message ? e.body.message : e.body : e); + } + })); +}; + +const argv = require('minimist')(process.argv.slice(2)); + +let NAMESPACE = argv.namespace ? argv.namespace.length > 0 ? argv.namespace : 'default' : 'default'; +let KIND_PLURAL = { + cluster: 'nificlusters', + dataflow: 'nifidataflows', + parametercontext: 'nifiparametercontexts', + registryclient: 'nifiregistryclients', + user: 'nifiusers', + usergroup: 'nifiusergroups', +}; + +if (!argv.type) { + console.error('Type not provided'); + process.exit(1); +} + +if (!KIND_PLURAL[argv.type]) { + console.error(`Type ${argv.type} is not one of the following types: ${Object.keys(KIND_PLURAL)}`); + process.exit(1); +} + +console.log(`########### START: ${KIND_PLURAL[argv.type]} ###########`); +call( ORANGE_GROUP, ORANGE_GROUP_VERSION, KONPYUTAIKA_GROUP, KONPYUTAIKA_GROUP_VERSION, KIND_PLURAL[argv.type], NAMESPACE) + .then(r => console.log('############ END ############')) + .catch(e => console.error(e)); +``` + +## Run script + +To migrate the resources, run the following command: + +```bash +npm start -- --type= --namespace= +``` + +with +- ``: NiFiKop resource type (cluster, dataflow, user, usergroup, parametercontext or registryclient) +- `:` Kubernetes namespace where the resources will be migrated \ No newline at end of file diff --git a/site/website/versioned_docs/version-v1.11.4/7_upgrade_guides/2_v0.14.1_to_v0.15.0.md b/site/website/versioned_docs/version-v1.11.4/7_upgrade_guides/2_v0.14.1_to_v0.15.0.md new file mode 100644 index 0000000000..387cc71fd2 --- /dev/null +++ b/site/website/versioned_docs/version-v1.11.4/7_upgrade_guides/2_v0.14.1_to_v0.15.0.md @@ -0,0 +1,35 @@ +--- +id: 2_v0.14.1_to_v0.15.0 +title: v0.14.1 to v0.15.0 +sidebar_label: v0.14.1 to v0.15.0 +--- + +[PR #189](https://github.com/konpyutaika/nifikop/pull/189) changed the default Zookeeper init container image changed from `busybox` to `bash`. If you have overridden the `NifiCluster.Spec.InitContainerImage` then you need to change it to `bash` or one that contains a bash shell. + +## Getting started + +If you haven't overridden the default `NifiCluster.Spec.InitContainerImage`, then there are no special upgrade instructions. If you have, like for example below: + +```yaml +apiVersion: nifi.konpyutaika.com/v1alpha1 +kind: NifiCluster +metadata: + name: mynifi +spec: + initContainerImage: + repository: busybox + tag: "1.34.0" +``` + +Then you must change it to `bash` or an image that contains a bash shell: + +```yaml +apiVersion: nifi.konpyutaika.com/v1alpha1 +kind: NifiCluster +metadata: + name: mynifi +spec: + initContainerImage: + repository: bash + tag: "5.2.2" +``` \ No newline at end of file diff --git a/site/website/versioned_docs/version-v1.11.4/7_upgrade_guides/3_v0.16.0_to_v1.0.0.md b/site/website/versioned_docs/version-v1.11.4/7_upgrade_guides/3_v0.16.0_to_v1.0.0.md new file mode 100644 index 0000000000..e5dd68678d --- /dev/null +++ b/site/website/versioned_docs/version-v1.11.4/7_upgrade_guides/3_v0.16.0_to_v1.0.0.md @@ -0,0 +1,47 @@ +--- +id: 3_v0.16.0_to_v1.0.0 +title: v0.16.0 to v1.0.0 +sidebar_label: v0.16.0 to v1.0.0 +--- + +# Getting started + +Moving from `v0.16.0` to `v1.0.0` version implies the migration of some CRD versions from `v1alpha1` to `v1`: + +- NifiCluster +- NifiDataflow +- NifiParameterContext +- NifiRegistryClient +- NifiUser +- NifiUserGroup + +To manage this resource version migration, you have to: + +1 - Upgrade your NifiKop helm chart release, that will enable conversion webhook in the operator pod (as the `webhook.enabled` values is set to `true` by default). +2 - Patch the CRDs associated to the resources with: + +```yaml +... +annotations: + cert-manager.io/inject-ca-from: ${namespace}/${certificate_name} +... +spec: + ... + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: ${namespace} + name: ${webhook_service_name} + path: /convert + conversionReviewVersions: + - v1 + - v1alpha1 + ... +``` + +Where: +- `namespace`: is the namespace in which you will deploy your helm chart. +- `certificate_name`: is `${helm release name}-webhook-cert` +- `webhook_service_name`: is `${helm release name}-webhook-cert` diff --git a/site/website/versioned_docs/version-v1.11.4/7_upgrade_guides/4_v1.3.1_to_v1.4.0.md b/site/website/versioned_docs/version-v1.11.4/7_upgrade_guides/4_v1.3.1_to_v1.4.0.md new file mode 100644 index 0000000000..137fa8c418 --- /dev/null +++ b/site/website/versioned_docs/version-v1.11.4/7_upgrade_guides/4_v1.3.1_to_v1.4.0.md @@ -0,0 +1,38 @@ +--- +id: 4_v1.3.1_to_v1.4.0 +title: v1.3.1 to v1.4.0 +sidebar_label: v1.3.1 to v1.4.0 +--- + +[PR #290](https://github.com/konpyutaika/nifikop/pull/290) changed the default NiFi sensitive algorithm from `PBEWITHMD5AND256BITAES-CBC-OPENSSL` to `NIFI_PBKDF2_AES_GCM_256`. If you have overridden the property `nifi.sensitive.props.algorithm` in the `nifi.properties` then you can ignore this guide. + +# Getting started + +If you have overridden the default `nifi.sensitive.props.algorithm`, then there are no special upgrade instructions. If you have, you need to upadate the flow configuration to recalculate the sensitive value beforehand. To do so, you need to use the [NiFi Encrypt-Config Tool](https://nifi.apache.org/docs/nifi-docs/html/toolkit-guide.html#encrypt_config_tool) to change the algorithm. + +```sh +encrypt-config.sh -n nifi.properties -f flow.xml.gz -x -s PROPERTIES_KEY -A NIFI_PBKDF2_AES_GCM_256 +encrypt-config.sh -n nifi.properties -f flow.json.gz -x -s PROPERTIES_KEY -A NIFI_PBKDF2_AES_GCM_256 +``` + +> Source: https://exceptionfactory.com/posts/2021/07/29/deciphering-apache-nifi-component-property-encryption/ + +You can do this automatically using an `initContainer`. To do this, you stop the operator, update the `NifiCluster` with this new `initContainer` and then upgrade and restart the operator. Finally, you can remove the `initContainer`. + +```yaml +initContainers: + - image: "apache/nifi-toolkit:latest" + name: nifi-toolkit + imagePullPolicy: Always + command: + - "sh" + - "-c" + - "NIFI_SENSITIVE_PROPS_KEY=$(grep 'nifi.sensitive.props.key' /opt/nifi/nifi-current/conf/nifi.properties | cut -d'=' -f2) && bin/encrypt-config.sh -n /opt/nifi/nifi-current/conf/nifi.properties -f /opt/nifi/data/flow.json.gz -x -A NIFI_PBKDF2_AES_GCM_256 -s $NIFI_SENSITIVE_PROPS_KEY; bin/encrypt-config.sh -n /opt/nifi/nifi-current/conf/nifi.properties -f /opt/nifi/data/flow.xml.gz -x -A NIFI_PBKDF2_AES_GCM_256 -s $NIFI_SENSITIVE_PROPS_KEY" + volumeMounts: + - name: data + mountPath: /opt/nifi/data + - name: conf + mountPath: /opt/nifi/nifi-current/conf +``` + +> Adapt the `volumeMounts` and `mountPath` to your needs. \ No newline at end of file diff --git a/site/website/versioned_sidebars/version-v1.11.4-sidebars.json b/site/website/versioned_sidebars/version-v1.11.4-sidebars.json new file mode 100644 index 0000000000..0d18360909 --- /dev/null +++ b/site/website/versioned_sidebars/version-v1.11.4-sidebars.json @@ -0,0 +1,154 @@ +{ + "docs": [ + { + "type": "category", + "label": "Concepts", + "items": [ + "1_concepts/1_start_here", + "1_concepts/2_design_principles", + "1_concepts/3_features", + "1_concepts/4_roadmap" + ] + }, + { + "type": "category", + "label": "Deploy NiFiKop", + "items": [ + "2_deploy_nifikop/1_quick_start", + "2_deploy_nifikop/2_customizable_install_with_helm", + "2_deploy_nifikop/3_kubectl_plugin" + ] + }, + { + "type": "category", + "label": "Manage NiFi", + "items": [ + { + "type": "category", + "label": "Manage cluster", + "items": [ + "3_manage_nifi/1_manage_clusters/0_design_principles", + { + "type": "category", + "label": "Deploy cluster", + "items": [ + "3_manage_nifi/1_manage_clusters/1_deploy_cluster/1_quick_start", + "3_manage_nifi/1_manage_clusters/1_deploy_cluster/2_nodes_configuration", + { + "type": "category", + "label": "Expose cluster", + "items": [ + "3_manage_nifi/1_manage_clusters/1_deploy_cluster/3_expose_cluster/1_kubernetes_service", + "3_manage_nifi/1_manage_clusters/1_deploy_cluster/3_expose_cluster/2_istio_service_mesh" + ] + }, + "3_manage_nifi/1_manage_clusters/1_deploy_cluster/4_ssl_configuration", + { + "type": "category", + "label": "Users authentication", + "items": [ + "3_manage_nifi/1_manage_clusters/1_deploy_cluster/5_users_authentication/1_oidc" + ] + }, + { + "type": "category", + "label": "Users authorization", + "items": [ + "3_manage_nifi/1_manage_clusters/1_deploy_cluster/6_users_authorization/1_custom_user_authorizer" + ] + } + ] + }, + { + "type": "category", + "label": "Cluster scaling", + "items": [ + "3_manage_nifi/1_manage_clusters/2_cluster_scaling/1_scaling_mechanism", + { + "type": "category", + "label": "Auto scaling", + "items": [ + "3_manage_nifi/1_manage_clusters/2_cluster_scaling/2_auto_scaling/0_design_principles", + "3_manage_nifi/1_manage_clusters/2_cluster_scaling/2_auto_scaling/1_using_keda" + ] + } + ] + }, + "3_manage_nifi/1_manage_clusters/3_external_cluster" + ] + }, + { + "type": "category", + "label": "Manage users & accesses", + "items": [ + "3_manage_nifi/2_manage_users_and_accesses/1_users_management", + "3_manage_nifi/2_manage_users_and_accesses/2_groups_management", + "3_manage_nifi/2_manage_users_and_accesses/3_managed_groups" + ] + }, + { + "type": "category", + "label": "Manage DataFlows", + "items": [ + "3_manage_nifi/3_manage_dataflows/0_design_principles", + "3_manage_nifi/3_manage_dataflows/1_deploy_dataflow" + ] + }, + { + "type": "category", + "label": "Manage Connections", + "items": [ + "3_manage_nifi/4_manage_connections/1_deploy_connection" + ] + } + ] + }, + "4_compatibility_versions", + { + "type": "category", + "label": "Reference", + "items": [ + { + "type": "category", + "label": "NiFi Cluster", + "items": [ + "5_references/1_nifi_cluster/1_nifi_cluster", + "5_references/1_nifi_cluster/2_read_only_config", + "5_references/1_nifi_cluster/3_node_config", + "5_references/1_nifi_cluster/4_node", + "5_references/1_nifi_cluster/5_node_state", + "5_references/1_nifi_cluster/6_listeners_config", + "5_references/1_nifi_cluster/7_external_service_config" + ] + }, + "5_references/2_nifi_user", + "5_references/3_nifi_registry_client", + "5_references/4_nifi_parameter_context", + "5_references/5_nifi_dataflow", + "5_references/6_nifi_usergroup", + "5_references/7_nifi_nodegroup_autoscaler", + "5_references/8_nifi_connection" + ] + }, + { + "type": "category", + "label": "Contributing", + "items": [ + "6_contributing/0_contribution_organization", + "6_contributing/1_developer_guide", + "6_contributing/2_reporting_bugs", + "6_contributing/3_credits" + ] + }, + { + "type": "category", + "label": "Upgrade Guides", + "items": [ + "7_upgrade_guides/1_v0.7.x_to_v0.8.0", + "7_upgrade_guides/2_v0.14.1_to_v0.15.0", + "7_upgrade_guides/3_v0.16.0_to_v1.0.0", + "7_upgrade_guides/4_v1.3.1_to_v1.4.0" + ] + } + ] +} diff --git a/site/website/versions.json b/site/website/versions.json index ea57428473..b56795b73a 100644 --- a/site/website/versions.json +++ b/site/website/versions.json @@ -1,4 +1,5 @@ [ + "v1.11.4", "v1.11.3", "v1.11.2", "v1.11.1", diff --git a/version/version.go b/version/version.go index c5768ae466..3262999148 100644 --- a/version/version.go +++ b/version/version.go @@ -1,5 +1,5 @@ package version var ( - Version = "1.11.3" + Version = "1.11.4" )