Upgraded NiFi to v1.20 (#22)

* Upgraded NiFi to v1.20 * Increased cluster protocol max threads * Enabled security script * Updated MAX threads and history size * Updated read me docs
saidsef · Mar 27, 2023 · 5036d50 · 5036d50
1 parent 74eb546
commit 5036d50
Show file tree

Hide file tree

Showing 4 changed files with 37 additions and 24 deletions.
diff --git a/README.md b/README.md
@@ -21,7 +21,7 @@ kubectl apply -k deployments/
 This will create:
 
 - 1x NiFi Namespace (all the items will be deployed here)
-- 3x Apache NiFi (each with it's own Service endpoint)
+- 2x Apache NiFi Instances (can be increased)
 - 1x Apache Zookeeper (accessible within the cluster only)
 - 1x Secrets (basic auth username/passowrd: `admin:admin`)
 - 1x Ingress (access endpoint)

diff --git a/deployment/kustomization.yml b/deployment/kustomization.yml
@@ -20,13 +20,13 @@ resources:
 images:
 - name: nifi
   newName: docker.io/apache/nifi
-  newTag: "1.12.1"
+  newTag: "1.20.0"
 - name: zookeeper
   newName: docker.io/zookeeper
   newTag: "3.8"
 - name: wait
   newName: docker.io/busybox
-  newTag: "1.35"
+  newTag: "1.36"
 
 vars:
 - fieldref: {}

diff --git a/deployment/nifi-configmap.yml b/deployment/nifi-configmap.yml
@@ -3,31 +3,36 @@ kind: ConfigMap
 metadata:
   name: nifi-cm
 data:
-  JAVA_OPTS: "-XX:UseAVX=0"
+  JAVA_OPTS: "-XX:UseAVX=0 -Djavax.net.debug=ssl,handshake"
+  KEYSTORE_PASSWORD: "th1s1s3up34e5r37"
+  KEYSTORE_PATH: "${NIFI_HOME}/keytool/keystore.p12"
+  KEYSTORE_TYPE: "PKCS12"
   NIFI_ANALYTICS_PREDICT_ENABLED: "true"
   NIFI_CLUSTER_IS_NODE: "true"
   NIFI_CLUSTER_LOAD_BALANCE_HOST: "nifi"
-  NIFI_CLUSTER_NODE_CONNECTION_TIMEOUT: "2 min"
+  NIFI_CLUSTER_NODE_CONNECTION_TIMEOUT: "5 min"
+  NIFI_CLUSTER_NODE_EVENT_HISTORY_SIZE: "25"
   NIFI_CLUSTER_NODE_LOAD_BALANCE_PORT: "6342"
-  NIFI_CLUSTER_NODE_PROTOCOL_MAX_THREADS: "15"
+  NIFI_CLUSTER_NODE_PROTOCOL_MAX_THREADS: "20"
   NIFI_CLUSTER_NODE_PROTOCOL_PORT: "11443"
-  NIFI_CLUSTER_NODE_READ_TIMEOUT: "2 min"
-  NIFI_CLUSTER_PROTOCOL_CONNECTION_HANDSHAKE_TIMEOUT: "2 min"
-  NIFI_CLUSTER_PROTOCOL_HEARTBEAT_INTERVAL: "1 min"
+  NIFI_CLUSTER_NODE_PROTOCOL_THREADS: "10"
+  NIFI_CLUSTER_NODE_READ_TIMEOUT: "5 min"
+  NIFI_CLUSTER_PROTOCOL_CONNECTION_HANDSHAKE_TIMEOUT: "3 min"
+  NIFI_CLUSTER_PROTOCOL_HEARTBEAT_INTERVAL: "2 min"
+  NIFI_CLUSTER_PROTOCOL_IS_SECURE: "true"
   NIFI_ELECTION_MAX_CANDIDATES: "1"
-  NIFI_ELECTION_MAX_WAIT: "1 min"
-  NIFI_JVM_HEAP_INIT: "1g"
+  NIFI_ELECTION_MAX_WAIT: "2 min"
+  NIFI_JVM_HEAP_INIT: "2g"
   NIFI_JVM_HEAP_MAX: "2g"
+  NIFI_SECURITY_AUTORELOAD_ENABLED: "true"
+  NIFI_SECURITY_AUTORELOAD_INTERVAL: "5 min"
+  NIFI_SECURITY_NEEDCLIENTAUTH: "true"
+  NIFI_SENSITIVE_PROPS_KEY_PROTECTED: "th1s1s3up34e5r37"
   NIFI_SENSITIVE_PROPS_KEY: "th1s1s3up34e5r37"
   NIFI_WEB_HTTP_PORT: "8080"
   NIFI_WEB_HTTPS_PORT: "8443"
   NIFI_ZK_CONNECT_STRING: "zookeeper:2181"
   NIFI_ZOOKEEPER_CONNECT_STRING: "zookeeper:2181"
-  # NIFI_SECURITY_AUTORELOAD_ENABLED: "true"
-  # NIFI_SECURITY_AUTORELOAD_INTERVAL: "1 min"
-  # NIFI_SECURITY_KEYSTORE: "${NIFI_HOME}/keytool/keystore.p12"
-  # NIFI_SECURITY_KEYSTORETYPE: "PKCS12"
-  # NIFI_SECURITY_TRUSTSTORE: "${NIFI_HOME}/keytool/truststore.jks"
-  # NIFI_SECURITY_TRUSTSTORETYPE: "jks"
-  # NIFI_SECURITY_NEEDCLIENTAUTH: "false"
-  # NIFI_CLUSTER_PROTOCOL_IS_SECURE: "false"
+  TRUSTSTORE_PASSWORD: "th1s1s3up34e5r37"
+  TRUSTSTORE_PATH: "${NIFI_HOME}/keytool/truststore.jks"
+  TRUSTSTORE_TYPE: "jks"
diff --git a/deployment/nifi.yml b/deployment/nifi.yml
@@ -32,7 +32,7 @@ spec:
           type: RuntimeDefault
       initContainers:
         - name: wait-for-zookeeper
-          image: docker.io/busybox:1.35
+          image: docker.io/busybox:1.36
           command:
           - sh
           - -c
@@ -62,7 +62,7 @@ spec:
                 - ALL
       containers:
       - name: nifi
-        image: docker.io/apache/nifi:1.19.1
+        image: docker.io/apache/nifi:1.20.0
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 8080
@@ -82,10 +82,18 @@ spec:
             valueFrom:
               fieldRef:
                 fieldPath: status.podIP
+          - name: NIFI_REMOTE_INPUT_HOST
+            valueFrom:
+              fieldRef:
+                fieldPath: status.podIP
           - name: HOSTNAME
             valueFrom:
               fieldRef:
                 fieldPath: status.podIP
+          - name: NODE_IDENTITY
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
         envFrom:
           - configMapRef:
               name: nifi-cm
@@ -113,7 +121,7 @@ spec:
             command:
               - pgrep
               - java
-          initialDelaySeconds: 30
+          initialDelaySeconds: 60
           periodSeconds: 30
           timeoutSeconds: 10
           failureThreshold: 3
@@ -129,10 +137,10 @@ spec:
         resources:
           requests:
             cpu: 400m
-            memory: 1Gi
+            memory: 2Gi
           limits:
             cpu: 400m
-            memory: 1Gi
+            memory: 2Gi
         securityContext:
           allowPrivilegeEscalation: false
           privileged: false