From 024d3a8398b90ef592989f8066563214d748c924 Mon Sep 17 00:00:00 2001
From: bradsawadye <bradsawadye@gmail.com>
Date: Tue, 19 Jul 2022 12:35:43 +0200
Subject: [PATCH 01/29] Configure kafka

This creates the kafka topics

PLAT-346
---
 .../importer/docker-compose.config.yml        | 24 +++++++++++
 message-bus-kafka/importer/kafka.js           | 40 +++++++++++++++++++
 2 files changed, 64 insertions(+)
 create mode 100644 message-bus-kafka/importer/docker-compose.config.yml
 create mode 100644 message-bus-kafka/importer/kafka.js

diff --git a/message-bus-kafka/importer/docker-compose.config.yml b/message-bus-kafka/importer/docker-compose.config.yml
new file mode 100644
index 00000000..f78d71ca
--- /dev/null
+++ b/message-bus-kafka/importer/docker-compose.config.yml
@@ -0,0 +1,24 @@
+version: '3.9'
+
+services:
+  kafka-config-importer:
+    image: jembi/instantohie-config-importer
+    deploy:
+      restart_policy:
+        condition: none
+    environment:
+      KAFDROP_HOST: ${KAFDROP_HOST:-kafdrop}
+      KAFDROP_PORT: ${KAFDROP_PORT:-9013}
+      KAFKA_PARTITIONS: ${KAFKA_PARTITIONS:-3}
+      KAFKA_TOPICS: ${KAFKA_TOPICS}
+    command: sh -c "wait-on -t 60000 http-get://kafdrop:9013/topic && node /kafka.js && echo 'success';"
+    configs:
+      - source: kafka.js
+        target: /kafka.js
+
+configs:
+  kafka.js:
+    file: ./kafka.js
+    name: kafka-config.js-${kafka_config_DIGEST:?err}
+    labels:
+      name: ethiopia
diff --git a/message-bus-kafka/importer/kafka.js b/message-bus-kafka/importer/kafka.js
new file mode 100644
index 00000000..76e38142
--- /dev/null
+++ b/message-bus-kafka/importer/kafka.js
@@ -0,0 +1,40 @@
+'use strict'
+
+const http = require('http')
+
+const KAFDROP_HOST = process.env.KAFDROP_HOST || 'kafdrop'
+const KAFDROP_PORT = process.env.KAFDROP_PORT || '9013'
+const KAFKA_PARTITIONS = process.env.KAFKA_PARTITIONS || 3
+const KAFKA_TOPICS= process.env.KAFKA_TOPICS
+
+const createKafkaTopic = topic => {
+  const options = {
+    protocol: 'http:',
+    hostname: KAFDROP_HOST,
+    path: '/topic',
+    port: KAFDROP_PORT,
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/x-www-form-urlencoded'
+    }
+  }
+
+  const req = http.request(options, res => {
+    if (res.statusCode != 200) {
+      throw Error(`Failed to create topic - ${topic}`)
+    }
+    console.log(`Created topic - ${topic}`)
+  })
+  req.write(`name=${topic}&partitionsNumber=${KAFKA_PARTITIONS}&replicationFactor=1`)
+  req.end()
+}
+
+console.log('Creating kafka topics......................');
+
+(() => {
+  if (KAFKA_TOPICS) {
+   KAFKA_TOPICS.split(',').forEach(topic => createKafkaTopic(topic))
+  } else {
+    console.log('Topics not created: KAFKA_TOPICS variable invalid')
+  }
+})();

From 1b3300bb4964c629e65e0feb8cc61b6c409e723f Mon Sep 17 00:00:00 2001
From: bradsawadye <bradsawadye@gmail.com>
Date: Tue, 19 Jul 2022 12:37:17 +0200
Subject: [PATCH 02/29] Add logic for configuring kafka

This adds the logic for running the importer scripts

PLAT-346
---
 message-bus-kafka/swarm.sh | 6 ++++++
 1 file changed, 6 insertions(+)
 mode change 100644 => 100755 message-bus-kafka/swarm.sh

diff --git a/message-bus-kafka/swarm.sh b/message-bus-kafka/swarm.sh
old mode 100644
new mode 100755
index 71db3192..aa512575
--- a/message-bus-kafka/swarm.sh
+++ b/message-bus-kafka/swarm.sh
@@ -30,7 +30,13 @@ else
 fi
 
 if [[ $1 == "init" ]] || [[ $1 == "up" ]]; then
+  config::set_config_digests "${COMPOSE_FILE_PATH}"/importer/docker-compose.config.yml
+
   try "docker stack deploy -c ${COMPOSE_FILE_PATH}/docker-compose.yml $kafkaClusterComposeParam $kafkaDevComposeParam instant" "Failed to deploy Message Bus Kafka"
+  try "docker stack deploy -c ${COMPOSE_FILE_PATH}/importer/docker-compose.config.yml instant" "Failed to deploy Message Bus Kafka"
+
+  config::remove_stale_service_configs "${COMPOSE_FILE_PATH}"/importer/docker-compose.config.yml "ethiopia"
+  config::remove_config_importer kafka-config-importer
 elif [[ $1 == "down" ]]; then
   try "docker service scale instant_zookeeper-1=0 instant_kafdrop=0" "Failed to scale down zookeeper and kafdrop"
   # You cannot scale a global service so we have to remove it

From e7180384d96f13bf84b08e521023de088ebb00cd Mon Sep 17 00:00:00 2001
From: bradsawadye <bradsawadye@gmail.com>
Date: Tue, 19 Jul 2022 12:40:28 +0200
Subject: [PATCH 03/29] Add the variables used in creating the kafka topics

PLAT-346
---
 .env.dev    |  3 +++
 .env.local  | 10 ++++++++++
 .env.remote | 10 ++++++++++
 3 files changed, 23 insertions(+)

diff --git a/.env.dev b/.env.dev
index d7a130c2..b8e1fd9b 100644
--- a/.env.dev
+++ b/.env.dev
@@ -123,6 +123,9 @@ KAFKA_CPU_LIMIT=0.8
 KAFKA_CPU_RESERVE=0.05
 KAFKA_MEMORY_LIMIT=3G
 KAFKA_MEMORY_RESERVE=500M
+KAFKA_PARTITIONS=3
+# Topics should comma seperated
+KAFKA_TOPICS=2xx,reprocess
 
 # Kafdrop
 
diff --git a/.env.local b/.env.local
index 7b72051e..395c4b73 100644
--- a/.env.local
+++ b/.env.local
@@ -67,3 +67,13 @@ JS_REPORT_LICENSE_KEY=
 # MAKE SURE YOU HAVE RUN 'set-permissions.sh' SCRIPT BEFORE AND AFTER RUNNING JS REPORT
 JS_REPORT_DEV_MOUNT=false
 JS_REPORT_PACKAGE_PATH=
+
+# Message Bus - Kafka
+
+KAFKA_CPU_LIMIT=0.8
+KAFKA_CPU_RESERVE=0.05
+KAFKA_MEMORY_LIMIT=3G
+KAFKA_MEMORY_RESERVE=500M
+KAFKA_PARTITIONS=3
+# Topics should comma seperated
+KAFKA_TOPICS=2xx,reprocess
diff --git a/.env.remote b/.env.remote
index a8491d16..413e6130 100644
--- a/.env.remote
+++ b/.env.remote
@@ -41,3 +41,13 @@ POSTGRES_REPLICA_SET=postgres-1:5432,postgres-2:5432,postgres-3:5432
 MONGO_SET_COUNT=3
 OPENHIM_MONGO_URL=mongodb://mongo-1:27017,mongo-2:27017,mongo-3:27017/openhim?replicaSet=mongo-set
 OPENHIM_MONGO_ATNAURL=mongodb://mongo-1:27017,mongo-2:27017,mongo-3:27017/openhim?replicaSet=mongo-set
+
+# Message Bus - Kafka
+
+KAFKA_CPU_LIMIT=0.8
+KAFKA_CPU_RESERVE=0.05
+KAFKA_MEMORY_LIMIT=3G
+KAFKA_MEMORY_RESERVE=500M
+KAFKA_PARTITIONS=3
+# Topics should comma seperated
+KAFKA_TOPICS=2xx,reprocess

From c675705a2a9424d002318d3ea8b5e765fd15b9ff Mon Sep 17 00:00:00 2001
From: bradsawadye <bradsawadye@gmail.com>
Date: Tue, 19 Jul 2022 12:55:23 +0200
Subject: [PATCH 04/29] Add the missing env variables

---
 .env.ci                                 | 19 +++++++++++++++++++
 .env.dev                                |  2 ++
 .env.local                              | 10 ++++++++++
 .env.remote                             |  9 +++++++++
 message-bus-kafka/package-metadata.json |  4 ++++
 5 files changed, 44 insertions(+)

diff --git a/.env.ci b/.env.ci
index 88b7c708..097a0728 100644
--- a/.env.ci
+++ b/.env.ci
@@ -62,3 +62,22 @@ JS_REPORT_USERNAME=admin
 JS_REPORT_SECRET=dev_secret_only
 JS_REPORT_SSL=false
 JS_REPORT_CONFIG_FILE=export.jsrexport
+
+# Message Bus - Kafka
+
+KAFKA_CPU_LIMIT=0.8
+KAFKA_CPU_RESERVE=0.05
+KAFKA_MEMORY_LIMIT=3G
+KAFKA_MEMORY_RESERVE=500M
+KAFKA_PARTITIONS=3
+# Topics should comma seperated
+KAFKA_TOPICS=2xx,reprocess
+
+# Kafdrop
+
+KAFDROP_CPU_LIMIT=0.8
+KAFDROP_CPU_RESERVE=0.05
+KAFDROP_MEMORY_LIMIT=3G
+KAFDROP_MEMORY_RESERVE=500M
+KAFDROP_HOST=Kafdrop
+KAFDROP_PORT=9013
diff --git a/.env.dev b/.env.dev
index b8e1fd9b..ab83fcb2 100644
--- a/.env.dev
+++ b/.env.dev
@@ -133,6 +133,8 @@ KAFDROP_CPU_LIMIT=0.8
 KAFDROP_CPU_RESERVE=0.05
 KAFDROP_MEMORY_LIMIT=3G
 KAFDROP_MEMORY_RESERVE=500M
+KAFDROP_HOST=Kafdrop
+KAFDROP_PORT=9013
 
 # Message Bus Helper - Hapi Proxy
 
diff --git a/.env.local b/.env.local
index 395c4b73..87d3b7d2 100644
--- a/.env.local
+++ b/.env.local
@@ -77,3 +77,13 @@ KAFKA_MEMORY_RESERVE=500M
 KAFKA_PARTITIONS=3
 # Topics should comma seperated
 KAFKA_TOPICS=2xx,reprocess
+
+# Kafdrop
+
+KAFDROP_CPU_LIMIT=0.8
+KAFDROP_CPU_RESERVE=0.05
+KAFDROP_MEMORY_LIMIT=3G
+KAFDROP_MEMORY_RESERVE=500M
+KAFDROP_HOST=Kafdrop
+KAFDROP_PORT=9013
+
diff --git a/.env.remote b/.env.remote
index 413e6130..db849751 100644
--- a/.env.remote
+++ b/.env.remote
@@ -51,3 +51,12 @@ KAFKA_MEMORY_RESERVE=500M
 KAFKA_PARTITIONS=3
 # Topics should comma seperated
 KAFKA_TOPICS=2xx,reprocess
+
+# Kafdrop
+
+KAFDROP_CPU_LIMIT=0.8
+KAFDROP_CPU_RESERVE=0.05
+KAFDROP_MEMORY_LIMIT=3G
+KAFDROP_MEMORY_RESERVE=500M
+KAFDROP_HOST=Kafdrop
+KAFDROP_PORT=9013
diff --git a/message-bus-kafka/package-metadata.json b/message-bus-kafka/package-metadata.json
index 07fb2298..f9c6d7e4 100644
--- a/message-bus-kafka/package-metadata.json
+++ b/message-bus-kafka/package-metadata.json
@@ -8,7 +8,11 @@
     "KAFKA_CPU_RESERVE": "0.05",
     "KAFKA_MEMORY_LIMIT": "3G",
     "KAFKA_MEMORY_RESERVE": "500M",
+    "KAFKA_PARTITIONS": "3",
+    "KAFKA_TOPICS": "2xx",
     "KAFDROP_CPU_LIMIT": "0.8",
+    "KAFDROP_HOST": "kafdrop",
+    "KAFDROP_PORT": "9013",
     "KAFDROP_CPU_RESERVE": "0.05",
     "KAFDROP_MEMORY_LIMIT": "3G",
     "KAFDROP_MEMORY_RESERVE": "500M",

From f1270ffcc5e15c62694eb95320a2a7a3221261f7 Mon Sep 17 00:00:00 2001
From: bradsawadye <bradsawadye@gmail.com>
Date: Thu, 21 Jul 2022 11:48:59 +0200
Subject: [PATCH 05/29] Update .env.remote

Co-authored-by: Mark-Labuschagne <78809924+Mark-Labuschagne@users.noreply.github.com>
---
 .env.remote | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.env.remote b/.env.remote
index db849751..60b9cefc 100644
--- a/.env.remote
+++ b/.env.remote
@@ -58,5 +58,5 @@ KAFDROP_CPU_LIMIT=0.8
 KAFDROP_CPU_RESERVE=0.05
 KAFDROP_MEMORY_LIMIT=3G
 KAFDROP_MEMORY_RESERVE=500M
-KAFDROP_HOST=Kafdrop
+KAFDROP_HOST=kafdrop
 KAFDROP_PORT=9013

From ccb6250b9947de54c48565f1d83bc5e26e9f372e Mon Sep 17 00:00:00 2001
From: bradsawadye <bradsawadye@gmail.com>
Date: Thu, 21 Jul 2022 11:49:52 +0200
Subject: [PATCH 06/29] Update .env.ci

Co-authored-by: Mark-Labuschagne <78809924+Mark-Labuschagne@users.noreply.github.com>
---
 .env.ci | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.env.ci b/.env.ci
index 097a0728..c96b348d 100644
--- a/.env.ci
+++ b/.env.ci
@@ -79,5 +79,5 @@ KAFDROP_CPU_LIMIT=0.8
 KAFDROP_CPU_RESERVE=0.05
 KAFDROP_MEMORY_LIMIT=3G
 KAFDROP_MEMORY_RESERVE=500M
-KAFDROP_HOST=Kafdrop
+KAFDROP_HOST=kafdrop
 KAFDROP_PORT=9013

From 284ff87d12925e4689c69c066bc00eb284abe3f2 Mon Sep 17 00:00:00 2001
From: bradsawadye <bradsawadye@gmail.com>
Date: Thu, 21 Jul 2022 11:50:08 +0200
Subject: [PATCH 07/29] Update message-bus-kafka/package-metadata.json

Co-authored-by: Mark-Labuschagne <78809924+Mark-Labuschagne@users.noreply.github.com>
---
 message-bus-kafka/package-metadata.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/message-bus-kafka/package-metadata.json b/message-bus-kafka/package-metadata.json
index f9c6d7e4..81b320f7 100644
--- a/message-bus-kafka/package-metadata.json
+++ b/message-bus-kafka/package-metadata.json
@@ -9,7 +9,7 @@
     "KAFKA_MEMORY_LIMIT": "3G",
     "KAFKA_MEMORY_RESERVE": "500M",
     "KAFKA_PARTITIONS": "3",
-    "KAFKA_TOPICS": "2xx",
+    "KAFKA_TOPICS": "",
     "KAFDROP_CPU_LIMIT": "0.8",
     "KAFDROP_HOST": "kafdrop",
     "KAFDROP_PORT": "9013",

From 704b325c4019f84b9e3fa91ed2f8fd4c6730907d Mon Sep 17 00:00:00 2001
From: bradsawadye <bradsawadye@gmail.com>
Date: Thu, 21 Jul 2022 11:54:34 +0200
Subject: [PATCH 08/29] Update .env.dev

Co-authored-by: Mark-Labuschagne <78809924+Mark-Labuschagne@users.noreply.github.com>
---
 .env.dev | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.env.dev b/.env.dev
index ab83fcb2..7389285c 100644
--- a/.env.dev
+++ b/.env.dev
@@ -133,7 +133,7 @@ KAFDROP_CPU_LIMIT=0.8
 KAFDROP_CPU_RESERVE=0.05
 KAFDROP_MEMORY_LIMIT=3G
 KAFDROP_MEMORY_RESERVE=500M
-KAFDROP_HOST=Kafdrop
+KAFDROP_HOST=kafdrop
 KAFDROP_PORT=9013
 
 # Message Bus Helper - Hapi Proxy

From eb6cfa741f35d6a58b7da12a90ca032096df4b63 Mon Sep 17 00:00:00 2001
From: bradsawadye <bradsawadye@gmail.com>
Date: Thu, 21 Jul 2022 11:54:42 +0200
Subject: [PATCH 09/29] Update .env.local

Co-authored-by: Mark-Labuschagne <78809924+Mark-Labuschagne@users.noreply.github.com>
---
 .env.local | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.env.local b/.env.local
index 87d3b7d2..916e3a1f 100644
--- a/.env.local
+++ b/.env.local
@@ -84,6 +84,6 @@ KAFDROP_CPU_LIMIT=0.8
 KAFDROP_CPU_RESERVE=0.05
 KAFDROP_MEMORY_LIMIT=3G
 KAFDROP_MEMORY_RESERVE=500M
-KAFDROP_HOST=Kafdrop
+KAFDROP_HOST=kafdrop
 KAFDROP_PORT=9013
 

From 7ec935c4487dacd9fb6b0ab80e1e006b3fb0e63c Mon Sep 17 00:00:00 2001
From: bradsawadye <bradsawadye@gmail.com>
Date: Thu, 21 Jul 2022 14:07:06 +0200
Subject: [PATCH 10/29] Make the config importer fail when topics are not
 specified

---
 message-bus-kafka/importer/kafka.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/message-bus-kafka/importer/kafka.js b/message-bus-kafka/importer/kafka.js
index 76e38142..6dfea0a9 100644
--- a/message-bus-kafka/importer/kafka.js
+++ b/message-bus-kafka/importer/kafka.js
@@ -36,5 +36,6 @@ console.log('Creating kafka topics......................');
    KAFKA_TOPICS.split(',').forEach(topic => createKafkaTopic(topic))
   } else {
     console.log('Topics not created: KAFKA_TOPICS variable invalid')
+    process.exit(1)
   }
 })();

From f6dfd4055bbf5baad98a649fa1c0f06d1b03119c Mon Sep 17 00:00:00 2001
From: bradsawadye <bradsawadye@gmail.com>
Date: Thu, 21 Jul 2022 14:07:58 +0200
Subject: [PATCH 11/29] Change name of the service

---
 message-bus-kafka/importer/docker-compose.config.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/message-bus-kafka/importer/docker-compose.config.yml b/message-bus-kafka/importer/docker-compose.config.yml
index f78d71ca..cf6b7487 100644
--- a/message-bus-kafka/importer/docker-compose.config.yml
+++ b/message-bus-kafka/importer/docker-compose.config.yml
@@ -1,7 +1,7 @@
 version: '3.9'
 
 services:
-  kafka-config-importer:
+  instant_kafka-config-importer:
     image: jembi/instantohie-config-importer
     deploy:
       restart_policy:

From 87e585382a2b9b0462a1020747940b8785b4a6a9 Mon Sep 17 00:00:00 2001
From: bradsawadye <bradsawadye@gmail.com>
Date: Thu, 21 Jul 2022 14:11:19 +0200
Subject: [PATCH 12/29] Wait for kafka to start up before configuring

The configuration can only be done once the service is up. THis ensures that the services for configuring dont fail
---
 .env.ci                                       |  1 +
 .env.dev                                      |  3 +-
 .env.local                                    |  1 +
 .env.remote                                   |  1 +
 .../docker-compose.await-helper.yml           | 10 +++++
 message-bus-kafka/swarm.sh                    | 37 +++++++++++++++++--
 6 files changed, 48 insertions(+), 5 deletions(-)
 create mode 100644 message-bus-kafka/docker-compose.await-helper.yml

diff --git a/.env.ci b/.env.ci
index c96b348d..a37a41cf 100644
--- a/.env.ci
+++ b/.env.ci
@@ -70,6 +70,7 @@ KAFKA_CPU_RESERVE=0.05
 KAFKA_MEMORY_LIMIT=3G
 KAFKA_MEMORY_RESERVE=500M
 KAFKA_PARTITIONS=3
+KAFKA_INSTANCES=1
 # Topics should comma seperated
 KAFKA_TOPICS=2xx,reprocess
 
diff --git a/.env.dev b/.env.dev
index 7389285c..4eef2dc3 100644
--- a/.env.dev
+++ b/.env.dev
@@ -124,8 +124,9 @@ KAFKA_CPU_RESERVE=0.05
 KAFKA_MEMORY_LIMIT=3G
 KAFKA_MEMORY_RESERVE=500M
 KAFKA_PARTITIONS=3
+KAFKA_INSTANCES=1
 # Topics should comma seperated
-KAFKA_TOPICS=2xx,reprocess
+KAFKA_TOPICS=2xx,reprocess,3xx
 
 # Kafdrop
 
diff --git a/.env.local b/.env.local
index 916e3a1f..cfbc46ee 100644
--- a/.env.local
+++ b/.env.local
@@ -75,6 +75,7 @@ KAFKA_CPU_RESERVE=0.05
 KAFKA_MEMORY_LIMIT=3G
 KAFKA_MEMORY_RESERVE=500M
 KAFKA_PARTITIONS=3
+KAFKA_INSTANCES=1
 # Topics should comma seperated
 KAFKA_TOPICS=2xx,reprocess
 
diff --git a/.env.remote b/.env.remote
index 60b9cefc..b5cd6275 100644
--- a/.env.remote
+++ b/.env.remote
@@ -49,6 +49,7 @@ KAFKA_CPU_RESERVE=0.05
 KAFKA_MEMORY_LIMIT=3G
 KAFKA_MEMORY_RESERVE=500M
 KAFKA_PARTITIONS=3
+KAFKA_INSTANCES=3
 # Topics should comma seperated
 KAFKA_TOPICS=2xx,reprocess
 
diff --git a/message-bus-kafka/docker-compose.await-helper.yml b/message-bus-kafka/docker-compose.await-helper.yml
new file mode 100644
index 00000000..674b4871
--- /dev/null
+++ b/message-bus-kafka/docker-compose.await-helper.yml
@@ -0,0 +1,10 @@
+version: '3.9'
+
+services:
+  await-helper:
+    image: jembi/await-helper:1.0.1
+    deploy:
+      replicas: 1
+      restart_policy:
+        condition: none
+    command: '-k http://kafdrop:9013'
diff --git a/message-bus-kafka/swarm.sh b/message-bus-kafka/swarm.sh
index aa512575..09f12582 100755
--- a/message-bus-kafka/swarm.sh
+++ b/message-bus-kafka/swarm.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-statefulNodes=${STATEFUL_NODES:-"cluster"}
+STATEFUL_NODES=${STATEFUL_NODES:-"cluster"}
 
 COMPOSE_FILE_PATH=$(
   cd "$(dirname "${BASH_SOURCE[0]}")" || exit
@@ -13,7 +13,7 @@ ROOT_PATH="${COMPOSE_FILE_PATH}/.."
 . "${ROOT_PATH}/utils/docker-utils.sh"
 . "${ROOT_PATH}/utils/log.sh"
 
-if [[ $statefulNodes == "cluster" ]]; then
+if [[ $STATEFUL_NODES == "cluster" ]]; then
   log info "Running Message Bus Kafka package in Cluster node mode"
   kafkaClusterComposeParam="-c ${COMPOSE_FILE_PATH}/docker-compose.cluster.yml"
 else
@@ -29,10 +29,39 @@ else
   kafkaDevComposeParam=""
 fi
 
+verify_kafka() {
+  local start_time
+  start_time=$(date +%s)
+  until [[ $(docker service ls -f name=instant_kafka --format "{{.Replicas}}") == *"${KAFKA_INSTANCES}/${KAFKA_INSTANCES}"* ]]; do
+    config::timeout_check "${start_time}" "kafka to start"
+    sleep 1
+  done
+
+  local await_helper_state
+  await_helper_state=$(docker service ps instant_await-helper --format "{{.CurrentState}}")
+  until [[ "${await_helper_state}" == *"Complete"* ]]; do
+    config::timeout_check "${start_time}" "kafka heartbeat check"
+    sleep 1
+
+    await_helper_state=$(docker service ps instant_await-helper --format "{{.CurrentState}}")
+    if [[ "${await_helper_state}" == *"Failed"* ]] || [[ "${await_helper_state}" == *"Rejected"* ]]; then
+      log error "Fatal: Received error when trying to verify state of kafka. Error:
+       $(docker service ps instant_await-helper --no-trunc --format '{{.Error}}')"
+      exit 1
+    fi
+  done
+
+  try "docker service rm instant_await-helper" "Failed to remove await helper"
+}
+
 if [[ $1 == "init" ]] || [[ $1 == "up" ]]; then
   config::set_config_digests "${COMPOSE_FILE_PATH}"/importer/docker-compose.config.yml
 
   try "docker stack deploy -c ${COMPOSE_FILE_PATH}/docker-compose.yml $kafkaClusterComposeParam $kafkaDevComposeParam instant" "Failed to deploy Message Bus Kafka"
+  try "docker stack deploy -c ${COMPOSE_FILE_PATH}/docker-compose.await-helper.yml instant" "Failed to deploy await-helper"
+
+  verify_kafka
+
   try "docker stack deploy -c ${COMPOSE_FILE_PATH}/importer/docker-compose.config.yml instant" "Failed to deploy Message Bus Kafka"
 
   config::remove_stale_service_configs "${COMPOSE_FILE_PATH}"/importer/docker-compose.config.yml "ethiopia"
@@ -41,7 +70,7 @@ elif [[ $1 == "down" ]]; then
   try "docker service scale instant_zookeeper-1=0 instant_kafdrop=0" "Failed to scale down zookeeper and kafdrop"
   # You cannot scale a global service so we have to remove it
   try "docker service rm instant_kafka" "Failed to remove kafka"
-  if [[ $statefulNodes == "cluster" ]]; then
+  if [[ $STATEFUL_NODES == "cluster" ]]; then
     try "docker service scale instant_zookeeper-2=0" "Failed to scale down zookeeper cluster"
     try "docker service scale instant_zookeeper-3=0" "Failed to scale down zookeeper cluster"
   fi
@@ -57,7 +86,7 @@ elif [[ $1 == "destroy" ]]; then
   try "docker volume rm instant_kafka-volume" "Failed to remove kafka volume"
   try "docker volume rm instant_zookeeper-1-volume" "Failed to remove zookeeper volume"
 
-  if [[ $statefulNodes == "cluster" ]]; then
+  if [[ $STATEFUL_NODES == "cluster" ]]; then
     try "docker service rm instant_zookeeper-2" "Failed to remove zookeeper cluster volumes"
     try "docker service rm instant_zookeeper-3" "Failed to remove zookeeper cluster volumes"
     log notice "Volumes are only deleted on the host on which the command is run. Kafka volumes on other nodes are not deleted"

From 4be2a81987357e92b6ed52343b8f702379c9579f Mon Sep 17 00:00:00 2001
From: bradsawadye <bradsawadye@gmail.com>
Date: Thu, 21 Jul 2022 14:47:03 +0200
Subject: [PATCH 13/29] Update message-bus-kafka/swarm.sh

Co-authored-by: Mark-Labuschagne <78809924+Mark-Labuschagne@users.noreply.github.com>
---
 message-bus-kafka/swarm.sh | 25 +------------------------
 1 file changed, 1 insertion(+), 24 deletions(-)

diff --git a/message-bus-kafka/swarm.sh b/message-bus-kafka/swarm.sh
index 09f12582..78b3a022 100755
--- a/message-bus-kafka/swarm.sh
+++ b/message-bus-kafka/swarm.sh
@@ -29,30 +29,7 @@ else
   kafkaDevComposeParam=""
 fi
 
-verify_kafka() {
-  local start_time
-  start_time=$(date +%s)
-  until [[ $(docker service ls -f name=instant_kafka --format "{{.Replicas}}") == *"${KAFKA_INSTANCES}/${KAFKA_INSTANCES}"* ]]; do
-    config::timeout_check "${start_time}" "kafka to start"
-    sleep 1
-  done
-
-  local await_helper_state
-  await_helper_state=$(docker service ps instant_await-helper --format "{{.CurrentState}}")
-  until [[ "${await_helper_state}" == *"Complete"* ]]; do
-    config::timeout_check "${start_time}" "kafka heartbeat check"
-    sleep 1
-
-    await_helper_state=$(docker service ps instant_await-helper --format "{{.CurrentState}}")
-    if [[ "${await_helper_state}" == *"Failed"* ]] || [[ "${await_helper_state}" == *"Rejected"* ]]; then
-      log error "Fatal: Received error when trying to verify state of kafka. Error:
-       $(docker service ps instant_await-helper --no-trunc --format '{{.Error}}')"
-      exit 1
-    fi
-  done
-
-  try "docker service rm instant_await-helper" "Failed to remove await helper"
-}
+config::await_service_running "kafka" "${COMPOSE_FILE_PATH}"/docker-compose.await-helper.yml "${KAFKA_INSTANCES}"
 
 if [[ $1 == "init" ]] || [[ $1 == "up" ]]; then
   config::set_config_digests "${COMPOSE_FILE_PATH}"/importer/docker-compose.config.yml

From e8772688be433e9129db0f8eea828919c2329962 Mon Sep 17 00:00:00 2001
From: bradsawadye <bradsawadye@gmail.com>
Date: Thu, 21 Jul 2022 14:47:11 +0200
Subject: [PATCH 14/29] Update
 message-bus-kafka/importer/docker-compose.config.yml

Co-authored-by: Mark-Labuschagne <78809924+Mark-Labuschagne@users.noreply.github.com>
---
 message-bus-kafka/importer/docker-compose.config.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/message-bus-kafka/importer/docker-compose.config.yml b/message-bus-kafka/importer/docker-compose.config.yml
index cf6b7487..0766444c 100644
--- a/message-bus-kafka/importer/docker-compose.config.yml
+++ b/message-bus-kafka/importer/docker-compose.config.yml
@@ -1,7 +1,7 @@
 version: '3.9'
 
 services:
-  instant_kafka-config-importer:
+  message-bus-kafka-config-importer:
     image: jembi/instantohie-config-importer
     deploy:
       restart_policy:

From c8c31921bb758972c9f2cd94901f8ce624ba3861 Mon Sep 17 00:00:00 2001
From: bradsawadye <bradsawadye@gmail.com>
Date: Thu, 21 Jul 2022 14:52:13 +0200
Subject: [PATCH 15/29] REmove the config importer once configuration is done

---
 message-bus-kafka/swarm.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/message-bus-kafka/swarm.sh b/message-bus-kafka/swarm.sh
index 78b3a022..584be37e 100755
--- a/message-bus-kafka/swarm.sh
+++ b/message-bus-kafka/swarm.sh
@@ -42,7 +42,7 @@ if [[ $1 == "init" ]] || [[ $1 == "up" ]]; then
   try "docker stack deploy -c ${COMPOSE_FILE_PATH}/importer/docker-compose.config.yml instant" "Failed to deploy Message Bus Kafka"
 
   config::remove_stale_service_configs "${COMPOSE_FILE_PATH}"/importer/docker-compose.config.yml "ethiopia"
-  config::remove_config_importer kafka-config-importer
+  config::remove_config_importer message-bus-kafka-config-importer
 elif [[ $1 == "down" ]]; then
   try "docker service scale instant_zookeeper-1=0 instant_kafdrop=0" "Failed to scale down zookeeper and kafdrop"
   # You cannot scale a global service so we have to remove it

From b886da6445a51140687401643818bd5537478f3f Mon Sep 17 00:00:00 2001
From: bradsawadye <bradsawadye@gmail.com>
Date: Fri, 22 Jul 2022 11:08:48 +0200
Subject: [PATCH 16/29] Move the logic for checking whether kafka is up

The logic was in the wrong place. It should only be invoked once loc=gic for starting up kafka has been run
---
 message-bus-kafka/swarm.sh | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/message-bus-kafka/swarm.sh b/message-bus-kafka/swarm.sh
index 584be37e..f0f8f2f2 100755
--- a/message-bus-kafka/swarm.sh
+++ b/message-bus-kafka/swarm.sh
@@ -29,15 +29,12 @@ else
   kafkaDevComposeParam=""
 fi
 
-config::await_service_running "kafka" "${COMPOSE_FILE_PATH}"/docker-compose.await-helper.yml "${KAFKA_INSTANCES}"
-
 if [[ $1 == "init" ]] || [[ $1 == "up" ]]; then
   config::set_config_digests "${COMPOSE_FILE_PATH}"/importer/docker-compose.config.yml
 
   try "docker stack deploy -c ${COMPOSE_FILE_PATH}/docker-compose.yml $kafkaClusterComposeParam $kafkaDevComposeParam instant" "Failed to deploy Message Bus Kafka"
-  try "docker stack deploy -c ${COMPOSE_FILE_PATH}/docker-compose.await-helper.yml instant" "Failed to deploy await-helper"
 
-  verify_kafka
+  config::await_service_running "kafka" "${COMPOSE_FILE_PATH}"/docker-compose.await-helper.yml "${KAFKA_INSTANCES}"
 
   try "docker stack deploy -c ${COMPOSE_FILE_PATH}/importer/docker-compose.config.yml instant" "Failed to deploy Message Bus Kafka"
 

From 3b9724b82e830a4f7fe7c3b99ae476a2de0f72d6 Mon Sep 17 00:00:00 2001
From: bradsawadye <bradsawadye@gmail.com>
Date: Fri, 22 Jul 2022 11:11:20 +0200
Subject: [PATCH 17/29] Remove config importer in the destroy action

For cases where the importer failed and was not removed in the init action
---
 message-bus-kafka/swarm.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/message-bus-kafka/swarm.sh b/message-bus-kafka/swarm.sh
index f0f8f2f2..5e594f09 100755
--- a/message-bus-kafka/swarm.sh
+++ b/message-bus-kafka/swarm.sh
@@ -57,14 +57,15 @@ elif [[ $1 == "destroy" ]]; then
   config::await_service_removed instant_kafka
   config::await_service_removed instant_kafdrop
 
-  try "docker volume rm instant_kafka-volume" "Failed to remove kafka volume"
-  try "docker volume rm instant_zookeeper-1-volume" "Failed to remove zookeeper volume"
+  try "docker volume rm instant_zookeeper-1-volume instant_kafka-volume" "Failed to remove zookeeper or kafka volume"
 
   if [[ $STATEFUL_NODES == "cluster" ]]; then
     try "docker service rm instant_zookeeper-2" "Failed to remove zookeeper cluster volumes"
     try "docker service rm instant_zookeeper-3" "Failed to remove zookeeper cluster volumes"
     log notice "Volumes are only deleted on the host on which the command is run. Kafka volumes on other nodes are not deleted"
   fi
+
+  config::remove_config_importer message-bus-kafka-config-importer
 else
   log error "Valid options are: init, up, down, or destroy"
 fi

From ce179ddd5fc0f32adcfc996e8d0d1e9427415926 Mon Sep 17 00:00:00 2001
From: bradsawadye <bradsawadye@gmail.com>
Date: Fri, 22 Jul 2022 11:13:33 +0200
Subject: [PATCH 18/29] Remove configs only if they exist

This ensures that we wont get errors when we try to remove a config importer that does not exist
---
 utils/config-utils.sh | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/utils/config-utils.sh b/utils/config-utils.sh
index a327cffd..193139a7 100755
--- a/utils/config-utils.sh
+++ b/utils/config-utils.sh
@@ -164,6 +164,12 @@ config::remove_config_importer() {
     local -r start_time=$(date +%s)
 
     local config_importer_state
+
+    if [[ -z $(docker service ps instant_"$config_importer_service_name") ]]; then
+        log info "instant_$config_importer_service_name service cannot be removed as it does not exist!"
+        exit 0
+    fi
+
     config_importer_state=$(docker service ps instant_"$config_importer_service_name" --format "{{.CurrentState}}")
     until [[ $config_importer_state == *"Complete"* ]]; do
         config::timeout_check "$start_time" "$config_importer_service_name to run" "$exit_time" "$warning_time"
@@ -192,6 +198,7 @@ config::await_service_removed() {
         config::timeout_check "$start_time" "${SERVICE_NAME} to be removed"
         sleep 1
     done
+    log info "Service $SERVICE_NAME successfully removed"
 }
 
 # Waits for the provided service to join the network

From 34e2915f718df1d130d6f84010fb6e3d7ab1db6b Mon Sep 17 00:00:00 2001
From: mark-labuschagne <marklabuschagne7@gmail.com>
Date: Fri, 22 Jul 2022 13:00:57 +0200
Subject: [PATCH 19/29] Added defaults for KAFKA_INSTANCES

PLAT-346
---
 message-bus-kafka/swarm.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/message-bus-kafka/swarm.sh b/message-bus-kafka/swarm.sh
index 5e594f09..31855b86 100755
--- a/message-bus-kafka/swarm.sh
+++ b/message-bus-kafka/swarm.sh
@@ -13,6 +13,9 @@ ROOT_PATH="${COMPOSE_FILE_PATH}/.."
 . "${ROOT_PATH}/utils/docker-utils.sh"
 . "${ROOT_PATH}/utils/log.sh"
 
+readonly KAFKA_INSTANCES=${KAFKA_INSTANCES:-1}
+export KAFKA_INSTANCES
+
 if [[ $STATEFUL_NODES == "cluster" ]]; then
   log info "Running Message Bus Kafka package in Cluster node mode"
   kafkaClusterComposeParam="-c ${COMPOSE_FILE_PATH}/docker-compose.cluster.yml"

From 4f066ed409ae8041477718a2d361543cb30fb4ac Mon Sep 17 00:00:00 2001
From: mark-labuschagne <marklabuschagne7@gmail.com>
Date: Fri, 22 Jul 2022 13:01:25 +0200
Subject: [PATCH 20/29] Updated kafka remove function

PLAT-346
---
 message-bus-kafka/swarm.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/message-bus-kafka/swarm.sh b/message-bus-kafka/swarm.sh
index 31855b86..7db18bef 100755
--- a/message-bus-kafka/swarm.sh
+++ b/message-bus-kafka/swarm.sh
@@ -68,7 +68,7 @@ elif [[ $1 == "destroy" ]]; then
     log notice "Volumes are only deleted on the host on which the command is run. Kafka volumes on other nodes are not deleted"
   fi
 
-  config::remove_config_importer message-bus-kafka-config-importer
+  log warn "docker service rm instant_message-bus-kafka-config-importer" ": message-bus-kafka-config-importer not removed... it's possible the service has already been removed"
 else
   log error "Valid options are: init, up, down, or destroy"
 fi

From f9979b0b55b1da63d2bab921cf6075a8c6ed0d07 Mon Sep 17 00:00:00 2001
From: mark-labuschagne <marklabuschagne7@gmail.com>
Date: Fri, 22 Jul 2022 14:53:44 +0200
Subject: [PATCH 21/29] Changed strategy for removing kafka services/volumes

PLAT-346
---
 message-bus-kafka/swarm.sh | 21 +++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/message-bus-kafka/swarm.sh b/message-bus-kafka/swarm.sh
index 7db18bef..c28cddb5 100755
--- a/message-bus-kafka/swarm.sh
+++ b/message-bus-kafka/swarm.sh
@@ -52,17 +52,26 @@ elif [[ $1 == "down" ]]; then
     try "docker service scale instant_zookeeper-3=0" "Failed to scale down zookeeper cluster"
   fi
 elif [[ $1 == "destroy" ]]; then
-  try "docker service rm instant_zookeeper-1 instant_kafka instant_kafdrop" "Failed to destroy kafka"
-
   log info "Allow services to shut down before deleting volumes"
 
-  config::await_service_removed instant_zookeeper-1
-  config::await_service_removed instant_kafka
-  config::await_service_removed instant_kafdrop
+  # The services are being scaled down first to ensure that the containers have been
+  # removed before removing the volumes. This is done because a service can be removed
+  # and the container can remain up for a few seconds, meaning the volumes can't be removed.
+  try "docker service scale instant_zookeeper-1=0" "Failed to scale down instant_zookeeper-1"
+  try "docker service scale instant_kafka=0" "Failed to scale down instant_kafka"
+  try "docker service scale instant_kafdrop=0" "Failed to scale down instant_kafdrop"
+
+  try "docker service rm instant_zookeeper-1" "Failed to destroy instant_zookeeper"
+  try "docker service rm instant_kafdrop" "Failed to destroy instant_kafdrop"
+  try "docker service rm instant_kafka" "Failed to destroy instant_kafka"
 
-  try "docker volume rm instant_zookeeper-1-volume instant_kafka-volume" "Failed to remove zookeeper or kafka volume"
+  try "docker volume rm instant_zookeeper-1-volume" "Failed to remove instant_zookeeper-1-volume volume"
+  try "docker volume rm instant_kafka-volume" "Failed to remove instant_kafka-volume volume"
 
   if [[ $STATEFUL_NODES == "cluster" ]]; then
+    try "docker service scale instant_zookeeper-2=0" "Failed to scale down instant_zookeeper-2"
+    try "docker service scale instant_zookeeper-3=0" "Failed to scale down instant_zookeeper-3"
+
     try "docker service rm instant_zookeeper-2" "Failed to remove zookeeper cluster volumes"
     try "docker service rm instant_zookeeper-3" "Failed to remove zookeeper cluster volumes"
     log notice "Volumes are only deleted on the host on which the command is run. Kafka volumes on other nodes are not deleted"

From 8e8adb61aec017a0580414828683d97c3d59df94 Mon Sep 17 00:00:00 2001
From: mark-labuschagne <marklabuschagne7@gmail.com>
Date: Tue, 26 Jul 2022 10:51:48 +0200
Subject: [PATCH 22/29] Improved service destroy strategy This strategy allows
 for safe and configurable removal of volumes, and is swarm compliant. So,
 when volumes can be removed over a swarm, this approach will still work
 PLAT-346

---
 message-bus-kafka/swarm.sh | 25 +++++++++----------------
 utils/docker-utils.sh      |  2 +-
 2 files changed, 10 insertions(+), 17 deletions(-)

diff --git a/message-bus-kafka/swarm.sh b/message-bus-kafka/swarm.sh
index c28cddb5..34d7eeb6 100755
--- a/message-bus-kafka/swarm.sh
+++ b/message-bus-kafka/swarm.sh
@@ -54,26 +54,19 @@ elif [[ $1 == "down" ]]; then
 elif [[ $1 == "destroy" ]]; then
   log info "Allow services to shut down before deleting volumes"
 
-  # The services are being scaled down first to ensure that the containers have been
-  # removed before removing the volumes. This is done because a service can be removed
-  # and the container can remain up for a few seconds, meaning the volumes can't be removed.
-  try "docker service scale instant_zookeeper-1=0" "Failed to scale down instant_zookeeper-1"
-  try "docker service scale instant_kafka=0" "Failed to scale down instant_kafka"
-  try "docker service scale instant_kafdrop=0" "Failed to scale down instant_kafdrop"
+  docker::service_destroy zookeeper-1
+  docker::service_destroy kafka
+  docker::service_destroy kafdrop
 
-  try "docker service rm instant_zookeeper-1" "Failed to destroy instant_zookeeper"
-  try "docker service rm instant_kafdrop" "Failed to destroy instant_kafdrop"
-  try "docker service rm instant_kafka" "Failed to destroy instant_kafka"
-
-  try "docker volume rm instant_zookeeper-1-volume" "Failed to remove instant_zookeeper-1-volume volume"
-  try "docker volume rm instant_kafka-volume" "Failed to remove instant_kafka-volume volume"
+  docker::try_remove_volume zookeeper-1-volume
+  docker::try_remove_volume kafka-volume
 
   if [[ $STATEFUL_NODES == "cluster" ]]; then
-    try "docker service scale instant_zookeeper-2=0" "Failed to scale down instant_zookeeper-2"
-    try "docker service scale instant_zookeeper-3=0" "Failed to scale down instant_zookeeper-3"
+    docker::service_destroy zookeeper-2
+    docker::service_destroy zookeeper-3
 
-    try "docker service rm instant_zookeeper-2" "Failed to remove zookeeper cluster volumes"
-    try "docker service rm instant_zookeeper-3" "Failed to remove zookeeper cluster volumes"
+    docker::try_remove_volume zookeeper-2-volume
+    docker::try_remove_volume zookeeper-3-volume
     log notice "Volumes are only deleted on the host on which the command is run. Kafka volumes on other nodes are not deleted"
   fi
 
diff --git a/utils/docker-utils.sh b/utils/docker-utils.sh
index 9979a76c..b86f6990 100644
--- a/utils/docker-utils.sh
+++ b/utils/docker-utils.sh
@@ -107,7 +107,7 @@ docker::try_remove_volume() {
     local start_time
     start_time=$(date +%s)
     until [[ -n "$(docker volume rm instant_"${VOLUME_NAME}" 2>/dev/null)" ]]; do
-        config::timeout_check "${start_time}" "${VOLUME_NAME} to be removed" "10" "20"
+        config::timeout_check "${start_time}" "${VOLUME_NAME} to be removed" "20" "10"
         sleep 1
     done
     overwrite "Waiting for volume ${VOLUME_NAME} to be removed... Done"

From be3ae816e921737ea08350dec72e5c3fb3143225 Mon Sep 17 00:00:00 2001
From: mark-labuschagne <marklabuschagne7@gmail.com>
Date: Tue, 26 Jul 2022 10:59:25 +0200
Subject: [PATCH 23/29] Hardcoded KAFDROP_HOST and KAFDROP_PORT where used This
 should never change, and is used in select enough places to only have to
 change it in one place should it ever change PLAT-346

---
 .env.dev                                             | 2 --
 .env.local                                           | 2 --
 .env.remote                                          | 2 --
 message-bus-kafka/importer/docker-compose.config.yml | 4 ++--
 message-bus-kafka/package-metadata.json              | 2 --
 5 files changed, 2 insertions(+), 10 deletions(-)

diff --git a/.env.dev b/.env.dev
index 4eef2dc3..dfe82647 100644
--- a/.env.dev
+++ b/.env.dev
@@ -134,8 +134,6 @@ KAFDROP_CPU_LIMIT=0.8
 KAFDROP_CPU_RESERVE=0.05
 KAFDROP_MEMORY_LIMIT=3G
 KAFDROP_MEMORY_RESERVE=500M
-KAFDROP_HOST=kafdrop
-KAFDROP_PORT=9013
 
 # Message Bus Helper - Hapi Proxy
 
diff --git a/.env.local b/.env.local
index 567d83cc..f6edaeb9 100644
--- a/.env.local
+++ b/.env.local
@@ -92,6 +92,4 @@ KAFDROP_CPU_LIMIT=0.8
 KAFDROP_CPU_RESERVE=0.05
 KAFDROP_MEMORY_LIMIT=3G
 KAFDROP_MEMORY_RESERVE=500M
-KAFDROP_HOST=kafdrop
-KAFDROP_PORT=9013
 
diff --git a/.env.remote b/.env.remote
index b5cd6275..6280d371 100644
--- a/.env.remote
+++ b/.env.remote
@@ -59,5 +59,3 @@ KAFDROP_CPU_LIMIT=0.8
 KAFDROP_CPU_RESERVE=0.05
 KAFDROP_MEMORY_LIMIT=3G
 KAFDROP_MEMORY_RESERVE=500M
-KAFDROP_HOST=kafdrop
-KAFDROP_PORT=9013
diff --git a/message-bus-kafka/importer/docker-compose.config.yml b/message-bus-kafka/importer/docker-compose.config.yml
index 0766444c..c3e2fed4 100644
--- a/message-bus-kafka/importer/docker-compose.config.yml
+++ b/message-bus-kafka/importer/docker-compose.config.yml
@@ -7,8 +7,8 @@ services:
       restart_policy:
         condition: none
     environment:
-      KAFDROP_HOST: ${KAFDROP_HOST:-kafdrop}
-      KAFDROP_PORT: ${KAFDROP_PORT:-9013}
+      KAFDROP_HOST: kafdrop
+      KAFDROP_PORT: 9013
       KAFKA_PARTITIONS: ${KAFKA_PARTITIONS:-3}
       KAFKA_TOPICS: ${KAFKA_TOPICS}
     command: sh -c "wait-on -t 60000 http-get://kafdrop:9013/topic && node /kafka.js && echo 'success';"
diff --git a/message-bus-kafka/package-metadata.json b/message-bus-kafka/package-metadata.json
index 81b320f7..20d1283b 100644
--- a/message-bus-kafka/package-metadata.json
+++ b/message-bus-kafka/package-metadata.json
@@ -11,8 +11,6 @@
     "KAFKA_PARTITIONS": "3",
     "KAFKA_TOPICS": "",
     "KAFDROP_CPU_LIMIT": "0.8",
-    "KAFDROP_HOST": "kafdrop",
-    "KAFDROP_PORT": "9013",
     "KAFDROP_CPU_RESERVE": "0.05",
     "KAFDROP_MEMORY_LIMIT": "3G",
     "KAFDROP_MEMORY_RESERVE": "500M",

From 6966a54a96d41e1f66ecf27a773387cf62f624ef Mon Sep 17 00:00:00 2001
From: mark-labuschagne <marklabuschagne7@gmail.com>
Date: Tue, 26 Jul 2022 11:08:04 +0200
Subject: [PATCH 24/29] Fixed incorrect usage of log warn

PLAT-346
---
 message-bus-kafka/swarm.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/message-bus-kafka/swarm.sh b/message-bus-kafka/swarm.sh
index 34d7eeb6..4635e997 100755
--- a/message-bus-kafka/swarm.sh
+++ b/message-bus-kafka/swarm.sh
@@ -70,7 +70,9 @@ elif [[ $1 == "destroy" ]]; then
     log notice "Volumes are only deleted on the host on which the command is run. Kafka volumes on other nodes are not deleted"
   fi
 
-  log warn "docker service rm instant_message-bus-kafka-config-importer" ": message-bus-kafka-config-importer not removed... it's possible the service has already been removed"
+  if ! docker service rm instant_message-bus-kafka-config-importer; then
+    log warn "message-bus-kafka-config-importer not removed... it's possible the service has already been removed"
+  fi
 else
   log error "Valid options are: init, up, down, or destroy"
 fi

From 9f0440ab6064318f4797b40d9351e7e63652a007 Mon Sep 17 00:00:00 2001
From: Mark-Labuschagne <78809924+Mark-Labuschagne@users.noreply.github.com>
Date: Tue, 26 Jul 2022 11:15:54 +0200
Subject: [PATCH 25/29] Update .env.local

Co-authored-by: Barry Dwyer <dwyerbaz@gmail.com>
---
 .env.local | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.env.local b/.env.local
index f6edaeb9..196fa95c 100644
--- a/.env.local
+++ b/.env.local
@@ -81,7 +81,7 @@ KAFKA_CPU_LIMIT=0.8
 KAFKA_CPU_RESERVE=0.05
 KAFKA_MEMORY_LIMIT=3G
 KAFKA_MEMORY_RESERVE=500M
-KAFKA_PARTITIONS=3
+KAFKA_PARTITIONS=1
 KAFKA_INSTANCES=1
 # Topics should comma seperated
 KAFKA_TOPICS=2xx,reprocess

From 541b2512553e034fdd9d8e6116fcbd213db8beef Mon Sep 17 00:00:00 2001
From: Mark-Labuschagne <78809924+Mark-Labuschagne@users.noreply.github.com>
Date: Tue, 26 Jul 2022 11:16:01 +0200
Subject: [PATCH 26/29] Update .env.local

Co-authored-by: Barry Dwyer <dwyerbaz@gmail.com>
---
 .env.local | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/.env.local b/.env.local
index 196fa95c..df41b112 100644
--- a/.env.local
+++ b/.env.local
@@ -88,8 +88,4 @@ KAFKA_TOPICS=2xx,reprocess
 
 # Kafdrop
 
-KAFDROP_CPU_LIMIT=0.8
-KAFDROP_CPU_RESERVE=0.05
-KAFDROP_MEMORY_LIMIT=3G
-KAFDROP_MEMORY_RESERVE=500M
 

From 004c0885d0036a5b4c46ead1a2d2a83134ba9a35 Mon Sep 17 00:00:00 2001
From: Mark-Labuschagne <78809924+Mark-Labuschagne@users.noreply.github.com>
Date: Tue, 26 Jul 2022 11:16:14 +0200
Subject: [PATCH 27/29] Update .env.remote

Co-authored-by: Barry Dwyer <dwyerbaz@gmail.com>
---
 .env.remote | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/.env.remote b/.env.remote
index 6280d371..ee7d5a04 100644
--- a/.env.remote
+++ b/.env.remote
@@ -44,10 +44,6 @@ OPENHIM_MONGO_ATNAURL=mongodb://mongo-1:27017,mongo-2:27017,mongo-3:27017/openhi
 
 # Message Bus - Kafka
 
-KAFKA_CPU_LIMIT=0.8
-KAFKA_CPU_RESERVE=0.05
-KAFKA_MEMORY_LIMIT=3G
-KAFKA_MEMORY_RESERVE=500M
 KAFKA_PARTITIONS=3
 KAFKA_INSTANCES=3
 # Topics should comma seperated

From 7941e99d056d459a7fb210b499dede5e68a4d525 Mon Sep 17 00:00:00 2001
From: Mark-Labuschagne <78809924+Mark-Labuschagne@users.noreply.github.com>
Date: Tue, 26 Jul 2022 11:16:19 +0200
Subject: [PATCH 28/29] Update .env.remote

Co-authored-by: Barry Dwyer <dwyerbaz@gmail.com>
---
 .env.remote | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/.env.remote b/.env.remote
index ee7d5a04..1b920be9 100644
--- a/.env.remote
+++ b/.env.remote
@@ -48,10 +48,3 @@ KAFKA_PARTITIONS=3
 KAFKA_INSTANCES=3
 # Topics should comma seperated
 KAFKA_TOPICS=2xx,reprocess
-
-# Kafdrop
-
-KAFDROP_CPU_LIMIT=0.8
-KAFDROP_CPU_RESERVE=0.05
-KAFDROP_MEMORY_LIMIT=3G
-KAFDROP_MEMORY_RESERVE=500M

From c81f15e46a2ced79de8bdc5a7cffae9c237b8519 Mon Sep 17 00:00:00 2001
From: Mark-Labuschagne <78809924+Mark-Labuschagne@users.noreply.github.com>
Date: Tue, 26 Jul 2022 11:16:24 +0200
Subject: [PATCH 29/29] Update .env.local

Co-authored-by: Barry Dwyer <dwyerbaz@gmail.com>
---
 .env.local | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/.env.local b/.env.local
index df41b112..d5614f23 100644
--- a/.env.local
+++ b/.env.local
@@ -77,10 +77,6 @@ JS_REPORT_PACKAGE_PATH=
 
 # Message Bus - Kafka
 
-KAFKA_CPU_LIMIT=0.8
-KAFKA_CPU_RESERVE=0.05
-KAFKA_MEMORY_LIMIT=3G
-KAFKA_MEMORY_RESERVE=500M
 KAFKA_PARTITIONS=1
 KAFKA_INSTANCES=1
 # Topics should comma seperated