86c0t11yc - Extract Minio from monitoring package into an independent package

config.yaml

@@ -27,6 +27,7 @@ packages:
   - database-postgres
   - reprocess-mediator
   - fhir-ig-importer
+  - datalake
 
 profiles:
   - name: cdr-dw
@@ -47,6 +48,7 @@ profiles:
       - kafka-unbundler-consumer
       - fhir-ig-importer
       - reprocess-mediator
+      - datalake
     envFiles:
       - cdr-dw.env
 

datalake/docker-compose.cluster.yml

@@ -0,0 +1,82 @@
+version: "3.9"
+
+services:
+  minio-01:
+    deploy:
+      placement:
+        constraints:
+          - "node.labels.name==${MINIO_01_PLACEMENT}"
+
+  minio-02:
+    image: ${MINIO_IMAGE}
+    entrypoint: sh
+    command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" http://minio-0{1...4}/data{1...2}'
+    environment:
+      MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER}
+      MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD}
+    healthcheck:
+      test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/live" ]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+    hostname: minio-02
+    volumes:
+      - minio-02-data1:/data1
+      - minio-02-data2:/data2
+    deploy:
+      placement:
+        constraints:
+          - "node.labels.name==${MINIO_02_PLACEMENT}"
+      replicas: 1
+
+  minio-03:
+    image: ${MINIO_IMAGE}
+    entrypoint: sh
+    command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" http://minio-0{1...4}/data{1...2}'
+    environment:
+      MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER}
+      MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD}
+    healthcheck:
+      test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/live" ]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+    hostname: minio-03
+    volumes:
+      - minio-03-data1:/data1
+      - minio-03-data2:/data2
+    deploy:
+      placement:
+        constraints:
+          - "node.labels.name==${MINIO_03_PLACEMENT}"
+      replicas: 1
+
+  minio-04:
+    image: ${MINIO_IMAGE}
+    entrypoint: sh
+    command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" http://minio-0{1...4}/data{1...2}'
+    environment:
+      MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER}
+      MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD}
+    healthcheck:
+      test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/live" ]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+    hostname: minio-04
+    volumes:
+      - minio-04-data1:/data1
+      - minio-04-data2:/data2
+    deploy:
+      placement:
+        constraints:
+          - "node.labels.name==${MINIO_04_PLACEMENT}"
+      replicas: 1
+
+volumes:
+  minio-02-data1:
+  minio-02-data2:
+  minio-03-data1:
+  minio-03-data2:
+  minio-04-data1:
+  minio-04-data2:

datalake/docker-compose.dev.yml

@@ -0,0 +1,12 @@
+version: '3.9'
+
+services:
+  minio-01:
+    ports:
+      - target: 9001
+        published: 9005
+        mode: host
+
+      - target: 9090
+        published: 9000
+        mode: host

datalake/docker-compose.yml

@@ -0,0 +1,40 @@
+version: '3.9'
+
+services:
+  minio-01:
+    image: ${MINIO_IMAGE}
+    entrypoint: sh
+    command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" http://minio-0{1...${NUM_MINIO_SERVERS}}/data{1...2}'
+    environment:
+      MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER}
+      MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD}
+      MINIO_BROWSER_REDIRECT_URL: ${MINIO_BROWSER_REDIRECT_URL}
+      MINIO_SERVER_URL: http://localhost:9000
+    healthcheck:
+      test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/live" ]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+    hostname: minio-01
+    volumes:
+      - minio-01-data1:/data1
+      - minio-01-data2:/data2
+    deploy:
+      replicas: 1
+      labels:
+        - traefik.enable=true
+        - traefik.docker.network=reverse-proxy-traefik_public
+        - traefik.http.routers.minio.rule=${DOMAIN_NAME_HOST_TRAEFIK} && PathPrefix(`/minio`)
+        - traefik.http.services.minio.loadbalancer.server.port=9001
+        - traefik.http.middlewares.minio-stripprefix.stripprefix.prefixes=/minio
+        - traefik.http.routers.minio.middlewares=minio-stripprefix
+    networks:
+      public:
+networks:
+  public:
+    name: minio_public
+    external: true
+
+volumes:
+  minio-01-data1:
+  minio-01-data2:

datalake/package-metadata.json

@@ -0,0 +1,21 @@
+{
+  "id": "datalake",
+  "name": "datalake",
+  "description": "Object storage for raw unstructured data, compatible with the s3 API",
+  "type": "infrastructure",
+  "version": "0.0.1",
+  "dependencies": [],
+  "environmentVariables": {
+    "NUM_MINIO_SERVERS": 1,
+    "MO_SECURITY_ADMIN_USER": "admin",
+    "MO_SECURITY_ADMIN_PASSWORD": "dev_password_only",
+    "MO_RETENTION_TIME": "15d",
+    "MINIO_BROWSER_REDIRECT_URL": "http://localhost:9001",
+    "DOMAIN_NAME_HOST_TRAEFIK": "localhost",
+    "MINIO_01_PLACEMENT": "minio-01",
+    "MINIO_02_PLACEMENT": "minio-02",
+    "MINIO_03_PLACEMENT": "minio-03",
+    "MINIO_04_PLACEMENT": "minio-04",
+    "MINIO_IMAGE": "minio/minio:RELEASE.2024-10-13T13-34-11Z.fips"
+  }
+}

datalake/swarm.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+
+declare ACTION=""
+declare MODE=""
+declare COMPOSE_FILE_PATH=""
+declare UTILS_PATH=""
+declare STACK="datalake"
+
+function init_vars() {
+  ACTION=$1
+  MODE=$2
+
+  COMPOSE_FILE_PATH=$(
+    cd "$(dirname "${BASH_SOURCE[0]}")" || exit
+    pwd -P
+  )
+
+  UTILS_PATH="${COMPOSE_FILE_PATH}/../utils"
+
+  readonly ACTION
+  readonly MODE
+  readonly COMPOSE_FILE_PATH
+  readonly UTILS_PATH
+  readonly STACK
+}
+
+# shellcheck disable=SC1091
+function import_sources() {
+  source "${UTILS_PATH}/docker-utils.sh"
+  source "${UTILS_PATH}/log.sh"
+}
+
+function initialize_package() {
+  local package_dev_compose_filename=""
+  local minio_cluster_compose_filename=""
+  if [[ "${CLUSTERED_MODE}" == "true" ]]; then
+    minio_cluster_compose_filename="docker-compose.cluster.yml"
+    export NUM_MINIO_SERVERS=4
+  else
+    export NUM_MINIO_SERVERS=1
+  fi
+  if [[ "${MODE}" == "dev" ]]; then
+    log info "Running package in DEV mode"
+    package_dev_compose_filename="docker-compose.dev.yml"
+  else
+    log info "Running package in PROD mode"
+  fi
+
+  (
+    docker::deploy_service "$STACK" "${COMPOSE_FILE_PATH}" "docker-compose.yml" "$package_dev_compose_filename" "$minio_cluster_compose_filename"
+  ) || {
+    log error "Failed to deploy package"
+    exit 1
+  }
+}
+
+function destroy_package() {
+  docker::stack_destroy "$STACK"
+}
+
+main() {
+  init_vars "$@"
+  import_sources
+
+  if [[ "${ACTION}" == "init" ]] || [[ "${ACTION}" == "up" ]]; then
+    log info "Running package in Single node mode"
+    initialize_package
+  elif [[ "${ACTION}" == "down" ]]; then
+    log info "Scaling down package"
+    docker::scale_services "$STACK" 0
+  elif [[ "${ACTION}" == "destroy" ]]; then
+    log info "Destroying package"
+    destroy_package
+  else
+    log error "Valid options are: init, up, down, or destroy"
+  fi
+}
+
+main "$@"

monitoring/docker-compose.cluster.yml

@@ -32,83 +32,6 @@ services:
       public:
       default:
 
-  minio-01:
-    deploy:
-      placement:
-        constraints:
-          - "node.labels.name==${MINIO_01_PLACEMENT}"
-
-  minio-02:
-    image: ${MINIO_IMAGE}
-    entrypoint: sh
-    command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" http://minio-0{1...4}/data{1...2}'
-    environment:
-      MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER}
-      MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD}
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
-      interval: 30s
-      timeout: 20s
-      retries: 3
-    hostname: minio-02
-    volumes:
-      - minio-02-data1:/data1
-      - minio-02-data2:/data2
-    deploy:
-      placement:
-        constraints:
-          - "node.labels.name==${MINIO_02_PLACEMENT}"
-      replicas: 1
-
-  minio-03:
-    image: ${MINIO_IMAGE}
-    entrypoint: sh
-    command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" http://minio-0{1...4}/data{1...2}'
-    environment:
-      MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER}
-      MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD}
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
-      interval: 30s
-      timeout: 20s
-      retries: 3
-    hostname: minio-03
-    volumes:
-      - minio-03-data1:/data1
-      - minio-03-data2:/data2
-    deploy:
-      placement:
-        constraints:
-          - "node.labels.name==${MINIO_03_PLACEMENT}"
-      replicas: 1
-
-  minio-04:
-    image: ${MINIO_IMAGE}
-    entrypoint: sh
-    command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" http://minio-0{1...4}/data{1...2}'
-    environment:
-      MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER}
-      MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD}
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
-      interval: 30s
-      timeout: 20s
-      retries: 3
-    hostname: minio-04
-    volumes:
-      - minio-04-data1:/data1
-      - minio-04-data2:/data2
-    deploy:
-      placement:
-        constraints:
-          - "node.labels.name==${MINIO_04_PLACEMENT}"
-      replicas: 1
 
 volumes:
   prometheus_data_backup:
-  minio-02-data1:
-  minio-02-data2:
-  minio-03-data1:
-  minio-03-data2:
-  minio-04-data1:
-  minio-04-data2:

monitoring/docker-compose.dev.yml

@@ -18,9 +18,3 @@ services:
       - target: 3100
         published: 3100
         mode: host
-
-  minio-01:
-    ports:
-      - target: 9001
-        published: 9005
-        mode: host