feat(mirror-node): Enhance mirror-node external database feature

Taskfile.helper.yml

@@ -297,7 +297,7 @@ tasks:
     status:
       - kind get clusters | grep -q "${SOLO_CLUSTER_NAME}"
     cmds:
-      - kind create cluster -n "${SOLO_CLUSTER_NAME}" --image "${KIND_IMAGE}"
+      - kind create cluster -n "${SOLO_CLUSTER_NAME}" --image "${KIND_IMAGE}" --config ~/Documents/kind-cluster.yaml
       - sleep 10 # wait for control plane to come up
       - kubectl config set-context kind-${SOLO_CLUSTER_NAME}
 
@@ -434,6 +434,15 @@ tasks:
           sleep 4
         fi
 
+    solo:mirror-node-only:
+      silent: true
+      desc: solo mirror-node deploy with port forward on explorer
+      deps:
+        - task: "init"
+      cmds:
+        - SOLO_HOME_DIR=${SOLO_HOME_DIR} npm run solo -- mirror-node deploy --namespace "${SOLO_NAMESPACE}" ${SOLO_CHARTS_DIR_FLAG} ${MIRROR_NODE_DEPLOY_EXTRA_FLAGS} --pinger -q --dev
+
+
   solo:destroy-mirror-node:
     silent: true
     desc: solo mirror-node destroy
@@ -464,3 +473,21 @@ tasks:
     cmds:
       - echo "Cleaning up temporary files..."
       - rm -f /tmp/solo-${USER}-* || true
+
+  solo:external-database:
+    silent: false
+    desc: setup external database PostgreSQL with helm
+    cmds:
+      - |
+        {{.solo_bin_dir}}/helm install my-postgresql bitnami/postgresql \
+          --namespace database --create-namespace \
+          --set global.postgresql.auth.postgresPassword={{.postgres_password}} \
+          --set primary.persistence.enabled=false --set secondary.enabled=false
+      - name: "Wait for PostgreSQL pod to be ready"
+        cmd: kubectl wait --for=condition=ready pod/my-postgresql-0 -n database --timeout=60s
+      - name: "Copy init.sql inside the database pod"
+        cmd: kubectl cp ../custom-mirror-node-database/scripts/init.sh my-postgresql-0:/tmp/init.sh -n database
+      - name: "Make init.sh executable"
+        cmd: kubectl exec -it my-postgresql-0 -n database -- chmod +x /tmp/init.sh
+      - name: "Execute init.sh inside the database pod"
+        cmd: kubectl exec -it my-postgresql-0 -n database -- /bin/bash /tmp/init.sh

examples/custom-mirror-node-database/scripts/init.sh

@@ -1,7 +1,3 @@
-# Example if initialization script for the external database,
-# values must match those inside values.yaml if used
-
-cat > init1.sh << 'EOF'
 #!/bin/bash
 set -e
 
@@ -139,6 +135,3 @@ if [[ -f "${PGHBACONF}.bak" ]]; then
   mv "${PGHBACONF}.bak" "${PGHBACONF}"
   pg_ctl reload
 fi
-EOF
-chmod +x init1.sh
-./init1.sh

examples/external-database-test/Taskfile.yml

@@ -0,0 +1,50 @@
+version: 3
+includes:
+  main:
+    taskfile: ../../Taskfile.helper.yml
+    flatten: true
+vars:
+  solo_home_override_dir: "/Users/{{.HOME}}/.solo"
+  use_port_forwards: "true"
+  postgres_username: "postgres"
+  postgres_password: "XXXXXXXXXXXX"
+  postgres_host: "my-postgresql-0.database.svc.cluster.local"
+env:
+  SOLO_NETWORK_SIZE: "1"
+  SOLO_NAMESPACE: "external-database-test"
+  MIRROR_NODE_DEPLOY_EXTRA_FLAGS: "--use-external-database --external-database-host {{.postgres_host}} --external-database-owner-username {{.postgres_username}} --external-database-owner-password {{.postgres_password}}"
+tasks:
+  default:
+    silent: true
+    desc: install Solo, create a kind cluster, deploy the network, set it up, and start it
+    deps:
+      - task: "init"
+    cmds:
+      - echo "This command is meant to deploy a Solo network to a Kind cluster on your local machine, "
+      - echo "ctrl-c if this is not what you want to do."
+      - sleep 5
+
+  install:
+    desc: create the cluster, solo init, solo cluster create, solo node keys, solo network deploy
+    deps:
+      - task: "init"
+    cmds:
+      - task: "cluster:create"
+      - task: "solo:init"
+      - task: "solo:cluster:setup"
+      - task: "solo:keys"
+      - task: "solo:network:deploy"
+      - task: "solo:node:setup"
+      - task: "solo:node:start"
+      - task: "solo:external-database"
+      - task: "solo:mirror-node-only"
+      - name: "Copy database-seeding-query.sql inside the database pod"
+        cmd: kubectl cp {{.HOME}}/.solo/cache/database-seeding-query.sql my-postgresql-0:/tmp/database-seeding-query.sql -n database
+      - name: "Execute the database-seeding–query.sql against the database"
+        cmd: kubectl exec -it my-postgresql-0 -n database -- env PGPASSWORD={{.postgres_password}} psql -U {{.postgres_username}} -d mirror_node -f /tmp/database-seeding-query.sql
+  destroy:
+    desc: destroy relay, mirror-node, and network
+    deps:
+      - task: "init"
+    cmds:
+      - task: "cluster:destroy"

examples/solo-gke-test/Taskfile.yml

@@ -27,3 +27,5 @@ env:
   NETWORK_DEPLOY_EXTRA_FLAGS: "--haproxy-ips node1=<ip-address>,node2=<ip-address>,node3=<ip-address>,node4=<ip-address> --pvcs"
   MIRROR_NODE_DEPLOY_EXTRA_FLAGS: "--values-file {{.USER_WORKING_DIR}}/mirror-and-explorer-values.yaml"
   RELAY_NODE_DEPLOY_EXTRA_FLAGS: "--values-file {{.USER_WORKING_DIR}}/relay-values.yaml"
+
+

src/commands/flags.ts

@@ -1513,15 +1513,76 @@ export class Flags {
     },
   };
 
-  static readonly customMirrorNodeDatabaseValuePath: CommandFlag = {
-    constName: 'customMirrorNodeDatabaseValuePath',
-    name: 'custom-mirror-node-database-values-path',
+  static readonly useExternalDatabase: CommandFlag = {
+    constName: 'useExternalDatabase',
+    name: 'use-external-database',
     definition: {
-      describe: 'Path to custom mirror node database values',
+      describe:
+        'Set to true if you have an external database to use instead of the database that the Mirror Node Helm chart supplies',
+      defaultValue: false,
+      type: 'boolean',
+    },
+    prompt: undefined,
+  };
+
+  static readonly externalDatabaseHost: CommandFlag = {
+    constName: 'externalDatabaseHost',
+    name: 'external-database-host',
+    definition: {
+      describe: "Use to provide the external database host if the '--use-external-database' is passed",
       defaultValue: '',
       type: 'string',
     },
-    prompt: undefined,
+    prompt: async function promptGrpcWebTlsKeyPath(task: ListrTaskWrapper<any, any, any>, input: any) {
+      return await Flags.promptText(
+        task,
+        input,
+        Flags.externalDatabaseHost.definition.defaultValue,
+        'Enter host of the external database',
+        null,
+        Flags.externalDatabaseHost.name,
+      );
+    },
+  };
+
+  static readonly externalDatabaseOwnerUsername: CommandFlag = {
+    constName: 'externalDatabaseOwnerUsername',
+    name: 'external-database-owner-username',
+    definition: {
+      describe: "Use to provide the external database owner's username if the '--use-external-database' is passed",
+      defaultValue: '',
+      type: 'string',
+    },
+    prompt: async function promptGrpcWebTlsKeyPath(task: ListrTaskWrapper<any, any, any>, input: any) {
+      return await Flags.promptText(
+        task,
+        input,
+        Flags.externalDatabaseOwnerUsername.definition.defaultValue,
+        'Enter username of the external database owner',
+        null,
+        Flags.externalDatabaseOwnerUsername.name,
+      );
+    },
+  };
+
+  static readonly externalDatabaseOwnerPassword: CommandFlag = {
+    constName: 'externalDatabaseOwnerPassword',
+    name: 'external-database-owner-password',
+    definition: {
+      describe: "Use to provide the external database owner's password if the '--use-external-database' is passed",
+      defaultValue: '',
+      type: 'string',
+    },
+    prompt: async function promptGrpcWebTlsKeyPath(task: ListrTaskWrapper<any, any, any>, input: any) {
+      return await Flags.promptText(
+        task,
+        input,
+        Flags.externalDatabaseOwnerPassword.definition.defaultValue,
+        'Enter password of the external database owner',
+        null,
+        Flags.externalDatabaseOwnerPassword.name,
+      );
+    },
   };
 
   static readonly grpcTlsKeyPath: CommandFlag = {
@@ -1807,7 +1868,10 @@ export class Flags {
     Flags.upgradeZipFile,
     Flags.userEmailAddress,
     Flags.valuesFile,
-    Flags.customMirrorNodeDatabaseValuePath,
+    Flags.useExternalDatabase,
+    Flags.externalDatabaseHost,
+    Flags.externalDatabaseOwnerUsername,
+    Flags.externalDatabaseOwnerPassword,
   ];
 
   /** Resets the definition.disablePrompt for all flags */