feat(mirror-node): Enhance mirror-node external database feature

Taskfile.helper.yml

@@ -464,3 +464,32 @@ tasks:
     cmds:
       - echo "Cleaning up temporary files..."
       - rm -f /tmp/solo-${USER}-* || true
+
+  solo:external-database:
+    silent: false
+    desc: setup external database PostgreSQL with helm
+    cmds:
+      - |
+        {{.solo_bin_dir}}/helm install {{.postgres_name}} bitnami/postgresql \
+          --set image.tag=16.4.0 \
+          --namespace {{.postgres_database_namespace}} --create-namespace \
+          --set global.postgresql.auth.postgresPassword={{.postgres_password}} \
+          --set primary.persistence.enabled=false --set secondary.enabled=false
+      - name: "Wait for PostgreSQL pod to be ready"
+        cmd: |
+          kubectl wait --for=condition=ready pod/{{.postgres_container_name}} \
+          -n {{.postgres_database_namespace}} --timeout=60s
+      - name: "Copy init.sql inside the database pod"
+        cmd: |
+          kubectl cp ../external-database-test/scripts/init.sh \
+          {{.postgres_container_name}}:/tmp/init.sh \
+          -n {{.postgres_database_namespace}}
+      - name: "Make init.sh executable"
+        cmd: |
+          kubectl exec -it {{.postgres_container_name}} \
+          -n {{.postgres_database_namespace}} -- chmod +x /tmp/init.sh
+      - name: "Execute init.sh inside the database pod"
+        cmd: |
+          kubectl exec -it {{.postgres_container_name}} \
+          -n {{.postgres_database_namespace}} \
+          -- /bin/bash /tmp/init.sh "{{.postgres_username}}" "{{.postgres_password}}"

examples/external-database-test/Taskfile.yml

@@ -0,0 +1,61 @@
+version: 3
+includes:
+  main:
+    taskfile: ../../Taskfile.helper.yml
+    flatten: true
+vars:
+  use_port_forwards: "true"
+  postgres_username: "postgres"
+  postgres_password: "XXXXXXXXX"
+  postgres_name: "my-postgresql"
+  postgres_container_name: "{{.postgres_name}}-0"
+  postgres_host_fqdn: "{{.postgres_name}}.database.svc.cluster.local"
+  postgres_container_fdqn: "{{.postgres_container_name}}.database.svc.cluster.local"
+  postgres_mirror_node_database_name: "mirror_node"
+  postgres_database_namespace: "database"
+env:
+  SOLO_NETWORK_SIZE: "1"
+  SOLO_NAMESPACE: "external-database-test"
+  MIRROR_NODE_DEPLOY_EXTRA_FLAGS: "--use-external-database --external-database-host {{.postgres_host_fqdn}} --external-database-owner-username {{.postgres_username}} --external-database-owner-password {{.postgres_password}}"
+tasks:
+  default:
+    silent: true
+    desc: install Solo, create a kind cluster, deploy the network, set it up, and start it
+    deps:
+      - task: "init"
+    cmds:
+      - echo "This command is meant to deploy a Solo network to a Kind cluster on your local machine, "
+      - echo "ctrl-c if this is not what you want to do."
+      - sleep 5
+
+  install:
+    desc: create the cluster, solo init, solo cluster create, solo node keys, solo network deploy
+    deps:
+      - task: "init"
+    cmds:
+      - task: "cluster:create"
+      - task: "solo:init"
+      - task: "solo:cluster:setup"
+      - task: "solo:keys"
+      - task: "solo:network:deploy"
+      - task: "solo:node:setup"
+      - task: "solo:node:start"
+      - task: "solo:external-database"
+      - task: "solo:mirror-node"
+      - name: "Copy database-seeding-query.sql inside the database pod"
+        cmd: |
+          kubectl cp {{.HOME}}/.solo/cache/database-seeding-query.sql {{.postgres_container_name}}:/tmp/database-seeding-query.sql \
+          -n {{.postgres_database_namespace}}
+      - name: "Execute the database-seeding–query.sql against the database"
+        cmd: |
+          kubectl exec -it {{.postgres_container_name}} -n {{.postgres_database_namespace}} -- env PGPASSWORD={{.postgres_password}} psql -U {{.postgres_username}} \
+            -f /tmp/database-seeding-query.sql \
+            -d {{.postgres_mirror_node_database_name}}
+      - name: "Run smoke test"
+        cmd: (cd ../../ && ./examples/external-database-test/scripts/solo_smoke_test.sh)
+  destroy:
+    desc: destroy relay, mirror-node, and network
+    deps:
+      - task: "init"
+    cmds:
+      - task: "cluster:destroy"

examples/external-database-test/scripts/init.sh

@@ -0,0 +1,84 @@
+#!/bin/bash
+set -e
+
+export HEDERA_MIRROR_DATABASE_NAME="mirror_node"
+HEDERA_MIRROR_OWNER="$1"
+HEDERA_MIRROR_OWNER_PASSWORD="$2"
+
+export HEDERA_MIRROR_GRPC_DB_HOST="localhost"
+
+export HEDERA_MIRROR_IMPORTER_DB_HOST="localhost"
+export HEDERA_MIRROR_IMPORTER_DB_NAME="${HEDERA_MIRROR_DATABASE_NAME}"
+export HEDERA_MIRROR_IMPORTER_DB_OWNER="${HEDERA_MIRROR_OWNER}"
+export HEDERA_MIRROR_IMPORTER_DB_SCHEMA="public"
+export HEDERA_MIRROR_IMPORTER_DB_TEMPSCHEMA="temporary"
+
+
+PGHBACONF="/opt/bitnami/postgresql/conf/pg_hba.conf"
+if [[ -f "${PGHBACONF}" ]]; then
+  cp "${PGHBACONF}" "${PGHBACONF}.bak"
+  echo "local all all trust" > "${PGHBACONF}"
+  pg_ctl reload
+fi
+
+psql -d "user=postgres connect_timeout=3" \
+  --set ON_ERROR_STOP=1 \
+  --set "dbName=${HEDERA_MIRROR_IMPORTER_DB_NAME}" \
+  --set "dbSchema=${HEDERA_MIRROR_IMPORTER_DB_SCHEMA}" \
+  --set "ownerUsername=${HEDERA_MIRROR_IMPORTER_DB_OWNER}" \
+  --set "tempSchema=${HEDERA_MIRROR_IMPORTER_DB_TEMPSCHEMA}" <<__SQL__
+
+-- Create database & owner
+create database :dbName with owner :ownerUsername;
+
+-- Create roles
+create role readonly;
+create role readwrite in role readonly;
+create role temporary_admin in role readwrite;
+
+-- Create users
+alter user :ownerUsername with createrole;
+
+-- Grant temp schema admin privileges
+grant temporary_admin to :ownerUsername;
+
+-- Add extensions
+\connect :dbName
+create extension if not exists btree_gist;
+create extension if not exists pg_stat_statements;
+create extension if not exists pg_trgm;
+
+-- Create schema
+\connect :dbName :ownerUsername
+create schema if not exists :dbSchema authorization :ownerUsername;
+grant usage on schema :dbSchema to public;
+revoke create on schema :dbSchema from public;
+
+-- Create temp table schema
+create schema if not exists :tempSchema authorization temporary_admin;
+grant usage on schema :tempSchema to public;
+revoke create on schema :tempSchema from public;
+
+-- Grant readonly privileges
+grant connect on database :dbName to readonly;
+grant select on all tables in schema :dbSchema, :tempSchema to readonly;
+grant select on all sequences in schema :dbSchema, :tempSchema to readonly;
+grant usage on schema :dbSchema, :tempSchema to readonly;
+alter default privileges in schema :dbSchema, :tempSchema grant select on tables to readonly;
+alter default privileges in schema :dbSchema, :tempSchema grant select on sequences to readonly;
+
+-- Grant readwrite privileges
+grant insert, update, delete on all tables in schema :dbSchema to readwrite;
+grant usage on all sequences in schema :dbSchema to readwrite;
+alter default privileges in schema :dbSchema grant insert, update, delete on tables to readwrite;
+alter default privileges in schema :dbSchema grant usage on sequences to readwrite;
+
+-- Alter search path
+\connect postgres postgres
+alter database :dbName set search_path = :dbSchema, public, :tempSchema;
+__SQL__
+
+if [[ -f "${PGHBACONF}.bak" ]]; then
+  mv "${PGHBACONF}.bak" "${PGHBACONF}"
+  pg_ctl reload
+fi