working values file, added scripts for initializing the database and …

…working on fixing database seeding

Signed-off-by: instamenta <[email protected]>

examples/custom-mirror-node-database/scripts/init-001.sh

@@ -0,0 +1,133 @@
+cat > init1.sh << 'EOF'
+#!/bin/bash
+set -e
+
+export HEDERA_MIRROR_GRAPHQL_DB_HOST="localhost"
+export HEDERA_MIRROR_GRAPHQL_DB_NAME="mirror_node"
+export HEDERA_MIRROR_GRAPHQL_DB_PASSWORD="XXXXXXXXXXXX"
+export HEDERA_MIRROR_GRAPHQL_DB_USERNAME="mirror_graphql"
+export HEDERA_MIRROR_GRPC_DB_HOST="localhost"
+export HEDERA_MIRROR_GRPC_DB_NAME="mirror_node"
+export HEDERA_MIRROR_GRPC_DB_PASSWORD="XXXXXXXXXXXX"
+export HEDERA_MIRROR_GRPC_DB_USERNAME="mirror_grpc"
+export HEDERA_MIRROR_IMPORTER_DB_HOST="localhost"
+export HEDERA_MIRROR_IMPORTER_DB_NAME="mirror_node"
+export HEDERA_MIRROR_IMPORTER_DB_OWNER="mirror_node"
+export HEDERA_MIRROR_IMPORTER_DB_OWNERPASSWORD="XXXXXXXXXXXX"
+export HEDERA_MIRROR_IMPORTER_DB_PASSWORD="XXXXXXXXXXXX"
+export HEDERA_MIRROR_IMPORTER_DB_RESTPASSWORD="XXXXXXXXXXXX"
+export HEDERA_MIRROR_IMPORTER_DB_RESTUSERNAME="mirror_rest"
+export HEDERA_MIRROR_IMPORTER_DB_SCHEMA="public"
+export HEDERA_MIRROR_IMPORTER_DB_TEMPSCHEMA="temporary"
+export HEDERA_MIRROR_IMPORTER_DB_USERNAME="mirror_importer"
+export HEDERA_MIRROR_RESTJAVA_DB_HOST="localhost"
+export HEDERA_MIRROR_RESTJAVA_DB_NAME="mirror_node"
+export HEDERA_MIRROR_RESTJAVA_DB_PASSWORD="XXXXXXXXXXXX"
+export HEDERA_MIRROR_RESTJAVA_DB_USERNAME="mirror_rest_java"
+export HEDERA_MIRROR_REST_DB_HOST="localhost"
+export HEDERA_MIRROR_REST_DB_NAME="mirror_node"
+export HEDERA_MIRROR_REST_DB_PASSWORD="XXXXXXXXXXXX"
+export HEDERA_MIRROR_REST_DB_USERNAME="mirror_rest"
+export HEDERA_MIRROR_ROSETTA_DB_HOST="localhost"
+export HEDERA_MIRROR_ROSETTA_DB_NAME="mirror_node"
+export HEDERA_MIRROR_ROSETTA_DB_PASSWORD="XXXXXXXXXXXX"
+export HEDERA_MIRROR_ROSETTA_DB_USERNAME="mirror_rosetta"
+export HEDERA_MIRROR_WEB3_DB_HOST="localhost"
+export HEDERA_MIRROR_WEB3_DB_NAME="mirror_node"
+export HEDERA_MIRROR_WEB3_DB_PASSWORD="XXXXXXXXXXXX"
+export HEDERA_MIRROR_WEB3_DB_USERNAME="mirror_web3"
+
+PGHBACONF="/opt/bitnami/postgresql/conf/pg_hba.conf"
+if [[ -f "${PGHBACONF}" ]]; then
+  cp "${PGHBACONF}" "${PGHBACONF}.bak"
+  echo "local all all trust" > "${PGHBACONF}"
+  pg_ctl reload
+fi
+
+psql -d "user=postgres connect_timeout=3" \
+  --set ON_ERROR_STOP=1 \
+  --set "dbName=${HEDERA_MIRROR_IMPORTER_DB_NAME}" \
+  --set "dbSchema=${HEDERA_MIRROR_IMPORTER_DB_SCHEMA}" \
+  --set "graphQLPassword=${HEDERA_MIRROR_GRAPHQL_DB_PASSWORD}" \
+  --set "graphQLUsername=${HEDERA_MIRROR_GRAPHQL_DB_USERNAME}" \
+  --set "grpcPassword=${HEDERA_MIRROR_GRPC_DB_PASSWORD}" \
+  --set "grpcUsername=${HEDERA_MIRROR_GRPC_DB_USERNAME}" \
+  --set "importerPassword=${HEDERA_MIRROR_IMPORTER_DB_PASSWORD}" \
+  --set "importerUsername=${HEDERA_MIRROR_IMPORTER_DB_USERNAME}" \
+  --set "ownerUsername=${HEDERA_MIRROR_IMPORTER_DB_OWNER}" \
+  --set "ownerPassword=${HEDERA_MIRROR_IMPORTER_DB_OWNERPASSWORD}" \
+  --set "restPassword=${HEDERA_MIRROR_IMPORTER_DB_RESTPASSWORD}" \
+  --set "restUsername=${HEDERA_MIRROR_IMPORTER_DB_RESTUSERNAME}" \
+  --set "restJavaPassword=${HEDERA_MIRROR_RESTJAVA_DB_PASSWORD}" \
+  --set "restJavaUsername=${HEDERA_MIRROR_RESTJAVA_DB_USERNAME}" \
+  --set "rosettaPassword=${HEDERA_MIRROR_ROSETTA_DB_PASSWORD}" \
+  --set "rosettaUsername=${HEDERA_MIRROR_ROSETTA_DB_USERNAME}" \
+  --set "web3Password=${HEDERA_MIRROR_WEB3_DB_PASSWORD}" \
+  --set "web3Username=${HEDERA_MIRROR_WEB3_DB_USERNAME}" \
+  --set "tempSchema=${HEDERA_MIRROR_IMPORTER_DB_TEMPSCHEMA}" <<__SQL__
+
+-- Create database & owner
+create user :ownerUsername with login password :'ownerPassword';
+create database :dbName with owner :ownerUsername;
+
+-- Create roles
+create role readonly;
+create role readwrite in role readonly;
+create role temporary_admin in role readwrite;
+
+-- Create users
+create user :graphQLUsername with login password :'graphQLPassword' in role readonly;
+create user :grpcUsername with login password :'grpcPassword' in role readonly;
+create user :importerUsername with login password :'importerPassword' in role readwrite admin :ownerUsername;
+create user :restJavaUsername with login password :'restJavaPassword' in role readonly;
+create user :rosettaUsername with login password :'rosettaPassword' in role readonly;
+create user :web3Username with login password :'web3Password' in role readonly;
+alter user :ownerUsername with createrole;
+
+-- Grant temp schema admin privileges
+grant temporary_admin to :ownerUsername;
+grant temporary_admin to :importerUsername;
+
+-- Add extensions
+\connect :dbName
+create extension if not exists btree_gist;
+create extension if not exists pg_stat_statements;
+create extension if not exists pg_trgm;
+
+-- Create schema
+\connect :dbName :ownerUsername
+create schema if not exists :dbSchema authorization :ownerUsername;
+grant usage on schema :dbSchema to public;
+revoke create on schema :dbSchema from public;
+
+-- Create temp table schema
+create schema if not exists :tempSchema authorization temporary_admin;
+grant usage on schema :tempSchema to public;
+revoke create on schema :tempSchema from public;
+
+-- Grant readonly privileges
+grant connect on database :dbName to readonly;
+grant select on all tables in schema :dbSchema, :tempSchema to readonly;
+grant select on all sequences in schema :dbSchema, :tempSchema to readonly;
+grant usage on schema :dbSchema, :tempSchema to readonly;
+alter default privileges in schema :dbSchema, :tempSchema grant select on tables to readonly;
+alter default privileges in schema :dbSchema, :tempSchema grant select on sequences to readonly;
+
+-- Grant readwrite privileges
+grant insert, update, delete on all tables in schema :dbSchema to readwrite;
+grant usage on all sequences in schema :dbSchema to readwrite;
+alter default privileges in schema :dbSchema grant insert, update, delete on tables to readwrite;
+alter default privileges in schema :dbSchema grant usage on sequences to readwrite;
+
+-- Alter search path
+\connect postgres postgres
+alter database :dbName set search_path = :dbSchema, public, :tempSchema;
+__SQL__
+
+if [[ -f "${PGHBACONF}.bak" ]]; then
+  mv "${PGHBACONF}.bak" "${PGHBACONF}"
+  pg_ctl reload
+fi
+EOF
+chmod +x init1.sh
+./init1.sh

examples/custom-mirror-node-database/scripts/init-002.sh

@@ -0,0 +1,88 @@
+cat > init2.sh << 'EOF'
+#!/bin/bash
+set -e
+
+# Define PostgreSQL host and credentials
+export PG_HOST="my-postgresql.database.svc.cluster.local"
+export PG_USER="postgres"
+export PG_PASSWORD="XXXXXXXXXXXX"
+export PG_DB="mirror_node"
+
+# Execute the SQL commands
+psql -h "$PG_HOST" -U "$PG_USER" -d "$PG_DB" <<EOF
+DO
+$$
+BEGIN
+  CREATE OR REPLACE FUNCTION if_role_not_exists(rolename name, sql text) RETURNS text AS
+  $$
+  BEGIN
+    IF NOT EXISTS (SELECT * FROM pg_roles WHERE rolname = rolename) THEN
+      EXECUTE sql;
+      RETURN format('Executed ''%s''', sql);
+    ELSE
+      RETURN format('Role ''%s'' already exists', rolename);
+    END IF;
+  END;
+  $$
+  LANGUAGE plpgsql;
+
+  -- Create owner
+  PERFORM if_role_not_exists('postgres', 'CREATE USER postgres WITH CREATEROLE LOGIN PASSWORD ''XXXXXXXXXXXX''');
+  ALTER DATABASE mirror_node SET search_path = public, public, temporary;
+  ALTER DATABASE mirror_node OWNER TO postgres;
+
+  -- Create roles
+  PERFORM if_role_not_exists('readonly', 'CREATE ROLE readonly');
+  PERFORM if_role_not_exists('readwrite', 'CREATE ROLE readwrite IN ROLE readonly');
+  PERFORM if_role_not_exists('temporary_admin', 'CREATE ROLE temporary_admin IN ROLE readwrite');
+
+  -- Create users
+  PERFORM if_role_not_exists('mirror_graphql', 'CREATE USER mirror_graphql WITH LOGIN PASSWORD ''XXXXXXXXXXXX'' IN ROLE readonly');
+  PERFORM if_role_not_exists('mirror_grpc', 'CREATE USER mirror_grpc WITH LOGIN PASSWORD ''XXXXXXXXXXXX'' IN ROLE readonly');
+  PERFORM if_role_not_exists('mirror_importer', 'CREATE USER mirror_importer WITH LOGIN PASSWORD ''XXXXXXXXXXXX'' IN ROLE readwrite');
+  PERFORM if_role_not_exists('mirror_rest', 'CREATE USER mirror_rest WITH LOGIN PASSWORD ''XXXXXXXXXXXX'' IN ROLE readonly');
+  PERFORM if_role_not_exists('mirror_rest_java', 'CREATE USER mirror_rest_java WITH LOGIN PASSWORD ''XXXXXXXXXXXX'' IN ROLE readonly');
+  PERFORM if_role_not_exists('mirror_rosetta', 'CREATE USER mirror_rosetta WITH LOGIN PASSWORD ''XXXXXXXXXXXX'' IN ROLE readonly');
+  PERFORM if_role_not_exists('mirror_web3', 'CREATE USER mirror_web3 WITH LOGIN PASSWORD ''XXXXXXXXXXXX'' IN ROLE readonly');
+
+  -- Set statement timeouts
+  ALTER USER mirror_graphql SET statement_timeout TO '10000';
+  ALTER USER mirror_grpc SET statement_timeout TO '10000';
+  ALTER USER mirror_rest SET statement_timeout TO '20000';
+  ALTER USER mirror_rest_java SET statement_timeout TO '20000';
+  ALTER USER mirror_rosetta SET statement_timeout TO '10000';
+  ALTER USER mirror_web3 SET statement_timeout TO '10000';
+
+  -- Grants
+  GRANT temporary_admin TO postgres;
+  GRANT temporary_admin TO mirror_importer;
+
+  -- Create schema and permissions
+  CREATE SCHEMA IF NOT EXISTS public AUTHORIZATION postgres;
+  GRANT USAGE ON SCHEMA public TO public;
+  REVOKE CREATE ON SCHEMA public FROM public;
+
+  CREATE SCHEMA IF NOT EXISTS temporary AUTHORIZATION temporary_admin;
+  GRANT USAGE ON SCHEMA temporary TO public;
+  REVOKE CREATE ON SCHEMA temporary FROM public;
+
+  -- Read-only privileges
+  GRANT CONNECT ON DATABASE mirror_node TO readonly;
+  GRANT SELECT ON ALL TABLES IN SCHEMA public, temporary TO readonly;
+  GRANT USAGE ON SCHEMA public, temporary TO readonly;
+  ALTER DEFAULT PRIVILEGES IN SCHEMA public, temporary GRANT SELECT ON TABLES TO readonly;
+
+  -- Read-write privileges
+  GRANT INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO readwrite;
+  ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT INSERT, UPDATE, DELETE ON TABLES TO readwrite;
+
+  -- Extensions
+  CREATE EXTENSION IF NOT EXISTS btree_gist;
+  CREATE EXTENSION IF NOT EXISTS pg_stat_statements;
+  CREATE EXTENSION IF NOT EXISTS pg_trgm;
+  ALTER TYPE timestamptz OWNER TO postgres;
+
+END;
+$$;
+EOF
+./init1.sh

examples/custom-mirror-node-database/values.yaml

@@ -1,29 +1,27 @@
 db:
-  host: "my-postgresql.database.svc.cluster.local"
+  host: "my-postgresql.mirror-db.svc.cluster.local"
   enabled: true
   owner:
     username: postgres
     password: "XXXXXXXXXXXX"
+
 stackgres:
   enabled: false
+
 postgresql:
-  enabled: true
-  pgpool:
-    adminPassword: "XXXXXXXXXXXX"
-#  postgresql:
-#    initdbScriptsCM: "{{ .Release.Name }}-init"
+  enabled: false
 
-fullnameOverride: "my-postgresql.database.svc.cluster.local"
+#fullnameOverride: ""
 
-graphql:
+importer:
   enabled: true
   db:
     password: "XXXXXXXXXXXX"
-grpc:
+graphql:
   enabled: true
   db:
     password: "XXXXXXXXXXXX"
-importer:
+grpc:
   enabled: true
   db:
     password: "XXXXXXXXXXXX"
@@ -42,4 +40,4 @@ web3:
 rosetta:
   enabled: false
   db:
-    password: "" # for disabled components, the password can be omitted
+    password: "XXXXXXXXXXXX" # for disabled components, the password can be omitted

src/commands/mirror_node.ts

@@ -24,13 +24,14 @@ import {BaseCommand} from './base.js';
 import {Flags as flags} from './flags.js';
 import {getEnvValue} from '../core/helpers.js';
 import {RemoteConfigTasks} from '../core/config/remote/remote_config_tasks.js';
-import {type CommandBuilder, type PodName} from '../types/aliases.js';
+import { type AnyObject, type CommandBuilder, type PodName } from '../types/aliases.js'
 import {type Opts} from '../types/command_types.js';
 import {ListrLease} from '../core/lease/listr_lease.js';
 import {ComponentType} from '../core/config/remote/enumerations.js';
 import {MirrorNodeComponent} from '../core/config/remote/components/mirror_node_component.js';
 import * as fs from 'node:fs';
 import * as path from 'node:path';
+import * as yaml from 'yaml';
 import type {Optional, SoloListrTask} from '../types/index.js';
 import type {Namespace} from '../core/config/remote/types.js';
 
@@ -375,6 +376,7 @@ export class MirrorNodeCommand extends BaseCommand {
                       constants.PODS_READY_MAX_ATTEMPTS,
                       constants.PODS_READY_DELAY,
                     ),
+                  skip: ctx => ctx.config.customMirrorNodeDatabaseValuePath,
                 },
                 {
                   title: 'Check REST API',
@@ -442,8 +444,8 @@ export class MirrorNodeCommand extends BaseCommand {
               [
                 {
                   title: 'Insert data in public.file_data',
-                  task: async () => {
-                    const namespace = self.configManager.getFlag<string>(flags.namespace) as string;
+                  task: async ctx => {
+                    let namespace = self.configManager.getFlag<Namespace>(flags.namespace) as Namespace;
 
                     const feesFileIdNum = 111;
                     const exchangeRatesFileIdNum = 112;
@@ -458,16 +460,27 @@ export class MirrorNodeCommand extends BaseCommand {
                     }, ${exchangeRatesFileIdNum}, 17);`;
                     const sqlQuery = [importFeesQuery, importExchangeRatesQuery].join('\n');
 
-                    const pods = await this.k8.getPodsByLabel(['app.kubernetes.io/name=postgres']);
+                    if (ctx.config.customMirrorNodeDatabaseValuePath) {
+                      const data = fs.readFileSync(ctx.config.customMirrorNodeDatabaseValuePath).toString();
+                      const mirrorNodeValues = yaml.parse(data) as Record<string, AnyObject>;
+                      if (mirrorNodeValues.db.host.split('.')?.[1]) {
+                        namespace = mirrorNodeValues.db.host.split('.')[1];
+                      }
+                    }
+
+                    const pods = await this.k8.getPodsByLabel(['app.kubernetes.io/name=postgres'], namespace);
                     if (pods.length === 0) {
                       throw new SoloError('postgres pod not found');
                     }
+
                     const postgresPodName = pods[0].metadata.name as PodName;
+
                     const postgresContainerName = 'postgresql';
                     const mirrorEnvVars = await self.k8.execContainer(
                       postgresPodName,
                       postgresContainerName,
                       '/bin/bash -c printenv',
+                      namespace,
                     );
                     const mirrorEnvVarsArray = mirrorEnvVars.split('\n');
                     const HEDERA_MIRROR_IMPORTER_DB_OWNER = getEnvValue(
@@ -483,11 +496,25 @@ export class MirrorNodeCommand extends BaseCommand {
                       'HEDERA_MIRROR_IMPORTER_DB_NAME',
                     );
 
+                    let connectionString: string;
+
+                    if (ctx.config.customMirrorNodeDatabaseValuePath) {
+                      const HEDERA_MIRROR_IMPORTER_DB_HOST = getEnvValue(
+                        mirrorEnvVarsArray,
+                        'HEDERA_MIRROR_IMPORTER_DB_HOST',
+                      );
+
+                      connectionString = `postgresql://${HEDERA_MIRROR_IMPORTER_DB_OWNER}:${HEDERA_MIRROR_IMPORTER_DB_OWNERPASSWORD}@${HEDERA_MIRROR_IMPORTER_DB_HOST}:5432/${HEDERA_MIRROR_IMPORTER_DB_NAME}`;
+                    } else {
+                      connectionString = `postgresql://${HEDERA_MIRROR_IMPORTER_DB_OWNER}:${HEDERA_MIRROR_IMPORTER_DB_OWNERPASSWORD}@localhost:5432/${HEDERA_MIRROR_IMPORTER_DB_NAME}`;
+                    }
+
                     await self.k8.execContainer(postgresPodName, postgresContainerName, [
                       'psql',
-                      `postgresql://${HEDERA_MIRROR_IMPORTER_DB_OWNER}:${HEDERA_MIRROR_IMPORTER_DB_OWNERPASSWORD}@localhost:5432/${HEDERA_MIRROR_IMPORTER_DB_NAME}`,
+                      connectionString,
                       '-c',
                       sqlQuery,
+                      namespace,
                     ]);
                   },
                 },