From a30a18e2d87e59549a3f254678197ef7de0fc1f5 Mon Sep 17 00:00:00 2001 From: instamenta Date: Thu, 9 Jan 2025 23:06:43 +0200 Subject: [PATCH] working values file, added scripts for initializing the database and working on fixing database seeding Signed-off-by: instamenta --- .../scripts/init-001.sh | 133 ++++++++++++++++++ .../scripts/init-002.sh | 88 ++++++++++++ .../custom-mirror-node-database/values.yaml | 20 ++- src/commands/mirror_node.ts | 37 ++++- src/core/k8.ts | 21 +-- 5 files changed, 274 insertions(+), 25 deletions(-) create mode 100644 examples/custom-mirror-node-database/scripts/init-001.sh create mode 100644 examples/custom-mirror-node-database/scripts/init-002.sh diff --git a/examples/custom-mirror-node-database/scripts/init-001.sh b/examples/custom-mirror-node-database/scripts/init-001.sh new file mode 100644 index 000000000..cde089412 --- /dev/null +++ b/examples/custom-mirror-node-database/scripts/init-001.sh @@ -0,0 +1,133 @@ +cat > init1.sh << 'EOF' +#!/bin/bash +set -e + +export HEDERA_MIRROR_GRAPHQL_DB_HOST="localhost" +export HEDERA_MIRROR_GRAPHQL_DB_NAME="mirror_node" +export HEDERA_MIRROR_GRAPHQL_DB_PASSWORD="XXXXXXXXXXXX" +export HEDERA_MIRROR_GRAPHQL_DB_USERNAME="mirror_graphql" +export HEDERA_MIRROR_GRPC_DB_HOST="localhost" +export HEDERA_MIRROR_GRPC_DB_NAME="mirror_node" +export HEDERA_MIRROR_GRPC_DB_PASSWORD="XXXXXXXXXXXX" +export HEDERA_MIRROR_GRPC_DB_USERNAME="mirror_grpc" +export HEDERA_MIRROR_IMPORTER_DB_HOST="localhost" +export HEDERA_MIRROR_IMPORTER_DB_NAME="mirror_node" +export HEDERA_MIRROR_IMPORTER_DB_OWNER="mirror_node" +export HEDERA_MIRROR_IMPORTER_DB_OWNERPASSWORD="XXXXXXXXXXXX" +export HEDERA_MIRROR_IMPORTER_DB_PASSWORD="XXXXXXXXXXXX" +export HEDERA_MIRROR_IMPORTER_DB_RESTPASSWORD="XXXXXXXXXXXX" +export HEDERA_MIRROR_IMPORTER_DB_RESTUSERNAME="mirror_rest" +export HEDERA_MIRROR_IMPORTER_DB_SCHEMA="public" +export HEDERA_MIRROR_IMPORTER_DB_TEMPSCHEMA="temporary" +export HEDERA_MIRROR_IMPORTER_DB_USERNAME="mirror_importer" +export HEDERA_MIRROR_RESTJAVA_DB_HOST="localhost" +export HEDERA_MIRROR_RESTJAVA_DB_NAME="mirror_node" +export HEDERA_MIRROR_RESTJAVA_DB_PASSWORD="XXXXXXXXXXXX" +export HEDERA_MIRROR_RESTJAVA_DB_USERNAME="mirror_rest_java" +export HEDERA_MIRROR_REST_DB_HOST="localhost" +export HEDERA_MIRROR_REST_DB_NAME="mirror_node" +export HEDERA_MIRROR_REST_DB_PASSWORD="XXXXXXXXXXXX" +export HEDERA_MIRROR_REST_DB_USERNAME="mirror_rest" +export HEDERA_MIRROR_ROSETTA_DB_HOST="localhost" +export HEDERA_MIRROR_ROSETTA_DB_NAME="mirror_node" +export HEDERA_MIRROR_ROSETTA_DB_PASSWORD="XXXXXXXXXXXX" +export HEDERA_MIRROR_ROSETTA_DB_USERNAME="mirror_rosetta" +export HEDERA_MIRROR_WEB3_DB_HOST="localhost" +export HEDERA_MIRROR_WEB3_DB_NAME="mirror_node" +export HEDERA_MIRROR_WEB3_DB_PASSWORD="XXXXXXXXXXXX" +export HEDERA_MIRROR_WEB3_DB_USERNAME="mirror_web3" + +PGHBACONF="/opt/bitnami/postgresql/conf/pg_hba.conf" +if [[ -f "${PGHBACONF}" ]]; then + cp "${PGHBACONF}" "${PGHBACONF}.bak" + echo "local all all trust" > "${PGHBACONF}" + pg_ctl reload +fi + +psql -d "user=postgres connect_timeout=3" \ + --set ON_ERROR_STOP=1 \ + --set "dbName=${HEDERA_MIRROR_IMPORTER_DB_NAME}" \ + --set "dbSchema=${HEDERA_MIRROR_IMPORTER_DB_SCHEMA}" \ + --set "graphQLPassword=${HEDERA_MIRROR_GRAPHQL_DB_PASSWORD}" \ + --set "graphQLUsername=${HEDERA_MIRROR_GRAPHQL_DB_USERNAME}" \ + --set "grpcPassword=${HEDERA_MIRROR_GRPC_DB_PASSWORD}" \ + --set "grpcUsername=${HEDERA_MIRROR_GRPC_DB_USERNAME}" \ + --set "importerPassword=${HEDERA_MIRROR_IMPORTER_DB_PASSWORD}" \ + --set "importerUsername=${HEDERA_MIRROR_IMPORTER_DB_USERNAME}" \ + --set "ownerUsername=${HEDERA_MIRROR_IMPORTER_DB_OWNER}" \ + --set "ownerPassword=${HEDERA_MIRROR_IMPORTER_DB_OWNERPASSWORD}" \ + --set "restPassword=${HEDERA_MIRROR_IMPORTER_DB_RESTPASSWORD}" \ + --set "restUsername=${HEDERA_MIRROR_IMPORTER_DB_RESTUSERNAME}" \ + --set "restJavaPassword=${HEDERA_MIRROR_RESTJAVA_DB_PASSWORD}" \ + --set "restJavaUsername=${HEDERA_MIRROR_RESTJAVA_DB_USERNAME}" \ + --set "rosettaPassword=${HEDERA_MIRROR_ROSETTA_DB_PASSWORD}" \ + --set "rosettaUsername=${HEDERA_MIRROR_ROSETTA_DB_USERNAME}" \ + --set "web3Password=${HEDERA_MIRROR_WEB3_DB_PASSWORD}" \ + --set "web3Username=${HEDERA_MIRROR_WEB3_DB_USERNAME}" \ + --set "tempSchema=${HEDERA_MIRROR_IMPORTER_DB_TEMPSCHEMA}" <<__SQL__ + +-- Create database & owner +create user :ownerUsername with login password :'ownerPassword'; +create database :dbName with owner :ownerUsername; + +-- Create roles +create role readonly; +create role readwrite in role readonly; +create role temporary_admin in role readwrite; + +-- Create users +create user :graphQLUsername with login password :'graphQLPassword' in role readonly; +create user :grpcUsername with login password :'grpcPassword' in role readonly; +create user :importerUsername with login password :'importerPassword' in role readwrite admin :ownerUsername; +create user :restJavaUsername with login password :'restJavaPassword' in role readonly; +create user :rosettaUsername with login password :'rosettaPassword' in role readonly; +create user :web3Username with login password :'web3Password' in role readonly; +alter user :ownerUsername with createrole; + +-- Grant temp schema admin privileges +grant temporary_admin to :ownerUsername; +grant temporary_admin to :importerUsername; + +-- Add extensions +\connect :dbName +create extension if not exists btree_gist; +create extension if not exists pg_stat_statements; +create extension if not exists pg_trgm; + +-- Create schema +\connect :dbName :ownerUsername +create schema if not exists :dbSchema authorization :ownerUsername; +grant usage on schema :dbSchema to public; +revoke create on schema :dbSchema from public; + +-- Create temp table schema +create schema if not exists :tempSchema authorization temporary_admin; +grant usage on schema :tempSchema to public; +revoke create on schema :tempSchema from public; + +-- Grant readonly privileges +grant connect on database :dbName to readonly; +grant select on all tables in schema :dbSchema, :tempSchema to readonly; +grant select on all sequences in schema :dbSchema, :tempSchema to readonly; +grant usage on schema :dbSchema, :tempSchema to readonly; +alter default privileges in schema :dbSchema, :tempSchema grant select on tables to readonly; +alter default privileges in schema :dbSchema, :tempSchema grant select on sequences to readonly; + +-- Grant readwrite privileges +grant insert, update, delete on all tables in schema :dbSchema to readwrite; +grant usage on all sequences in schema :dbSchema to readwrite; +alter default privileges in schema :dbSchema grant insert, update, delete on tables to readwrite; +alter default privileges in schema :dbSchema grant usage on sequences to readwrite; + +-- Alter search path +\connect postgres postgres +alter database :dbName set search_path = :dbSchema, public, :tempSchema; +__SQL__ + +if [[ -f "${PGHBACONF}.bak" ]]; then + mv "${PGHBACONF}.bak" "${PGHBACONF}" + pg_ctl reload +fi +EOF +chmod +x init1.sh +./init1.sh \ No newline at end of file diff --git a/examples/custom-mirror-node-database/scripts/init-002.sh b/examples/custom-mirror-node-database/scripts/init-002.sh new file mode 100644 index 000000000..7592e7d0c --- /dev/null +++ b/examples/custom-mirror-node-database/scripts/init-002.sh @@ -0,0 +1,88 @@ +cat > init2.sh << 'EOF' +#!/bin/bash +set -e + +# Define PostgreSQL host and credentials +export PG_HOST="my-postgresql.database.svc.cluster.local" +export PG_USER="postgres" +export PG_PASSWORD="XXXXXXXXXXXX" +export PG_DB="mirror_node" + +# Execute the SQL commands +psql -h "$PG_HOST" -U "$PG_USER" -d "$PG_DB" < ctx.config.customMirrorNodeDatabaseValuePath, }, { title: 'Check REST API', @@ -442,8 +444,8 @@ export class MirrorNodeCommand extends BaseCommand { [ { title: 'Insert data in public.file_data', - task: async () => { - const namespace = self.configManager.getFlag(flags.namespace) as string; + task: async ctx => { + let namespace = self.configManager.getFlag(flags.namespace) as Namespace; const feesFileIdNum = 111; const exchangeRatesFileIdNum = 112; @@ -458,16 +460,27 @@ export class MirrorNodeCommand extends BaseCommand { }, ${exchangeRatesFileIdNum}, 17);`; const sqlQuery = [importFeesQuery, importExchangeRatesQuery].join('\n'); - const pods = await this.k8.getPodsByLabel(['app.kubernetes.io/name=postgres']); + if (ctx.config.customMirrorNodeDatabaseValuePath) { + const data = fs.readFileSync(ctx.config.customMirrorNodeDatabaseValuePath).toString(); + const mirrorNodeValues = yaml.parse(data) as Record; + if (mirrorNodeValues.db.host.split('.')?.[1]) { + namespace = mirrorNodeValues.db.host.split('.')[1]; + } + } + + const pods = await this.k8.getPodsByLabel(['app.kubernetes.io/name=postgres'], namespace); if (pods.length === 0) { throw new SoloError('postgres pod not found'); } + const postgresPodName = pods[0].metadata.name as PodName; + const postgresContainerName = 'postgresql'; const mirrorEnvVars = await self.k8.execContainer( postgresPodName, postgresContainerName, '/bin/bash -c printenv', + namespace, ); const mirrorEnvVarsArray = mirrorEnvVars.split('\n'); const HEDERA_MIRROR_IMPORTER_DB_OWNER = getEnvValue( @@ -483,11 +496,25 @@ export class MirrorNodeCommand extends BaseCommand { 'HEDERA_MIRROR_IMPORTER_DB_NAME', ); + let connectionString: string; + + if (ctx.config.customMirrorNodeDatabaseValuePath) { + const HEDERA_MIRROR_IMPORTER_DB_HOST = getEnvValue( + mirrorEnvVarsArray, + 'HEDERA_MIRROR_IMPORTER_DB_HOST', + ); + + connectionString = `postgresql://${HEDERA_MIRROR_IMPORTER_DB_OWNER}:${HEDERA_MIRROR_IMPORTER_DB_OWNERPASSWORD}@${HEDERA_MIRROR_IMPORTER_DB_HOST}:5432/${HEDERA_MIRROR_IMPORTER_DB_NAME}`; + } else { + connectionString = `postgresql://${HEDERA_MIRROR_IMPORTER_DB_OWNER}:${HEDERA_MIRROR_IMPORTER_DB_OWNERPASSWORD}@localhost:5432/${HEDERA_MIRROR_IMPORTER_DB_NAME}`; + } + await self.k8.execContainer(postgresPodName, postgresContainerName, [ 'psql', - `postgresql://${HEDERA_MIRROR_IMPORTER_DB_OWNER}:${HEDERA_MIRROR_IMPORTER_DB_OWNERPASSWORD}@localhost:5432/${HEDERA_MIRROR_IMPORTER_DB_NAME}`, + connectionString, '-c', sqlQuery, + namespace, ]); }, }, diff --git a/src/core/k8.ts b/src/core/k8.ts index d2ee3694a..badc7bdc0 100644 --- a/src/core/k8.ts +++ b/src/core/k8.ts @@ -195,12 +195,13 @@ export class K8 { /** * Get a podName by name * @param name - podName name + * @param namespace - optional namespace overrides the one provided via commandline */ - async getPodByName(name: string): Promise { - const ns = this._getNamespace(); + async getPodByName(name: string, namespace?: Namespace): Promise { + if (!namespace) namespace = this._getNamespace(); const fieldSelector = `metadata.name=${name}`; const resp = await this.kubeClient.listNamespacedPod( - ns, + namespace, undefined, undefined, undefined, @@ -219,12 +220,13 @@ export class K8 { /** * Get pods by labels * @param labels - list of labels + * @param namespace - overrides namespace provided via command line */ - async getPodsByLabel(labels: string[] = []) { - const ns = this._getNamespace(); + async getPodsByLabel(labels: string[] = [], namespace?: Namespace): Promise { + if (!namespace) namespace = this._getNamespace(); const labelSelector = labels.join(','); const result = await this.kubeClient.listNamespacedPod( - ns, + namespace, undefined, undefined, undefined, @@ -801,15 +803,16 @@ export class K8 { * @param podName * @param containerName * @param command - sh commands as an array to be run within the containerName (e.g 'ls -la /opt/hgcapp') + * @param namespace - optional namespace overrides the one provided from config manager * @returns console output as string */ - async execContainer(podName: string, containerName: string, command: string | string[]) { + async execContainer(podName: string, containerName: string, command: string | string[], namespace?: Namespace) { const self = this; - const namespace = self._getNamespace(); + if (!namespace) namespace = self._getNamespace(); const guid = uuid4(); const messagePrefix = `execContainer[${podName},${guid}]:`; - if (!(await self.getPodByName(podName))) throw new IllegalArgumentError(`Invalid pod ${podName}`); + if (!(await self.getPodByName(podName, namespace))) throw new IllegalArgumentError(`Invalid pod ${podName}`); if (!command) throw new MissingArgumentError('command cannot be empty'); if (!Array.isArray(command)) {