From 4984a6664f92979a627f4810fc2d87fb2084afd0 Mon Sep 17 00:00:00 2001 From: instamenta Date: Wed, 29 Jan 2025 11:35:08 +0200 Subject: [PATCH 01/12] added new flags, and placed them inside the mirror_node command to be used, together with a helper for them Signed-off-by: instamenta --- src/commands/flags.ts | 47 ++++++++++++++-- src/commands/mirror_node.ts | 104 +++++++++++++++++++++++++++--------- src/core/helpers.ts | 10 ++++ 3 files changed, 131 insertions(+), 30 deletions(-) diff --git a/src/commands/flags.ts b/src/commands/flags.ts index d218a6075..436d876d3 100644 --- a/src/commands/flags.ts +++ b/src/commands/flags.ts @@ -1515,11 +1515,45 @@ export class Flags { }, }; - static readonly customMirrorNodeDatabaseValuePath: CommandFlag = { - constName: 'customMirrorNodeDatabaseValuePath', - name: 'custom-mirror-node-database-values-path', + static readonly useExternalDatabase: CommandFlag = { + constName: 'useExternalDatabase', + name: 'use-external-database', definition: { - describe: 'Path to custom mirror node database values', + describe: + 'Set to true if you have an external database to use instead of the database that the Mirror Node Helm chart supplies', + defaultValue: false, + type: 'boolean', + }, + prompt: undefined, + }; + + static readonly externalDatabaseHost: CommandFlag = { + constName: 'externalDatabaseHost', + name: 'external-database-host', + definition: { + describe: '', + defaultValue: '', + type: 'string', + }, + prompt: undefined, + }; + + static readonly externalDatabaseOwnerUsername: CommandFlag = { + constName: 'externalDatabaseOwnerUsername', + name: 'external-database-owner-username', + definition: { + describe: '', + defaultValue: '', + type: 'string', + }, + prompt: undefined, + }; + + static readonly externalDatabaseOwnerPassword: CommandFlag = { + constName: 'externalDatabaseOwnerPassword', + name: 'external-database-owner-password', + definition: { + describe: '', defaultValue: '', type: 'string', }, @@ -1807,7 +1841,10 @@ export class Flags { Flags.upgradeZipFile, Flags.userEmailAddress, Flags.valuesFile, - Flags.customMirrorNodeDatabaseValuePath, + Flags.useExternalDatabase, + Flags.externalDatabaseHost, + Flags.externalDatabaseOwnerUsername, + Flags.externalDatabaseOwnerPassword, ]; /** Resets the definition.disablePrompt for all flags */ diff --git a/src/commands/mirror_node.ts b/src/commands/mirror_node.ts index 2fe6f67f8..b12dffe5c 100644 --- a/src/commands/mirror_node.ts +++ b/src/commands/mirror_node.ts @@ -22,7 +22,7 @@ import type {AccountManager} from '../core/account_manager.js'; import type {ProfileManager} from '../core/profile_manager.js'; import {BaseCommand} from './base.js'; import {Flags as flags} from './flags.js'; -import {getEnvValue} from '../core/helpers.js'; +import * as helpers from '../core/helpers.js'; import type {CommandBuilder, PodName} from '../types/aliases.js'; import type {Opts} from '../types/command_types.js'; import {ListrLease} from '../core/lease/listr_lease.js'; @@ -46,13 +46,16 @@ interface MirrorNodeDeployConfigClass { pinger: boolean; operatorId: string; operatorKey: string; - customMirrorNodeDatabaseValuePath: Optional; + useExternalDatabase: boolean; storageType: constants.StorageType; storageAccessKey: string; storageSecrets: string; storageEndpoint: string; storageBucket: string; storageBucketPrefix: string; + externalDatabaseHost: Optional; + externalDatabaseOwnerUsername: Optional; + externalDatabaseOwnerPassword: Optional; } interface Context { @@ -89,7 +92,7 @@ export class MirrorNodeCommand extends BaseCommand { flags.valuesFile, flags.mirrorNodeVersion, flags.pinger, - flags.customMirrorNodeDatabaseValuePath, + flags.useExternalDatabase, flags.operatorId, flags.operatorKey, flags.storageType, @@ -98,6 +101,9 @@ export class MirrorNodeCommand extends BaseCommand { flags.storageEndpoint, flags.storageBucket, flags.storageBucketPrefix, + flags.externalDatabaseHost, + flags.externalDatabaseOwnerUsername, + flags.externalDatabaseOwnerPassword, ]; } @@ -140,6 +146,42 @@ export class MirrorNodeCommand extends BaseCommand { valuesArg += ` --set importer.env.HEDERA_MIRROR_IMPORTER_DOWNLOADER_SOURCES_0_CREDENTIALS_ACCESSKEY=${config.storageAccessKey}`; valuesArg += ` --set importer.env.HEDERA_MIRROR_IMPORTER_DOWNLOADER_SOURCES_0_CREDENTIALS_SECRETKEY=${config.storageSecrets}`; } + + // if the useExternalDatabase populate all the required values before installing the chart + if (config.useExternalDatabase) { + const { + externalDatabaseHost: host, + externalDatabaseOwnerUsername: username, + externalDatabaseOwnerPassword: password, + } = config; + + config.valuesArg += helpers.populateHelmArgs({ + // Disable default database deployment + 'stackgres.enabled': false, + 'postgresql.enabled': false, + + // Set the host and name + 'db.host': host, + 'db.name': 'mirror_node', + + // set the usernames + 'db.owner.username': username, + 'importer.db.username': username, + 'grpc.db.username': username, + 'rest.db.username': username, + 'restjava.db.username': username, + 'web3.db.username': username, + + // set the passwords + 'db.owner.password': password, + 'importer.db.password': password, + 'grpc.db.password': password, + 'rest.db.password': password, + 'restjava.db.password': password, + 'web3.db.password': password, + }); + } + return valuesArg; } @@ -165,6 +207,15 @@ export class MirrorNodeCommand extends BaseCommand { await self.configManager.executePrompt(task, MirrorNodeCommand.DEPLOY_FLAGS_LIST); + // In case the useExternalDatabase is set, prompt for the rest of the required data + if (self.configManager.getFlag(flags.useExternalDatabase)) { + await self.configManager.executePrompt(task, [ + flags.externalDatabaseHost, + flags.externalDatabaseOwnerUsername, + flags.externalDatabaseOwnerPassword, + ]); + } + ctx.config = this.getConfig(MirrorNodeCommand.DEPLOY_CONFIGS_NAME, MirrorNodeCommand.DEPLOY_FLAGS_LIST, [ 'chartPath', 'valuesArg', @@ -210,7 +261,7 @@ export class MirrorNodeCommand extends BaseCommand { const operatorKeyFromK8 = Base64.decode(secrets[0].data.privateKey); ctx.config.valuesArg += ` --set monitor.config.hedera.mirror.monitor.operator.privateKey=${operatorKeyFromK8}`; } - } catch (e: Error | any) { + } catch (e) { throw new SoloError(`Error getting operator key: ${e.message}`, e); } } @@ -239,20 +290,6 @@ export class MirrorNodeCommand extends BaseCommand { { title: 'Deploy mirror-node', task: async ctx => { - if (ctx.config.customMirrorNodeDatabaseValuePath) { - if (!fs.existsSync(ctx.config.customMirrorNodeDatabaseValuePath)) { - throw new SoloError('Path provided for custom mirror node database value is not found'); - } - - // Check if the file has a .yaml or .yml extension - const fileExtension = path.extname(ctx.config.customMirrorNodeDatabaseValuePath); - if (fileExtension !== '.yaml' && fileExtension !== '.yml') { - throw new SoloError('The provided file is not a valid YAML file (.yaml or .yml)'); - } - - ctx.config.valuesArg += ` --values ${ctx.config.customMirrorNodeDatabaseValuePath}`; - } - await self.chartManager.install( ctx.config.namespace, constants.MIRROR_NODE_RELEASE_NAME, @@ -284,7 +321,7 @@ export class MirrorNodeCommand extends BaseCommand { constants.PODS_READY_MAX_ATTEMPTS, constants.PODS_READY_DELAY, ), - skip: ctx => !!ctx.config.customMirrorNodeDatabaseValuePath, + skip: ctx => !!ctx.config.useExternalDatabase, }, { title: 'Check REST API', @@ -357,9 +394,26 @@ export class MirrorNodeCommand extends BaseCommand { }, ${exchangeRatesFileIdNum}, 17);`; const sqlQuery = [importFeesQuery, importExchangeRatesQuery].join('\n'); - if (ctx.config.customMirrorNodeDatabaseValuePath) { - fs.writeFileSync(path.join(constants.SOLO_CACHE_DIR, 'database-seeding-query.sql'), sqlQuery); - return; + // When useExternalDatabase flag is enabled, the query is not executed, + // but exported to the specified path inside the cache directory, + // and the user has the responsibility to execute it manually on his own + if (ctx.config.useExternalDatabase) { + // Build the path + const databaseSeedingQueryPath = path.join( + constants.SOLO_CACHE_DIR, + 'database-seeding-query.sql', + ); + + // Write the file database seeding query inside the cache + fs.writeFileSync(databaseSeedingQueryPath, sqlQuery); + + // Notify the user + self.logger.showUser( + 'Please run the following SQL script against the external database' + + `to enable Mirror Node to function correctly: ${databaseSeedingQueryPath}`, + ); + + return; //! stop the execution } const pods = await this.k8.getPodsByLabel(['app.kubernetes.io/name=postgres']); @@ -374,15 +428,15 @@ export class MirrorNodeCommand extends BaseCommand { '/bin/bash -c printenv', ); const mirrorEnvVarsArray = mirrorEnvVars.split('\n'); - const HEDERA_MIRROR_IMPORTER_DB_OWNER = getEnvValue( + const HEDERA_MIRROR_IMPORTER_DB_OWNER = helpers.getEnvValue( mirrorEnvVarsArray, 'HEDERA_MIRROR_IMPORTER_DB_OWNER', ); - const HEDERA_MIRROR_IMPORTER_DB_OWNERPASSWORD = getEnvValue( + const HEDERA_MIRROR_IMPORTER_DB_OWNERPASSWORD = helpers.getEnvValue( mirrorEnvVarsArray, 'HEDERA_MIRROR_IMPORTER_DB_OWNERPASSWORD', ); - const HEDERA_MIRROR_IMPORTER_DB_NAME = getEnvValue( + const HEDERA_MIRROR_IMPORTER_DB_NAME = helpers.getEnvValue( mirrorEnvVarsArray, 'HEDERA_MIRROR_IMPORTER_DB_NAME', ); diff --git a/src/core/helpers.ts b/src/core/helpers.ts index 201b5109c..9cdb14ca1 100644 --- a/src/core/helpers.ts +++ b/src/core/helpers.ts @@ -384,3 +384,13 @@ export function resolveValidJsonFilePath(filePath: string, defaultPath?: string) throw new SoloError(`Invalid JSON data in file: ${filePath}`); } } + +export function populateHelmArgs(valuesMapping: Record): string { + let valuesArg = ''; + + for (const [key, value] of Object.entries(valuesMapping)) { + valuesArg += ` --set ${key}=${value}`; + } + + return valuesArg; +} From 0e6bf460651a222a4286f9092c575a5688bba1b2 Mon Sep 17 00:00:00 2001 From: instamenta Date: Wed, 29 Jan 2025 15:08:11 +0200 Subject: [PATCH 02/12] add color to the warming about the sql query for the external database Signed-off-by: instamenta --- src/commands/mirror_node.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/commands/mirror_node.ts b/src/commands/mirror_node.ts index b12dffe5c..6c104af46 100644 --- a/src/commands/mirror_node.ts +++ b/src/commands/mirror_node.ts @@ -32,6 +32,7 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; import type {Optional, SoloListrTask} from '../types/index.js'; import * as Base64 from 'js-base64'; +import chalk from 'chalk'; interface MirrorNodeDeployConfigClass { chartDirectory: string; @@ -409,8 +410,11 @@ export class MirrorNodeCommand extends BaseCommand { // Notify the user self.logger.showUser( - 'Please run the following SQL script against the external database' + - `to enable Mirror Node to function correctly: ${databaseSeedingQueryPath}`, + chalk.cyan( + 'Please run the following SQL script against the external database ' + + 'to enable Mirror Node to function correctly:', + ), + chalk.yellow(databaseSeedingQueryPath), ); return; //! stop the execution From a8f09c8f83d49116bf84de13bb71e42e290db40d Mon Sep 17 00:00:00 2001 From: instamenta Date: Wed, 29 Jan 2025 15:18:15 +0200 Subject: [PATCH 03/12] add descriptions to the flags and prompt's Signed-off-by: instamenta --- src/commands/flags.ts | 39 +++++++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/src/commands/flags.ts b/src/commands/flags.ts index 436d876d3..6ac2c9f57 100644 --- a/src/commands/flags.ts +++ b/src/commands/flags.ts @@ -1531,33 +1531,60 @@ export class Flags { constName: 'externalDatabaseHost', name: 'external-database-host', definition: { - describe: '', + describe: "Used to provide the external database host if the '--use-external-database' is passed", defaultValue: '', type: 'string', }, - prompt: undefined, + prompt: async function promptGrpcWebTlsKeyPath(task: ListrTaskWrapper, input: any) { + return await Flags.promptText( + task, + input, + Flags.externalDatabaseHost.definition.defaultValue, + 'Enter host of the external database', + null, + Flags.externalDatabaseHost.name, + ); + }, }; static readonly externalDatabaseOwnerUsername: CommandFlag = { constName: 'externalDatabaseOwnerUsername', name: 'external-database-owner-username', definition: { - describe: '', + describe: "Used to provide the external database owner's username if the '--use-external-database' is passed", defaultValue: '', type: 'string', }, - prompt: undefined, + prompt: async function promptGrpcWebTlsKeyPath(task: ListrTaskWrapper, input: any) { + return await Flags.promptText( + task, + input, + Flags.externalDatabaseOwnerUsername.definition.defaultValue, + 'Enter username of the external database owner', + null, + Flags.externalDatabaseOwnerUsername.name, + ); + }, }; static readonly externalDatabaseOwnerPassword: CommandFlag = { constName: 'externalDatabaseOwnerPassword', name: 'external-database-owner-password', definition: { - describe: '', + describe: "Used to provide the external database owner's password if the '--use-external-database' is passed", defaultValue: '', type: 'string', }, - prompt: undefined, + prompt: async function promptGrpcWebTlsKeyPath(task: ListrTaskWrapper, input: any) { + return await Flags.promptText( + task, + input, + Flags.externalDatabaseOwnerPassword.definition.defaultValue, + 'Enter password of the external database owner', + null, + Flags.externalDatabaseOwnerPassword.name, + ); + }, }; static readonly grpcTlsKeyPath: CommandFlag = { From 82fa778549f01ee7cfa8b5765872c4c40a48da63 Mon Sep 17 00:00:00 2001 From: instamenta Date: Wed, 29 Jan 2025 15:19:20 +0200 Subject: [PATCH 04/12] change wording for description Signed-off-by: instamenta --- src/commands/flags.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/commands/flags.ts b/src/commands/flags.ts index 6ac2c9f57..7db9d0c1e 100644 --- a/src/commands/flags.ts +++ b/src/commands/flags.ts @@ -1531,7 +1531,7 @@ export class Flags { constName: 'externalDatabaseHost', name: 'external-database-host', definition: { - describe: "Used to provide the external database host if the '--use-external-database' is passed", + describe: "Use to provide the external database host if the '--use-external-database' is passed", defaultValue: '', type: 'string', }, @@ -1551,7 +1551,7 @@ export class Flags { constName: 'externalDatabaseOwnerUsername', name: 'external-database-owner-username', definition: { - describe: "Used to provide the external database owner's username if the '--use-external-database' is passed", + describe: "Use to provide the external database owner's username if the '--use-external-database' is passed", defaultValue: '', type: 'string', }, @@ -1571,7 +1571,7 @@ export class Flags { constName: 'externalDatabaseOwnerPassword', name: 'external-database-owner-password', definition: { - describe: "Used to provide the external database owner's password if the '--use-external-database' is passed", + describe: "Use to provide the external database owner's password if the '--use-external-database' is passed", defaultValue: '', type: 'string', }, From 204c6a567a44ccc8bf58f4d0953d832911c968a9 Mon Sep 17 00:00:00 2001 From: instamenta Date: Wed, 29 Jan 2025 16:20:47 +0200 Subject: [PATCH 05/12] fix mirror-node command logic for --quiet-mode Signed-off-by: instamenta --- src/commands/mirror_node.ts | 40 +++++++++++++++++++++------ test/e2e/commands/mirror_node.test.ts | 3 ++ 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/src/commands/mirror_node.ts b/src/commands/mirror_node.ts index 6c104af46..2b8f3e7ba 100644 --- a/src/commands/mirror_node.ts +++ b/src/commands/mirror_node.ts @@ -33,6 +33,7 @@ import * as path from 'node:path'; import type {Optional, SoloListrTask} from '../types/index.js'; import * as Base64 from 'js-base64'; import chalk from 'chalk'; +import {type CommandFlag} from '../types/flag_types.js'; interface MirrorNodeDeployConfigClass { chartDirectory: string; @@ -42,6 +43,7 @@ interface MirrorNodeDeployConfigClass { valuesFile: string; chartPath: string; valuesArg: string; + quiet: boolean; mirrorNodeVersion: string; getUnusedConfigs: () => string[]; pinger: boolean; @@ -208,15 +210,6 @@ export class MirrorNodeCommand extends BaseCommand { await self.configManager.executePrompt(task, MirrorNodeCommand.DEPLOY_FLAGS_LIST); - // In case the useExternalDatabase is set, prompt for the rest of the required data - if (self.configManager.getFlag(flags.useExternalDatabase)) { - await self.configManager.executePrompt(task, [ - flags.externalDatabaseHost, - flags.externalDatabaseOwnerUsername, - flags.externalDatabaseOwnerPassword, - ]); - } - ctx.config = this.getConfig(MirrorNodeCommand.DEPLOY_CONFIGS_NAME, MirrorNodeCommand.DEPLOY_FLAGS_LIST, [ 'chartPath', 'valuesArg', @@ -269,6 +262,35 @@ export class MirrorNodeCommand extends BaseCommand { } } + const isQuiet = ctx.config.quiet; + + // In case the useExternalDatabase is set, prompt for the rest of the required data + if (ctx.config.useExternalDatabase && !isQuiet) { + await self.configManager.executePrompt(task, [ + flags.externalDatabaseHost, + flags.externalDatabaseOwnerUsername, + flags.externalDatabaseOwnerPassword, + ]); + } else if (ctx.config.useExternalDatabase) { + if ( + !ctx.config.externalDatabaseHost || + !ctx.config.externalDatabaseOwnerUsername || + !ctx.config.externalDatabaseOwnerPassword + ) { + const missingFlags: CommandFlag[] = []; + if (!ctx.config.externalDatabaseHost) missingFlags.push(flags.externalDatabaseHost); + if (!ctx.config.externalDatabaseOwnerUsername) missingFlags.push(flags.externalDatabaseOwnerUsername); + if (!ctx.config.externalDatabaseOwnerPassword) missingFlags.push(flags.externalDatabaseOwnerPassword); + if (missingFlags.length) { + const errorMessage = + 'There are missing values that need to be provided when' + + `${chalk.cyan(`--${flags.useExternalDatabase.name}`)} is provided: `; + + throw new SoloError(`${errorMessage} ${missingFlags.map(flag => `--${flag.name}`).join(', ')}`); + } + } + } + if (!(await self.k8.hasNamespace(ctx.config.namespace))) { throw new SoloError(`namespace ${ctx.config.namespace} does not exist`); } diff --git a/test/e2e/commands/mirror_node.test.ts b/test/e2e/commands/mirror_node.test.ts index a0f946744..8982a4e9c 100644 --- a/test/e2e/commands/mirror_node.test.ts +++ b/test/e2e/commands/mirror_node.test.ts @@ -102,6 +102,9 @@ e2eTestSuite(testName, argv, undefined, undefined, undefined, undefined, undefin flags.quiet.constName, flags.storageSecrets.constName, flags.storageEndpoint.constName, + flags.externalDatabaseHost.constName, + flags.externalDatabaseOwnerUsername.constName, + flags.externalDatabaseOwnerPassword.constName, ]); expect(explorerCommand.getUnusedConfigs(MirrorNodeCommand.DEPLOY_CONFIGS_NAME)).to.deep.equal([ flags.profileFile.constName, From 86b21281e7481d546a510ebd28f171ea91cb1392 Mon Sep 17 00:00:00 2001 From: instamenta Date: Fri, 31 Jan 2025 16:27:53 +0200 Subject: [PATCH 06/12] working on adding the taskfiles for the external database testing and added the new flags to igrone prompts Signed-off-by: instamenta --- Taskfile.helper.yml | 44 +++++++++++----- .../scripts/init.sh | 7 --- .../solo-external-database-test/Taskfile.yml | 52 +++++++++++++++++++ examples/solo-gke-test/Taskfile.yml | 2 + 4 files changed, 85 insertions(+), 20 deletions(-) create mode 100644 examples/solo-external-database-test/Taskfile.yml diff --git a/Taskfile.helper.yml b/Taskfile.helper.yml index c78a91e53..e3798db91 100644 --- a/Taskfile.helper.yml +++ b/Taskfile.helper.yml @@ -297,7 +297,7 @@ tasks: status: - kind get clusters | grep -q "${SOLO_CLUSTER_NAME}" cmds: - - kind create cluster -n "${SOLO_CLUSTER_NAME}" --image "${KIND_IMAGE}" + - kind create cluster -n "${SOLO_CLUSTER_NAME}" --image "${KIND_IMAGE}" --config ~/Documents/kind-cluster.yaml - sleep 10 # wait for control plane to come up - kubectl config set-context kind-${SOLO_CLUSTER_NAME} @@ -421,18 +421,18 @@ tasks: - task: "init" cmds: - SOLO_HOME_DIR=${SOLO_HOME_DIR} npm run solo -- mirror-node deploy --namespace "${SOLO_NAMESPACE}" ${SOLO_CHARTS_DIR_FLAG} ${MIRROR_NODE_DEPLOY_EXTRA_FLAGS} --pinger -q --dev - - SOLO_HOME_DIR=${SOLO_HOME_DIR} npm run solo -- explorer deploy --namespace "${SOLO_NAMESPACE}" -s ${SOLO_CLUSTER_SETUP_NAMESPACE} ${SOLO_CHARTS_DIR_FLAG} ${ENABLE_EXPLORER_TLS_FLAG} ${TLS_CLUSTER_ISSUER_TYPE_FLAG} -q --dev - - | - if [[ "{{ .use_port_forwards }}" == "true" ]];then - echo "Enable port forwarding for Hedera Explorer & Mirror Node Network" - echo "Port forwarding for Hedera Explorer: http://localhost:8080" - explorer_svc="$(kubectl get svc -l app.kubernetes.io/component=hedera-explorer -n ${SOLO_NAMESPACE} --output json | jq -r '.items[].metadata.name')" - /bin/bash -c "nohup kubectl port-forward -n \"${SOLO_NAMESPACE}\" \"svc/${explorer_svc}\" 8080:80 > /dev/null 2>&1 &" - echo "Port forwarding for Mirror Node Network: grpc:5600, rest:5551" - /bin/bash -c "nohup kubectl port-forward -n \"${SOLO_NAMESPACE}\" svc/mirror-grpc 5600:5600 > /dev/null 2>&1 &" - /bin/bash -c "nohup kubectl port-forward -n \"${SOLO_NAMESPACE}\" svc/mirror-rest 5551:80 > /dev/null 2>&1 &" - sleep 4 - fi +# - SOLO_HOME_DIR=${SOLO_HOME_DIR} npm run solo -- explorer deploy --namespace "${SOLO_NAMESPACE}" -s ${SOLO_CLUSTER_SETUP_NAMESPACE} ${SOLO_CHARTS_DIR_FLAG} ${ENABLE_EXPLORER_TLS_FLAG} ${TLS_CLUSTER_ISSUER_TYPE_FLAG} -q --dev +# - | +# if [[ "{{ .use_port_forwards }}" == "true" ]];then +# echo "Enable port forwarding for Hedera Explorer & Mirror Node Network" +# echo "Port forwarding for Hedera Explorer: http://localhost:8080" +# explorer_svc="$(kubectl get svc -l app.kubernetes.io/component=hedera-explorer -n ${SOLO_NAMESPACE} --output json | jq -r '.items[].metadata.name')" +# /bin/bash -c "nohup kubectl port-forward -n \"${SOLO_NAMESPACE}\" \"svc/${explorer_svc}\" 8080:80 > /dev/null 2>&1 &" +# echo "Port forwarding for Mirror Node Network: grpc:5600, rest:5551" +# /bin/bash -c "nohup kubectl port-forward -n \"${SOLO_NAMESPACE}\" svc/mirror-grpc 5600:5600 > /dev/null 2>&1 &" +# /bin/bash -c "nohup kubectl port-forward -n \"${SOLO_NAMESPACE}\" svc/mirror-rest 5551:80 > /dev/null 2>&1 &" +# sleep 4 +# fi solo:destroy-mirror-node: silent: true @@ -464,3 +464,21 @@ tasks: cmds: - echo "Cleaning up temporary files..." - rm -f /tmp/solo-${USER}-* || true + + solo:external-database: + silent: false + desc: setup external database PostgreSQL with helm + cmds: + - | + {{.solo_bin_dir}}/helm install my-postgresql bitnami/postgresql \ + --namespace database --create-namespace \ + --set global.postgresql.auth.postgresPassword={{.postgres_password}} \ + --set primary.persistence.enabled=false --set secondary.enabled=false + - name: "Wait for PostgreSQL pod to be ready" + cmd: kubectl wait --for=condition=ready pod/my-postgresql-0 -n database --timeout=60s + - name: "Copy init.sql inside the database pod" + cmd: kubectl cp ../custom-mirror-node-database/scripts/init.sh my-postgresql-0:/tmp/init.sh -n database + - name: "Make init.sh executable" + cmd: kubectl exec -it my-postgresql-0 -n database -- chmod +x /tmp/init.sh + - name: "Execute init.sh inside the database pod" + cmd: kubectl exec -it my-postgresql-0 -n database -- /bin/bash /tmp/init.sh diff --git a/examples/custom-mirror-node-database/scripts/init.sh b/examples/custom-mirror-node-database/scripts/init.sh index 70c27b821..ff68e1e1d 100644 --- a/examples/custom-mirror-node-database/scripts/init.sh +++ b/examples/custom-mirror-node-database/scripts/init.sh @@ -1,7 +1,3 @@ -# Example if initialization script for the external database, -# values must match those inside values.yaml if used - -cat > init1.sh << 'EOF' #!/bin/bash set -e @@ -139,6 +135,3 @@ if [[ -f "${PGHBACONF}.bak" ]]; then mv "${PGHBACONF}.bak" "${PGHBACONF}" pg_ctl reload fi -EOF -chmod +x init1.sh -./init1.sh diff --git a/examples/solo-external-database-test/Taskfile.yml b/examples/solo-external-database-test/Taskfile.yml new file mode 100644 index 000000000..a61350dd2 --- /dev/null +++ b/examples/solo-external-database-test/Taskfile.yml @@ -0,0 +1,52 @@ +version: 3 +includes: + main: + taskfile: ../../Taskfile.helper.yml + flatten: true +vars: + solo_home_override_dir: "/Users/zhanmilenkov/.solo" + use_port_forwards: "true" + postgres_username: "postgres" + postgres_password: "XXXXXXXXXXXX" + postgres_host: "my-postgresql-0.database.svc.cluster.local" +env: + SOLO_NETWORK_SIZE: "1" + SOLO_NAMESPACE: "solo-external-database-test" + MIRROR_NODE_DEPLOY_EXTRA_FLAGS: "--use-external-database --external-database-host {{.postgres_host}} --external-database-owner-username {{.postgres_username}} --external-database-owner-password {{.postgres_password}}" +tasks: + default: + silent: true + desc: install Solo, create a kind cluster, deploy the network, set it up, and start it + deps: + - task: "init" + cmds: + - echo "This command is meant to deploy a Solo network to a Kind cluster on your local machine, " + - echo "ctrl-c if this is not what you want to do." + - sleep 5 + + install: + desc: create the cluster, solo init, solo cluster create, solo node keys, solo network deploy + deps: + - task: "init" + cmds: + - task: "cluster:create" + - task: "solo:init" + - task: "solo:cluster:setup" + - task: "solo:keys" + - task: "solo:network:deploy" + - task: "solo:node:setup" + - task: "solo:node:start" + - task: "solo:external-database" + - task: "solo:mirror-node" + - name: "Copy database-seeding-query.sql inside the database pod" + cmd: kubectl cp /Users/zhanmilenkov/.solo/cache/database-seeding-query.sql my-postgresql-0:/tmp/database-seeding-query.sql -n database + - ls -la /tmp/database-seeding-query.sql + - name: "Execute the database-seeding–query.sql against the database" + cmd: kubectl exec -it my-postgresql-0 -n database -- env PGPASSWORD={{.postgres_password}} psql -U {{.postgres_username}} -d mirror_node -c "$(cat /tmp/database-seeding-query.sql)" + + destroy: + desc: destroy relay, mirror-node, and network + deps: + - task: "init" + cmds: + - task: "cluster:destroy" diff --git a/examples/solo-gke-test/Taskfile.yml b/examples/solo-gke-test/Taskfile.yml index 2bd1340f0..5e36ab091 100644 --- a/examples/solo-gke-test/Taskfile.yml +++ b/examples/solo-gke-test/Taskfile.yml @@ -27,3 +27,5 @@ env: NETWORK_DEPLOY_EXTRA_FLAGS: "--haproxy-ips node1=,node2=,node3=,node4= --pvcs" MIRROR_NODE_DEPLOY_EXTRA_FLAGS: "--values-file {{.USER_WORKING_DIR}}/mirror-and-explorer-values.yaml" RELAY_NODE_DEPLOY_EXTRA_FLAGS: "--values-file {{.USER_WORKING_DIR}}/relay-values.yaml" + + From b52e77239bbc8bd12d96971616b237bbfafb6bed Mon Sep 17 00:00:00 2001 From: instamenta Date: Fri, 31 Jan 2025 16:32:05 +0200 Subject: [PATCH 07/12] dont prompt for the optional flags Signed-off-by: instamenta --- src/commands/mirror_node.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/commands/mirror_node.ts b/src/commands/mirror_node.ts index 2b8f3e7ba..01c594388 100644 --- a/src/commands/mirror_node.ts +++ b/src/commands/mirror_node.ts @@ -206,6 +206,10 @@ export class MirrorNodeCommand extends BaseCommand { flags.pinger, flags.operatorId, flags.operatorKey, + flags.useExternalDatabase, + flags.externalDatabaseHost, + flags.externalDatabaseOwnerUsername, + flags.externalDatabaseOwnerPassword, ]); await self.configManager.executePrompt(task, MirrorNodeCommand.DEPLOY_FLAGS_LIST); From 451d5b6c219ca0cdd8f6405031e7d7f58865a8b4 Mon Sep 17 00:00:00 2001 From: instamenta Date: Mon, 3 Feb 2025 16:55:22 +0200 Subject: [PATCH 08/12] rename dir and fix the logic for the query execution post the 'mirror-node deploy command' Signed-off-by: instamenta --- .../Taskfile.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) rename examples/{solo-external-database-test => external-database-test}/Taskfile.yml (92%) diff --git a/examples/solo-external-database-test/Taskfile.yml b/examples/external-database-test/Taskfile.yml similarity index 92% rename from examples/solo-external-database-test/Taskfile.yml rename to examples/external-database-test/Taskfile.yml index a61350dd2..d4c933cfd 100644 --- a/examples/solo-external-database-test/Taskfile.yml +++ b/examples/external-database-test/Taskfile.yml @@ -11,7 +11,7 @@ vars: postgres_host: "my-postgresql-0.database.svc.cluster.local" env: SOLO_NETWORK_SIZE: "1" - SOLO_NAMESPACE: "solo-external-database-test" + SOLO_NAMESPACE: "external-database-test" MIRROR_NODE_DEPLOY_EXTRA_FLAGS: "--use-external-database --external-database-host {{.postgres_host}} --external-database-owner-username {{.postgres_username}} --external-database-owner-password {{.postgres_password}}" tasks: default: @@ -40,10 +40,8 @@ tasks: - task: "solo:mirror-node" - name: "Copy database-seeding-query.sql inside the database pod" cmd: kubectl cp /Users/zhanmilenkov/.solo/cache/database-seeding-query.sql my-postgresql-0:/tmp/database-seeding-query.sql -n database - - ls -la /tmp/database-seeding-query.sql - name: "Execute the database-seeding–query.sql against the database" - cmd: kubectl exec -it my-postgresql-0 -n database -- env PGPASSWORD={{.postgres_password}} psql -U {{.postgres_username}} -d mirror_node -c "$(cat /tmp/database-seeding-query.sql)" - + cmd: kubectl exec -it my-postgresql-0 -n database -- env PGPASSWORD={{.postgres_password}} psql -U {{.postgres_username}} -d mirror_node -f /tmp/database-seeding-query.sql destroy: desc: destroy relay, mirror-node, and network deps: From 9abc741c8d610bd46198070dbfb1d5c49a0d4dfd Mon Sep 17 00:00:00 2001 From: instamenta Date: Mon, 3 Feb 2025 16:59:11 +0200 Subject: [PATCH 09/12] fix paths Signed-off-by: instamenta --- examples/external-database-test/Taskfile.yml | 22 ++++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/examples/external-database-test/Taskfile.yml b/examples/external-database-test/Taskfile.yml index d4c933cfd..08738287b 100644 --- a/examples/external-database-test/Taskfile.yml +++ b/examples/external-database-test/Taskfile.yml @@ -4,7 +4,7 @@ includes: taskfile: ../../Taskfile.helper.yml flatten: true vars: - solo_home_override_dir: "/Users/zhanmilenkov/.solo" + solo_home_override_dir: "/Users/{{.HOME}}/.solo" use_port_forwards: "true" postgres_username: "postgres" postgres_password: "XXXXXXXXXXXX" @@ -29,17 +29,17 @@ tasks: deps: - task: "init" cmds: - - task: "cluster:create" - - task: "solo:init" - - task: "solo:cluster:setup" - - task: "solo:keys" - - task: "solo:network:deploy" - - task: "solo:node:setup" - - task: "solo:node:start" - - task: "solo:external-database" - - task: "solo:mirror-node" +# - task: "cluster:create" +# - task: "solo:init" +# - task: "solo:cluster:setup" +# - task: "solo:keys" +# - task: "solo:network:deploy" +# - task: "solo:node:setup" +# - task: "solo:node:start" +# - task: "solo:external-database" +# - task: "solo:mirror-node" - name: "Copy database-seeding-query.sql inside the database pod" - cmd: kubectl cp /Users/zhanmilenkov/.solo/cache/database-seeding-query.sql my-postgresql-0:/tmp/database-seeding-query.sql -n database + cmd: kubectl cp {{.HOME}}/.solo/cache/database-seeding-query.sql my-postgresql-0:/tmp/database-seeding-query.sql -n database - name: "Execute the database-seeding–query.sql against the database" cmd: kubectl exec -it my-postgresql-0 -n database -- env PGPASSWORD={{.postgres_password}} psql -U {{.postgres_username}} -d mirror_node -f /tmp/database-seeding-query.sql destroy: From 63c00504426b27fa95d1015c8ad480898067e758 Mon Sep 17 00:00:00 2001 From: instamenta Date: Mon, 3 Feb 2025 17:22:58 +0200 Subject: [PATCH 10/12] gits Signed-off-by: instamenta --- examples/external-database-test/Taskfile.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/examples/external-database-test/Taskfile.yml b/examples/external-database-test/Taskfile.yml index 08738287b..395a0d958 100644 --- a/examples/external-database-test/Taskfile.yml +++ b/examples/external-database-test/Taskfile.yml @@ -29,15 +29,15 @@ tasks: deps: - task: "init" cmds: -# - task: "cluster:create" -# - task: "solo:init" -# - task: "solo:cluster:setup" -# - task: "solo:keys" -# - task: "solo:network:deploy" -# - task: "solo:node:setup" -# - task: "solo:node:start" -# - task: "solo:external-database" -# - task: "solo:mirror-node" + - task: "cluster:create" + - task: "solo:init" + - task: "solo:cluster:setup" + - task: "solo:keys" + - task: "solo:network:deploy" + - task: "solo:node:setup" + - task: "solo:node:start" + - task: "solo:external-database" + - task: "solo:mirror-node" - name: "Copy database-seeding-query.sql inside the database pod" cmd: kubectl cp {{.HOME}}/.solo/cache/database-seeding-query.sql my-postgresql-0:/tmp/database-seeding-query.sql -n database - name: "Execute the database-seeding–query.sql against the database" From cceb1b52cbb482d2e5baaa5429c608f3da3653e2 Mon Sep 17 00:00:00 2001 From: instamenta Date: Mon, 3 Feb 2025 17:23:31 +0200 Subject: [PATCH 11/12] revert code that should be unnafected Signed-off-by: instamenta --- Taskfile.helper.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Taskfile.helper.yml b/Taskfile.helper.yml index e3798db91..e5b5305fb 100644 --- a/Taskfile.helper.yml +++ b/Taskfile.helper.yml @@ -421,18 +421,18 @@ tasks: - task: "init" cmds: - SOLO_HOME_DIR=${SOLO_HOME_DIR} npm run solo -- mirror-node deploy --namespace "${SOLO_NAMESPACE}" ${SOLO_CHARTS_DIR_FLAG} ${MIRROR_NODE_DEPLOY_EXTRA_FLAGS} --pinger -q --dev -# - SOLO_HOME_DIR=${SOLO_HOME_DIR} npm run solo -- explorer deploy --namespace "${SOLO_NAMESPACE}" -s ${SOLO_CLUSTER_SETUP_NAMESPACE} ${SOLO_CHARTS_DIR_FLAG} ${ENABLE_EXPLORER_TLS_FLAG} ${TLS_CLUSTER_ISSUER_TYPE_FLAG} -q --dev -# - | -# if [[ "{{ .use_port_forwards }}" == "true" ]];then -# echo "Enable port forwarding for Hedera Explorer & Mirror Node Network" -# echo "Port forwarding for Hedera Explorer: http://localhost:8080" -# explorer_svc="$(kubectl get svc -l app.kubernetes.io/component=hedera-explorer -n ${SOLO_NAMESPACE} --output json | jq -r '.items[].metadata.name')" -# /bin/bash -c "nohup kubectl port-forward -n \"${SOLO_NAMESPACE}\" \"svc/${explorer_svc}\" 8080:80 > /dev/null 2>&1 &" -# echo "Port forwarding for Mirror Node Network: grpc:5600, rest:5551" -# /bin/bash -c "nohup kubectl port-forward -n \"${SOLO_NAMESPACE}\" svc/mirror-grpc 5600:5600 > /dev/null 2>&1 &" -# /bin/bash -c "nohup kubectl port-forward -n \"${SOLO_NAMESPACE}\" svc/mirror-rest 5551:80 > /dev/null 2>&1 &" -# sleep 4 -# fi + - SOLO_HOME_DIR=${SOLO_HOME_DIR} npm run solo -- explorer deploy --namespace "${SOLO_NAMESPACE}" -s ${SOLO_CLUSTER_SETUP_NAMESPACE} ${SOLO_CHARTS_DIR_FLAG} ${ENABLE_EXPLORER_TLS_FLAG} ${TLS_CLUSTER_ISSUER_TYPE_FLAG} -q --dev + - | + if [[ "{{ .use_port_forwards }}" == "true" ]];then + echo "Enable port forwarding for Hedera Explorer & Mirror Node Network" + echo "Port forwarding for Hedera Explorer: http://localhost:8080" + explorer_svc="$(kubectl get svc -l app.kubernetes.io/component=hedera-explorer -n ${SOLO_NAMESPACE} --output json | jq -r '.items[].metadata.name')" + /bin/bash -c "nohup kubectl port-forward -n \"${SOLO_NAMESPACE}\" \"svc/${explorer_svc}\" 8080:80 > /dev/null 2>&1 &" + echo "Port forwarding for Mirror Node Network: grpc:5600, rest:5551" + /bin/bash -c "nohup kubectl port-forward -n \"${SOLO_NAMESPACE}\" svc/mirror-grpc 5600:5600 > /dev/null 2>&1 &" + /bin/bash -c "nohup kubectl port-forward -n \"${SOLO_NAMESPACE}\" svc/mirror-rest 5551:80 > /dev/null 2>&1 &" + sleep 4 + fi solo:destroy-mirror-node: silent: true From f0869c6e40d4aeece2dd0270e15ea1e4260a293f Mon Sep 17 00:00:00 2001 From: instamenta Date: Mon, 3 Feb 2025 17:25:14 +0200 Subject: [PATCH 12/12] make new command for deploying only the mirror-node Signed-off-by: instamenta --- Taskfile.helper.yml | 9 +++++++++ examples/external-database-test/Taskfile.yml | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Taskfile.helper.yml b/Taskfile.helper.yml index e5b5305fb..fadd964c3 100644 --- a/Taskfile.helper.yml +++ b/Taskfile.helper.yml @@ -434,6 +434,15 @@ tasks: sleep 4 fi + solo:mirror-node-only: + silent: true + desc: solo mirror-node deploy with port forward on explorer + deps: + - task: "init" + cmds: + - SOLO_HOME_DIR=${SOLO_HOME_DIR} npm run solo -- mirror-node deploy --namespace "${SOLO_NAMESPACE}" ${SOLO_CHARTS_DIR_FLAG} ${MIRROR_NODE_DEPLOY_EXTRA_FLAGS} --pinger -q --dev + + solo:destroy-mirror-node: silent: true desc: solo mirror-node destroy diff --git a/examples/external-database-test/Taskfile.yml b/examples/external-database-test/Taskfile.yml index 395a0d958..496e0920b 100644 --- a/examples/external-database-test/Taskfile.yml +++ b/examples/external-database-test/Taskfile.yml @@ -37,7 +37,7 @@ tasks: - task: "solo:node:setup" - task: "solo:node:start" - task: "solo:external-database" - - task: "solo:mirror-node" + - task: "solo:mirror-node-only" - name: "Copy database-seeding-query.sql inside the database pod" cmd: kubectl cp {{.HOME}}/.solo/cache/database-seeding-query.sql my-postgresql-0:/tmp/database-seeding-query.sql -n database - name: "Execute the database-seeding–query.sql against the database"