From 276b59e7bde447034aa0c90acd3299fd7265e00f Mon Sep 17 00:00:00 2001 From: Dave Myler Date: Thu, 12 Sep 2024 10:46:19 +0100 Subject: [PATCH 1/6] Allow connections to external MongoDB (from PR4016) --- common/src/helpers/db-helper.ts | 3 ++- common/src/helpers/fix-connection-string.ts | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 common/src/helpers/fix-connection-string.ts diff --git a/common/src/helpers/db-helper.ts b/common/src/helpers/db-helper.ts index 7bb2625bf3..07abf5312f 100644 --- a/common/src/helpers/db-helper.ts +++ b/common/src/helpers/db-helper.ts @@ -3,6 +3,7 @@ import { MongoDriver, MongoEntityManager, MongoEntityRepository, ObjectId } from import { BaseEntity } from '../models/index.js'; import { DataBaseNamingStrategy } from './db-naming-strategy.js'; import { GridFSBucket } from 'mongodb'; +import fixConnectionString from './fix-connection-string.js'; /** * Common connection config @@ -13,7 +14,7 @@ export const COMMON_CONNECTION_CONFIG: any = { dbName: (process.env.GUARDIAN_ENV || (process.env.HEDERA_NET !== process.env.PREUSED_HEDERA_NET)) ? `${process.env.GUARDIAN_ENV}_${process.env.HEDERA_NET}_${process.env.DB_DATABASE}` : process.env.DB_DATABASE, - clientUrl: `mongodb://${process.env.DB_HOST}`, + clientUrl: fixConnectionString(process.env.DB_HOST), entities: [ 'dist/entity/*.js' ] diff --git a/common/src/helpers/fix-connection-string.ts b/common/src/helpers/fix-connection-string.ts new file mode 100644 index 0000000000..5fc73aeab2 --- /dev/null +++ b/common/src/helpers/fix-connection-string.ts @@ -0,0 +1,8 @@ +/** + * Fix connection string + * @param cs Connection string + * @returns Fixed connection string + */ +export default function fixConnectionString(cs: string) { + return /.+\:\/\/.+/.test(cs) ? cs : `mongodb://${cs}`; +} \ No newline at end of file From 8468ce9720cc0d45856547e11616ee80be9ebade Mon Sep 17 00:00:00 2001 From: Dave Myler Date: Thu, 12 Sep 2024 10:50:36 +0100 Subject: [PATCH 2/6] Add secret entity --- common/src/entity/secret.ts | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 common/src/entity/secret.ts diff --git a/common/src/entity/secret.ts b/common/src/entity/secret.ts new file mode 100644 index 0000000000..3168cc42c4 --- /dev/null +++ b/common/src/entity/secret.ts @@ -0,0 +1,21 @@ +import { Entity, Property, Unique } from '@mikro-orm/core'; +import { BaseEntity } from '../models/index.js'; + +/** + * Secrets collection + */ +@Entity() +@Unique({ properties: ['key'], options: { partialFilterExpression: { key: { $type: 'string' }}}}) +export class Secret extends BaseEntity { + /** + * Secret name + */ + @Property({ nullable: true }) + key?: string; + + /** + * Secret value + */ + @Property({ nullable: true }) + value?: string; +} From 4774dbf311647152a6582cddd84419f6e53dc3e9 Mon Sep 17 00:00:00 2001 From: Dave Myler Date: Thu, 12 Sep 2024 10:56:13 +0100 Subject: [PATCH 3/6] Expose secret entity type --- common/src/entity/index.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/common/src/entity/index.ts b/common/src/entity/index.ts index 2eb5c02d06..15a8768368 100644 --- a/common/src/entity/index.ts +++ b/common/src/entity/index.ts @@ -40,4 +40,5 @@ export * from './mint-transaction.js'; export * from './dry-run-files.js'; export * from './policy-cache-data.js'; export * from './policy-cache.js'; -export * from './assign-entity.js'; \ No newline at end of file +export * from './assign-entity.js'; +export * from './secret.js'; From 0762037677869a488139d60ec6470248437da22f Mon Sep 17 00:00:00 2001 From: Dave Myler Date: Fri, 13 Sep 2024 22:44:54 +0100 Subject: [PATCH 4/6] Store secrets encrypted in the database --- common/src/entity/secret.ts | 6 +- .../src/secret-manager/mongodb/encryptor.ts | 51 ++++++++++++++ .../mongodb/mongodb-secret-manager.ts | 67 +++++++++++++++++++ .../secret-manager/secret-manager-config.ts | 6 ++ common/src/secret-manager/secret-manager.ts | 3 + 5 files changed, 130 insertions(+), 3 deletions(-) create mode 100644 common/src/secret-manager/mongodb/encryptor.ts create mode 100644 common/src/secret-manager/mongodb/mongodb-secret-manager.ts diff --git a/common/src/entity/secret.ts b/common/src/entity/secret.ts index 3168cc42c4..c53390aabf 100644 --- a/common/src/entity/secret.ts +++ b/common/src/entity/secret.ts @@ -5,17 +5,17 @@ import { BaseEntity } from '../models/index.js'; * Secrets collection */ @Entity() -@Unique({ properties: ['key'], options: { partialFilterExpression: { key: { $type: 'string' }}}}) +@Unique({ properties: ['path'], options: { partialFilterExpression: { path: { $type: 'string' }}}}) export class Secret extends BaseEntity { /** * Secret name */ @Property({ nullable: true }) - key?: string; + path?: string; /** * Secret value */ @Property({ nullable: true }) - value?: string; + data?: string; } diff --git a/common/src/secret-manager/mongodb/encryptor.ts b/common/src/secret-manager/mongodb/encryptor.ts new file mode 100644 index 0000000000..58234d5d09 --- /dev/null +++ b/common/src/secret-manager/mongodb/encryptor.ts @@ -0,0 +1,51 @@ +import * as crypto from 'crypto'; + +class Encryptor { + private readonly algorithm = 'aes-256-cbc'; + private readonly key: Buffer | null; + private readonly encryptionEnabled: boolean; + private readonly encryptedPrefix = 'ENC:'; + public static readonly KEYNAME = "MONGO_ENCRYPTION_KEY"; + + constructor() { + const envKey = process.env[Encryptor.KEYNAME]; + if (!envKey) { + this.key = null; + this.encryptionEnabled = false; + } else { + this.key = crypto.scryptSync(envKey, 'salt', 32); + this.encryptionEnabled = true; + } + } + + encrypt(text: string): string { + if (!this.encryptionEnabled) { + return text; + } + const iv = crypto.randomBytes(16); + const cipher = crypto.createCipheriv(this.algorithm, this.key!, iv); + let encrypted = cipher.update(text, 'utf8', 'base64'); + encrypted += cipher.final('base64'); + const salt = iv.toString('base64'); + return this.encryptedPrefix + `${salt}~${encrypted}`; + } + + decrypt(text: string): string { + if (!text.startsWith(this.encryptedPrefix)) { + return text; + } + if (!this.encryptionEnabled) { + console.warn('Attempted to decrypt, but encryption is disabled. Returning original text.'); + return text.slice(this.encryptedPrefix.length); + } + const encryptedText = text.slice(this.encryptedPrefix.length); + const [salt, encrypted] = encryptedText.split('~'); + const iv = Buffer.from(salt, 'base64'); + const decipher = crypto.createDecipheriv(this.algorithm, this.key!, iv); + let decrypted = decipher.update(encrypted, 'base64', 'utf8'); + decrypted += decipher.final('utf8'); + return decrypted; + } +} + +export default Encryptor; diff --git a/common/src/secret-manager/mongodb/mongodb-secret-manager.ts b/common/src/secret-manager/mongodb/mongodb-secret-manager.ts new file mode 100644 index 0000000000..6c1457e1fa --- /dev/null +++ b/common/src/secret-manager/mongodb/mongodb-secret-manager.ts @@ -0,0 +1,67 @@ +import { SecretManagerBase } from '../secret-manager-base.js'; +import { DataBaseHelper } from '../../helpers/db-helper.js'; +import { Secret } from '../../entity/secret.js'; +import Encryptor from './encryptor.js'; + +export class MongoDbSecretManager implements SecretManagerBase { + + // Bypass DB for secrets that are stored in environment variables + // as these are accessed during the environment verification + private static secretsFromEnvironment: Record = { + "secretkey/auth": { + "ACCESS_TOKEN_SECRET": process.env.ACCESS_TOKEN_SECRET, + }, + "keys/operator": { + "OPERATOR_ID": process.env.OPERATOR_ID, + "OPERATOR_KEY": process.env.OPERATOR_KEY + }, + "apikey/ipfs": { + "IPFS_STORAGE_API_KEY": process.env.IPFS_STORAGE_API_KEY + } + } + + public async getSecrets(path: string, _?: any): Promise { + if (MongoDbSecretManager.secretsFromEnvironment.hasOwnProperty(path)) { + return MongoDbSecretManager.secretsFromEnvironment[path]; + } + const helper = new DataBaseHelper(Secret); + const secrets = await helper.findOne({ path }); + if (!secrets) { + return null; + } + return JSON.parse(new Encryptor().decrypt(secrets.data)); + } + + public async setSecrets(path: string, data: any, _?: any): Promise { + if (MongoDbSecretManager.secretsFromEnvironment.hasOwnProperty(path)) { + return; + } + console.log(`MONGODB-SECRETS: Setting secret ${path}`); + const secret = await this.getSecrets(path); + const helper = new DataBaseHelper(Secret); + const secretData = new Encryptor().encrypt(JSON.stringify(data)); + if (secret) { + console.log(`MONGODB-SECRETS: Existing secret ${path}`); + secret.data = secretData; + await helper.update({ path }, secret); + } else { + console.log(`MONGODB-SECRETS: New secret ${path}`); + const row = helper.create({ path, data: secretData }); + await helper.create({ path, data: secretData }); + await helper.save(row) + } + console.log(`MONGODB-SECRETS: Completed setting secret ${path}`); + + { + // TEST: Verification + const verify = await this.getSecrets(path); + if (JSON.stringify(verify) !== JSON.stringify(data)) { + console.log('Secrets verification failed'); + console.log('Original:', data); + console.log('Returned:', verify); + throw new Error('Secrets verification failed'); + } + console.log(`MONGODB-SECRETS: VERIFIED secret ${path}`); + } + } +} diff --git a/common/src/secret-manager/secret-manager-config.ts b/common/src/secret-manager/secret-manager-config.ts index 949243cfd0..f8116576a9 100644 --- a/common/src/secret-manager/secret-manager-config.ts +++ b/common/src/secret-manager/secret-manager-config.ts @@ -24,6 +24,10 @@ export enum SecretManagerType { * Azure Secrets Manager */ AZURE = 'azure', + /** + * MongoDB Secrets Manager + */ + MONGODB = 'mongodb', /** * Old style secrets */ @@ -52,6 +56,8 @@ export class SecretManagerConfigs { return GcpSecretManagerConfigs.getConfigs() case SecretManagerType.AZURE: return AzureSecretManagerConfigs.getConfigs() + case SecretManagerType.MONGODB: + return case SecretManagerType.OLD_STYLE: return default: diff --git a/common/src/secret-manager/secret-manager.ts b/common/src/secret-manager/secret-manager.ts index 866f4f5c51..4fd2e049fb 100644 --- a/common/src/secret-manager/secret-manager.ts +++ b/common/src/secret-manager/secret-manager.ts @@ -9,6 +9,7 @@ import { AzureSecretManager } from './azure/azure-secret-manager.js'; import { IAzureSecretManagerConfigs } from './azure/azure-secret-manager-configs.js'; import { IGcpSecretManagerConfigs } from './gcp/gcp-secret-manager-configs.js'; import { GcpSecretManager } from './gcp/gcp-secret-manager.js'; +import { MongoDbSecretManager } from './mongodb/mongodb-secret-manager.js'; /** * Class to get secret manager @@ -62,6 +63,8 @@ export class SecretManager { return new GcpSecretManager(configs as IGcpSecretManagerConfigs) case SecretManagerType.AZURE: return new AzureSecretManager(configs as IAzureSecretManagerConfigs) + case SecretManagerType.MONGODB: + return new MongoDbSecretManager() case SecretManagerType.OLD_STYLE: return new OldSecretManager() default: From 4d9ce7394ccea3a631730319b7994565b09bf2c9 Mon Sep 17 00:00:00 2001 From: Dave Myler Date: Fri, 13 Sep 2024 22:50:55 +0100 Subject: [PATCH 5/6] Make secret encrypt/decrypt verification configurable --- common/src/secret-manager/mongodb/mongodb-secret-manager.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/common/src/secret-manager/mongodb/mongodb-secret-manager.ts b/common/src/secret-manager/mongodb/mongodb-secret-manager.ts index 6c1457e1fa..e92768161f 100644 --- a/common/src/secret-manager/mongodb/mongodb-secret-manager.ts +++ b/common/src/secret-manager/mongodb/mongodb-secret-manager.ts @@ -52,11 +52,12 @@ export class MongoDbSecretManager implements SecretManagerBase { } console.log(`MONGODB-SECRETS: Completed setting secret ${path}`); + // TEMP: Verify encrypt/decrypt symetry + if ("Y" === process.env.VERIFY_SECRETS) { - // TEST: Verification const verify = await this.getSecrets(path); if (JSON.stringify(verify) !== JSON.stringify(data)) { - console.log('Secrets verification failed'); + console.log(`MONGODB-SECRETS: VERIFICATION FAILED secret ${path}`); console.log('Original:', data); console.log('Returned:', verify); throw new Error('Secrets verification failed'); From 7a2549d0fd04fdcce052c7154487c9b0ed480aeb Mon Sep 17 00:00:00 2001 From: Dave Myler Date: Thu, 19 Sep 2024 15:44:38 +0100 Subject: [PATCH 6/6] Add script to build arm/amd multi-architecture containers --- multi-build.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100755 multi-build.sh diff --git a/multi-build.sh b/multi-build.sh new file mode 100755 index 0000000000..1895310f62 --- /dev/null +++ b/multi-build.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +docker build -t guardian-notification-service:latest -f ./notification-service/Dockerfile --platform linux/amd64,linux/arm64 . +docker build -t guardian-logger-service:latest -f ./logger-service/Dockerfile --platform linux/amd64,linux/arm64 . +docker build -t guardian-worker-service:latest -f ./worker-service/Dockerfile --platform linux/amd64,linux/arm64 . +docker build -t guardian-auth-service:latest -f ./auth-service/Dockerfile.demo --platform linux/amd64,linux/arm64 . +docker build -t guardian-api-gateway:latest -f ./api-gateway/Dockerfile.demo --platform linux/amd64,linux/arm64 . +docker build -t guardian-policy-service:latest -f ./policy-service/Dockerfile --platform linux/amd64,linux/arm64 . +docker build -t guardian-mrv-sender:latest -f ./mrv-sender/Dockerfile --platform linux/amd64,linux/arm64 . +docker build -t guardian-guardian-service:latest -f ./guardian-service/Dockerfile --platform linux/amd64,linux/arm64 . +docker build -t guardian-web-proxy:latest -f ./web-proxy/Dockerfile.demo --platform linux/amd64,linux/arm64 . +docker build -t guardian-application-events:latest -f ./application-events/Dockerfile --platform linux/amd64,linux/arm64 . +docker build -t guardian-topic-viewer:latest -f ./topic-viewer/Dockerfile --platform linux/amd64,linux/arm64 .