External MongoDB and Secrets in MongoDB

common/src/entity/index.ts

@@ -40,4 +40,5 @@ export * from './mint-transaction.js';
 export * from './dry-run-files.js';
 export * from './policy-cache-data.js';
 export * from './policy-cache.js';
-export * from './assign-entity.js';
+export * from './assign-entity.js';
+export * from './secret.js';

common/src/entity/secret.ts

@@ -0,0 +1,21 @@
+import { Entity, Property, Unique } from '@mikro-orm/core';
+import { BaseEntity } from '../models/index.js';
+
+/**
+ * Secrets collection
+ */
+@Entity()
+@Unique({ properties: ['path'], options: { partialFilterExpression: { path: { $type: 'string' }}}})
+export class Secret extends BaseEntity {
+    /**
+     * Secret name
+     */
+    @Property({ nullable: true })
+    path?: string;
+
+    /**
+     * Secret value
+     */
+    @Property({ nullable: true })
+    data?: string;
+}

common/src/helpers/db-helper.ts

@@ -3,6 +3,7 @@ import { MongoDriver, MongoEntityManager, MongoEntityRepository, ObjectId } from
 import { BaseEntity } from '../models/index.js';
 import { DataBaseNamingStrategy } from './db-naming-strategy.js';
 import { GridFSBucket } from 'mongodb';
+import fixConnectionString from './fix-connection-string.js';
 
 /**
  * Common connection config
@@ -13,7 +14,7 @@ export const COMMON_CONNECTION_CONFIG: any = {
     dbName: (process.env.GUARDIAN_ENV || (process.env.HEDERA_NET !== process.env.PREUSED_HEDERA_NET)) ?
         `${process.env.GUARDIAN_ENV}_${process.env.HEDERA_NET}_${process.env.DB_DATABASE}` :
         process.env.DB_DATABASE,
-    clientUrl: `mongodb://${process.env.DB_HOST}`,
+    clientUrl: fixConnectionString(process.env.DB_HOST),
     entities: [
         'dist/entity/*.js'
     ]

common/src/helpers/fix-connection-string.ts

@@ -0,0 +1,8 @@
+/**
+ * Fix connection string
+ * @param cs Connection string
+ * @returns Fixed connection string
+ */
+export default function fixConnectionString(cs: string) {
+    return /.+\:\/\/.+/.test(cs) ? cs : `mongodb://${cs}`;
+}

common/src/secret-manager/mongodb/encryptor.ts

@@ -0,0 +1,51 @@
+import * as crypto from 'crypto';
+
+class Encryptor {
+    private readonly algorithm = 'aes-256-cbc';
+    private readonly key: Buffer | null;
+    private readonly encryptionEnabled: boolean;
+    private readonly encryptedPrefix = 'ENC:';
+    public static readonly KEYNAME = "MONGO_ENCRYPTION_KEY";
+
+    constructor() {
+        const envKey = process.env[Encryptor.KEYNAME];
+        if (!envKey) {
+            this.key = null;
+            this.encryptionEnabled = false;
+        } else {
+            this.key = crypto.scryptSync(envKey, 'salt', 32);
+            this.encryptionEnabled = true;
+        }
+    }
+
+    encrypt(text: string): string {
+        if (!this.encryptionEnabled) {
+            return text;
+        }
+        const iv = crypto.randomBytes(16);
+        const cipher = crypto.createCipheriv(this.algorithm, this.key!, iv);
+        let encrypted = cipher.update(text, 'utf8', 'base64');
+        encrypted += cipher.final('base64');
+        const salt = iv.toString('base64');
+        return this.encryptedPrefix + `${salt}~${encrypted}`;
+    }
+
+    decrypt(text: string): string {
+        if (!text.startsWith(this.encryptedPrefix)) {
+            return text;
+        }
+        if (!this.encryptionEnabled) {
+            console.warn('Attempted to decrypt, but encryption is disabled. Returning original text.');
+            return text.slice(this.encryptedPrefix.length);
+        }
+        const encryptedText = text.slice(this.encryptedPrefix.length);
+        const [salt, encrypted] = encryptedText.split('~');
+        const iv = Buffer.from(salt, 'base64');
+        const decipher = crypto.createDecipheriv(this.algorithm, this.key!, iv);
+        let decrypted = decipher.update(encrypted, 'base64', 'utf8');
+        decrypted += decipher.final('utf8');
+        return decrypted;
+    }
+}
+
+export default Encryptor;

common/src/secret-manager/mongodb/mongodb-secret-manager.ts

@@ -0,0 +1,68 @@
+import { SecretManagerBase } from '../secret-manager-base.js';
+import { DataBaseHelper } from '../../helpers/db-helper.js';
+import { Secret } from '../../entity/secret.js';
+import Encryptor from './encryptor.js';
+
+export class MongoDbSecretManager implements SecretManagerBase {
+
+    // Bypass DB for secrets that are stored in environment variables
+    // as these are accessed during the environment verification
+    private static secretsFromEnvironment: Record<string, any> = {
+        "secretkey/auth": {
+            "ACCESS_TOKEN_SECRET": process.env.ACCESS_TOKEN_SECRET,
+        },
+        "keys/operator": {
+            "OPERATOR_ID": process.env.OPERATOR_ID,
+            "OPERATOR_KEY": process.env.OPERATOR_KEY
+        },
+        "apikey/ipfs": {
+            "IPFS_STORAGE_API_KEY": process.env.IPFS_STORAGE_API_KEY
+        }
+    }
+
+    public async getSecrets(path: string, _?: any): Promise<any> {
+        if (MongoDbSecretManager.secretsFromEnvironment.hasOwnProperty(path)) {
+            return MongoDbSecretManager.secretsFromEnvironment[path];
+        }
+        const helper = new DataBaseHelper(Secret);
+        const secrets = await helper.findOne({ path });
+        if (!secrets) {
+            return null;
+        }
+        return JSON.parse(new Encryptor().decrypt(secrets.data));
+    }
+
+    public async setSecrets(path: string, data: any, _?: any): Promise<void> {
+        if (MongoDbSecretManager.secretsFromEnvironment.hasOwnProperty(path)) {
+            return;
+        }
+        console.log(`MONGODB-SECRETS: Setting secret ${path}`);
+        const secret = await this.getSecrets(path);
+        const helper = new DataBaseHelper(Secret);
+        const secretData = new Encryptor().encrypt(JSON.stringify(data));
+        if (secret) {
+            console.log(`MONGODB-SECRETS: Existing secret ${path}`);
+            secret.data = secretData;
+            await helper.update({ path }, secret);
+        } else {
+            console.log(`MONGODB-SECRETS: New secret ${path}`);
+            const row = helper.create({ path, data: secretData });
+            await helper.create({ path, data: secretData });
+            await helper.save(row)
+        }
+        console.log(`MONGODB-SECRETS: Completed setting secret ${path}`);
+
+        // TEMP: Verify encrypt/decrypt symetry
+        if ("Y" === process.env.VERIFY_SECRETS)
+        {
+            const verify = await this.getSecrets(path);
+            if (JSON.stringify(verify) !== JSON.stringify(data)) {
+                console.log(`MONGODB-SECRETS: VERIFICATION FAILED secret ${path}`);
+                console.log('Original:', data);
+                console.log('Returned:', verify);
+                throw new Error('Secrets verification failed');
+            }
+            console.log(`MONGODB-SECRETS: VERIFIED secret ${path}`);
+        }
+    }
+}

common/src/secret-manager/secret-manager-config.ts

@@ -24,6 +24,10 @@ export enum SecretManagerType {
    * Azure Secrets Manager
    */
   AZURE = 'azure',
+  /**
+   * MongoDB Secrets Manager
+   */
+  MONGODB = 'mongodb',
   /**
    * Old style secrets
    */
@@ -52,6 +56,8 @@ export class SecretManagerConfigs {
         return GcpSecretManagerConfigs.getConfigs()
       case SecretManagerType.AZURE:
         return AzureSecretManagerConfigs.getConfigs()
+      case SecretManagerType.MONGODB:
+        return
       case SecretManagerType.OLD_STYLE:
         return
       default:

common/src/secret-manager/secret-manager.ts

@@ -9,6 +9,7 @@ import { AzureSecretManager } from './azure/azure-secret-manager.js';
 import { IAzureSecretManagerConfigs } from './azure/azure-secret-manager-configs.js';
 import { IGcpSecretManagerConfigs } from './gcp/gcp-secret-manager-configs.js';
 import { GcpSecretManager } from './gcp/gcp-secret-manager.js';
+import { MongoDbSecretManager } from './mongodb/mongodb-secret-manager.js';
 
 /**
  * Class to get secret manager
@@ -62,6 +63,8 @@ export class SecretManager {
         return new GcpSecretManager(configs as IGcpSecretManagerConfigs)
       case SecretManagerType.AZURE:
         return new AzureSecretManager(configs as IAzureSecretManagerConfigs)
+      case SecretManagerType.MONGODB:
+        return new MongoDbSecretManager()
       case SecretManagerType.OLD_STYLE:
         return new OldSecretManager()
       default:

multi-build.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+docker build -t guardian-notification-service:latest -f ./notification-service/Dockerfile --platform linux/amd64,linux/arm64 .
+docker build -t guardian-logger-service:latest -f ./logger-service/Dockerfile --platform linux/amd64,linux/arm64 .
+docker build -t guardian-worker-service:latest -f ./worker-service/Dockerfile --platform linux/amd64,linux/arm64 .
+docker build -t guardian-auth-service:latest -f ./auth-service/Dockerfile.demo --platform linux/amd64,linux/arm64 .
+docker build -t guardian-api-gateway:latest -f ./api-gateway/Dockerfile.demo --platform linux/amd64,linux/arm64 .
+docker build -t guardian-policy-service:latest -f ./policy-service/Dockerfile --platform linux/amd64,linux/arm64 .
+docker build -t guardian-mrv-sender:latest -f ./mrv-sender/Dockerfile --platform linux/amd64,linux/arm64 .
+docker build -t guardian-guardian-service:latest -f ./guardian-service/Dockerfile --platform linux/amd64,linux/arm64 .
+docker build -t guardian-web-proxy:latest -f ./web-proxy/Dockerfile.demo --platform linux/amd64,linux/arm64 .
+docker build -t guardian-application-events:latest -f ./application-events/Dockerfile --platform linux/amd64,linux/arm64 .
+docker build -t guardian-topic-viewer:latest -f ./topic-viewer/Dockerfile --platform linux/amd64,linux/arm64 .