postgres-manager: Initiate postgres-manager (#126)

.github/workflows/prbuild.yml

@@ -31,7 +31,7 @@ jobs:
       # TODO see whether we can leverage https://github.com/pre-commit/action
       - name: Install system-wide tools dependencies
         run: |
-          pip3 install pre-commit detect-secrets black ggshield awscli-local
+          pip3 install pre-commit detect-secrets black ggshield
 
       - name: Checkout code
         uses: actions/checkout@v3

config/constants.ts

@@ -6,10 +6,13 @@ import {
 } from '../lib/workload/orcabus-stateless-stack';
 import { Duration, aws_lambda, RemovalPolicy } from 'aws-cdk-lib';
 import { EventSourceProps } from '../lib/workload/stateful/event_source/component';
+import { DbAuthType } from '../lib/workload/stateless/postgres_manager/function/utils';
 
 const regName = 'OrcaBusSchemaRegistry';
 const eventBusName = 'OrcaBusMain';
 const lambdaSecurityGroupName = 'OrcaBusLambdaSecurityGroup';
+const dbClusterIdentifier = 'orcabus-db';
+const dbClusterResourceIdParameterName = '/orcabus/db-cluster-resource-id';
 
 // Note, this should not end with a hyphen and 6 characters, otherwise secrets manager won't be
 // able to find the secret using a partial ARN.
@@ -27,7 +30,7 @@ const orcaBusStatefulConfig = {
     archiveRetention: 365,
   },
   databaseProps: {
-    clusterIdentifier: 'orcabus-db',
+    clusterIdentifier: dbClusterIdentifier,
     defaultDatabaseName: 'orcabus',
     version: AuroraPostgresEngineVersion.VER_15_4,
     parameterGroupName: 'default.aurora-postgresql15',
@@ -37,6 +40,7 @@ const orcaBusStatefulConfig = {
     monitoring: {
       cloudwatchLogsExports: ['orcabus-postgresql'],
     },
+    clusterResourceIdParameterName: dbClusterResourceIdParameterName,
   },
   securityGroupProps: {
     securityGroupName: lambdaSecurityGroupName,
@@ -67,6 +71,18 @@ const orcaBusStatelessConfig = {
   lambdaRuntimePythonVersion: aws_lambda.Runtime.PYTHON_3_10,
   bclConvertFunctionName: 'orcabus_bcl_convert',
   rdsMasterSecretName: rdsMasterSecretName,
+  postgresManagerConfig: {
+    masterSecretName: rdsMasterSecretName,
+    dbClusterIdentifier: dbClusterIdentifier,
+    clusterResourceIdParameterName: dbClusterResourceIdParameterName,
+    microserviceDbConfig: [
+      {
+        name: 'metadata_manager',
+        authType: DbAuthType.USERNAME_PASSWORD,
+      },
+      { name: 'filemanager', authType: DbAuthType.RDS_IAM },
+    ],
+  },
 };
 
 const eventSourceConfig: EventSourceProps = {

lib/workload/orcabus-stateless-stack.ts

@@ -3,10 +3,14 @@ import { Arn, aws_lambda } from 'aws-cdk-lib';
 import { Construct } from 'constructs';
 import { getVpc } from './stateful/vpc/component';
 import { MultiSchemaConstructProps } from './stateless/schema/component';
-import { IVpc, SecurityGroup } from 'aws-cdk-lib/aws-ec2';
+import { IVpc, ISecurityGroup, SecurityGroup } from 'aws-cdk-lib/aws-ec2';
 import { Filemanager } from './stateless/filemanager/deploy/lib/filemanager';
 import { Queue } from 'aws-cdk-lib/aws-sqs';
 import { Secret } from 'aws-cdk-lib/aws-secretsmanager';
+import {
+  PostgresManager,
+  PostgresManagerConfig,
+} from './stateless/postgres_manager/construct/postgresManager';
 
 export interface OrcaBusStatelessConfig {
   multiSchemaConstructProps: MultiSchemaConstructProps;
@@ -15,6 +19,7 @@ export interface OrcaBusStatelessConfig {
   lambdaRuntimePythonVersion: aws_lambda.Runtime;
   bclConvertFunctionName: string;
   rdsMasterSecretName: string;
+  postgresManagerConfig: PostgresManagerConfig;
   filemanagerDependencies?: FilemanagerDependencies;
 }
 
@@ -35,21 +40,20 @@ export interface FilemanagerDependencies {
 
 export class OrcaBusStatelessStack extends cdk.Stack {
   private vpc: IVpc;
+  private lambdaSecurityGroup: ISecurityGroup;
   constructor(scope: Construct, id: string, props: cdk.StackProps & OrcaBusStatelessConfig) {
     super(scope, id, props);
 
     // --- Constructs from Stateful stack or pre-existing resources
 
     this.vpc = getVpc(this);
 
-    // const securityGroups = [
-    //   aws_ec2.SecurityGroup.fromLookupByName(
-    //     this,
-    //     'LambdaSecurityGroup',
-    //     props.lambdaSecurityGroupName,
-    //     vpc
-    //   ),
-    // ];
+    this.lambdaSecurityGroup = SecurityGroup.fromLookupByName(
+      this,
+      'OrcaBusLambdaSecurityGroup',
+      props.lambdaSecurityGroupName,
+      this.vpc
+    );
 
     // const mainBus = EventBus.fromEventBusName(this, 'OrcaBusMain', props.eventBusName);
 
@@ -60,6 +64,8 @@ export class OrcaBusStatelessStack extends cdk.Stack {
     // hook microservice construct components here
     this.createSequenceRunManager();
 
+    this.createPostgresManager(props.postgresManagerConfig);
+
     if (props.filemanagerDependencies) {
       this.createFilemanager({
         ...props.filemanagerDependencies,
@@ -73,6 +79,14 @@ export class OrcaBusStatelessStack extends cdk.Stack {
     //   However, the implementation is still incomplete...
   }
 
+  private createPostgresManager(config: PostgresManagerConfig) {
+    new PostgresManager(this, 'PostgresManager', {
+      ...config,
+      vpc: this.vpc,
+      lambdaSecurityGroup: this.lambdaSecurityGroup,
+    });
+  }
+
   private createFilemanager(
     dependencies: FilemanagerDependencies & { lambdaSecurityGroupName: string }
   ) {

lib/workload/stateful/database/component.ts

@@ -2,6 +2,7 @@ import { Construct } from 'constructs';
 import { RemovalPolicy, Duration } from 'aws-cdk-lib';
 import * as rds from 'aws-cdk-lib/aws-rds';
 import * as ec2 from 'aws-cdk-lib/aws-ec2';
+import * as ssm from 'aws-cdk-lib/aws-ssm';
 import { SecurityGroup } from 'aws-cdk-lib/aws-ec2';
 import { DatabaseCluster } from 'aws-cdk-lib/aws-rds';
 
@@ -75,6 +76,10 @@ export type ConfigurableDatabaseProps = MonitoringProps & {
    * The database removal policy.
    */
   removalPolicy: RemovalPolicy;
+  /**
+   * The ssm parameter name to store the cluster resource id
+   */
+  clusterResourceIdParameterName: string;
 };
 
 /**
@@ -148,5 +153,13 @@ export class Database extends Construct {
         enablePerformanceInsights: props.enablePerformanceInsights,
       }),
     });
+
+    // saving the cluster id to be used in stateless stack on rds-iam
+    // ref: https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/UsingWithRDS.IAMDBAuth.IAMPolicy.html
+    new ssm.StringParameter(this, 'DbClusterResourceIdSSM', {
+      stringValue: this.cluster.clusterResourceIdentifier,
+      description: 'cluster resource id at the orcabus rds cluster',
+      parameterName: props.clusterResourceIdParameterName,
+    });
   }
 }

lib/workload/stateless/postgres_manager/.gitignore

@@ -0,0 +1,11 @@
+# https://yarnpkg.com/getting-started/qa#which-files-should-be-gitignored
+.pnp.*
+.yarn/*
+!.yarn/patches
+!.yarn/plugins
+!.yarn/releases
+!.yarn/sdks
+!.yarn/versions
+
+out.txt
+response.json

lib/workload/stateless/postgres_manager/README.md

@@ -0,0 +1,98 @@
+# RDS Postgres Manager
+
+This will deploy lambdas that will connect to the RDS instance with the master credential. This microservice is
+responsible for admin Postgres activity that requires superuser access.
+
+Before using this microservice you would need to configure this lambda by passing it as props from stateless config
+(`./config/constants.ts`) at the postgresManagerConfig.
+
+The config should register the microservice name and the connection type to the RDS. The connection can
+make use of `rds_iam` or the conventional `user-password` connection string.
+
+The microservice config should look as follows:
+
+```ts
+    microserviceDbConfig: [
+      {
+        name: 'metadata_manager',
+        authType: DbAuthType.USERNAME_PASSWORD,
+      },
+      { 
+        name: 'filemanager', 
+        authType: DbAuthType.RDS_IAM 
+      },
+    ]
+```
+
+The DbAuthType is defined at the [./function/utils.ts](./function/utils.ts) in this project and it is as follows:
+
+```ts
+export enum DbAuthType {
+  RDS_IAM,
+  USERNAME_PASSWORD,
+}
+```
+
+There are 4 lambdas in this stack:
+
+1. `orcabus-create-pg-db`
+
+    This aimed to create a new database name for the microservice. The database name will be the same as the
+    microservice name.
+
+    ```sh
+    aws lambda invoke \
+      --function-name orcabus-create-pg-db \
+      --payload '{ "microserviceName": "microservice_name" }' \
+      --cli-binary-format raw-in-base64-out \
+      response.json
+    ```
+
+2. `orcabus-create-pg-login-role`
+
+    Create a role with login credentials used for this microservice.
+    The name of the role would be the microservice name itself, and the credential will be saved into the secret
+    manager. The secret manager name is saved to `orcabus/microservice/${microserviceName}`.
+
+    Note: this will only work if the DbAuthType is configured to `USERNAME_PASSWORD`.
+
+    ```sh
+    aws lambda invoke \
+      --function-name orcabus-create-pg-login-role \
+      --payload '{ "microserviceName": "microservice_name" }' \
+      --cli-binary-format raw-in-base64-out \
+      response.json
+    ```
+
+3. `orcabus-create-pg-iam-role`
+
+    Create a new role and assign `rds_iam` role to this role to be able to connect over IAM database authentication.
+
+    A new managed policy will be created and the policy name
+    will be `orcabus-rds-connect-${microservice_name}`. This could be attached to your compute role for access to the RDS and the token needed. Follow the documentation from AWS to connect to the RDS [here](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/UsingWithRDS.IAMDBAuth.Connecting.html).
+
+    Note: this will only work if the microservice DbAuthType is configured to `RDS_IAM`.
+
+    Ref:
+    [aws-rds-iam-postgres](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/UsingWithRDS.IAMDBAuth.DBAccounts.html#UsingWithRDS.IAMDBAuth.DBAccounts.PostgreSQL)
+
+    ```sh
+    aws lambda invoke \
+      --function-name orcabus-create-pg-iam-role \
+      --payload '{ "microserviceName": "microservice_name" }' \
+      --cli-binary-format raw-in-base64-out \
+      response.json
+    ```
+
+4. `orcabus-alter-pg-db-owner`
+
+    Alter existing db to its respective role. The respective role will be the microservice user role created in either
+    lambda number 2 or 3.
+
+    ```sh
+    aws lambda invoke \
+      --function-name orcabus-alter-pg-db-owner \
+      --payload '{ "microserviceName": "microservice_name" }' \
+      --cli-binary-format raw-in-base64-out \
+      response.json
+    ```