From b89a1a3fb1cf72ddd9e3b7c8c75e93fd4129dee5 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Wed, 7 Feb 2024 09:19:28 +1100 Subject: [PATCH 01/13] feat(filemanager): add makefile --- Makefile | 2 +- .../stateful/filemanager/.env.example | 2 +- lib/workload/stateful/filemanager/Makefile | 49 ++++++++++++++++++ lib/workload/stateful/filemanager/README.md | 51 +++++++++---------- .../stateful/filemanager/docker-compose.yml | 18 ++----- 5 files changed, 78 insertions(+), 44 deletions(-) create mode 100644 lib/workload/stateful/filemanager/Makefile diff --git a/Makefile b/Makefile index 7c516fc13..4b594c98d 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ test: suite: test @(cd lib/workload/stateless/sequence_run_manager && $(MAKE) test) @(cd lib/workload/stateless/metadata_manager && $(MAKE) test) - @#(cd lib/workload/stateless/filemanager && $(MAKE) test) # FIXME uncomment when ready @Marko + @(cd lib/workload/stateful/filemanager && $(MAKE) test) clean: @yarn clean diff --git a/lib/workload/stateful/filemanager/.env.example b/lib/workload/stateful/filemanager/.env.example index 2f3c4f0b9..a5a86c8df 100644 --- a/lib/workload/stateful/filemanager/.env.example +++ b/lib/workload/stateful/filemanager/.env.example @@ -1,2 +1,2 @@ # Example env file which sets the DATABASE_URL to a local postgres instance. -DATABASE_URL=postgresql://filemanager:filemanager@localhost:5432/filemanager #pragma: allowlist secret +DATABASE_URL=postgresql://filemanager:filemanager@localhost:4321/filemanager #pragma: allowlist secret diff --git a/lib/workload/stateful/filemanager/Makefile b/lib/workload/stateful/filemanager/Makefile new file mode 100644 index 000000000..ff026dbba --- /dev/null +++ b/lib/workload/stateful/filemanager/Makefile @@ -0,0 +1,49 @@ +POSTGRES_DATABASE_URL ?= postgresql://filemanager:filemanager@localhost:4321/filemanager #pragma: allowlist secret + +export DATABASE_URL=$(POSTGRES_DATABASE_URL) + +## Default target +default: help + +install: build +all: build + +## Docker related targets +up: + @docker compose up --wait -d +down: + @docker compose down +docker-postgres: + @docker compose up --wait -d postgres +docker-clean: + @docker compose down --volumes + +## Build related commands +build: docker-postgres + @cargo build --all-features --all-targets +watch: build + @cargo watch -w *.toml -w *.rs +test: build + @cargo test --all-features --all-targets +lint: + @cargo fmt --check +lint-fix: + @cargo fmt +clippy: build + cargo clippy --all-targets --all-features +check: lint clippy +check-fix: lint-fix clippy + + +## Clean +clean: docker-clean + @cargo clean + +## Connect to the database +psql: + @docker exec -it filemanager_db psql filemanager -U filemanager + +## Help text +help: + @printf "The filemanager Makefile.\n\n" + @printf "Usage: make [POSTGRES_DATABASE_URL=]\n" \ No newline at end of file diff --git a/lib/workload/stateful/filemanager/README.md b/lib/workload/stateful/filemanager/README.md index 0cc85da94..810c80395 100644 --- a/lib/workload/stateful/filemanager/README.md +++ b/lib/workload/stateful/filemanager/README.md @@ -12,21 +12,23 @@ This project is split up into multiple crates in a workspace. For development, d ## Rust code development -Start the postgres database and ensure that an `.env` file is set containing the `DATABASE_URL`, e.g. see [`.env.example`][env-example]: +The filemanager uses docker to run a local postgres database to track objects, and sqlx, which connects to the database +at compile time to ensure that queries are valid. Compilation will emit errors if a query cannot successfully be run +on postgres database. + +Makefile is used to simplify development. To get started run: ```sh -docker compose up +make build ``` -The filemanager uses sqlx to check if queries succeed against a database at compile time. - -A `.env` file ensures that the sqlx code can check queries at compile time by providing a `DATABASE_URL`. If `.env` is not present and there's no active database running and ready to be connected to, this project will fail to compile at all (preventing unnecessary runtime errors). +This will spin up the docker postgres database, and compile the code using `cargo build`. -Filemanager uses docker to run a postgres database to track objects. This means that sqlx connects to the postgres server -running inside the docker compose container. If there are additional postgres installations locally (outside of docker), -this might interfere and complain about non-existing roles and users. +Note, that this sets an environment variable containing the `DATABASE_URL` to the local postgres database. This is variable +is expected by sqlx to check queries at compile time. If running `cargo` commands directly, this variable can be sourced +from an `.env` file. E.g. see [`.env.example`][env-example]. -### Tooling prerequisites, testing and building the code +### Tooling pre-requisites, testing and building the code For development of the rust workspace, it's recommended to install a build cache (sccache) to improve compilation speeds: @@ -34,52 +36,47 @@ For development of the rust workspace, it's recommended to install a build cache brew install sccache && export RUSTC_WRAPPER=`which sccache` ``` -or +or ```sh cargo install sccache && export RUSTC_WRAPPER=`which sccache` ``` -Then install build prerequisites to build: +Then, cargo-watch can be used to recompile files as they change: ```sh cargo install cargo-watch sqlx-cli -cargo build --all-targets --all-features +make watch ``` -## Local development +## Linting and testing Unit tests can be run with: ```sh -cargo test --all-targets --all-features +make test ``` -See the [deploy][deploy] directory for the cdk infrastructure code. +Which runs `cargo test`. -In a nutshell, a filemanager developer only needs to run the following to automatically recompile changes and re-deploy the changes: +To lint the code and format it, run: ```sh -./scripts/watch.sh +make check-fix ``` +This will run `cargo clippy` and `cargo fmt`. -Please don't use `scripts/deploy.sh` on production deployments, it is only meant for development. - -Formatting and clippy should also be run before committing changes: - -```sh -cargo clippy --all-targets --all-features -cargo fmt -``` +Testing and linting should be run before committing changes to the repository. +See the [deploy][deploy] directory for the cdk infrastructure code. ## Database -A shortcut for connecting to the docker database and inspecting its contents: +To connect to the local postgres database, run: ```bash -docker exec -it filemanager_db psql filemanager -U filemanager +make psql ``` Alternatively, just `brew install dbeaver-community` to easily browse the database contents (or any other DB viewer you prefer). diff --git a/lib/workload/stateful/filemanager/docker-compose.yml b/lib/workload/stateful/filemanager/docker-compose.yml index 132744106..ee8cfc40f 100644 --- a/lib/workload/stateful/filemanager/docker-compose.yml +++ b/lib/workload/stateful/filemanager/docker-compose.yml @@ -1,7 +1,7 @@ version: '3.1' services: - db: + postgres: build: database container_name: filemanager_db restart: always @@ -9,18 +9,6 @@ services: - POSTGRES_DATABASE=filemanager - POSTGRES_USER=filemanager - POSTGRES_PASSWORD=filemanager + - PGPORT=4321 ports: - - "5432:5432" - -# localstack: -# container_name: localstack-main -# image: localstack/localstack:3.0.1 -# ports: -# - "127.0.0.1:4566:4566" # LocalStack Gateway -# - "127.0.0.1:4510-4559:4510-4559" # external services port range -# environment: -# - DEBUG=${DEBUG-} -# - DOCKER_HOST=unix:///var/run/docker.sock -# volumes: -# - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack" -# - "/var/run/docker.sock:/var/run/docker.sock" + - "4321:4321" \ No newline at end of file From 21405a190a899e9df557536cc0e841adb6f3a3f2 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Wed, 7 Feb 2024 13:06:55 +1100 Subject: [PATCH 02/13] wip: removing database from filemanager --- lib/workload/orcabus-stateful-stack.ts | 6 ++-- lib/workload/stateful/database/component.ts | 31 ++++++++++++++----- .../stateful/databaseConstruct.test.ts | 6 ++-- 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/lib/workload/orcabus-stateful-stack.ts b/lib/workload/orcabus-stateful-stack.ts index 95e8e7347..12b0ba816 100644 --- a/lib/workload/orcabus-stateful-stack.ts +++ b/lib/workload/orcabus-stateful-stack.ts @@ -2,7 +2,7 @@ import * as cdk from 'aws-cdk-lib'; import { Construct } from 'constructs'; import { getVpc } from './stateful/vpc/component'; import { EventBusConstruct, EventBusProps } from './stateful/eventbridge/component'; -import { DatabaseConstruct, DatabaseProps } from './stateful/database/component'; +import { Database, DatabaseProps } from './stateful/database/component'; import { SecurityGroupConstruct, SecurityGroupProps } from './stateful/securitygroup/component'; import { SchemaRegistryConstruct, SchemaRegistryProps } from './stateful/schemaregistry/component'; @@ -15,7 +15,7 @@ export interface OrcaBusStatefulConfig { export class OrcaBusStatefulStack extends cdk.Stack { readonly eventBus: EventBusConstruct; - readonly database: DatabaseConstruct; + readonly database: Database; readonly securityGroup: SecurityGroupConstruct; readonly schemaRegistry: SchemaRegistryConstruct; @@ -37,7 +37,7 @@ export class OrcaBusStatefulStack extends cdk.Stack { props.securityGroupProps ); - this.database = new DatabaseConstruct(this, 'OrcaBusDatabaseConstruct', vpc, { + this.database = new Database(this, 'OrcaBusDatabaseConstruct', vpc, { ...props.databaseProps, }); diff --git a/lib/workload/stateful/database/component.ts b/lib/workload/stateful/database/component.ts index 423b62c4f..ce37339cf 100644 --- a/lib/workload/stateful/database/component.ts +++ b/lib/workload/stateful/database/component.ts @@ -39,9 +39,16 @@ export type DatabaseProps = MonitoringProps & { allowedInboundSG?: ec2.SecurityGroup; }; -export class DatabaseConstruct extends Construct { - readonly dbSecurityGroup: ec2.SecurityGroup; - readonly dbCluster: rds.DatabaseCluster; +export interface IDatabase { + readonly securityGroup: ec2.SecurityGroup; + readonly cluster: rds.DatabaseCluster; + readonly unsafeConnection: string; +} + +export class Database extends Construct implements IDatabase { + readonly securityGroup: ec2.SecurityGroup; + readonly cluster: rds.DatabaseCluster; + readonly unsafeConnection: string; constructor(scope: Construct, id: string, vpc: ec2.IVpc, props: DatabaseProps) { super(scope, id); @@ -51,7 +58,7 @@ export class DatabaseConstruct extends Construct { secretName: props.masterSecretName, }); - this.dbSecurityGroup = new ec2.SecurityGroup(this, 'DbSecurityGroup', { + this.securityGroup = new ec2.SecurityGroup(this, 'DbSecurityGroup', { vpc: vpc, allowAllOutbound: false, allowAllIpv6Outbound: false, @@ -60,14 +67,14 @@ export class DatabaseConstruct extends Construct { // give compute sg to access the rds if (props.allowedInboundSG) { - this.dbSecurityGroup.addIngressRule( + this.securityGroup.addIngressRule( props.allowedInboundSG, ec2.Port.tcp(props.dbPort), 'allow the OrcaBus compute sg to access db' ); } - this.dbCluster = new rds.DatabaseCluster(this, id + 'Cluster', { + this.cluster = new rds.DatabaseCluster(this, id + 'Cluster', { engine: rds.DatabaseClusterEngine.auroraPostgres({ version: props.version }), clusterIdentifier: props.clusterIdentifier, credentials: rds.Credentials.fromSecret(dbSecret), @@ -79,7 +86,7 @@ export class DatabaseConstruct extends Construct { ), port: props.dbPort, removalPolicy: RemovalPolicy.DESTROY, - securityGroups: [this.dbSecurityGroup], + securityGroups: [this.securityGroup], serverlessV2MaxCapacity: props.maxACU, serverlessV2MinCapacity: props.minACU, vpc: vpc, @@ -94,5 +101,15 @@ export class DatabaseConstruct extends Construct { enablePerformanceInsights: props.enablePerformanceInsights, }), }); + + this.unsafeConnection = + `postgres://` + + `${props.secret.secretValueFromJson('username').unsafeUnwrap()}` + + `:` + + `${props.secret.secretValueFromJson('password').unsafeUnwrap()}` + + `@` + + `${this._cluster.clusterEndpoint.socketAddress}` + + `/` + + `${props.databaseName}`; } } diff --git a/test/workload/stateful/databaseConstruct.test.ts b/test/workload/stateful/databaseConstruct.test.ts index 53e89e9bc..09fbefee0 100644 --- a/test/workload/stateful/databaseConstruct.test.ts +++ b/test/workload/stateful/databaseConstruct.test.ts @@ -1,6 +1,6 @@ import * as cdk from 'aws-cdk-lib'; import { Template } from 'aws-cdk-lib/assertions'; -import { DatabaseConstruct } from '../../../lib/workload/stateful/database/component'; +import { Database } from '../../../lib/workload/stateful/database/component'; import * as ec2 from 'aws-cdk-lib/aws-ec2'; import { getEnvironmentConfig } from '../../../config/constants'; @@ -21,7 +21,7 @@ beforeEach(() => { }); test('Test DBCluster created props', () => { - new DatabaseConstruct(stack, 'TestDatabaseConstruct', vpc, { + new Database(stack, 'TestDatabaseConstruct', vpc, { ...constructConfig.stackProps.orcaBusStatefulConfig.databaseProps, }); const template = Template.fromStack(stack); @@ -44,7 +44,7 @@ test('Test other SG Allow Ingress to DB SG', () => { }); const sgLogicalId = stack.getLogicalId(allowedSG.node.defaultChild as ec2.CfnSecurityGroup); - new DatabaseConstruct(stack, 'TestDatabaseConstruct', vpc, { + new Database(stack, 'TestDatabaseConstruct', vpc, { ...constructConfig.stackProps.orcaBusStatefulConfig.databaseProps, allowedInboundSG: allowedSG, }); From ee302b67299366d481558485c9fb23d81072dab0 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Thu, 8 Feb 2024 08:44:47 +1100 Subject: [PATCH 03/13] refactor: remove stateful components from filemanager --- lib/workload/orcabus-stateful-stack.ts | 7 +- lib/workload/orcabus-stateless-stack.ts | 21 ++ lib/workload/stateful/database/component.ts | 78 +++++++- .../filemanager/deploy/bin/filemanager.ts | 37 ---- .../deploy/constructs/cdk_resource_invoke.ts | 4 +- .../filemanager/deploy/constructs/database.ts | 185 ------------------ .../deploy/constructs/functions/function.ts | 24 +-- .../deploy/constructs/functions/ingest.ts | 22 ++- .../filemanager/deploy/lib/filemanager.ts | 77 ++++++++ .../deploy/lib/filemanager_stack.ts | 126 ------------ .../stateful/databaseConstruct.test.ts | 6 +- 11 files changed, 206 insertions(+), 381 deletions(-) delete mode 100644 lib/workload/stateful/filemanager/deploy/bin/filemanager.ts delete mode 100644 lib/workload/stateful/filemanager/deploy/constructs/database.ts create mode 100644 lib/workload/stateful/filemanager/deploy/lib/filemanager.ts delete mode 100644 lib/workload/stateful/filemanager/deploy/lib/filemanager_stack.ts diff --git a/lib/workload/orcabus-stateful-stack.ts b/lib/workload/orcabus-stateful-stack.ts index 12b0ba816..9bd3245c4 100644 --- a/lib/workload/orcabus-stateful-stack.ts +++ b/lib/workload/orcabus-stateful-stack.ts @@ -2,14 +2,14 @@ import * as cdk from 'aws-cdk-lib'; import { Construct } from 'constructs'; import { getVpc } from './stateful/vpc/component'; import { EventBusConstruct, EventBusProps } from './stateful/eventbridge/component'; -import { Database, DatabaseProps } from './stateful/database/component'; +import { Database, DatabasePropsNoVPC } from './stateful/database/component'; import { SecurityGroupConstruct, SecurityGroupProps } from './stateful/securitygroup/component'; import { SchemaRegistryConstruct, SchemaRegistryProps } from './stateful/schemaregistry/component'; export interface OrcaBusStatefulConfig { schemaRegistryProps: SchemaRegistryProps; eventBusProps: EventBusProps; - databaseProps: DatabaseProps; + databaseProps: DatabasePropsNoVPC; securityGroupProps: SecurityGroupProps; } @@ -37,7 +37,8 @@ export class OrcaBusStatefulStack extends cdk.Stack { props.securityGroupProps ); - this.database = new Database(this, 'OrcaBusDatabaseConstruct', vpc, { + this.database = new Database(this, 'OrcaBusDatabaseConstruct', { + vpc, ...props.databaseProps, }); diff --git a/lib/workload/orcabus-stateless-stack.ts b/lib/workload/orcabus-stateless-stack.ts index 2bb3e089b..c97c14142 100644 --- a/lib/workload/orcabus-stateless-stack.ts +++ b/lib/workload/orcabus-stateless-stack.ts @@ -4,6 +4,7 @@ import { Construct } from 'constructs'; import { getVpc } from './stateful/vpc/component'; import { MultiSchemaConstructProps } from './stateless/schema/component'; import { IVpc } from 'aws-cdk-lib/aws-ec2'; +import { Filemanager } from './stateful/filemanager/deploy/lib/filemanager'; export interface OrcaBusStatelessConfig { multiSchemaConstructProps: MultiSchemaConstructProps; @@ -40,10 +41,30 @@ export class OrcaBusStatelessStack extends cdk.Stack { // hook microservice construct components here this.createSequenceRunManager(); + + this.createFilemanager(); } private createSequenceRunManager() { // TODO new SequenceRunManagerConstruct() from lib/workload/stateless/sequence_run_manager/deploy/component.ts // However, the implementation is still incomplete... } + + private createFilemanager() { + // todo, implement after https://github.com/umccr/orcabus/issues/86 + // new Filemanager( + // this, + // "Filemanager", + // { + // buckets: [], + // buildEnvironment: {}, + // database: undefined, + // eventSources: [], + // migrateDatabase: false, + // onFailure: undefined, + // rustLog: '', + // vpc: undefined + // } + // ); + } } diff --git a/lib/workload/stateful/database/component.ts b/lib/workload/stateful/database/component.ts index ce37339cf..d84d44918 100644 --- a/lib/workload/stateful/database/component.ts +++ b/lib/workload/stateful/database/component.ts @@ -16,32 +16,90 @@ type MonitoringProps = { */ readonly enablePerformanceInsights?: boolean; /** - * performance insights retention period + * performance insights retention period. */ readonly performanceInsightsRetention?: rds.PerformanceInsightRetention; /** - * Enable enhanced monitoring by specifying the interval + * Enable enhanced monitoring by specifying the interval. */ readonly enhancedMonitoringInterval?: Duration; }; -export type DatabaseProps = MonitoringProps & { +/** + * Database props without a VPC. + */ +export type DatabasePropsNoVPC = MonitoringProps & { + /** + * The cluster identifier. + */ clusterIdentifier: string; + /** + * The initial database name created. + */ defaultDatabaseName: string; + /** + * Parameter group for the database. + */ parameterGroupName: string; + /** + * Database username. + */ username: string; + /** + * Database secret name. + */ masterSecretName: string; + /** + * Database engine version. + */ version: rds.AuroraPostgresEngineVersion; + /** + * Number of database instances. + */ numberOfInstance: number; + /** + * Min ACU for serverless database. + */ minACU: number; + /** + * Max ACU for serverless database. + */ maxACU: number; + /** + * Port to run the database on. + */ dbPort: number; + /** + * Inbound security groups that are allowed to connect to the database. + */ allowedInboundSG?: ec2.SecurityGroup; }; +/** + * Database props with a vpc. + */ +export type DatabaseProps = DatabasePropsNoVPC & { + /** + * The database VPC. + */ + vpc: ec2.IVpc; +}; + +/** + * Interface representing the database. + */ export interface IDatabase { + /** + * The database security group. + */ readonly securityGroup: ec2.SecurityGroup; + /** + * The database cluster. + */ readonly cluster: rds.DatabaseCluster; + /** + * A connection string to the database. + */ readonly unsafeConnection: string; } @@ -50,7 +108,7 @@ export class Database extends Construct implements IDatabase { readonly cluster: rds.DatabaseCluster; readonly unsafeConnection: string; - constructor(scope: Construct, id: string, vpc: ec2.IVpc, props: DatabaseProps) { + constructor(scope: Construct, id: string, props: DatabaseProps) { super(scope, id); const dbSecret = new rds.DatabaseSecret(this, id + 'DbSecret', { @@ -59,7 +117,7 @@ export class Database extends Construct implements IDatabase { }); this.securityGroup = new ec2.SecurityGroup(this, 'DbSecurityGroup', { - vpc: vpc, + vpc: props.vpc, allowAllOutbound: false, allowAllIpv6Outbound: false, description: 'security group for OrcaBus RDS', @@ -89,7 +147,7 @@ export class Database extends Construct implements IDatabase { securityGroups: [this.securityGroup], serverlessV2MaxCapacity: props.maxACU, serverlessV2MinCapacity: props.minACU, - vpc: vpc, + vpc: props.vpc, vpcSubnets: { subnetType: ec2.SubnetType.PRIVATE_ISOLATED, }, @@ -104,12 +162,12 @@ export class Database extends Construct implements IDatabase { this.unsafeConnection = `postgres://` + - `${props.secret.secretValueFromJson('username').unsafeUnwrap()}` + + `${dbSecret.secretValueFromJson('username').unsafeUnwrap()}` + `:` + - `${props.secret.secretValueFromJson('password').unsafeUnwrap()}` + + `${dbSecret.secretValueFromJson('password').unsafeUnwrap()}` + `@` + - `${this._cluster.clusterEndpoint.socketAddress}` + + `${this.cluster.clusterEndpoint.socketAddress}` + `/` + - `${props.databaseName}`; + `${props.defaultDatabaseName}`; } } diff --git a/lib/workload/stateful/filemanager/deploy/bin/filemanager.ts b/lib/workload/stateful/filemanager/deploy/bin/filemanager.ts deleted file mode 100644 index 061692463..000000000 --- a/lib/workload/stateful/filemanager/deploy/bin/filemanager.ts +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env node - -import 'source-map-support/register'; -import * as cdk from 'aws-cdk-lib'; -import { FilemanagerStack } from '../lib/filemanager_stack'; -import { Tags } from 'aws-cdk-lib'; - -export const STACK_NAME = 'FilemanagerStack'; -const STACK_DESCRIPTION = 'A stack deploying filemanager to dev.'; - -const app = new cdk.App(); -new FilemanagerStack( - app, - STACK_NAME, - { - stackName: STACK_NAME, - description: STACK_DESCRIPTION, - tags: { - Stack: STACK_NAME, - }, - env: { - region: 'ap-southeast-2', - }, - }, - { - destroyOnRemove: true, - enableMonitoring: { - enablePerformanceInsights: true, - }, - public: [ - // Put your IP here if you want the database to be reachable. - ], - migrateDatabase: process.env.FILEMANAGER_DEPLOY_MIGRATE_DATABASE == 'true', - } -); - -Tags.of(app).add('Stack', STACK_NAME); diff --git a/lib/workload/stateful/filemanager/deploy/constructs/cdk_resource_invoke.ts b/lib/workload/stateful/filemanager/deploy/constructs/cdk_resource_invoke.ts index 78f5fa9ea..5796a41c3 100644 --- a/lib/workload/stateful/filemanager/deploy/constructs/cdk_resource_invoke.ts +++ b/lib/workload/stateful/filemanager/deploy/constructs/cdk_resource_invoke.ts @@ -5,10 +5,10 @@ import { AwsSdkCall, PhysicalResourceId, } from 'aws-cdk-lib/custom-resources'; -import { IVpc, SubnetType } from 'aws-cdk-lib/aws-ec2'; import * as fn from './functions/function'; -import { ManagedPolicy, PolicyStatement, Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam'; import { CfnOutput, Stack, Token } from 'aws-cdk-lib'; +import { IVpc, SubnetType } from 'aws-cdk-lib/aws-ec2'; +import { ManagedPolicy, PolicyStatement, Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam'; /** * Props for the resource invoke construct. diff --git a/lib/workload/stateful/filemanager/deploy/constructs/database.ts b/lib/workload/stateful/filemanager/deploy/constructs/database.ts deleted file mode 100644 index 1b8530808..000000000 --- a/lib/workload/stateful/filemanager/deploy/constructs/database.ts +++ /dev/null @@ -1,185 +0,0 @@ -import { Construct } from 'constructs'; -import { IVpc, SecurityGroup, SubnetType } from 'aws-cdk-lib/aws-ec2'; -import { ISecret } from 'aws-cdk-lib/aws-secretsmanager'; -import { - AuroraPostgresEngineVersion, - ClusterInstance, - DatabaseCluster, - DatabaseClusterEngine, -} from 'aws-cdk-lib/aws-rds'; -import { aws_ec2 as ec2, aws_rds as rds, Duration, RemovalPolicy } from 'aws-cdk-lib'; -import { ManagedPolicy, Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam'; - -/** - * Props for enabling enhanced monitoring. - */ -export type EnableMonitoringProps = { - /** - * Add cloud watch exports. - */ - readonly cloudwatchLogsExports?: string[]; - /** - * Enable performance insights. - */ - readonly enablePerformanceInsights?: boolean; - /** - * The interval for monitoring, defaults to 60 seconds. - */ - readonly monitoringInterval?: Duration; -}; - -/** - * Settable database props that can be configured. - */ -export type DatabaseSettings = { - /** - * If present, specifies the database as public and adds additional inbound CIDRs to the security group. - */ - readonly public?: string[]; - /** - * Whether to destroy the database on stack removal. Defaults to keeping a snapshot. - */ - readonly destroyOnRemove?: boolean; - /** - * Enable enhanced monitoring. - */ - readonly enableMonitoring?: EnableMonitoringProps; - /** - * Minimum ACU capacity, defaults to 0.5. - */ - readonly minCapacity?: number; - /** - * Maximum ACU capacity, defaults to 4. - */ - readonly maxCapacity?: number; - /** - * Port to use for the database. The default for the engine is used if not specified. - */ - readonly port?: number; -}; - -/** - * Props for the database. - */ -export type DatabaseProps = DatabaseSettings & { - /** - * Vpc for the database. - */ - readonly vpc: IVpc; - /** - * Secret for database credentials - */ - readonly secret: ISecret; - /** - * The name of the database initially created. - */ - readonly databaseName: string; -}; - -/** - * A construct for the postgres database used with filemanager. - */ -export class Database extends Construct { - private readonly _securityGroup: SecurityGroup; - private readonly _cluster: DatabaseCluster; - private readonly _unsafeConnection: string; - - constructor(scope: Construct, id: string, props: DatabaseProps) { - super(scope, id); - - // Create security group with no outbound connections, because outbound connections - // shouldn't be very useful for a database anyway. - this._securityGroup = new SecurityGroup(this, 'SecurityGroup', { - vpc: props.vpc, - allowAllOutbound: false, - allowAllIpv6Outbound: false, - description: 'Security group for communicating with the filemanager RDS instance', - }); - - // Creates roles for enhanced RDS monitoring, if enabled. - let enableMonitoring; - if (props.enableMonitoring) { - const monitoringRole = new Role(this, 'DatabaseMonitoringRole', { - assumedBy: new ServicePrincipal('monitoring.rds.amazonaws.com'), - }); - monitoringRole.addManagedPolicy( - ManagedPolicy.fromAwsManagedPolicyName('service-role/AmazonRDSEnhancedMonitoringRole') - ); - - enableMonitoring = { - enablePerformanceInsights: props.enableMonitoring.enablePerformanceInsights, - cloudwatchLogsExports: props.enableMonitoring.cloudwatchLogsExports, - monitoringInterval: props.enableMonitoring.monitoringInterval?.toSeconds() ?? 60, - monitoringRoleArn: monitoringRole.roleArn, - }; - } - - // Serverless V2 Cluster. - this._cluster = new DatabaseCluster(this, 'Cluster', { - vpc: props.vpc, - vpcSubnets: { - subnetType: props.public ? SubnetType.PUBLIC : SubnetType.PRIVATE_ISOLATED, - }, - securityGroups: [this._securityGroup], - credentials: rds.Credentials.fromSecret(props.secret), - removalPolicy: props.destroyOnRemove ? RemovalPolicy.DESTROY : RemovalPolicy.SNAPSHOT, - defaultDatabaseName: props.databaseName, - port: props.port, - engine: DatabaseClusterEngine.auroraPostgres({ - version: AuroraPostgresEngineVersion.VER_15_4, - }), - serverlessV2MinCapacity: props.minCapacity ?? 0.5, - serverlessV2MaxCapacity: props.maxCapacity ?? 4, - writer: ClusterInstance.serverlessV2('Writer', { - ...(enableMonitoring && { ...enableMonitoring }), - }), - }); - - if (props.public) { - // If it's public, set the CIDRs from the config. - props.public.forEach((cidr) => { - this._securityGroup.addIngressRule( - ec2.Peer.ipv4(cidr), - ec2.Port.tcp(this._cluster.clusterEndpoint.port) - ); - }); - } else { - // Any inbound connections within the same security group are allowed access to the database port. - this._securityGroup.addIngressRule( - this._securityGroup, - ec2.Port.tcp(this._cluster.clusterEndpoint.port) - ); - } - - this._unsafeConnection = - `postgres://` + - `${props.secret.secretValueFromJson('username').unsafeUnwrap()}` + - `:` + - `${props.secret.secretValueFromJson('password').unsafeUnwrap()}` + - `@` + - `${this._cluster.clusterEndpoint.socketAddress}` + - `/` + - `${props.databaseName}`; - } - - /** - * Get the serverless cluster. - */ - get cluster(): DatabaseCluster { - return this._cluster; - } - - /** - * Get the security group for the database. - */ - get securityGroup(): SecurityGroup { - return this._securityGroup; - } - - /** - * Get the connection string. Unsafe because it contains the username and password. - */ - get unsafeConnection(): string { - return this._unsafeConnection; - } -} diff --git a/lib/workload/stateful/filemanager/deploy/constructs/functions/function.ts b/lib/workload/stateful/filemanager/deploy/constructs/functions/function.ts index 06346026b..5ab0b7a57 100644 --- a/lib/workload/stateful/filemanager/deploy/constructs/functions/function.ts +++ b/lib/workload/stateful/filemanager/deploy/constructs/functions/function.ts @@ -1,11 +1,11 @@ -import { IVpc, SecurityGroup, SubnetType } from 'aws-cdk-lib/aws-ec2'; -import { Database } from '../database'; -import { Architecture, IDestination, Version } from 'aws-cdk-lib/aws-lambda'; -import { ManagedPolicy, PolicyStatement, Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam'; import { Construct } from 'constructs'; import { RustFunction } from 'rust.aws-cdk-lambda'; import { Duration } from 'aws-cdk-lib'; import { Settings as CargoSettings } from 'rust.aws-cdk-lambda/dist/settings'; +import { IDatabase } from '../../../../database/component'; +import { IVpc, SecurityGroup, SubnetType } from 'aws-cdk-lib/aws-ec2'; +import { Architecture, IDestination, Version } from 'aws-cdk-lib/aws-lambda'; +import { ManagedPolicy, PolicyStatement, Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam'; /** * Settable values for a Rust function. @@ -32,15 +32,11 @@ export type FunctionPropsNoPackage = FunctionSettings & { /** * Database that the function uses. */ - readonly database: Database; + readonly database: IDatabase; /** * The destination to post failed invocations to. */ readonly onFailure?: IDestination; - /** - * Additional policies to add to the Lambda role. - */ - readonly policies?: PolicyStatement[]; /** * Name of the Lambda function resource. */ @@ -74,9 +70,6 @@ export class Function extends Construct { }); // Lambda needs VPC access if it is created in a VPC. this.addManagedPolicy('service-role/AWSLambdaVPCAccessExecutionRole'); - props.policies?.forEach((policy) => { - this._role.addToPolicy(policy); - }); // Lambda needs to be able to reach out to access S3, security manager (eventually), etc. // Could this use an endpoint instead? @@ -124,6 +117,13 @@ export class Function extends Construct { this._role.addManagedPolicy(ManagedPolicy.fromAwsManagedPolicyName(policyName)); } + /** + * Add a policy statement to this function's role. + */ + addToPolicy(policyStatement: PolicyStatement) { + this._role.addToPolicy(policyStatement) + } + /** * Get the function name. */ diff --git a/lib/workload/stateful/filemanager/deploy/constructs/functions/ingest.ts b/lib/workload/stateful/filemanager/deploy/constructs/functions/ingest.ts index 4f4426495..967ee3a90 100644 --- a/lib/workload/stateful/filemanager/deploy/constructs/functions/ingest.ts +++ b/lib/workload/stateful/filemanager/deploy/constructs/functions/ingest.ts @@ -1,7 +1,8 @@ import { Construct } from 'constructs'; import { IQueue } from 'aws-cdk-lib/aws-sqs'; -import * as lambdaEventSources from 'aws-cdk-lib/aws-lambda-event-sources'; import * as fn from './function'; +import { PolicyStatement } from 'aws-cdk-lib/aws-iam'; +import { SqsEventSource } from 'aws-cdk-lib/aws-lambda-event-sources'; /** * Settable values for the ingest function. @@ -16,7 +17,12 @@ export type IngestFunctionProps = IngestFunctionSettings & /** * The SQS queue URL to receive events from. */ - readonly queue: IQueue; + readonly eventSources: IQueue[]; + /** + * The buckets that the filemanager is expected to process. This will add policies to access the buckets via + * 's3:List*' and 's3:Get*'. + */ + readonly buckets: string[]; }; /** @@ -28,7 +34,15 @@ export class IngestFunction extends fn.Function { this.addManagedPolicy('service-role/AWSLambdaSQSQueueExecutionRole'); - const eventSource = new lambdaEventSources.SqsEventSource(props.queue); - this.function.addEventSource(eventSource); + props.eventSources.forEach((source) => { + const eventSource = new SqsEventSource(source); + this.function.addEventSource(eventSource); + }); + props.buckets.map((bucket) => { + this.addToPolicy(new PolicyStatement({ + actions: ['s3:List*', 's3:Get*'], + resources: [`arn:aws:s3:::${bucket}/*`], + })); + }) } } diff --git a/lib/workload/stateful/filemanager/deploy/lib/filemanager.ts b/lib/workload/stateful/filemanager/deploy/lib/filemanager.ts new file mode 100644 index 000000000..df27f921a --- /dev/null +++ b/lib/workload/stateful/filemanager/deploy/lib/filemanager.ts @@ -0,0 +1,77 @@ +import { Stack, StackProps } from 'aws-cdk-lib'; +import { Construct } from 'constructs'; +import { IngestFunction, IngestFunctionSettings } from '../constructs/functions/ingest'; +import { CdkResourceInvoke } from '../constructs/cdk_resource_invoke'; +import { MigrateFunction } from '../constructs/functions/migrate'; +import * as fn from '../constructs/functions/function'; +import { IDatabase } from '../../../database/component'; +import { IVpc } from 'aws-cdk-lib/aws-ec2'; +import { IQueue } from 'aws-cdk-lib/aws-sqs'; +import { IDestination } from 'aws-cdk-lib/aws-lambda'; + +/** + * Common settings for the filemanager stack. + */ +type Settings = IngestFunctionSettings & { + /** + * VPC to use for filemanager. + */ + readonly vpc: IVpc, + /** + * The database for filemanager. + */ + readonly database: IDatabase; + /** + * Whether to initialize a database migration. + */ + readonly migrateDatabase?: boolean; + /** + * Event sources to use for the filemanager + */ + readonly eventSources: IQueue[]; + /** + * Location to send any events that could not be processed. + */ + readonly onFailure: IDestination; + /** + * The buckets that the filemanager is expected to process. This will add policies to access the buckets via + * 's3:List*' and 's3:Get*'. + */ + readonly buckets: string[]; +}; + +/** + * Construct used to configure the filemanager. + */ +export class Filemanager extends Construct { + constructor(scope: Construct, id: string, settings: Settings) { + super(scope, id); + + if (settings?.migrateDatabase) { + new CdkResourceInvoke(this, 'MigrateDatabase', { + vpc: settings.vpc, + createFunction: (scope: Construct, id: string, props: fn.FunctionPropsNoPackage) => { + return new MigrateFunction(scope, id, props); + }, + functionProps: { + vpc: settings.vpc, + database: settings.database, + buildEnvironment: settings?.buildEnvironment, + rustLog: settings?.rustLog, + }, + id: 'MigrateFunction', + dependencies: [settings.database.cluster], + }); + } + + new IngestFunction(this, 'IngestLambda', { + vpc: settings.vpc, + database: settings.database, + eventSources: settings.eventSources, + onFailure: settings.onFailure, + buckets: settings.buckets, + buildEnvironment: settings?.buildEnvironment, + rustLog: settings?.rustLog, + }); + } +} diff --git a/lib/workload/stateful/filemanager/deploy/lib/filemanager_stack.ts b/lib/workload/stateful/filemanager/deploy/lib/filemanager_stack.ts deleted file mode 100644 index 3f91c0816..000000000 --- a/lib/workload/stateful/filemanager/deploy/lib/filemanager_stack.ts +++ /dev/null @@ -1,126 +0,0 @@ -import { RemovalPolicy, Stack, StackProps } from 'aws-cdk-lib'; -import { Construct } from 'constructs'; -import * as s3 from 'aws-cdk-lib/aws-s3'; -import * as lambdaDestinations from 'aws-cdk-lib/aws-lambda-destinations'; -import * as ec2 from 'aws-cdk-lib/aws-ec2'; -import { IngestFunction, IngestFunctionSettings } from '../constructs/functions/ingest'; -import { Database, DatabaseSettings } from '../constructs/database'; -import { SubnetType } from 'aws-cdk-lib/aws-ec2'; -import { SqsDestination } from 'aws-cdk-lib/aws-s3-notifications'; -import { Bucket, EventType } from 'aws-cdk-lib/aws-s3'; -import { PolicyStatement } from 'aws-cdk-lib/aws-iam'; -import { Queue } from 'aws-cdk-lib/aws-sqs'; -import { Secret } from 'aws-cdk-lib/aws-secretsmanager'; -import { CdkResourceInvoke } from '../constructs/cdk_resource_invoke'; -import { MigrateFunction } from '../constructs/functions/migrate'; -import * as fn from '../constructs/functions/function'; - -/** - * Common settings for the filemanager stack. - */ -type Settings = DatabaseSettings & - IngestFunctionSettings & { - /** - * The name of the database. Defaults to `filemanager`. - */ - databaseName?: string; - /** - * Whether to initialize a database migration. - */ - migrateDatabase?: boolean; - }; - -/** - * Stack used to deploy filemanager. - */ -export class FilemanagerStack extends Stack { - constructor(scope: Construct, id: string, props: StackProps, settings?: Settings) { - super(scope, id, props); - - const queue = new Queue(this, 'Queue'); - - const testBucket = new Bucket(this, 'Bucket', { - bucketName: 'filemanager-test-ingest', - blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL, - encryption: s3.BucketEncryption.S3_MANAGED, - enforceSSL: true, - removalPolicy: RemovalPolicy.DESTROY, - }); - const testBucketPolicy = new PolicyStatement({ - actions: ['s3:List*', 's3:Get*'], - resources: ['arn:aws:s3:::*'], - }); - testBucket.addEventNotification(EventType.OBJECT_CREATED, new SqsDestination(queue)); - testBucket.addEventNotification(EventType.OBJECT_REMOVED, new SqsDestination(queue)); - - const deadLetterQueue = new Queue(this, 'DeadLetterQueue'); - const deadLetterQueueDestination = new lambdaDestinations.SqsDestination(deadLetterQueue); - - const vpc = new ec2.Vpc(this, 'Vpc', { - maxAzs: 99, // As many as there are available in the region - natGateways: 1, - subnetConfiguration: [ - { - name: 'ingress', - subnetType: SubnetType.PUBLIC, - }, - { - name: 'application', - subnetType: SubnetType.PRIVATE_WITH_EGRESS, - }, - { - name: 'database', - subnetType: SubnetType.PRIVATE_ISOLATED, - }, - ], - }); - - const secret = new Secret(this, 'FilemanagerDatabaseSecret', { - secretName: 'FilemanagerDatabaseSecret', // pragma: allowlist secret - generateSecretString: { - secretStringTemplate: JSON.stringify({ username: 'filemanager' }), - excludePunctuation: true, - generateStringKey: 'password', - }, - }); - - const database = new Database(this, 'Database', { - vpc, - databaseName: settings?.databaseName ?? 'filemanager', - secret, - destroyOnRemove: settings?.destroyOnRemove, - enableMonitoring: settings?.enableMonitoring, - minCapacity: settings?.minCapacity, - maxCapacity: settings?.maxCapacity, - port: settings?.port, - public: settings?.public, - }); - - if (settings?.migrateDatabase) { - new CdkResourceInvoke(this, 'MigrateDatabase', { - vpc, - createFunction: (scope: Construct, id: string, props: fn.FunctionPropsNoPackage) => { - return new MigrateFunction(scope, id, props); - }, - functionProps: { - vpc, - database, - buildEnvironment: settings?.buildEnvironment, - rustLog: settings?.rustLog, - }, - id: 'MigrateFunction', - dependencies: [database.cluster], - }); - } - - new IngestFunction(this, 'IngestLambda', { - vpc, - database, - queue, - onFailure: deadLetterQueueDestination, - policies: [testBucketPolicy], - buildEnvironment: settings?.buildEnvironment, - rustLog: settings?.rustLog, - }); - } -} diff --git a/test/workload/stateful/databaseConstruct.test.ts b/test/workload/stateful/databaseConstruct.test.ts index 09fbefee0..6f309ba93 100644 --- a/test/workload/stateful/databaseConstruct.test.ts +++ b/test/workload/stateful/databaseConstruct.test.ts @@ -21,7 +21,8 @@ beforeEach(() => { }); test('Test DBCluster created props', () => { - new Database(stack, 'TestDatabaseConstruct', vpc, { + new Database(stack, 'TestDatabaseConstruct', { + vpc, ...constructConfig.stackProps.orcaBusStatefulConfig.databaseProps, }); const template = Template.fromStack(stack); @@ -44,7 +45,8 @@ test('Test other SG Allow Ingress to DB SG', () => { }); const sgLogicalId = stack.getLogicalId(allowedSG.node.defaultChild as ec2.CfnSecurityGroup); - new Database(stack, 'TestDatabaseConstruct', vpc, { + new Database(stack, 'TestDatabaseConstruct', { + vpc, ...constructConfig.stackProps.orcaBusStatefulConfig.databaseProps, allowedInboundSG: allowedSG, }); From 9e74dac00923804e67b696422e318bdd3465ccfd Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Thu, 8 Feb 2024 09:49:33 +1100 Subject: [PATCH 04/13] refactor: move filemanager from stateful to stateless --- lib/workload/orcabus-stateless-stack.ts | 1 - lib/workload/stateful/database/component.ts | 1 + .../filemanager/deploy/package-lock.json | 2545 ----------------- .../filemanager/.env.example | 0 .../filemanager/.gitignore | 0 .../filemanager/Cargo.lock | 0 .../filemanager/Cargo.toml | 0 .../filemanager/Makefile | 0 .../filemanager/README.md | 0 .../filemanager/build.rs | 0 .../filemanager/database/Dockerfile | 0 .../migrations/0001_add_object_table.sql | 0 .../migrations/0002_add_s3_object_table.sql | 0 .../aws/insert_s3_created_objects.sql | 0 .../aws/insert_s3_deleted_objects.sql | 0 .../queries/ingester/aws/update_deleted.sql | 0 .../queries/ingester/insert_objects.sql | 0 .../filemanager/deploy/README.md | 0 .../filemanager/deploy/cdk.json | 0 .../deploy/constructs/cdk_resource_invoke.ts | 0 .../deploy/constructs/functions/function.ts | 2 +- .../deploy/constructs/functions/ingest.ts | 0 .../deploy/constructs/functions/migrate.ts | 0 .../filemanager/deploy/lib/filemanager.ts | 3 +- .../filemanager/deploy/package.json | 0 .../filemanager/deploy/tsconfig.json | 0 .../filemanager/docker-compose.yml | 0 .../filemanager-http-lambda/Cargo.toml | 0 .../filemanager-http-lambda/README.md | 0 .../filemanager-http-lambda/src/main.rs | 0 .../filemanager-ingest-lambda/Cargo.toml | 0 .../filemanager-ingest-lambda/README.md | 0 .../filemanager-ingest-lambda/src/main.rs | 0 .../filemanager-migrate-lambda/Cargo.toml | 0 .../filemanager-migrate-lambda/src/main.rs | 0 .../filemanager/filemanager/Cargo.lock | 0 .../filemanager/filemanager/Cargo.toml | 0 .../filemanager/filemanager/README.md | 0 .../filemanager/src/clients/aws/config.rs | 0 .../filemanager/src/clients/aws/mod.rs | 0 .../filemanager/src/clients/aws/s3.rs | 0 .../filemanager/src/clients/aws/sqs.rs | 0 .../filemanager/src/clients/mod.rs | 0 .../filemanager/src/database/aws/ingester.rs | 0 .../filemanager/src/database/aws/migration.rs | 0 .../filemanager/src/database/aws/mod.rs | 0 .../filemanager/src/database/mod.rs | 0 .../filemanager/filemanager/src/env.rs | 0 .../filemanager/filemanager/src/error.rs | 0 .../filemanager/src/events/aws/collecter.rs | 0 .../src/events/aws/collector_builder.rs | 0 .../filemanager/src/events/aws/mod.rs | 0 .../filemanager/filemanager/src/events/mod.rs | 0 .../filemanager/src/handlers/aws.rs | 0 .../filemanager/src/handlers/mod.rs | 0 .../filemanager/filemanager/src/lib.rs | 0 .../filemanager/scripts/deploy.sh | 0 .../filemanager/scripts/logs.sh | 0 .../filemanager/scripts/watch.sh | 0 59 files changed, 3 insertions(+), 2549 deletions(-) delete mode 100644 lib/workload/stateful/filemanager/deploy/package-lock.json rename lib/workload/{stateful => stateless}/filemanager/.env.example (100%) rename lib/workload/{stateful => stateless}/filemanager/.gitignore (100%) rename lib/workload/{stateful => stateless}/filemanager/Cargo.lock (100%) rename lib/workload/{stateful => stateless}/filemanager/Cargo.toml (100%) rename lib/workload/{stateful => stateless}/filemanager/Makefile (100%) rename lib/workload/{stateful => stateless}/filemanager/README.md (100%) rename lib/workload/{stateful => stateless}/filemanager/build.rs (100%) rename lib/workload/{stateful => stateless}/filemanager/database/Dockerfile (100%) rename lib/workload/{stateful => stateless}/filemanager/database/migrations/0001_add_object_table.sql (100%) rename lib/workload/{stateful => stateless}/filemanager/database/migrations/0002_add_s3_object_table.sql (100%) rename lib/workload/{stateful => stateless}/filemanager/database/queries/ingester/aws/insert_s3_created_objects.sql (100%) rename lib/workload/{stateful => stateless}/filemanager/database/queries/ingester/aws/insert_s3_deleted_objects.sql (100%) rename lib/workload/{stateful => stateless}/filemanager/database/queries/ingester/aws/update_deleted.sql (100%) rename lib/workload/{stateful => stateless}/filemanager/database/queries/ingester/insert_objects.sql (100%) rename lib/workload/{stateful => stateless}/filemanager/deploy/README.md (100%) rename lib/workload/{stateful => stateless}/filemanager/deploy/cdk.json (100%) rename lib/workload/{stateful => stateless}/filemanager/deploy/constructs/cdk_resource_invoke.ts (100%) rename lib/workload/{stateful => stateless}/filemanager/deploy/constructs/functions/function.ts (98%) rename lib/workload/{stateful => stateless}/filemanager/deploy/constructs/functions/ingest.ts (100%) rename lib/workload/{stateful => stateless}/filemanager/deploy/constructs/functions/migrate.ts (100%) rename lib/workload/{stateful => stateless}/filemanager/deploy/lib/filemanager.ts (95%) rename lib/workload/{stateful => stateless}/filemanager/deploy/package.json (100%) rename lib/workload/{stateful => stateless}/filemanager/deploy/tsconfig.json (100%) rename lib/workload/{stateful => stateless}/filemanager/docker-compose.yml (100%) rename lib/workload/{stateful => stateless}/filemanager/filemanager-http-lambda/Cargo.toml (100%) rename lib/workload/{stateful => stateless}/filemanager/filemanager-http-lambda/README.md (100%) rename lib/workload/{stateful => stateless}/filemanager/filemanager-http-lambda/src/main.rs (100%) rename lib/workload/{stateful => stateless}/filemanager/filemanager-ingest-lambda/Cargo.toml (100%) rename lib/workload/{stateful => stateless}/filemanager/filemanager-ingest-lambda/README.md (100%) rename lib/workload/{stateful => stateless}/filemanager/filemanager-ingest-lambda/src/main.rs (100%) rename lib/workload/{stateful => stateless}/filemanager/filemanager-migrate-lambda/Cargo.toml (100%) rename lib/workload/{stateful => stateless}/filemanager/filemanager-migrate-lambda/src/main.rs (100%) rename lib/workload/{stateful => stateless}/filemanager/filemanager/Cargo.lock (100%) rename lib/workload/{stateful => stateless}/filemanager/filemanager/Cargo.toml (100%) rename lib/workload/{stateful => stateless}/filemanager/filemanager/README.md (100%) rename lib/workload/{stateful => stateless}/filemanager/filemanager/src/clients/aws/config.rs (100%) rename lib/workload/{stateful => stateless}/filemanager/filemanager/src/clients/aws/mod.rs (100%) rename lib/workload/{stateful => stateless}/filemanager/filemanager/src/clients/aws/s3.rs (100%) rename lib/workload/{stateful => stateless}/filemanager/filemanager/src/clients/aws/sqs.rs (100%) rename lib/workload/{stateful => stateless}/filemanager/filemanager/src/clients/mod.rs (100%) rename lib/workload/{stateful => stateless}/filemanager/filemanager/src/database/aws/ingester.rs (100%) rename lib/workload/{stateful => stateless}/filemanager/filemanager/src/database/aws/migration.rs (100%) rename lib/workload/{stateful => stateless}/filemanager/filemanager/src/database/aws/mod.rs (100%) rename lib/workload/{stateful => stateless}/filemanager/filemanager/src/database/mod.rs (100%) rename lib/workload/{stateful => stateless}/filemanager/filemanager/src/env.rs (100%) rename lib/workload/{stateful => stateless}/filemanager/filemanager/src/error.rs (100%) rename lib/workload/{stateful => stateless}/filemanager/filemanager/src/events/aws/collecter.rs (100%) rename lib/workload/{stateful => stateless}/filemanager/filemanager/src/events/aws/collector_builder.rs (100%) rename lib/workload/{stateful => stateless}/filemanager/filemanager/src/events/aws/mod.rs (100%) rename lib/workload/{stateful => stateless}/filemanager/filemanager/src/events/mod.rs (100%) rename lib/workload/{stateful => stateless}/filemanager/filemanager/src/handlers/aws.rs (100%) rename lib/workload/{stateful => stateless}/filemanager/filemanager/src/handlers/mod.rs (100%) rename lib/workload/{stateful => stateless}/filemanager/filemanager/src/lib.rs (100%) rename lib/workload/{stateful => stateless}/filemanager/scripts/deploy.sh (100%) rename lib/workload/{stateful => stateless}/filemanager/scripts/logs.sh (100%) rename lib/workload/{stateful => stateless}/filemanager/scripts/watch.sh (100%) diff --git a/lib/workload/orcabus-stateless-stack.ts b/lib/workload/orcabus-stateless-stack.ts index c97c14142..1b3ce0c84 100644 --- a/lib/workload/orcabus-stateless-stack.ts +++ b/lib/workload/orcabus-stateless-stack.ts @@ -4,7 +4,6 @@ import { Construct } from 'constructs'; import { getVpc } from './stateful/vpc/component'; import { MultiSchemaConstructProps } from './stateless/schema/component'; import { IVpc } from 'aws-cdk-lib/aws-ec2'; -import { Filemanager } from './stateful/filemanager/deploy/lib/filemanager'; export interface OrcaBusStatelessConfig { multiSchemaConstructProps: MultiSchemaConstructProps; diff --git a/lib/workload/stateful/database/component.ts b/lib/workload/stateful/database/component.ts index d84d44918..7470712df 100644 --- a/lib/workload/stateful/database/component.ts +++ b/lib/workload/stateful/database/component.ts @@ -160,6 +160,7 @@ export class Database extends Construct implements IDatabase { }), }); + // Todo use secrets manager for this and query for password within Lambda functions. this.unsafeConnection = `postgres://` + `${dbSecret.secretValueFromJson('username').unsafeUnwrap()}` + diff --git a/lib/workload/stateful/filemanager/deploy/package-lock.json b/lib/workload/stateful/filemanager/deploy/package-lock.json deleted file mode 100644 index 9f8b1741f..000000000 --- a/lib/workload/stateful/filemanager/deploy/package-lock.json +++ /dev/null @@ -1,2545 +0,0 @@ -{ - "name": "filemanager", - "version": "0.1", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "filemanager", - "version": "0.1", - "dependencies": { - "@aws-cdk/aws-apigatewayv2-alpha": "^2.114.1-alpha.0", - "@aws-cdk/aws-apigatewayv2-authorizers-alpha": "^2.114.1-alpha.0", - "@aws-cdk/aws-apigatewayv2-integrations-alpha": "^2.114.1-alpha.0", - "aws-cdk-lib": "^2.114.1", - "constructs": "^10.3.0", - "dotenv": "^16.3.1", - "rust.aws-cdk-lambda": "^1.2.1", - "source-map-support": "^0.5.21" - }, - "bin": { - "filemanager": "bin/filemanager.js" - }, - "devDependencies": { - "@types/node": "^20.5.9", - "@typescript-eslint/eslint-plugin": "^6.14.0", - "aws-cdk": "^2.114.1", - "cross-env": "^7.0.3", - "prettier": "3.0.3", - "typescript": "^5.3.3" - } - }, - "node_modules/@aashutoshrathi/word-wrap": { - "version": "1.2.6", - "resolved": "https://registry.npmjs.org/@aashutoshrathi/word-wrap/-/word-wrap-1.2.6.tgz", - "integrity": "sha512-1Yjs2SvM8TflER/OD3cOjhWWOZb58A2t7wpE2S9XfBYTiIl+XFhQG2bjy4Pu1I+EAlCNUzRDYDdFwFYUKvXcIA==", - "dev": true, - "peer": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/@aws-cdk/asset-awscli-v1": { - "version": "2.2.201", - "resolved": "https://registry.npmjs.org/@aws-cdk/asset-awscli-v1/-/asset-awscli-v1-2.2.201.tgz", - "integrity": "sha512-INZqcwDinNaIdb5CtW3ez5s943nX5stGBQS6VOP2JDlOFP81hM3fds/9NDknipqfUkZM43dx+HgVvkXYXXARCQ==" - }, - "node_modules/@aws-cdk/asset-kubectl-v20": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/@aws-cdk/asset-kubectl-v20/-/asset-kubectl-v20-2.1.2.tgz", - "integrity": "sha512-3M2tELJOxQv0apCIiuKQ4pAbncz9GuLwnKFqxifWfe77wuMxyTRPmxssYHs42ePqzap1LT6GDcPygGs+hHstLg==" - }, - "node_modules/@aws-cdk/asset-node-proxy-agent-v6": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/@aws-cdk/asset-node-proxy-agent-v6/-/asset-node-proxy-agent-v6-2.0.1.tgz", - "integrity": "sha512-DDt4SLdLOwWCjGtltH4VCST7hpOI5DzieuhGZsBpZ+AgJdSI2GCjklCXm0GCTwJG/SolkL5dtQXyUKgg9luBDg==" - }, - "node_modules/@aws-cdk/aws-apigatewayv2-alpha": { - "version": "2.114.1-alpha.0", - "resolved": "https://registry.npmjs.org/@aws-cdk/aws-apigatewayv2-alpha/-/aws-apigatewayv2-alpha-2.114.1-alpha.0.tgz", - "integrity": "sha512-+urpw7rGrtdGvnHQlDXVfpI3TmQJpjuT9jTOeuuG5dNDczLJrUokBvQdj6H6KsngdmBC07WfWU+yL2MBp71ozA==", - "engines": { - "node": ">= 14.15.0" - }, - "peerDependencies": { - "aws-cdk-lib": "^2.114.1", - "constructs": "^10.0.0" - } - }, - "node_modules/@aws-cdk/aws-apigatewayv2-authorizers-alpha": { - "version": "2.114.1-alpha.0", - "resolved": "https://registry.npmjs.org/@aws-cdk/aws-apigatewayv2-authorizers-alpha/-/aws-apigatewayv2-authorizers-alpha-2.114.1-alpha.0.tgz", - "integrity": "sha512-wvQ7l+VsDf8SgcOylLnp/6SirDcE5D3WNyQMl5ynxtmzuK5OKSnlC07bruKtKpYVT4A/6tBq/VVILfCBphyMJg==", - "engines": { - "node": ">= 14.15.0" - }, - "peerDependencies": { - "@aws-cdk/aws-apigatewayv2-alpha": "2.114.1-alpha.0", - "aws-cdk-lib": "^2.114.1", - "constructs": "^10.0.0" - } - }, - "node_modules/@aws-cdk/aws-apigatewayv2-integrations-alpha": { - "version": "2.114.1-alpha.0", - "resolved": "https://registry.npmjs.org/@aws-cdk/aws-apigatewayv2-integrations-alpha/-/aws-apigatewayv2-integrations-alpha-2.114.1-alpha.0.tgz", - "integrity": "sha512-iB7vHoTguDKLeatJS4p/8OBqH4oLCG2xBn1toUFwMD9iWoRGahGiK0pYuq24baab3SuNS1LFThJw5Zu0R+cZGA==", - "engines": { - "node": ">= 14.15.0" - }, - "peerDependencies": { - "@aws-cdk/aws-apigatewayv2-alpha": "2.114.1-alpha.0", - "aws-cdk-lib": "^2.114.1", - "constructs": "^10.0.0" - } - }, - "node_modules/@eslint-community/eslint-utils": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz", - "integrity": "sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA==", - "dev": true, - "dependencies": { - "eslint-visitor-keys": "^3.3.0" - }, - "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" - }, - "peerDependencies": { - "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" - } - }, - "node_modules/@eslint-community/regexpp": { - "version": "4.10.0", - "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.10.0.tgz", - "integrity": "sha512-Cu96Sd2By9mCNTx2iyKOmq10v22jUVQv0lQnlGNy16oE9589yE+QADPbrMGCkA51cKZSg3Pu/aTJVTGfL/qjUA==", - "dev": true, - "engines": { - "node": "^12.0.0 || ^14.0.0 || >=16.0.0" - } - }, - "node_modules/@eslint/eslintrc": { - "version": "2.1.4", - "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.1.4.tgz", - "integrity": "sha512-269Z39MS6wVJtsoUl10L60WdkhJVdPG24Q4eZTH3nnF6lpvSShEK3wQjDX9JRWAUPvPh7COouPpU9IrqaZFvtQ==", - "dev": true, - "peer": true, - "dependencies": { - "ajv": "^6.12.4", - "debug": "^4.3.2", - "espree": "^9.6.0", - "globals": "^13.19.0", - "ignore": "^5.2.0", - "import-fresh": "^3.2.1", - "js-yaml": "^4.1.0", - "minimatch": "^3.1.2", - "strip-json-comments": "^3.1.1" - }, - "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "node_modules/@eslint/js": { - "version": "8.55.0", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.55.0.tgz", - "integrity": "sha512-qQfo2mxH5yVom1kacMtZZJFVdW+E70mqHMJvVg6WTLo+VBuQJ4TojZlfWBjK0ve5BdEeNAVxOsl/nvNMpJOaJA==", - "dev": true, - "peer": true, - "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" - } - }, - "node_modules/@humanwhocodes/config-array": { - "version": "0.11.13", - "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.13.tgz", - "integrity": "sha512-JSBDMiDKSzQVngfRjOdFXgFfklaXI4K9nLF49Auh21lmBWRLIK3+xTErTWD4KU54pb6coM6ESE7Awz/FNU3zgQ==", - "dev": true, - "peer": true, - "dependencies": { - "@humanwhocodes/object-schema": "^2.0.1", - "debug": "^4.1.1", - "minimatch": "^3.0.5" - }, - "engines": { - "node": ">=10.10.0" - } - }, - "node_modules/@humanwhocodes/module-importer": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", - "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", - "dev": true, - "peer": true, - "engines": { - "node": ">=12.22" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/nzakas" - } - }, - "node_modules/@humanwhocodes/object-schema": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-2.0.1.tgz", - "integrity": "sha512-dvuCeX5fC9dXgJn9t+X5atfmgQAzUOWqS1254Gh0m6i8wKd10ebXkfNKiRK+1GWi/yTvvLDHpoxLr0xxxeslWw==", - "dev": true, - "peer": true - }, - "node_modules/@nodelib/fs.scandir": { - "version": "2.1.5", - "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", - "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", - "devOptional": true, - "dependencies": { - "@nodelib/fs.stat": "2.0.5", - "run-parallel": "^1.1.9" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/@nodelib/fs.stat": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", - "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", - "devOptional": true, - "engines": { - "node": ">= 8" - } - }, - "node_modules/@nodelib/fs.walk": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", - "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", - "devOptional": true, - "dependencies": { - "@nodelib/fs.scandir": "2.1.5", - "fastq": "^1.6.0" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/@types/fs-extra": { - "version": "9.0.13", - "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-9.0.13.tgz", - "integrity": "sha512-nEnwB++1u5lVDM2UI4c1+5R+FYaKfaAzS4OococimjVm3nQw3TuzH5UNsocrcTBbhnerblyHj4A49qXbIiZdpA==", - "optional": true, - "dependencies": { - "@types/node": "*" - } - }, - "node_modules/@types/json-schema": { - "version": "7.0.15", - "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", - "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", - "dev": true - }, - "node_modules/@types/minimist": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-hov8bUuiLiyFPGyFPE1lwWhmzYbirOXQNNo40+y3zow8aFVTeyn3VWL0VFFfdNddA8S4Vf0Tc062rzyNr7Paag==", - "optional": true - }, - "node_modules/@types/node": { - "version": "20.10.4", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.10.4.tgz", - "integrity": "sha512-D08YG6rr8X90YB56tSIuBaddy/UXAA9RKJoFvrsnogAum/0pmjkgi4+2nx96A330FmioegBWmEYQ+syqCFaveg==", - "devOptional": true, - "dependencies": { - "undici-types": "~5.26.4" - } - }, - "node_modules/@types/ps-tree": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/@types/ps-tree/-/ps-tree-1.1.6.tgz", - "integrity": "sha512-PtrlVaOaI44/3pl3cvnlK+GxOM3re2526TJvPvh7W+keHIXdV4TE0ylpPBAcvFQCbGitaTXwL9u+RF7qtVeazQ==", - "optional": true - }, - "node_modules/@types/semver": { - "version": "7.5.6", - "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.5.6.tgz", - "integrity": "sha512-dn1l8LaMea/IjDoHNd9J52uBbInB796CDffS6VdIxvqYCPSG0V0DzHp76GpaWnlhg88uYyPbXCDIowa86ybd5A==", - "dev": true - }, - "node_modules/@types/which": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/@types/which/-/which-2.0.2.tgz", - "integrity": "sha512-113D3mDkZDjo+EeUEHCFy0qniNc1ZpecGiAU7WSo7YDoSzolZIQKpYFHrPpjkB2nuyahcKfrmLXeQlh7gqJYdw==", - "optional": true - }, - "node_modules/@typescript-eslint/eslint-plugin": { - "version": "6.14.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-6.14.0.tgz", - "integrity": "sha512-1ZJBykBCXaSHG94vMMKmiHoL0MhNHKSVlcHVYZNw+BKxufhqQVTOawNpwwI1P5nIFZ/4jLVop0mcY6mJJDFNaw==", - "dev": true, - "dependencies": { - "@eslint-community/regexpp": "^4.5.1", - "@typescript-eslint/scope-manager": "6.14.0", - "@typescript-eslint/type-utils": "6.14.0", - "@typescript-eslint/utils": "6.14.0", - "@typescript-eslint/visitor-keys": "6.14.0", - "debug": "^4.3.4", - "graphemer": "^1.4.0", - "ignore": "^5.2.4", - "natural-compare": "^1.4.0", - "semver": "^7.5.4", - "ts-api-utils": "^1.0.1" - }, - "engines": { - "node": "^16.0.0 || >=18.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "@typescript-eslint/parser": "^6.0.0 || ^6.0.0-alpha", - "eslint": "^7.0.0 || ^8.0.0" - }, - "peerDependenciesMeta": { - "typescript": { - "optional": true - } - } - }, - "node_modules/@typescript-eslint/parser": { - "version": "6.14.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-6.14.0.tgz", - "integrity": "sha512-QjToC14CKacd4Pa7JK4GeB/vHmWFJckec49FR4hmIRf97+KXole0T97xxu9IFiPxVQ1DBWrQ5wreLwAGwWAVQA==", - "dev": true, - "peer": true, - "dependencies": { - "@typescript-eslint/scope-manager": "6.14.0", - "@typescript-eslint/types": "6.14.0", - "@typescript-eslint/typescript-estree": "6.14.0", - "@typescript-eslint/visitor-keys": "6.14.0", - "debug": "^4.3.4" - }, - "engines": { - "node": "^16.0.0 || >=18.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "eslint": "^7.0.0 || ^8.0.0" - }, - "peerDependenciesMeta": { - "typescript": { - "optional": true - } - } - }, - "node_modules/@typescript-eslint/scope-manager": { - "version": "6.14.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-6.14.0.tgz", - "integrity": "sha512-VT7CFWHbZipPncAZtuALr9y3EuzY1b1t1AEkIq2bTXUPKw+pHoXflGNG5L+Gv6nKul1cz1VH8fz16IThIU0tdg==", - "dev": true, - "dependencies": { - "@typescript-eslint/types": "6.14.0", - "@typescript-eslint/visitor-keys": "6.14.0" - }, - "engines": { - "node": "^16.0.0 || >=18.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - } - }, - "node_modules/@typescript-eslint/type-utils": { - "version": "6.14.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-6.14.0.tgz", - "integrity": "sha512-x6OC9Q7HfYKqjnuNu5a7kffIYs3No30isapRBJl1iCHLitD8O0lFbRcVGiOcuyN837fqXzPZ1NS10maQzZMKqw==", - "dev": true, - "dependencies": { - "@typescript-eslint/typescript-estree": "6.14.0", - "@typescript-eslint/utils": "6.14.0", - "debug": "^4.3.4", - "ts-api-utils": "^1.0.1" - }, - "engines": { - "node": "^16.0.0 || >=18.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "eslint": "^7.0.0 || ^8.0.0" - }, - "peerDependenciesMeta": { - "typescript": { - "optional": true - } - } - }, - "node_modules/@typescript-eslint/types": { - "version": "6.14.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-6.14.0.tgz", - "integrity": "sha512-uty9H2K4Xs8E47z3SnXEPRNDfsis8JO27amp2GNCnzGETEW3yTqEIVg5+AI7U276oGF/tw6ZA+UesxeQ104ceA==", - "dev": true, - "engines": { - "node": "^16.0.0 || >=18.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - } - }, - "node_modules/@typescript-eslint/typescript-estree": { - "version": "6.14.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-6.14.0.tgz", - "integrity": "sha512-yPkaLwK0yH2mZKFE/bXkPAkkFgOv15GJAUzgUVonAbv0Hr4PK/N2yaA/4XQbTZQdygiDkpt5DkxPELqHguNvyw==", - "dev": true, - "dependencies": { - "@typescript-eslint/types": "6.14.0", - "@typescript-eslint/visitor-keys": "6.14.0", - "debug": "^4.3.4", - "globby": "^11.1.0", - "is-glob": "^4.0.3", - "semver": "^7.5.4", - "ts-api-utils": "^1.0.1" - }, - "engines": { - "node": "^16.0.0 || >=18.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependenciesMeta": { - "typescript": { - "optional": true - } - } - }, - "node_modules/@typescript-eslint/typescript-estree/node_modules/globby": { - "version": "11.1.0", - "resolved": "https://registry.npmjs.org/globby/-/globby-11.1.0.tgz", - "integrity": "sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==", - "dev": true, - "dependencies": { - "array-union": "^2.1.0", - "dir-glob": "^3.0.1", - "fast-glob": "^3.2.9", - "ignore": "^5.2.0", - "merge2": "^1.4.1", - "slash": "^3.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/@typescript-eslint/typescript-estree/node_modules/slash": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", - "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", - "dev": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/@typescript-eslint/utils": { - "version": "6.14.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-6.14.0.tgz", - "integrity": "sha512-XwRTnbvRr7Ey9a1NT6jqdKX8y/atWG+8fAIu3z73HSP8h06i3r/ClMhmaF/RGWGW1tHJEwij1uEg2GbEmPYvYg==", - "dev": true, - "dependencies": { - "@eslint-community/eslint-utils": "^4.4.0", - "@types/json-schema": "^7.0.12", - "@types/semver": "^7.5.0", - "@typescript-eslint/scope-manager": "6.14.0", - "@typescript-eslint/types": "6.14.0", - "@typescript-eslint/typescript-estree": "6.14.0", - "semver": "^7.5.4" - }, - "engines": { - "node": "^16.0.0 || >=18.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "eslint": "^7.0.0 || ^8.0.0" - } - }, - "node_modules/@typescript-eslint/visitor-keys": { - "version": "6.14.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-6.14.0.tgz", - "integrity": "sha512-fB5cw6GRhJUz03MrROVuj5Zm/Q+XWlVdIsFj+Zb1Hvqouc8t+XP2H5y53QYU/MGtd2dPg6/vJJlhoX3xc2ehfw==", - "dev": true, - "dependencies": { - "@typescript-eslint/types": "6.14.0", - "eslint-visitor-keys": "^3.4.1" - }, - "engines": { - "node": "^16.0.0 || >=18.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - } - }, - "node_modules/@ungap/structured-clone": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz", - "integrity": "sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==", - "dev": true, - "peer": true - }, - "node_modules/@ziglang/cli": { - "version": "0.0.11", - "resolved": "https://registry.npmjs.org/@ziglang/cli/-/cli-0.0.11.tgz", - "integrity": "sha512-PCeyxchKF1nG9tr4sHUJlJiDjSFP0D+YS7jS0qREaAsa0cjdXYCqGXltFrEEbSMgWWkxJr9KnWviJgnHnnSFeg==", - "hasInstallScript": true, - "optional": true, - "dependencies": { - "zx": "^6.0.7" - }, - "bin": { - "zig": "bin/zig", - "zig-cli": "zig.js", - "zig-install": "install.js", - "zig-postinstall": "postinstall.js", - "zig-reinstall": "reinstall.js", - "zig-uninstall": "uninstall.js" - } - }, - "node_modules/acorn": { - "version": "8.11.2", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.2.tgz", - "integrity": "sha512-nc0Axzp/0FILLEVsm4fNwLCwMttvhEI263QtVPQcbpfZZ3ts0hLsZGOpE6czNlid7CJ9MlyH8reXkpsf3YUY4w==", - "dev": true, - "peer": true, - "bin": { - "acorn": "bin/acorn" - }, - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/acorn-jsx": { - "version": "5.3.2", - "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", - "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", - "dev": true, - "peer": true, - "peerDependencies": { - "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" - } - }, - "node_modules/ajv": { - "version": "6.12.6", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", - "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", - "dev": true, - "peer": true, - "dependencies": { - "fast-deep-equal": "^3.1.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", - "uri-js": "^4.2.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, - "node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, - "peer": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dev": true, - "peer": true, - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "dev": true, - "peer": true - }, - "node_modules/array-union": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", - "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==", - "dev": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/aws-cdk": { - "version": "2.114.1", - "resolved": "https://registry.npmjs.org/aws-cdk/-/aws-cdk-2.114.1.tgz", - "integrity": "sha512-iLOCPb3WAJOgVYQ4GvAnrjtScJfPwcczlB4995h3nUYQdHbus0jNffFv13zBShdWct3cuX+bqLuZ4JyEmJ9+rg==", - "dev": true, - "bin": { - "cdk": "bin/cdk" - }, - "engines": { - "node": ">= 14.15.0" - }, - "optionalDependencies": { - "fsevents": "2.3.2" - } - }, - "node_modules/aws-cdk-lib": { - "version": "2.114.1", - "resolved": "https://registry.npmjs.org/aws-cdk-lib/-/aws-cdk-lib-2.114.1.tgz", - "integrity": "sha512-pJy+Sa3+s6K9I0CXYGU8J5jumw9uQEbl8zPK8EMA+A6hP9qb1JN+a8ohyw6a1O1cb4D5S6gwH+hE7Fq7hGPY3A==", - "bundleDependencies": [ - "@balena/dockerignore", - "case", - "fs-extra", - "ignore", - "jsonschema", - "minimatch", - "punycode", - "semver", - "table", - "yaml" - ], - "dependencies": { - "@aws-cdk/asset-awscli-v1": "^2.2.201", - "@aws-cdk/asset-kubectl-v20": "^2.1.2", - "@aws-cdk/asset-node-proxy-agent-v6": "^2.0.1", - "@balena/dockerignore": "^1.0.2", - "case": "1.6.3", - "fs-extra": "^11.1.1", - "ignore": "^5.3.0", - "jsonschema": "^1.4.1", - "minimatch": "^3.1.2", - "punycode": "^2.3.1", - "semver": "^7.5.4", - "table": "^6.8.1", - "yaml": "1.10.2" - }, - "engines": { - "node": ">= 14.15.0" - }, - "peerDependencies": { - "constructs": "^10.0.0" - } - }, - "node_modules/aws-cdk-lib/node_modules/@balena/dockerignore": { - "version": "1.0.2", - "inBundle": true, - "license": "Apache-2.0" - }, - "node_modules/aws-cdk-lib/node_modules/ajv": { - "version": "8.12.0", - "inBundle": true, - "license": "MIT", - "dependencies": { - "fast-deep-equal": "^3.1.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2", - "uri-js": "^4.2.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, - "node_modules/aws-cdk-lib/node_modules/ansi-regex": { - "version": "5.0.1", - "inBundle": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/aws-cdk-lib/node_modules/ansi-styles": { - "version": "4.3.0", - "inBundle": true, - "license": "MIT", - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/aws-cdk-lib/node_modules/astral-regex": { - "version": "2.0.0", - "inBundle": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/aws-cdk-lib/node_modules/balanced-match": { - "version": "1.0.2", - "inBundle": true, - "license": "MIT" - }, - "node_modules/aws-cdk-lib/node_modules/brace-expansion": { - "version": "1.1.11", - "inBundle": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, - "node_modules/aws-cdk-lib/node_modules/case": { - "version": "1.6.3", - "inBundle": true, - "license": "(MIT OR GPL-3.0-or-later)", - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/aws-cdk-lib/node_modules/color-convert": { - "version": "2.0.1", - "inBundle": true, - "license": "MIT", - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "node_modules/aws-cdk-lib/node_modules/color-name": { - "version": "1.1.4", - "inBundle": true, - "license": "MIT" - }, - "node_modules/aws-cdk-lib/node_modules/concat-map": { - "version": "0.0.1", - "inBundle": true, - "license": "MIT" - }, - "node_modules/aws-cdk-lib/node_modules/emoji-regex": { - "version": "8.0.0", - "inBundle": true, - "license": "MIT" - }, - "node_modules/aws-cdk-lib/node_modules/fast-deep-equal": { - "version": "3.1.3", - "inBundle": true, - "license": "MIT" - }, - "node_modules/aws-cdk-lib/node_modules/fs-extra": { - "version": "11.1.1", - "inBundle": true, - "license": "MIT", - "dependencies": { - "graceful-fs": "^4.2.0", - "jsonfile": "^6.0.1", - "universalify": "^2.0.0" - }, - "engines": { - "node": ">=14.14" - } - }, - "node_modules/aws-cdk-lib/node_modules/graceful-fs": { - "version": "4.2.11", - "inBundle": true, - "license": "ISC" - }, - "node_modules/aws-cdk-lib/node_modules/ignore": { - "version": "5.3.0", - "inBundle": true, - "license": "MIT", - "engines": { - "node": ">= 4" - } - }, - "node_modules/aws-cdk-lib/node_modules/is-fullwidth-code-point": { - "version": "3.0.0", - "inBundle": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/aws-cdk-lib/node_modules/json-schema-traverse": { - "version": "1.0.0", - "inBundle": true, - "license": "MIT" - }, - "node_modules/aws-cdk-lib/node_modules/jsonfile": { - "version": "6.1.0", - "inBundle": true, - "license": "MIT", - "dependencies": { - "universalify": "^2.0.0" - }, - "optionalDependencies": { - "graceful-fs": "^4.1.6" - } - }, - "node_modules/aws-cdk-lib/node_modules/jsonschema": { - "version": "1.4.1", - "inBundle": true, - "license": "MIT", - "engines": { - "node": "*" - } - }, - "node_modules/aws-cdk-lib/node_modules/lodash.truncate": { - "version": "4.4.2", - "inBundle": true, - "license": "MIT" - }, - "node_modules/aws-cdk-lib/node_modules/lru-cache": { - "version": "6.0.0", - "inBundle": true, - "license": "ISC", - "dependencies": { - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/aws-cdk-lib/node_modules/minimatch": { - "version": "3.1.2", - "inBundle": true, - "license": "ISC", - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": "*" - } - }, - "node_modules/aws-cdk-lib/node_modules/punycode": { - "version": "2.3.1", - "inBundle": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/aws-cdk-lib/node_modules/require-from-string": { - "version": "2.0.2", - "inBundle": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/aws-cdk-lib/node_modules/semver": { - "version": "7.5.4", - "inBundle": true, - "license": "ISC", - "dependencies": { - "lru-cache": "^6.0.0" - }, - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/aws-cdk-lib/node_modules/slice-ansi": { - "version": "4.0.0", - "inBundle": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.0.0", - "astral-regex": "^2.0.0", - "is-fullwidth-code-point": "^3.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/slice-ansi?sponsor=1" - } - }, - "node_modules/aws-cdk-lib/node_modules/string-width": { - "version": "4.2.3", - "inBundle": true, - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/aws-cdk-lib/node_modules/strip-ansi": { - "version": "6.0.1", - "inBundle": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/aws-cdk-lib/node_modules/table": { - "version": "6.8.1", - "inBundle": true, - "license": "BSD-3-Clause", - "dependencies": { - "ajv": "^8.0.1", - "lodash.truncate": "^4.4.2", - "slice-ansi": "^4.0.0", - "string-width": "^4.2.3", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=10.0.0" - } - }, - "node_modules/aws-cdk-lib/node_modules/universalify": { - "version": "2.0.1", - "inBundle": true, - "license": "MIT", - "engines": { - "node": ">= 10.0.0" - } - }, - "node_modules/aws-cdk-lib/node_modules/uri-js": { - "version": "4.4.1", - "inBundle": true, - "license": "BSD-2-Clause", - "dependencies": { - "punycode": "^2.1.0" - } - }, - "node_modules/aws-cdk-lib/node_modules/yallist": { - "version": "4.0.0", - "inBundle": true, - "license": "ISC" - }, - "node_modules/aws-cdk-lib/node_modules/yaml": { - "version": "1.10.2", - "inBundle": true, - "license": "ISC", - "engines": { - "node": ">= 6" - } - }, - "node_modules/balanced-match": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "dev": true, - "peer": true - }, - "node_modules/brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "dev": true, - "peer": true, - "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, - "node_modules/braces": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", - "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", - "devOptional": true, - "dependencies": { - "fill-range": "^7.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/buffer-from": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", - "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==" - }, - "node_modules/callsites": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", - "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", - "dev": true, - "peer": true, - "engines": { - "node": ">=6" - } - }, - "node_modules/chalk": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.3.0.tgz", - "integrity": "sha512-dLitG79d+GV1Nb/VYcCDFivJeK1hiukt9QjRNVOsUtTy1rR1YJsmpGGTZ3qJos+uw7WmWF4wUwBd9jxjocFC2w==", - "optional": true, - "engines": { - "node": "^12.17.0 || ^14.13 || >=16.0.0" - }, - "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" - } - }, - "node_modules/color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true, - "peer": true, - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "node_modules/color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true, - "peer": true - }, - "node_modules/concat-map": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", - "dev": true, - "peer": true - }, - "node_modules/constructs": { - "version": "10.3.0", - "resolved": "https://registry.npmjs.org/constructs/-/constructs-10.3.0.tgz", - "integrity": "sha512-vbK8i3rIb/xwZxSpTjz3SagHn1qq9BChLEfy5Hf6fB3/2eFbrwt2n9kHwQcS0CPTRBesreeAcsJfMq2229FnbQ==", - "engines": { - "node": ">= 16.14.0" - } - }, - "node_modules/cross-env": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-7.0.3.tgz", - "integrity": "sha512-+/HKd6EgcQCJGh2PSjZuUitQBQynKor4wrFbRg4DtAgS1aWO+gU52xpH7M9ScGgXSYmAVS9bIJ8EzuaGw0oNAw==", - "dev": true, - "dependencies": { - "cross-spawn": "^7.0.1" - }, - "bin": { - "cross-env": "src/bin/cross-env.js", - "cross-env-shell": "src/bin/cross-env-shell.js" - }, - "engines": { - "node": ">=10.14", - "npm": ">=6", - "yarn": ">=1" - } - }, - "node_modules/cross-spawn": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", - "dev": true, - "dependencies": { - "path-key": "^3.1.0", - "shebang-command": "^2.0.0", - "which": "^2.0.1" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/data-uri-to-buffer": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", - "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", - "optional": true, - "engines": { - "node": ">= 12" - } - }, - "node_modules/debug": { - "version": "4.3.4", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", - "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", - "dev": true, - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/deep-is": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", - "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", - "dev": true, - "peer": true - }, - "node_modules/dir-glob": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", - "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==", - "devOptional": true, - "dependencies": { - "path-type": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/doctrine": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", - "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==", - "dev": true, - "peer": true, - "dependencies": { - "esutils": "^2.0.2" - }, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/dotenv": { - "version": "16.3.1", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.3.1.tgz", - "integrity": "sha512-IPzF4w4/Rd94bA9imS68tZBaYyBWSCE47V1RGuMrB94iyTOIEwRmVL2x/4An+6mETpLrKJ5hQkB8W4kFAadeIQ==", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/motdotla/dotenv?sponsor=1" - } - }, - "node_modules/duplexer": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.2.tgz", - "integrity": "sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==", - "optional": true - }, - "node_modules/escape-string-regexp": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", - "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", - "dev": true, - "peer": true, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/eslint": { - "version": "8.55.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.55.0.tgz", - "integrity": "sha512-iyUUAM0PCKj5QpwGfmCAG9XXbZCWsqP/eWAWrG/W0umvjuLRBECwSFdt+rCntju0xEH7teIABPwXpahftIaTdA==", - "dev": true, - "peer": true, - "dependencies": { - "@eslint-community/eslint-utils": "^4.2.0", - "@eslint-community/regexpp": "^4.6.1", - "@eslint/eslintrc": "^2.1.4", - "@eslint/js": "8.55.0", - "@humanwhocodes/config-array": "^0.11.13", - "@humanwhocodes/module-importer": "^1.0.1", - "@nodelib/fs.walk": "^1.2.8", - "@ungap/structured-clone": "^1.2.0", - "ajv": "^6.12.4", - "chalk": "^4.0.0", - "cross-spawn": "^7.0.2", - "debug": "^4.3.2", - "doctrine": "^3.0.0", - "escape-string-regexp": "^4.0.0", - "eslint-scope": "^7.2.2", - "eslint-visitor-keys": "^3.4.3", - "espree": "^9.6.1", - "esquery": "^1.4.2", - "esutils": "^2.0.2", - "fast-deep-equal": "^3.1.3", - "file-entry-cache": "^6.0.1", - "find-up": "^5.0.0", - "glob-parent": "^6.0.2", - "globals": "^13.19.0", - "graphemer": "^1.4.0", - "ignore": "^5.2.0", - "imurmurhash": "^0.1.4", - "is-glob": "^4.0.0", - "is-path-inside": "^3.0.3", - "js-yaml": "^4.1.0", - "json-stable-stringify-without-jsonify": "^1.0.1", - "levn": "^0.4.1", - "lodash.merge": "^4.6.2", - "minimatch": "^3.1.2", - "natural-compare": "^1.4.0", - "optionator": "^0.9.3", - "strip-ansi": "^6.0.1", - "text-table": "^0.2.0" - }, - "bin": { - "eslint": "bin/eslint.js" - }, - "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "node_modules/eslint-scope": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.2.2.tgz", - "integrity": "sha512-dOt21O7lTMhDM+X9mB4GX+DZrZtCUJPL/wlcTqxyrx5IvO0IYtILdtrQGQp+8n5S0gwSVmOf9NQrjMOgfQZlIg==", - "dev": true, - "peer": true, - "dependencies": { - "esrecurse": "^4.3.0", - "estraverse": "^5.2.0" - }, - "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "node_modules/eslint-visitor-keys": { - "version": "3.4.3", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", - "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", - "dev": true, - "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "node_modules/eslint/node_modules/chalk": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", - "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "dev": true, - "peer": true, - "dependencies": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" - } - }, - "node_modules/eslint/node_modules/glob-parent": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", - "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", - "dev": true, - "peer": true, - "dependencies": { - "is-glob": "^4.0.3" - }, - "engines": { - "node": ">=10.13.0" - } - }, - "node_modules/espree": { - "version": "9.6.1", - "resolved": "https://registry.npmjs.org/espree/-/espree-9.6.1.tgz", - "integrity": "sha512-oruZaFkjorTpF32kDSI5/75ViwGeZginGGy2NoOSg3Q9bnwlnmDm4HLnkl0RE3n+njDXR037aY1+x58Z/zFdwQ==", - "dev": true, - "peer": true, - "dependencies": { - "acorn": "^8.9.0", - "acorn-jsx": "^5.3.2", - "eslint-visitor-keys": "^3.4.1" - }, - "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "node_modules/esquery": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.5.0.tgz", - "integrity": "sha512-YQLXUplAwJgCydQ78IMJywZCceoqk1oH01OERdSAJc/7U2AylwjhSCLDEtqwg811idIS/9fIU5GjG73IgjKMVg==", - "dev": true, - "peer": true, - "dependencies": { - "estraverse": "^5.1.0" - }, - "engines": { - "node": ">=0.10" - } - }, - "node_modules/esrecurse": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", - "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", - "dev": true, - "peer": true, - "dependencies": { - "estraverse": "^5.2.0" - }, - "engines": { - "node": ">=4.0" - } - }, - "node_modules/estraverse": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", - "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", - "dev": true, - "peer": true, - "engines": { - "node": ">=4.0" - } - }, - "node_modules/esutils": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", - "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", - "dev": true, - "peer": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/event-stream": { - "version": "3.3.4", - "resolved": "https://registry.npmjs.org/event-stream/-/event-stream-3.3.4.tgz", - "integrity": "sha512-QHpkERcGsR0T7Qm3HNJSyXKEEj8AHNxkY3PK8TS2KJvQ7NiSHe3DDpwVKKtoYprL/AreyzFBeIkBIWChAqn60g==", - "optional": true, - "dependencies": { - "duplexer": "~0.1.1", - "from": "~0", - "map-stream": "~0.1.0", - "pause-stream": "0.0.11", - "split": "0.3", - "stream-combiner": "~0.0.4", - "through": "~2.3.1" - } - }, - "node_modules/fast-deep-equal": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", - "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", - "dev": true, - "peer": true - }, - "node_modules/fast-glob": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.2.tgz", - "integrity": "sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow==", - "devOptional": true, - "dependencies": { - "@nodelib/fs.stat": "^2.0.2", - "@nodelib/fs.walk": "^1.2.3", - "glob-parent": "^5.1.2", - "merge2": "^1.3.0", - "micromatch": "^4.0.4" - }, - "engines": { - "node": ">=8.6.0" - } - }, - "node_modules/fast-json-stable-stringify": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", - "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", - "dev": true, - "peer": true - }, - "node_modules/fast-levenshtein": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", - "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", - "dev": true, - "peer": true - }, - "node_modules/fastq": { - "version": "1.15.0", - "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz", - "integrity": "sha512-wBrocU2LCXXa+lWBt8RoIRD89Fi8OdABODa/kEnyeyjS5aZO5/GNvI5sEINADqP/h8M29UHTHUb53sUu5Ihqdw==", - "devOptional": true, - "dependencies": { - "reusify": "^1.0.4" - } - }, - "node_modules/fetch-blob": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", - "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "paypal", - "url": "https://paypal.me/jimmywarting" - } - ], - "optional": true, - "dependencies": { - "node-domexception": "^1.0.0", - "web-streams-polyfill": "^3.0.3" - }, - "engines": { - "node": "^12.20 || >= 14.13" - } - }, - "node_modules/file-entry-cache": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", - "integrity": "sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==", - "dev": true, - "peer": true, - "dependencies": { - "flat-cache": "^3.0.4" - }, - "engines": { - "node": "^10.12.0 || >=12.0.0" - } - }, - "node_modules/fill-range": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", - "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", - "devOptional": true, - "dependencies": { - "to-regex-range": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/find-up": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", - "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", - "dev": true, - "peer": true, - "dependencies": { - "locate-path": "^6.0.0", - "path-exists": "^4.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/flat-cache": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.2.0.tgz", - "integrity": "sha512-CYcENa+FtcUKLmhhqyctpclsq7QF38pKjZHsGNiSQF5r4FtoKDWabFDl3hzaEQMvT1LHEysw5twgLvpYYb4vbw==", - "dev": true, - "peer": true, - "dependencies": { - "flatted": "^3.2.9", - "keyv": "^4.5.3", - "rimraf": "^3.0.2" - }, - "engines": { - "node": "^10.12.0 || >=12.0.0" - } - }, - "node_modules/flatted": { - "version": "3.2.9", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.2.9.tgz", - "integrity": "sha512-36yxDn5H7OFZQla0/jFJmbIKTdZAQHngCedGxiMmpNfEZM0sdEeT+WczLQrjK6D7o2aiyLYDnkw0R3JK0Qv1RQ==", - "dev": true, - "peer": true - }, - "node_modules/formdata-polyfill": { - "version": "4.0.10", - "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", - "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", - "optional": true, - "dependencies": { - "fetch-blob": "^3.1.2" - }, - "engines": { - "node": ">=12.20.0" - } - }, - "node_modules/from": { - "version": "0.1.7", - "resolved": "https://registry.npmjs.org/from/-/from-0.1.7.tgz", - "integrity": "sha512-twe20eF1OxVxp/ML/kq2p1uc6KvFK/+vs8WjEbeKmV2He22MKm7YF2ANIt+EOqhJ5L3K/SuuPhk0hWQDjOM23g==", - "optional": true - }, - "node_modules/fs-extra": { - "version": "10.1.0", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.1.0.tgz", - "integrity": "sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==", - "optional": true, - "dependencies": { - "graceful-fs": "^4.2.0", - "jsonfile": "^6.0.1", - "universalify": "^2.0.0" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/fs.realpath": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", - "dev": true, - "peer": true - }, - "node_modules/fsevents": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", - "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", - "dev": true, - "hasInstallScript": true, - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" - } - }, - "node_modules/glob": { - "version": "7.2.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", - "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", - "dev": true, - "peer": true, - "dependencies": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.1.1", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - }, - "engines": { - "node": "*" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/glob-parent": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", - "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", - "devOptional": true, - "dependencies": { - "is-glob": "^4.0.1" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/globals": { - "version": "13.24.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-13.24.0.tgz", - "integrity": "sha512-AhO5QUcj8llrbG09iWhPU2B204J1xnPeL8kQmVorSsy+Sjj1sk8gIyh6cUocGmH4L0UuhAJy+hJMRA4mgA4mFQ==", - "dev": true, - "peer": true, - "dependencies": { - "type-fest": "^0.20.2" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/globby": { - "version": "13.2.2", - "resolved": "https://registry.npmjs.org/globby/-/globby-13.2.2.tgz", - "integrity": "sha512-Y1zNGV+pzQdh7H39l9zgB4PJqjRNqydvdYCDG4HFXM4XuvSaQQlEc91IU1yALL8gUTDomgBAfz3XJdmUS+oo0w==", - "optional": true, - "dependencies": { - "dir-glob": "^3.0.1", - "fast-glob": "^3.3.0", - "ignore": "^5.2.4", - "merge2": "^1.4.1", - "slash": "^4.0.0" - }, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/graceful-fs": { - "version": "4.2.11", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", - "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", - "optional": true - }, - "node_modules/graphemer": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/graphemer/-/graphemer-1.4.0.tgz", - "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==", - "dev": true - }, - "node_modules/has-flag": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", - "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", - "dev": true, - "peer": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/ignore": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.0.tgz", - "integrity": "sha512-g7dmpshy+gD7mh88OC9NwSGTKoc3kyLAZQRU1mt53Aw/vnvfXnbC+F/7F7QoYVKbV+KNvJx8wArewKy1vXMtlg==", - "devOptional": true, - "engines": { - "node": ">= 4" - } - }, - "node_modules/import-fresh": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz", - "integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==", - "dev": true, - "peer": true, - "dependencies": { - "parent-module": "^1.0.0", - "resolve-from": "^4.0.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/imurmurhash": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", - "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", - "dev": true, - "peer": true, - "engines": { - "node": ">=0.8.19" - } - }, - "node_modules/inflight": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", - "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", - "dev": true, - "peer": true, - "dependencies": { - "once": "^1.3.0", - "wrappy": "1" - } - }, - "node_modules/inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "dev": true, - "peer": true - }, - "node_modules/is-extglob": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", - "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", - "devOptional": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/is-glob": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", - "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", - "devOptional": true, - "dependencies": { - "is-extglob": "^2.1.1" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/is-number": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", - "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", - "devOptional": true, - "engines": { - "node": ">=0.12.0" - } - }, - "node_modules/is-path-inside": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz", - "integrity": "sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==", - "dev": true, - "peer": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/isexe": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", - "devOptional": true - }, - "node_modules/js-yaml": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", - "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", - "dev": true, - "peer": true, - "dependencies": { - "argparse": "^2.0.1" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, - "node_modules/json-buffer": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", - "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", - "dev": true, - "peer": true - }, - "node_modules/json-schema-traverse": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", - "dev": true, - "peer": true - }, - "node_modules/json-stable-stringify-without-jsonify": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", - "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", - "dev": true, - "peer": true - }, - "node_modules/jsonfile": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", - "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", - "optional": true, - "dependencies": { - "universalify": "^2.0.0" - }, - "optionalDependencies": { - "graceful-fs": "^4.1.6" - } - }, - "node_modules/keyv": { - "version": "4.5.4", - "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", - "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", - "dev": true, - "peer": true, - "dependencies": { - "json-buffer": "3.0.1" - } - }, - "node_modules/levn": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", - "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", - "dev": true, - "peer": true, - "dependencies": { - "prelude-ls": "^1.2.1", - "type-check": "~0.4.0" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/locate-path": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", - "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", - "dev": true, - "peer": true, - "dependencies": { - "p-locate": "^5.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/lodash.merge": { - "version": "4.6.2", - "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", - "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", - "dev": true, - "peer": true - }, - "node_modules/lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "dev": true, - "dependencies": { - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/map-stream": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/map-stream/-/map-stream-0.1.0.tgz", - "integrity": "sha512-CkYQrPYZfWnu/DAmVCpTSX/xHpKZ80eKh2lAkyA6AJTef6bW+6JpbQZN5rofum7da+SyN1bi5ctTm+lTfcCW3g==", - "optional": true - }, - "node_modules/merge2": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", - "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", - "devOptional": true, - "engines": { - "node": ">= 8" - } - }, - "node_modules/micromatch": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", - "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==", - "devOptional": true, - "dependencies": { - "braces": "^3.0.2", - "picomatch": "^2.3.1" - }, - "engines": { - "node": ">=8.6" - } - }, - "node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dev": true, - "peer": true, - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": "*" - } - }, - "node_modules/minimist": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", - "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", - "optional": true, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true - }, - "node_modules/natural-compare": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", - "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", - "dev": true - }, - "node_modules/node-domexception": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", - "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "github", - "url": "https://paypal.me/jimmywarting" - } - ], - "optional": true, - "engines": { - "node": ">=10.5.0" - } - }, - "node_modules/node-fetch": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", - "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", - "optional": true, - "dependencies": { - "data-uri-to-buffer": "^4.0.0", - "fetch-blob": "^3.1.4", - "formdata-polyfill": "^4.0.10" - }, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/node-fetch" - } - }, - "node_modules/once": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "dev": true, - "peer": true, - "dependencies": { - "wrappy": "1" - } - }, - "node_modules/optionator": { - "version": "0.9.3", - "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.3.tgz", - "integrity": "sha512-JjCoypp+jKn1ttEFExxhetCKeJt9zhAgAve5FXHixTvFDW/5aEktX9bufBKLRRMdU7bNtpLfcGu94B3cdEJgjg==", - "dev": true, - "peer": true, - "dependencies": { - "@aashutoshrathi/word-wrap": "^1.2.3", - "deep-is": "^0.1.3", - "fast-levenshtein": "^2.0.6", - "levn": "^0.4.1", - "prelude-ls": "^1.2.1", - "type-check": "^0.4.0" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/p-limit": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", - "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", - "dev": true, - "peer": true, - "dependencies": { - "yocto-queue": "^0.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/p-locate": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", - "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", - "dev": true, - "peer": true, - "dependencies": { - "p-limit": "^3.0.2" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/parent-module": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", - "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", - "dev": true, - "peer": true, - "dependencies": { - "callsites": "^3.0.0" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/path-exists": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", - "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", - "dev": true, - "peer": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/path-is-absolute": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", - "dev": true, - "peer": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/path-key": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", - "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", - "dev": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/path-type": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", - "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", - "devOptional": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/pause-stream": { - "version": "0.0.11", - "resolved": "https://registry.npmjs.org/pause-stream/-/pause-stream-0.0.11.tgz", - "integrity": "sha512-e3FBlXLmN/D1S+zHzanP4E/4Z60oFAa3O051qt1pxa7DEJWKAyil6upYVXCWadEnuoqa4Pkc9oUx9zsxYeRv8A==", - "optional": true, - "dependencies": { - "through": "~2.3" - } - }, - "node_modules/picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", - "devOptional": true, - "engines": { - "node": ">=8.6" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, - "node_modules/prelude-ls": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", - "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", - "dev": true, - "peer": true, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/prettier": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.0.3.tgz", - "integrity": "sha512-L/4pUDMxcNa8R/EthV08Zt42WBO4h1rarVtK0K+QJG0X187OLo7l699jWw0GKuwzkPQ//jMFA/8Xm6Fh3J/DAg==", - "dev": true, - "bin": { - "prettier": "bin/prettier.cjs" - }, - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/prettier/prettier?sponsor=1" - } - }, - "node_modules/ps-tree": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/ps-tree/-/ps-tree-1.2.0.tgz", - "integrity": "sha512-0VnamPPYHl4uaU/nSFeZZpR21QAWRz+sRv4iW9+v/GS/J5U5iZB5BNN6J0RMoOvdx2gWM2+ZFMIm58q24e4UYA==", - "optional": true, - "dependencies": { - "event-stream": "=3.3.4" - }, - "bin": { - "ps-tree": "bin/ps-tree.js" - }, - "engines": { - "node": ">= 0.10" - } - }, - "node_modules/punycode": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", - "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", - "dev": true, - "peer": true, - "engines": { - "node": ">=6" - } - }, - "node_modules/queue-microtask": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", - "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", - "devOptional": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ] - }, - "node_modules/resolve-from": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", - "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", - "dev": true, - "peer": true, - "engines": { - "node": ">=4" - } - }, - "node_modules/reusify": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", - "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", - "devOptional": true, - "engines": { - "iojs": ">=1.0.0", - "node": ">=0.10.0" - } - }, - "node_modules/rimraf": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", - "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", - "dev": true, - "peer": true, - "dependencies": { - "glob": "^7.1.3" - }, - "bin": { - "rimraf": "bin.js" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/run-parallel": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", - "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", - "devOptional": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "dependencies": { - "queue-microtask": "^1.2.2" - } - }, - "node_modules/rust.aws-cdk-lambda": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/rust.aws-cdk-lambda/-/rust.aws-cdk-lambda-1.2.1.tgz", - "integrity": "sha512-BlkV7rVRQUkLVQuwP1A89C8f9yuDA1St7B/+EsckxBu/fUER8KKLydoe/hMfskCt6LQaIIAQqGqZ6HUr1DHjnw==", - "dependencies": { - "aws-cdk-lib": "^2.*", - "constructs": "^10.*", - "toml": "~3.0.0" - }, - "engines": { - "node": ">= 10.3.0" - }, - "optionalDependencies": { - "@ziglang/cli": "^0.0.11" - }, - "peerDependencies": { - "aws-cdk-lib": "^2.*", - "constructs": "^10.*" - } - }, - "node_modules/semver": { - "version": "7.5.4", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", - "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", - "dev": true, - "dependencies": { - "lru-cache": "^6.0.0" - }, - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/shebang-command": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", - "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", - "dev": true, - "dependencies": { - "shebang-regex": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/shebang-regex": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", - "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", - "dev": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/slash": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/slash/-/slash-4.0.0.tgz", - "integrity": "sha512-3dOsAHXXUkQTpOYcoAxLIorMTp4gIQr5IW3iVb7A7lFIp0VHhnynm9izx6TssdrIcVIESAlVjtnO2K8bg+Coew==", - "optional": true, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/source-map": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/source-map-support": { - "version": "0.5.21", - "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz", - "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==", - "dependencies": { - "buffer-from": "^1.0.0", - "source-map": "^0.6.0" - } - }, - "node_modules/split": { - "version": "0.3.3", - "resolved": "https://registry.npmjs.org/split/-/split-0.3.3.tgz", - "integrity": "sha512-wD2AeVmxXRBoX44wAycgjVpMhvbwdI2aZjCkvfNcH1YqHQvJVa1duWc73OyVGJUc05fhFaTZeQ/PYsrmyH0JVA==", - "optional": true, - "dependencies": { - "through": "2" - }, - "engines": { - "node": "*" - } - }, - "node_modules/stream-combiner": { - "version": "0.0.4", - "resolved": "https://registry.npmjs.org/stream-combiner/-/stream-combiner-0.0.4.tgz", - "integrity": "sha512-rT00SPnTVyRsaSz5zgSPma/aHSOic5U1prhYdRy5HS2kTZviFpmDgzilbtsJsxiroqACmayynDN/9VzIbX5DOw==", - "optional": true, - "dependencies": { - "duplexer": "~0.1.1" - } - }, - "node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, - "peer": true, - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-json-comments": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", - "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", - "dev": true, - "peer": true, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/supports-color": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", - "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "dev": true, - "peer": true, - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/text-table": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", - "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==", - "dev": true, - "peer": true - }, - "node_modules/through": { - "version": "2.3.8", - "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", - "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==", - "optional": true - }, - "node_modules/to-regex-range": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", - "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "devOptional": true, - "dependencies": { - "is-number": "^7.0.0" - }, - "engines": { - "node": ">=8.0" - } - }, - "node_modules/toml": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/toml/-/toml-3.0.0.tgz", - "integrity": "sha512-y/mWCZinnvxjTKYhJ+pYxwD0mRLVvOtdS2Awbgxln6iEnt4rk0yBxeSBHkGJcPucRiG0e55mwWp+g/05rsrd6w==" - }, - "node_modules/ts-api-utils": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-1.0.3.tgz", - "integrity": "sha512-wNMeqtMz5NtwpT/UZGY5alT+VoKdSsOOP/kqHFcUW1P/VRhH2wJ48+DN2WwUliNbQ976ETwDL0Ifd2VVvgonvg==", - "dev": true, - "engines": { - "node": ">=16.13.0" - }, - "peerDependencies": { - "typescript": ">=4.2.0" - } - }, - "node_modules/type-check": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", - "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", - "dev": true, - "peer": true, - "dependencies": { - "prelude-ls": "^1.2.1" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/type-fest": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", - "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==", - "dev": true, - "peer": true, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/typescript": { - "version": "5.3.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.3.tgz", - "integrity": "sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==", - "dev": true, - "bin": { - "tsc": "bin/tsc", - "tsserver": "bin/tsserver" - }, - "engines": { - "node": ">=14.17" - } - }, - "node_modules/undici-types": { - "version": "5.26.5", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", - "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", - "devOptional": true - }, - "node_modules/universalify": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", - "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==", - "optional": true, - "engines": { - "node": ">= 10.0.0" - } - }, - "node_modules/uri-js": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", - "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", - "dev": true, - "peer": true, - "dependencies": { - "punycode": "^2.1.0" - } - }, - "node_modules/web-streams-polyfill": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz", - "integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==", - "optional": true, - "engines": { - "node": ">= 8" - } - }, - "node_modules/which": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", - "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "devOptional": true, - "dependencies": { - "isexe": "^2.0.0" - }, - "bin": { - "node-which": "bin/node-which" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/wrappy": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "dev": true, - "peer": true - }, - "node_modules/yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true - }, - "node_modules/yaml": { - "version": "2.3.4", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.3.4.tgz", - "integrity": "sha512-8aAvwVUSHpfEqTQ4w/KMlf3HcRdt50E5ODIQJBw1fQ5RL34xabzxtUlzTXVqc4rkZsPbvrXKWnABCD7kWSmocA==", - "optional": true, - "engines": { - "node": ">= 14" - } - }, - "node_modules/yocto-queue": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", - "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", - "dev": true, - "peer": true, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/zx": { - "version": "6.2.5", - "resolved": "https://registry.npmjs.org/zx/-/zx-6.2.5.tgz", - "integrity": "sha512-aTEZSL8sp1IhMUhxlXRTPH6MwzcuTBA3G4LLuH68NIj8kENfcaVGV7cn8LQXXHwG1M9LDI6ST6Qcw0EOrhRAVA==", - "optional": true, - "dependencies": { - "@types/fs-extra": "^9.0.13", - "@types/minimist": "^1.2.2", - "@types/node": "^17.0", - "@types/ps-tree": "^1.1.2", - "@types/which": "^2.0.1", - "chalk": "^5.0.1", - "fs-extra": "^10.1.0", - "globby": "^13.1.1", - "ignore": "^5.2.0", - "minimist": "^1.2.6", - "node-fetch": "^3.2.5", - "ps-tree": "^1.2.0", - "which": "^2.0.2", - "yaml": "^2.1.1" - }, - "bin": { - "zx": "build/cli.js" - }, - "engines": { - "node": ">= 16.0.0" - } - }, - "node_modules/zx/node_modules/@types/node": { - "version": "17.0.45", - "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.45.tgz", - "integrity": "sha512-w+tIMs3rq2afQdsPJlODhoUEKzFP1ayaoyl1CcnwtIlsVe7K7bA1NGm4s3PraqTLlXnbIN84zuBlxBWo1u9BLw==", - "optional": true - } - } -} diff --git a/lib/workload/stateful/filemanager/.env.example b/lib/workload/stateless/filemanager/.env.example similarity index 100% rename from lib/workload/stateful/filemanager/.env.example rename to lib/workload/stateless/filemanager/.env.example diff --git a/lib/workload/stateful/filemanager/.gitignore b/lib/workload/stateless/filemanager/.gitignore similarity index 100% rename from lib/workload/stateful/filemanager/.gitignore rename to lib/workload/stateless/filemanager/.gitignore diff --git a/lib/workload/stateful/filemanager/Cargo.lock b/lib/workload/stateless/filemanager/Cargo.lock similarity index 100% rename from lib/workload/stateful/filemanager/Cargo.lock rename to lib/workload/stateless/filemanager/Cargo.lock diff --git a/lib/workload/stateful/filemanager/Cargo.toml b/lib/workload/stateless/filemanager/Cargo.toml similarity index 100% rename from lib/workload/stateful/filemanager/Cargo.toml rename to lib/workload/stateless/filemanager/Cargo.toml diff --git a/lib/workload/stateful/filemanager/Makefile b/lib/workload/stateless/filemanager/Makefile similarity index 100% rename from lib/workload/stateful/filemanager/Makefile rename to lib/workload/stateless/filemanager/Makefile diff --git a/lib/workload/stateful/filemanager/README.md b/lib/workload/stateless/filemanager/README.md similarity index 100% rename from lib/workload/stateful/filemanager/README.md rename to lib/workload/stateless/filemanager/README.md diff --git a/lib/workload/stateful/filemanager/build.rs b/lib/workload/stateless/filemanager/build.rs similarity index 100% rename from lib/workload/stateful/filemanager/build.rs rename to lib/workload/stateless/filemanager/build.rs diff --git a/lib/workload/stateful/filemanager/database/Dockerfile b/lib/workload/stateless/filemanager/database/Dockerfile similarity index 100% rename from lib/workload/stateful/filemanager/database/Dockerfile rename to lib/workload/stateless/filemanager/database/Dockerfile diff --git a/lib/workload/stateful/filemanager/database/migrations/0001_add_object_table.sql b/lib/workload/stateless/filemanager/database/migrations/0001_add_object_table.sql similarity index 100% rename from lib/workload/stateful/filemanager/database/migrations/0001_add_object_table.sql rename to lib/workload/stateless/filemanager/database/migrations/0001_add_object_table.sql diff --git a/lib/workload/stateful/filemanager/database/migrations/0002_add_s3_object_table.sql b/lib/workload/stateless/filemanager/database/migrations/0002_add_s3_object_table.sql similarity index 100% rename from lib/workload/stateful/filemanager/database/migrations/0002_add_s3_object_table.sql rename to lib/workload/stateless/filemanager/database/migrations/0002_add_s3_object_table.sql diff --git a/lib/workload/stateful/filemanager/database/queries/ingester/aws/insert_s3_created_objects.sql b/lib/workload/stateless/filemanager/database/queries/ingester/aws/insert_s3_created_objects.sql similarity index 100% rename from lib/workload/stateful/filemanager/database/queries/ingester/aws/insert_s3_created_objects.sql rename to lib/workload/stateless/filemanager/database/queries/ingester/aws/insert_s3_created_objects.sql diff --git a/lib/workload/stateful/filemanager/database/queries/ingester/aws/insert_s3_deleted_objects.sql b/lib/workload/stateless/filemanager/database/queries/ingester/aws/insert_s3_deleted_objects.sql similarity index 100% rename from lib/workload/stateful/filemanager/database/queries/ingester/aws/insert_s3_deleted_objects.sql rename to lib/workload/stateless/filemanager/database/queries/ingester/aws/insert_s3_deleted_objects.sql diff --git a/lib/workload/stateful/filemanager/database/queries/ingester/aws/update_deleted.sql b/lib/workload/stateless/filemanager/database/queries/ingester/aws/update_deleted.sql similarity index 100% rename from lib/workload/stateful/filemanager/database/queries/ingester/aws/update_deleted.sql rename to lib/workload/stateless/filemanager/database/queries/ingester/aws/update_deleted.sql diff --git a/lib/workload/stateful/filemanager/database/queries/ingester/insert_objects.sql b/lib/workload/stateless/filemanager/database/queries/ingester/insert_objects.sql similarity index 100% rename from lib/workload/stateful/filemanager/database/queries/ingester/insert_objects.sql rename to lib/workload/stateless/filemanager/database/queries/ingester/insert_objects.sql diff --git a/lib/workload/stateful/filemanager/deploy/README.md b/lib/workload/stateless/filemanager/deploy/README.md similarity index 100% rename from lib/workload/stateful/filemanager/deploy/README.md rename to lib/workload/stateless/filemanager/deploy/README.md diff --git a/lib/workload/stateful/filemanager/deploy/cdk.json b/lib/workload/stateless/filemanager/deploy/cdk.json similarity index 100% rename from lib/workload/stateful/filemanager/deploy/cdk.json rename to lib/workload/stateless/filemanager/deploy/cdk.json diff --git a/lib/workload/stateful/filemanager/deploy/constructs/cdk_resource_invoke.ts b/lib/workload/stateless/filemanager/deploy/constructs/cdk_resource_invoke.ts similarity index 100% rename from lib/workload/stateful/filemanager/deploy/constructs/cdk_resource_invoke.ts rename to lib/workload/stateless/filemanager/deploy/constructs/cdk_resource_invoke.ts diff --git a/lib/workload/stateful/filemanager/deploy/constructs/functions/function.ts b/lib/workload/stateless/filemanager/deploy/constructs/functions/function.ts similarity index 98% rename from lib/workload/stateful/filemanager/deploy/constructs/functions/function.ts rename to lib/workload/stateless/filemanager/deploy/constructs/functions/function.ts index 5ab0b7a57..36b1acbc5 100644 --- a/lib/workload/stateful/filemanager/deploy/constructs/functions/function.ts +++ b/lib/workload/stateless/filemanager/deploy/constructs/functions/function.ts @@ -2,7 +2,7 @@ import { Construct } from 'constructs'; import { RustFunction } from 'rust.aws-cdk-lambda'; import { Duration } from 'aws-cdk-lib'; import { Settings as CargoSettings } from 'rust.aws-cdk-lambda/dist/settings'; -import { IDatabase } from '../../../../database/component'; +import { IDatabase } from '../../../../../stateful/database/component'; import { IVpc, SecurityGroup, SubnetType } from 'aws-cdk-lib/aws-ec2'; import { Architecture, IDestination, Version } from 'aws-cdk-lib/aws-lambda'; import { ManagedPolicy, PolicyStatement, Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam'; diff --git a/lib/workload/stateful/filemanager/deploy/constructs/functions/ingest.ts b/lib/workload/stateless/filemanager/deploy/constructs/functions/ingest.ts similarity index 100% rename from lib/workload/stateful/filemanager/deploy/constructs/functions/ingest.ts rename to lib/workload/stateless/filemanager/deploy/constructs/functions/ingest.ts diff --git a/lib/workload/stateful/filemanager/deploy/constructs/functions/migrate.ts b/lib/workload/stateless/filemanager/deploy/constructs/functions/migrate.ts similarity index 100% rename from lib/workload/stateful/filemanager/deploy/constructs/functions/migrate.ts rename to lib/workload/stateless/filemanager/deploy/constructs/functions/migrate.ts diff --git a/lib/workload/stateful/filemanager/deploy/lib/filemanager.ts b/lib/workload/stateless/filemanager/deploy/lib/filemanager.ts similarity index 95% rename from lib/workload/stateful/filemanager/deploy/lib/filemanager.ts rename to lib/workload/stateless/filemanager/deploy/lib/filemanager.ts index df27f921a..92fa59dcd 100644 --- a/lib/workload/stateful/filemanager/deploy/lib/filemanager.ts +++ b/lib/workload/stateless/filemanager/deploy/lib/filemanager.ts @@ -1,10 +1,9 @@ -import { Stack, StackProps } from 'aws-cdk-lib'; import { Construct } from 'constructs'; import { IngestFunction, IngestFunctionSettings } from '../constructs/functions/ingest'; import { CdkResourceInvoke } from '../constructs/cdk_resource_invoke'; import { MigrateFunction } from '../constructs/functions/migrate'; import * as fn from '../constructs/functions/function'; -import { IDatabase } from '../../../database/component'; +import { IDatabase } from '../../../../stateful/database/component'; import { IVpc } from 'aws-cdk-lib/aws-ec2'; import { IQueue } from 'aws-cdk-lib/aws-sqs'; import { IDestination } from 'aws-cdk-lib/aws-lambda'; diff --git a/lib/workload/stateful/filemanager/deploy/package.json b/lib/workload/stateless/filemanager/deploy/package.json similarity index 100% rename from lib/workload/stateful/filemanager/deploy/package.json rename to lib/workload/stateless/filemanager/deploy/package.json diff --git a/lib/workload/stateful/filemanager/deploy/tsconfig.json b/lib/workload/stateless/filemanager/deploy/tsconfig.json similarity index 100% rename from lib/workload/stateful/filemanager/deploy/tsconfig.json rename to lib/workload/stateless/filemanager/deploy/tsconfig.json diff --git a/lib/workload/stateful/filemanager/docker-compose.yml b/lib/workload/stateless/filemanager/docker-compose.yml similarity index 100% rename from lib/workload/stateful/filemanager/docker-compose.yml rename to lib/workload/stateless/filemanager/docker-compose.yml diff --git a/lib/workload/stateful/filemanager/filemanager-http-lambda/Cargo.toml b/lib/workload/stateless/filemanager/filemanager-http-lambda/Cargo.toml similarity index 100% rename from lib/workload/stateful/filemanager/filemanager-http-lambda/Cargo.toml rename to lib/workload/stateless/filemanager/filemanager-http-lambda/Cargo.toml diff --git a/lib/workload/stateful/filemanager/filemanager-http-lambda/README.md b/lib/workload/stateless/filemanager/filemanager-http-lambda/README.md similarity index 100% rename from lib/workload/stateful/filemanager/filemanager-http-lambda/README.md rename to lib/workload/stateless/filemanager/filemanager-http-lambda/README.md diff --git a/lib/workload/stateful/filemanager/filemanager-http-lambda/src/main.rs b/lib/workload/stateless/filemanager/filemanager-http-lambda/src/main.rs similarity index 100% rename from lib/workload/stateful/filemanager/filemanager-http-lambda/src/main.rs rename to lib/workload/stateless/filemanager/filemanager-http-lambda/src/main.rs diff --git a/lib/workload/stateful/filemanager/filemanager-ingest-lambda/Cargo.toml b/lib/workload/stateless/filemanager/filemanager-ingest-lambda/Cargo.toml similarity index 100% rename from lib/workload/stateful/filemanager/filemanager-ingest-lambda/Cargo.toml rename to lib/workload/stateless/filemanager/filemanager-ingest-lambda/Cargo.toml diff --git a/lib/workload/stateful/filemanager/filemanager-ingest-lambda/README.md b/lib/workload/stateless/filemanager/filemanager-ingest-lambda/README.md similarity index 100% rename from lib/workload/stateful/filemanager/filemanager-ingest-lambda/README.md rename to lib/workload/stateless/filemanager/filemanager-ingest-lambda/README.md diff --git a/lib/workload/stateful/filemanager/filemanager-ingest-lambda/src/main.rs b/lib/workload/stateless/filemanager/filemanager-ingest-lambda/src/main.rs similarity index 100% rename from lib/workload/stateful/filemanager/filemanager-ingest-lambda/src/main.rs rename to lib/workload/stateless/filemanager/filemanager-ingest-lambda/src/main.rs diff --git a/lib/workload/stateful/filemanager/filemanager-migrate-lambda/Cargo.toml b/lib/workload/stateless/filemanager/filemanager-migrate-lambda/Cargo.toml similarity index 100% rename from lib/workload/stateful/filemanager/filemanager-migrate-lambda/Cargo.toml rename to lib/workload/stateless/filemanager/filemanager-migrate-lambda/Cargo.toml diff --git a/lib/workload/stateful/filemanager/filemanager-migrate-lambda/src/main.rs b/lib/workload/stateless/filemanager/filemanager-migrate-lambda/src/main.rs similarity index 100% rename from lib/workload/stateful/filemanager/filemanager-migrate-lambda/src/main.rs rename to lib/workload/stateless/filemanager/filemanager-migrate-lambda/src/main.rs diff --git a/lib/workload/stateful/filemanager/filemanager/Cargo.lock b/lib/workload/stateless/filemanager/filemanager/Cargo.lock similarity index 100% rename from lib/workload/stateful/filemanager/filemanager/Cargo.lock rename to lib/workload/stateless/filemanager/filemanager/Cargo.lock diff --git a/lib/workload/stateful/filemanager/filemanager/Cargo.toml b/lib/workload/stateless/filemanager/filemanager/Cargo.toml similarity index 100% rename from lib/workload/stateful/filemanager/filemanager/Cargo.toml rename to lib/workload/stateless/filemanager/filemanager/Cargo.toml diff --git a/lib/workload/stateful/filemanager/filemanager/README.md b/lib/workload/stateless/filemanager/filemanager/README.md similarity index 100% rename from lib/workload/stateful/filemanager/filemanager/README.md rename to lib/workload/stateless/filemanager/filemanager/README.md diff --git a/lib/workload/stateful/filemanager/filemanager/src/clients/aws/config.rs b/lib/workload/stateless/filemanager/filemanager/src/clients/aws/config.rs similarity index 100% rename from lib/workload/stateful/filemanager/filemanager/src/clients/aws/config.rs rename to lib/workload/stateless/filemanager/filemanager/src/clients/aws/config.rs diff --git a/lib/workload/stateful/filemanager/filemanager/src/clients/aws/mod.rs b/lib/workload/stateless/filemanager/filemanager/src/clients/aws/mod.rs similarity index 100% rename from lib/workload/stateful/filemanager/filemanager/src/clients/aws/mod.rs rename to lib/workload/stateless/filemanager/filemanager/src/clients/aws/mod.rs diff --git a/lib/workload/stateful/filemanager/filemanager/src/clients/aws/s3.rs b/lib/workload/stateless/filemanager/filemanager/src/clients/aws/s3.rs similarity index 100% rename from lib/workload/stateful/filemanager/filemanager/src/clients/aws/s3.rs rename to lib/workload/stateless/filemanager/filemanager/src/clients/aws/s3.rs diff --git a/lib/workload/stateful/filemanager/filemanager/src/clients/aws/sqs.rs b/lib/workload/stateless/filemanager/filemanager/src/clients/aws/sqs.rs similarity index 100% rename from lib/workload/stateful/filemanager/filemanager/src/clients/aws/sqs.rs rename to lib/workload/stateless/filemanager/filemanager/src/clients/aws/sqs.rs diff --git a/lib/workload/stateful/filemanager/filemanager/src/clients/mod.rs b/lib/workload/stateless/filemanager/filemanager/src/clients/mod.rs similarity index 100% rename from lib/workload/stateful/filemanager/filemanager/src/clients/mod.rs rename to lib/workload/stateless/filemanager/filemanager/src/clients/mod.rs diff --git a/lib/workload/stateful/filemanager/filemanager/src/database/aws/ingester.rs b/lib/workload/stateless/filemanager/filemanager/src/database/aws/ingester.rs similarity index 100% rename from lib/workload/stateful/filemanager/filemanager/src/database/aws/ingester.rs rename to lib/workload/stateless/filemanager/filemanager/src/database/aws/ingester.rs diff --git a/lib/workload/stateful/filemanager/filemanager/src/database/aws/migration.rs b/lib/workload/stateless/filemanager/filemanager/src/database/aws/migration.rs similarity index 100% rename from lib/workload/stateful/filemanager/filemanager/src/database/aws/migration.rs rename to lib/workload/stateless/filemanager/filemanager/src/database/aws/migration.rs diff --git a/lib/workload/stateful/filemanager/filemanager/src/database/aws/mod.rs b/lib/workload/stateless/filemanager/filemanager/src/database/aws/mod.rs similarity index 100% rename from lib/workload/stateful/filemanager/filemanager/src/database/aws/mod.rs rename to lib/workload/stateless/filemanager/filemanager/src/database/aws/mod.rs diff --git a/lib/workload/stateful/filemanager/filemanager/src/database/mod.rs b/lib/workload/stateless/filemanager/filemanager/src/database/mod.rs similarity index 100% rename from lib/workload/stateful/filemanager/filemanager/src/database/mod.rs rename to lib/workload/stateless/filemanager/filemanager/src/database/mod.rs diff --git a/lib/workload/stateful/filemanager/filemanager/src/env.rs b/lib/workload/stateless/filemanager/filemanager/src/env.rs similarity index 100% rename from lib/workload/stateful/filemanager/filemanager/src/env.rs rename to lib/workload/stateless/filemanager/filemanager/src/env.rs diff --git a/lib/workload/stateful/filemanager/filemanager/src/error.rs b/lib/workload/stateless/filemanager/filemanager/src/error.rs similarity index 100% rename from lib/workload/stateful/filemanager/filemanager/src/error.rs rename to lib/workload/stateless/filemanager/filemanager/src/error.rs diff --git a/lib/workload/stateful/filemanager/filemanager/src/events/aws/collecter.rs b/lib/workload/stateless/filemanager/filemanager/src/events/aws/collecter.rs similarity index 100% rename from lib/workload/stateful/filemanager/filemanager/src/events/aws/collecter.rs rename to lib/workload/stateless/filemanager/filemanager/src/events/aws/collecter.rs diff --git a/lib/workload/stateful/filemanager/filemanager/src/events/aws/collector_builder.rs b/lib/workload/stateless/filemanager/filemanager/src/events/aws/collector_builder.rs similarity index 100% rename from lib/workload/stateful/filemanager/filemanager/src/events/aws/collector_builder.rs rename to lib/workload/stateless/filemanager/filemanager/src/events/aws/collector_builder.rs diff --git a/lib/workload/stateful/filemanager/filemanager/src/events/aws/mod.rs b/lib/workload/stateless/filemanager/filemanager/src/events/aws/mod.rs similarity index 100% rename from lib/workload/stateful/filemanager/filemanager/src/events/aws/mod.rs rename to lib/workload/stateless/filemanager/filemanager/src/events/aws/mod.rs diff --git a/lib/workload/stateful/filemanager/filemanager/src/events/mod.rs b/lib/workload/stateless/filemanager/filemanager/src/events/mod.rs similarity index 100% rename from lib/workload/stateful/filemanager/filemanager/src/events/mod.rs rename to lib/workload/stateless/filemanager/filemanager/src/events/mod.rs diff --git a/lib/workload/stateful/filemanager/filemanager/src/handlers/aws.rs b/lib/workload/stateless/filemanager/filemanager/src/handlers/aws.rs similarity index 100% rename from lib/workload/stateful/filemanager/filemanager/src/handlers/aws.rs rename to lib/workload/stateless/filemanager/filemanager/src/handlers/aws.rs diff --git a/lib/workload/stateful/filemanager/filemanager/src/handlers/mod.rs b/lib/workload/stateless/filemanager/filemanager/src/handlers/mod.rs similarity index 100% rename from lib/workload/stateful/filemanager/filemanager/src/handlers/mod.rs rename to lib/workload/stateless/filemanager/filemanager/src/handlers/mod.rs diff --git a/lib/workload/stateful/filemanager/filemanager/src/lib.rs b/lib/workload/stateless/filemanager/filemanager/src/lib.rs similarity index 100% rename from lib/workload/stateful/filemanager/filemanager/src/lib.rs rename to lib/workload/stateless/filemanager/filemanager/src/lib.rs diff --git a/lib/workload/stateful/filemanager/scripts/deploy.sh b/lib/workload/stateless/filemanager/scripts/deploy.sh similarity index 100% rename from lib/workload/stateful/filemanager/scripts/deploy.sh rename to lib/workload/stateless/filemanager/scripts/deploy.sh diff --git a/lib/workload/stateful/filemanager/scripts/logs.sh b/lib/workload/stateless/filemanager/scripts/logs.sh similarity index 100% rename from lib/workload/stateful/filemanager/scripts/logs.sh rename to lib/workload/stateless/filemanager/scripts/logs.sh diff --git a/lib/workload/stateful/filemanager/scripts/watch.sh b/lib/workload/stateless/filemanager/scripts/watch.sh similarity index 100% rename from lib/workload/stateful/filemanager/scripts/watch.sh rename to lib/workload/stateless/filemanager/scripts/watch.sh From be5aec106b47798c57a015f98a05dd15677283f6 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Thu, 8 Feb 2024 09:51:50 +1100 Subject: [PATCH 05/13] refactor(filemanager): remove deploy-level package info --- .../stateless/filemanager/deploy/cdk.json | 29 ----------------- .../stateless/filemanager/deploy/package.json | 31 ------------------- .../filemanager/deploy/tsconfig.json | 23 -------------- 3 files changed, 83 deletions(-) delete mode 100644 lib/workload/stateless/filemanager/deploy/cdk.json delete mode 100644 lib/workload/stateless/filemanager/deploy/package.json delete mode 100644 lib/workload/stateless/filemanager/deploy/tsconfig.json diff --git a/lib/workload/stateless/filemanager/deploy/cdk.json b/lib/workload/stateless/filemanager/deploy/cdk.json deleted file mode 100644 index a795fe39a..000000000 --- a/lib/workload/stateless/filemanager/deploy/cdk.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "app": "npx ts-node --prefer-ts-exts bin/filemanager.ts", - "watch": { - "include": ["**"], - "exclude": [ - "README.md", - "cdk*.json", - "**/*.d.ts", - "**/*.js", - "tsconfig.json", - "package*.json", - "yarn.lock", - "node_modules", - "test" - ] - }, - "context": { - "@aws-cdk/aws-apigateway:usagePlanKeyOrderInsensitiveId": true, - "@aws-cdk/core:stackRelativeExports": true, - "@aws-cdk/aws-rds:lowercaseDbIdentifier": true, - "@aws-cdk/aws-lambda:recognizeVersionProps": true, - "@aws-cdk/aws-cloudfront:defaultSecurityPolicyTLSv1.2_2021": true, - "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true, - "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true, - "@aws-cdk/core:checkSecretUsage": true, - "@aws-cdk/aws-iam:minimizePolicies": true, - "@aws-cdk/core:target-partitions": ["aws", "aws-cn"] - } -} diff --git a/lib/workload/stateless/filemanager/deploy/package.json b/lib/workload/stateless/filemanager/deploy/package.json deleted file mode 100644 index f92fb23fb..000000000 --- a/lib/workload/stateless/filemanager/deploy/package.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "name": "filemanager", - "version": "0.1", - "bin": { - "filemanager": "bin/filemanager.js" - }, - "scripts": { - "build": "tsc", - "watch": "tsc -w", - "cdk": "cdk", - "migrate": "cross-env FILEMANAGER_DEPLOY_MIGRATE_DATABASE=true" - }, - "devDependencies": { - "@types/node": "^20.5.9", - "@typescript-eslint/eslint-plugin": "^6.14.0", - "aws-cdk": "^2.114.1", - "cross-env": "^7.0.3", - "prettier": "3.0.3", - "typescript": "^5.3.3" - }, - "dependencies": { - "@aws-cdk/aws-apigatewayv2-alpha": "^2.114.1-alpha.0", - "@aws-cdk/aws-apigatewayv2-authorizers-alpha": "^2.114.1-alpha.0", - "@aws-cdk/aws-apigatewayv2-integrations-alpha": "^2.114.1-alpha.0", - "aws-cdk-lib": "^2.114.1", - "constructs": "^10.3.0", - "dotenv": "^16.3.1", - "rust.aws-cdk-lambda": "^1.2.1", - "source-map-support": "^0.5.21" - } -} diff --git a/lib/workload/stateless/filemanager/deploy/tsconfig.json b/lib/workload/stateless/filemanager/deploy/tsconfig.json deleted file mode 100644 index 810149e3c..000000000 --- a/lib/workload/stateless/filemanager/deploy/tsconfig.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "compilerOptions": { - "target": "ES2021", - "module": "commonjs", - "lib": ["ES2021"], - "declaration": true, - "strict": true, - "noImplicitAny": true, - "strictNullChecks": true, - "noImplicitThis": true, - "alwaysStrict": true, - "noUnusedLocals": false, - "noUnusedParameters": false, - "noImplicitReturns": true, - "noFallthroughCasesInSwitch": false, - "inlineSourceMap": true, - "inlineSources": true, - "experimentalDecorators": true, - "strictPropertyInitialization": false, - "typeRoots": ["./node_modules/@types"] - }, - "exclude": ["node_modules", "cdk.out"] -} From 3a0c8db32ded0112d40054fd29faaea3a98dd532 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Thu, 8 Feb 2024 09:59:12 +1100 Subject: [PATCH 06/13] docs(filemanager): update deploy README.md --- .../stateless/filemanager/deploy/README.md | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/lib/workload/stateless/filemanager/deploy/README.md b/lib/workload/stateless/filemanager/deploy/README.md index 6c30a6215..9780cfffb 100644 --- a/lib/workload/stateless/filemanager/deploy/README.md +++ b/lib/workload/stateless/filemanager/deploy/README.md @@ -1,21 +1,23 @@ -# Deployment of filemanager +# CDK constructs for filemanager -This folder contains CDK deployment code for filemanager. The CDK code can be deployed using `cdk`: +This folder contains CDK constructs for filemanager which is used by the outer project +to instantiate and deploy filemanager. -```sh -npm install -cdk bootstrap -cdk deploy -``` +## Overview -By default, the stack does not perform database migration. To migrate the database, use the script inside `package.json`: +The primary component that is deployed in filemanager is Lambda functions for each of the lambda +crates within this project. This, alongside the shared database in Orcabus allows the filemanager +to perform file ingestion, querying, etc. -```sh -npm run migrate -- cdk deploy -``` +### Inputs -or set `FILEMANAGER_DEPLOY_DATABASE_MIGRATION`: +The filemanage operates on S3 buckets, and expects queues as input which can recieve S3 events. It also requires +the bucket names to create policies that allow S3 's3:List*' and 's3:Get*' operations. These operations are used to +fetch additional object data such as storage classes. -```sh -export FILEMANAGER_DEPLOY_DATABASE_MIGRATION="true" -``` +### Migration + +The filemanager expects a dedicated database within the shared database cluster, which it can use to store data. +Initially, the filemanager-migrate-lambda function is deployed to perform database table migrations using the +cdk_resource_invoke.ts construct. Then, the other Lambda functions are deployed normally within the filemanager +construct. From 1e95b11fff3f70c15412ff637db1c49c5684a0fc Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Thu, 8 Feb 2024 12:47:17 +1100 Subject: [PATCH 07/13] refactor: make the cdk_resource_invoke construct generic over lambda function type --- .../deploy/constructs/functions/function.ts | 5 +- .../filemanager/deploy/lib/filemanager.ts | 2 +- .../cdk_resource_invoke.ts | 57 +++++++++++++++---- 3 files changed, 49 insertions(+), 15 deletions(-) rename lib/workload/stateless/{filemanager/deploy/constructs => functions}/cdk_resource_invoke.ts (67%) diff --git a/lib/workload/stateless/filemanager/deploy/constructs/functions/function.ts b/lib/workload/stateless/filemanager/deploy/constructs/functions/function.ts index 36b1acbc5..0ab13e5c4 100644 --- a/lib/workload/stateless/filemanager/deploy/constructs/functions/function.ts +++ b/lib/workload/stateless/filemanager/deploy/constructs/functions/function.ts @@ -7,6 +7,7 @@ import { IVpc, SecurityGroup, SubnetType } from 'aws-cdk-lib/aws-ec2'; import { Architecture, IDestination, Version } from 'aws-cdk-lib/aws-lambda'; import { ManagedPolicy, PolicyStatement, Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam'; + /** * Settable values for a Rust function. */ @@ -127,14 +128,14 @@ export class Function extends Construct { /** * Get the function name. */ - functionName(): string { + get functionName(): string { return this.function.functionName; } /** * Get the function version. */ - currentVersion(): Version { + get currentVersion(): Version { return this.function.currentVersion; } diff --git a/lib/workload/stateless/filemanager/deploy/lib/filemanager.ts b/lib/workload/stateless/filemanager/deploy/lib/filemanager.ts index 92fa59dcd..603d7cade 100644 --- a/lib/workload/stateless/filemanager/deploy/lib/filemanager.ts +++ b/lib/workload/stateless/filemanager/deploy/lib/filemanager.ts @@ -1,6 +1,6 @@ import { Construct } from 'constructs'; import { IngestFunction, IngestFunctionSettings } from '../constructs/functions/ingest'; -import { CdkResourceInvoke } from '../constructs/cdk_resource_invoke'; +import { CdkResourceInvoke } from '../../../functions/cdk_resource_invoke'; import { MigrateFunction } from '../constructs/functions/migrate'; import * as fn from '../constructs/functions/function'; import { IDatabase } from '../../../../stateful/database/component'; diff --git a/lib/workload/stateless/filemanager/deploy/constructs/cdk_resource_invoke.ts b/lib/workload/stateless/functions/cdk_resource_invoke.ts similarity index 67% rename from lib/workload/stateless/filemanager/deploy/constructs/cdk_resource_invoke.ts rename to lib/workload/stateless/functions/cdk_resource_invoke.ts index 5796a41c3..94577c469 100644 --- a/lib/workload/stateless/filemanager/deploy/constructs/cdk_resource_invoke.ts +++ b/lib/workload/stateless/functions/cdk_resource_invoke.ts @@ -5,29 +5,60 @@ import { AwsSdkCall, PhysicalResourceId, } from 'aws-cdk-lib/custom-resources'; -import * as fn from './functions/function'; +import * as lambda from 'aws-cdk-lib/aws-lambda'; import { CfnOutput, Stack, Token } from 'aws-cdk-lib'; import { IVpc, SubnetType } from 'aws-cdk-lib/aws-ec2'; import { ManagedPolicy, PolicyStatement, Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam'; +import { Version } from 'aws-cdk-lib/aws-lambda'; + +/** + * The interface by which the generic type of `CdkResourceInvoke` is constrained by. + * Note that the standard `lambda.Function` meets this constraint, which means it can + * be used directly with the `CdkResourceInvoke`, rather than having to wrap it in + * another class. + */ +export interface InvokeFunction { + /** + * The function name to be used when constructing the invoke function. + */ + functionName?: string; + /** + * The current version of the function. + */ + currentVersion: Version; +} + +/** + * An interface representing the function name that is created when using `CdkResourceInvoke`. + */ +export type FunctionName = { + /** + * Function name. + */ + functionName: string; +} /** * Props for the resource invoke construct. */ -export type CdkResourceInvokeProps = { +export type CdkResourceInvokeProps = { /** * Vpc for the function. */ vpc: IVpc; /** - * The function to create. This will override the function name to ensure that it remains + * The function to create. This will override any `functionName` property to ensure that it remains * callable using the singleton function created by `AwsCustomResource`. See * https://github.com/aws-samples/amazon-rds-init-cdk/blob/239626632f399ebe4928410a49d5ac5d009a6502/lib/resource-initializer.ts#L69-L71. + * + * It is expected that this creates a Lambda function with the `functionName`. This allows `CdkResourceInvoke` + * to call the function after it is created. */ - createFunction: (scope: Construct, id: string, props: fn.FunctionPropsNoPackage) => fn.Function; + createFunction: (scope: Construct, id: string, props: FunctionName & P) => F; /** * Function props when creating the Lambda function. */ - functionProps: fn.FunctionPropsNoPackage; + functionProps: P; /** * Name to use when creating the function. */ @@ -39,14 +70,16 @@ export type CdkResourceInvokeProps = { }; /** - * A construct for invoking a Lambda function for resource initialization. + * A construct for invoking a Lambda function for resource initialization. This is useful for performing + * database actions such as migrations during CloudFormation stack creation, to ensure that a database is + * in the expected state before the stack succeeds. */ -export class CdkResourceInvoke extends Construct { +export class CdkResourceInvoke extends Construct { private readonly _response: string; private readonly _customResource: AwsCustomResource; - private readonly _function: fn.Function; + private readonly _function: F; - constructor(scope: Construct, id: string, props: CdkResourceInvokeProps) { + constructor(scope: Construct, id: string, props: CdkResourceInvokeProps) { super(scope, id); const stack = Stack.of(this); @@ -60,10 +93,10 @@ export class CdkResourceInvoke extends Construct { service: 'Lambda', action: 'invoke', parameters: { - FunctionName: this.function.functionName(), + FunctionName: this.function.functionName, }, physicalResourceId: PhysicalResourceId.of( - `${id}-AwsSdkCall-${this.function.currentVersion()}` + `${id}-AwsSdkCall-${this.function.currentVersion}` ), }; @@ -130,7 +163,7 @@ export class CdkResourceInvoke extends Construct { /** * Get the function. */ - get function(): fn.Function { + get function(): F { return this._function; } } From 463b10b8eb13aa5a0022acede5668d11e312d5f6 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Thu, 8 Feb 2024 12:55:10 +1100 Subject: [PATCH 08/13] style: line changes and imports --- Makefile | 2 +- lib/workload/stateless/filemanager/Makefile | 2 +- lib/workload/stateless/filemanager/deploy/README.md | 2 +- lib/workload/stateless/filemanager/docker-compose.yml | 2 +- lib/workload/stateless/functions/cdk_resource_invoke.ts | 1 - 5 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 4b594c98d..22df9a164 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ test: suite: test @(cd lib/workload/stateless/sequence_run_manager && $(MAKE) test) @(cd lib/workload/stateless/metadata_manager && $(MAKE) test) - @(cd lib/workload/stateful/filemanager && $(MAKE) test) + @(cd lib/workload/stateless/filemanager && $(MAKE) test) clean: @yarn clean diff --git a/lib/workload/stateless/filemanager/Makefile b/lib/workload/stateless/filemanager/Makefile index ff026dbba..d41242e12 100644 --- a/lib/workload/stateless/filemanager/Makefile +++ b/lib/workload/stateless/filemanager/Makefile @@ -46,4 +46,4 @@ psql: ## Help text help: @printf "The filemanager Makefile.\n\n" - @printf "Usage: make [POSTGRES_DATABASE_URL=]\n" \ No newline at end of file + @printf "Usage: make [POSTGRES_DATABASE_URL=]\n" diff --git a/lib/workload/stateless/filemanager/deploy/README.md b/lib/workload/stateless/filemanager/deploy/README.md index 9780cfffb..caea31a55 100644 --- a/lib/workload/stateless/filemanager/deploy/README.md +++ b/lib/workload/stateless/filemanager/deploy/README.md @@ -11,7 +11,7 @@ to perform file ingestion, querying, etc. ### Inputs -The filemanage operates on S3 buckets, and expects queues as input which can recieve S3 events. It also requires +The filemanager operates on S3 buckets, and expects queues as input which can recieve S3 events. It also requires the bucket names to create policies that allow S3 's3:List*' and 's3:Get*' operations. These operations are used to fetch additional object data such as storage classes. diff --git a/lib/workload/stateless/filemanager/docker-compose.yml b/lib/workload/stateless/filemanager/docker-compose.yml index ee8cfc40f..fb6a3c82f 100644 --- a/lib/workload/stateless/filemanager/docker-compose.yml +++ b/lib/workload/stateless/filemanager/docker-compose.yml @@ -11,4 +11,4 @@ services: - POSTGRES_PASSWORD=filemanager - PGPORT=4321 ports: - - "4321:4321" \ No newline at end of file + - "4321:4321" diff --git a/lib/workload/stateless/functions/cdk_resource_invoke.ts b/lib/workload/stateless/functions/cdk_resource_invoke.ts index 94577c469..a274530e2 100644 --- a/lib/workload/stateless/functions/cdk_resource_invoke.ts +++ b/lib/workload/stateless/functions/cdk_resource_invoke.ts @@ -5,7 +5,6 @@ import { AwsSdkCall, PhysicalResourceId, } from 'aws-cdk-lib/custom-resources'; -import * as lambda from 'aws-cdk-lib/aws-lambda'; import { CfnOutput, Stack, Token } from 'aws-cdk-lib'; import { IVpc, SubnetType } from 'aws-cdk-lib/aws-ec2'; import { ManagedPolicy, PolicyStatement, Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam'; From 0b3996e368b97a7bbed5d54ac05d7bab2e8202a7 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Fri, 16 Feb 2024 13:02:56 +1100 Subject: [PATCH 09/13] build(filemanager): add PHONY target and ignored tests to Makefile --- lib/workload/stateless/filemanager/Makefile | 5 +++++ lib/workload/stateless/filemanager/README.md | 6 ++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/workload/stateless/filemanager/Makefile b/lib/workload/stateless/filemanager/Makefile index d41242e12..483123c4c 100644 --- a/lib/workload/stateless/filemanager/Makefile +++ b/lib/workload/stateless/filemanager/Makefile @@ -25,6 +25,8 @@ watch: build @cargo watch -w *.toml -w *.rs test: build @cargo test --all-features --all-targets +test-ignored: build + @cargo test --all-features --all-targets -- --ignored lint: @cargo fmt --check lint-fix: @@ -47,3 +49,6 @@ psql: help: @printf "The filemanager Makefile.\n\n" @printf "Usage: make [POSTGRES_DATABASE_URL=]\n" + +# No targets are files, so they are all PHONY. +.PHONY: * diff --git a/lib/workload/stateless/filemanager/README.md b/lib/workload/stateless/filemanager/README.md index 0685c3801..0a795176d 100644 --- a/lib/workload/stateless/filemanager/README.md +++ b/lib/workload/stateless/filemanager/README.md @@ -72,12 +72,14 @@ Testing and linting should be run before committing changes to the repository. See the [deploy][deploy] directory for the cdk infrastructure code. Note, some tests can take a while to run so they are ignored by default. It is still useful to run these tests sometimes, -especially if changing code related to them. To do so, run the ignored tests with the `--ignored` flag: +especially if changing code related to them. To do so, run the ignored tests with: ```sh -cargo test --all-targets --all-features -- --ignored +make test-ignored ``` +Which runs `cargo test -- --ignored`. + ## Database To connect to the local postgres database, run: From 7bb8e244ca4211f888583e45c2db84e6805b8800 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Fri, 16 Feb 2024 16:21:58 +1100 Subject: [PATCH 10/13] refactor: link up stateless stack resources and filemanager --- config/constants.ts | 16 +++- lib/workload/orcabus-stateless-stack.ts | 82 ++++++++++++++----- lib/workload/stateful/database/component.ts | 37 ++------- .../deploy/constructs/functions/function.ts | 55 +++++++------ .../deploy/constructs/functions/ingest.ts | 8 +- .../deploy/constructs/functions/migrate.ts | 7 +- .../filemanager/deploy/lib/filemanager.ts | 34 ++++---- 7 files changed, 131 insertions(+), 108 deletions(-) diff --git a/config/constants.ts b/config/constants.ts index 417d13d88..d5137f110 100644 --- a/config/constants.ts +++ b/config/constants.ts @@ -1,6 +1,9 @@ import { OrcaBusStatefulConfig } from '../lib/workload/orcabus-stateful-stack'; import { AuroraPostgresEngineVersion } from 'aws-cdk-lib/aws-rds'; -import { OrcaBusStatelessConfig } from '../lib/workload/orcabus-stateless-stack'; +import { + FilemanagerDependencies, + OrcaBusStatelessConfig, +} from '../lib/workload/orcabus-stateless-stack'; import { Duration, aws_lambda, RemovalPolicy } from 'aws-cdk-lib'; import { EventSourceProps } from '../lib/workload/stateful/event_source/component'; @@ -28,9 +31,11 @@ const orcaBusStatefulConfig = { username: 'postgres', dbPort: 5432, masterSecretName: rdsMasterSecretName, + securityGroupName: 'orcabus-database-security-group', monitoring: { cloudwatchLogsExports: ['orcabus-postgresql'], }, + databaseSecurityGroupName: 'database-security-group', }, securityGroupProps: { securityGroupName: lambdaSecurityGroupName, @@ -73,6 +78,13 @@ const eventSourceConfig: EventSourceProps = { ], }; +const filemanagerDependencies: FilemanagerDependencies = { + eventSourceBuckets: ['umccr-temp-dev'], + eventSourceQueueName: eventSourceConfig.queueName, + databaseSecretName: orcaBusStatefulConfig.databaseProps.masterSecretName, + databaseSecurityGroupName: orcaBusStatefulConfig.databaseProps.databaseSecurityGroupName, +}; + interface EnvironmentConfig { name: string; accountId: string; @@ -113,7 +125,7 @@ export const getEnvironmentConfig = ( }, orcaBusStatelessConfig: { ...orcaBusStatelessConfig, - eventSourceQueueName: eventSourceConfig.queueName, + filemanagerDependencies: filemanagerDependencies, }, }, }; diff --git a/lib/workload/orcabus-stateless-stack.ts b/lib/workload/orcabus-stateless-stack.ts index 4ee0ea8a3..59f920e36 100644 --- a/lib/workload/orcabus-stateless-stack.ts +++ b/lib/workload/orcabus-stateless-stack.ts @@ -1,9 +1,12 @@ import * as cdk from 'aws-cdk-lib'; -import { aws_lambda } from 'aws-cdk-lib'; +import { Arn, aws_lambda } from 'aws-cdk-lib'; import { Construct } from 'constructs'; import { getVpc } from './stateful/vpc/component'; import { MultiSchemaConstructProps } from './stateless/schema/component'; -import { IVpc } from 'aws-cdk-lib/aws-ec2'; +import { IVpc, SecurityGroup } from 'aws-cdk-lib/aws-ec2'; +import { Filemanager } from './stateless/filemanager/deploy/lib/filemanager'; +import { Queue } from 'aws-cdk-lib/aws-sqs'; +import { Secret } from 'aws-cdk-lib/aws-secretsmanager'; export interface OrcaBusStatelessConfig { multiSchemaConstructProps: MultiSchemaConstructProps; @@ -12,7 +15,26 @@ export interface OrcaBusStatelessConfig { lambdaRuntimePythonVersion: aws_lambda.Runtime; bclConvertFunctionName: string; rdsMasterSecretName: string; - eventSourceQueueName?: string; + filemanagerDependencies?: FilemanagerDependencies; +} + +export interface FilemanagerDependencies { + /** + * Queue name used by the EventSource construct. + */ + eventSourceQueueName: string; + /** + * Buckets defined by the EventSource construct. + */ + eventSourceBuckets: string[]; + /** + * Database secret name for the filemanager. + */ + databaseSecretName: string; + /** + * Database security group name to allow the filemanager Lambda to connect. + */ + databaseSecurityGroupName: string; } export class OrcaBusStatelessStack extends cdk.Stack { @@ -42,7 +64,9 @@ export class OrcaBusStatelessStack extends cdk.Stack { // hook microservice construct components here this.createSequenceRunManager(); - this.createFilemanager(); + if (props.filemanagerDependencies) { + this.createFilemanager(props.filemanagerDependencies, this.vpc); + } } private createSequenceRunManager() { @@ -50,21 +74,39 @@ export class OrcaBusStatelessStack extends cdk.Stack { // However, the implementation is still incomplete... } - private createFilemanager() { - // todo, implement after https://github.com/umccr/orcabus/issues/86 - // new Filemanager( - // this, - // "Filemanager", - // { - // buckets: [], - // buildEnvironment: {}, - // database: undefined, - // eventSources: [], - // migrateDatabase: false, - // onFailure: undefined, - // rustLog: '', - // vpc: undefined - // } - // ); + private createFilemanager(dependencies: FilemanagerDependencies, vpc: IVpc) { + // Opting to reconstruct the dependencies here, and pass them into the service as constructs. + const queue = Queue.fromQueueArn( + this, + 'Filemanager Queue', + Arn.format( + { + resource: dependencies.eventSourceQueueName, + service: 'Queue', + }, + this + ) + ); + const securityGroup = SecurityGroup.fromLookupByName( + this, + 'Filemanager Database Security Group', + dependencies.databaseSecurityGroupName, + vpc + ); + const databaseSecret = Secret.fromSecretNameV2( + this, + 'Filemanager Database Secret', + dependencies.databaseSecretName + ); + + new Filemanager(this, 'Filemanager', { + buckets: dependencies.eventSourceBuckets, + buildEnvironment: {}, + databaseSecret: databaseSecret, + databaseSecurityGroup: securityGroup, + eventSources: [queue], + migrateDatabase: true, + vpc: vpc, + }); } } diff --git a/lib/workload/stateful/database/component.ts b/lib/workload/stateful/database/component.ts index 3054a0778..abc3f6290 100644 --- a/lib/workload/stateful/database/component.ts +++ b/lib/workload/stateful/database/component.ts @@ -45,6 +45,10 @@ export type DatabasePropsNoVPC = MonitoringProps & { * Database username. */ username: string; + /** + * Database security group name. + */ + securityGroupName: string; /** * Database secret name. */ @@ -89,28 +93,9 @@ export type DatabaseProps = DatabasePropsNoVPC & { vpc: ec2.IVpc; }; -/** - * Interface representing the database. - */ -export interface IDatabase { - /** - * The database security group. - */ +export class Database extends Construct { readonly securityGroup: ec2.SecurityGroup; - /** - * The database cluster. - */ readonly cluster: rds.DatabaseCluster; - /** - * A connection string to the database. - */ - readonly unsafeConnection: string; -} - -export class Database extends Construct implements IDatabase { - readonly securityGroup: ec2.SecurityGroup; - readonly cluster: rds.DatabaseCluster; - readonly unsafeConnection: string; constructor(scope: Construct, id: string, props: DatabaseProps) { super(scope, id); @@ -124,6 +109,7 @@ export class Database extends Construct implements IDatabase { vpc: props.vpc, allowAllOutbound: false, allowAllIpv6Outbound: false, + securityGroupName: props.securityGroupName, description: 'security group for OrcaBus RDS', }); @@ -165,16 +151,5 @@ export class Database extends Construct implements IDatabase { enablePerformanceInsights: props.enablePerformanceInsights, }), }); - - // Todo use secrets manager for this and query for password within Lambda functions. - this.unsafeConnection = - `postgres://` + - `${dbSecret.secretValueFromJson('username').unsafeUnwrap()}` + - `:` + - `${dbSecret.secretValueFromJson('password').unsafeUnwrap()}` + - `@` + - `${this.cluster.clusterEndpoint.socketAddress}` + - `/` + - `${props.defaultDatabaseName}`; } } diff --git a/lib/workload/stateless/filemanager/deploy/constructs/functions/function.ts b/lib/workload/stateless/filemanager/deploy/constructs/functions/function.ts index 0ab13e5c4..80c77bda0 100644 --- a/lib/workload/stateless/filemanager/deploy/constructs/functions/function.ts +++ b/lib/workload/stateless/filemanager/deploy/constructs/functions/function.ts @@ -2,16 +2,15 @@ import { Construct } from 'constructs'; import { RustFunction } from 'rust.aws-cdk-lambda'; import { Duration } from 'aws-cdk-lib'; import { Settings as CargoSettings } from 'rust.aws-cdk-lambda/dist/settings'; -import { IDatabase } from '../../../../../stateful/database/component'; -import { IVpc, SecurityGroup, SubnetType } from 'aws-cdk-lib/aws-ec2'; -import { Architecture, IDestination, Version } from 'aws-cdk-lib/aws-lambda'; +import { ISecurityGroup, IVpc, SecurityGroup, SubnetType } from 'aws-cdk-lib/aws-ec2'; +import { Architecture, Version } from 'aws-cdk-lib/aws-lambda'; import { ManagedPolicy, PolicyStatement, Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam'; - +import { ISecret } from 'aws-cdk-lib/aws-secretsmanager'; /** - * Settable values for a Rust function. + * Props for a Rust function without the package. */ -export type FunctionSettings = { +export type FunctionPropsNoPackage = { /** * Additional build environment variables when building the Lambda function. */ @@ -20,28 +19,18 @@ export type FunctionSettings = { * RUST_LOG string, defaults to trace on local crates and info everywhere else. */ readonly rustLog?: string; -}; - -/** - * Props for a Rust function without the package. - */ -export type FunctionPropsNoPackage = FunctionSettings & { /** * Vpc for the function. */ readonly vpc: IVpc; /** - * Database that the function uses. - */ - readonly database: IDatabase; - /** - * The destination to post failed invocations to. + * The database secret. */ - readonly onFailure?: IDestination; + readonly databaseSecret: ISecret; /** - * Name of the Lambda function resource. + * The database security group. */ - readonly functionName?: string; + readonly databaseSecurityGroup: ISecurityGroup; }; /** @@ -52,6 +41,10 @@ export type FunctionProps = FunctionPropsNoPackage & { * The package to build for this function. */ readonly package: string; + /** + * Name of the Lambda function resource. + */ + readonly functionName?: string; }; /** @@ -82,28 +75,40 @@ export class Function extends Construct { CargoSettings.BUILD_INDIVIDUALLY = true; + // Todo use secrets manager for this and query for password within Lambda functions. + const unsafeConnection = + `postgres://` + + `${props.databaseSecret.secretValueFromJson('username').unsafeUnwrap()}` + + `:` + + `${props.databaseSecret.secretValueFromJson('password').unsafeUnwrap()}` + + `@` + + `${props.databaseSecret.secretValueFromJson('host').unsafeUnwrap()}` + + ':' + + `${props.databaseSecret.secretValueFromJson('port').unsafeUnwrap()}` + + `/` + + `${props.databaseSecret.secretValueFromJson('dbname').unsafeUnwrap()}`; + this._function = new RustFunction(this, 'RustFunction', { package: props.package, target: 'aarch64-unknown-linux-gnu', memorySize: 128, timeout: Duration.seconds(28), environment: { - // Todo use security manager to get connection string rather than passing it in as an environment variable - DATABASE_URL: props.database.unsafeConnection, + DATABASE_URL: unsafeConnection, RUST_LOG: props.rustLog ?? `info,${props.package.replace('-', '_')}=trace,filemanager=trace`, }, buildEnvironment: props.buildEnvironment, - extraBuildArgs: ['--manifest-path', `../${props.package}/Cargo.toml`], + // Todo, fix this so that it's not relying on the path. + extraBuildArgs: ['--manifest-path', `lib/workload/stateless/filemanager/${props.package}/Cargo.toml`], architecture: Architecture.ARM_64, role: this._role, - onFailure: props.onFailure, vpc: props.vpc, vpcSubnets: { subnetType: SubnetType.PRIVATE_WITH_EGRESS }, securityGroups: [ securityGroup, // Allow access to database. - props.database.securityGroup, + props.databaseSecurityGroup ], functionName: props.functionName, }); diff --git a/lib/workload/stateless/filemanager/deploy/constructs/functions/ingest.ts b/lib/workload/stateless/filemanager/deploy/constructs/functions/ingest.ts index 967ee3a90..9af453e2f 100644 --- a/lib/workload/stateless/filemanager/deploy/constructs/functions/ingest.ts +++ b/lib/workload/stateless/filemanager/deploy/constructs/functions/ingest.ts @@ -4,16 +4,10 @@ import * as fn from './function'; import { PolicyStatement } from 'aws-cdk-lib/aws-iam'; import { SqsEventSource } from 'aws-cdk-lib/aws-lambda-event-sources'; -/** - * Settable values for the ingest function. - */ -export type IngestFunctionSettings = fn.FunctionSettings; - /** * Props for the ingest function. */ -export type IngestFunctionProps = IngestFunctionSettings & - fn.FunctionPropsNoPackage & { +export type IngestFunctionProps = fn.FunctionPropsNoPackage & { /** * The SQS queue URL to receive events from. */ diff --git a/lib/workload/stateless/filemanager/deploy/constructs/functions/migrate.ts b/lib/workload/stateless/filemanager/deploy/constructs/functions/migrate.ts index 45982ca2a..990b5f567 100644 --- a/lib/workload/stateless/filemanager/deploy/constructs/functions/migrate.ts +++ b/lib/workload/stateless/filemanager/deploy/constructs/functions/migrate.ts @@ -1,15 +1,10 @@ import { Construct } from 'constructs'; import * as fn from './function'; -/** - * Settable values for the migrate function. - */ -export type MigrateFunctionSettings = fn.FunctionSettings; - /** * Props for the migrate function. */ -export type MigrateFunctionProps = MigrateFunctionSettings & fn.FunctionPropsNoPackage; +export type MigrateFunctionProps = fn.FunctionPropsNoPackage; /** * A construct for the Lambda migrate function. diff --git a/lib/workload/stateless/filemanager/deploy/lib/filemanager.ts b/lib/workload/stateless/filemanager/deploy/lib/filemanager.ts index 603d7cade..c4a943586 100644 --- a/lib/workload/stateless/filemanager/deploy/lib/filemanager.ts +++ b/lib/workload/stateless/filemanager/deploy/lib/filemanager.ts @@ -1,25 +1,28 @@ import { Construct } from 'constructs'; -import { IngestFunction, IngestFunctionSettings } from '../constructs/functions/ingest'; +import { IngestFunction, IngestFunctionProps } from '../constructs/functions/ingest'; import { CdkResourceInvoke } from '../../../functions/cdk_resource_invoke'; import { MigrateFunction } from '../constructs/functions/migrate'; import * as fn from '../constructs/functions/function'; -import { IDatabase } from '../../../../stateful/database/component'; -import { IVpc } from 'aws-cdk-lib/aws-ec2'; +import { ISecurityGroup, IVpc } from 'aws-cdk-lib/aws-ec2'; import { IQueue } from 'aws-cdk-lib/aws-sqs'; -import { IDestination } from 'aws-cdk-lib/aws-lambda'; +import { ISecret } from 'aws-cdk-lib/aws-secretsmanager'; /** - * Common settings for the filemanager stack. + * Props for the filemanager stack. */ -type Settings = IngestFunctionSettings & { +type FilemanagerProps = IngestFunctionProps & { /** * VPC to use for filemanager. */ readonly vpc: IVpc, /** - * The database for filemanager. + * The database secret. */ - readonly database: IDatabase; + readonly databaseSecret: ISecret; + /** + * The database security group. + */ + readonly databaseSecurityGroup: ISecurityGroup; /** * Whether to initialize a database migration. */ @@ -28,10 +31,6 @@ type Settings = IngestFunctionSettings & { * Event sources to use for the filemanager */ readonly eventSources: IQueue[]; - /** - * Location to send any events that could not be processed. - */ - readonly onFailure: IDestination; /** * The buckets that the filemanager is expected to process. This will add policies to access the buckets via * 's3:List*' and 's3:Get*'. @@ -43,7 +42,7 @@ type Settings = IngestFunctionSettings & { * Construct used to configure the filemanager. */ export class Filemanager extends Construct { - constructor(scope: Construct, id: string, settings: Settings) { + constructor(scope: Construct, id: string, settings: FilemanagerProps) { super(scope, id); if (settings?.migrateDatabase) { @@ -54,20 +53,21 @@ export class Filemanager extends Construct { }, functionProps: { vpc: settings.vpc, - database: settings.database, + databaseSecret: settings.databaseSecret, + databaseSecurityGroup: settings.databaseSecurityGroup, buildEnvironment: settings?.buildEnvironment, rustLog: settings?.rustLog, }, id: 'MigrateFunction', - dependencies: [settings.database.cluster], + // Assuming no dependencies because the database will already exist. }); } new IngestFunction(this, 'IngestLambda', { vpc: settings.vpc, - database: settings.database, + databaseSecret: settings.databaseSecret, + databaseSecurityGroup: settings.databaseSecurityGroup, eventSources: settings.eventSources, - onFailure: settings.onFailure, buckets: settings.buckets, buildEnvironment: settings?.buildEnvironment, rustLog: settings?.rustLog, From 17569605a5b7764ed9ece663f2823e20bc4d83bf Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Sun, 18 Feb 2024 14:27:53 +1100 Subject: [PATCH 11/13] refactor(filemanager): database property links using names, security groups and pre-compiled sqlx queries --- .secrets.baseline | 8 +- config/constants.ts | 3 +- lib/workload/orcabus-stateful-stack.ts | 4 +- lib/workload/orcabus-stateless-stack.ts | 12 +- lib/workload/stateful/database/component.ts | 37 +++-- ...0c8eec41a9bd4bcc72a06d36a5d0eed482115.json | 117 ++++++++++++++ ...f43b6f0d78fd3a31ae2517ba889ed50496dff.json | 64 ++++++++ ...df247af518d4fafe42761f1488d1927d616c6.json | 65 ++++++++ ...91338f4623a239c677301c9d13bcbd5e65e78.json | 14 ++ ...e2130db9c1526f858159de63c3af383f4a3b6.json | 147 ++++++++++++++++++ lib/workload/stateless/filemanager/Makefile | 7 +- lib/workload/stateless/filemanager/README.md | 42 ++++- .../deploy/constructs/functions/function.ts | 24 +-- .../filemanager/deploy/lib/filemanager.ts | 44 +++--- .../filemanager/docs/ARCHITECTURE.md | 0 .../functions/cdk_resource_invoke.ts | 23 ++- test/stateful/databaseConstruct.test.ts | 21 ++- 17 files changed, 557 insertions(+), 75 deletions(-) create mode 100644 lib/workload/stateless/filemanager/.sqlx/query-6515aa29536fb79eb777151f8650c8eec41a9bd4bcc72a06d36a5d0eed482115.json create mode 100644 lib/workload/stateless/filemanager/.sqlx/query-a21baafb9a67774ec404c4ef80ff43b6f0d78fd3a31ae2517ba889ed50496dff.json create mode 100644 lib/workload/stateless/filemanager/.sqlx/query-b93388efd27ae9253929e157bc8df247af518d4fafe42761f1488d1927d616c6.json create mode 100644 lib/workload/stateless/filemanager/.sqlx/query-bfa37a3a6325c7bf5449aba02be91338f4623a239c677301c9d13bcbd5e65e78.json create mode 100644 lib/workload/stateless/filemanager/.sqlx/query-c0649cd0af246c20b2ee60a3b27e2130db9c1526f858159de63c3af383f4a3b6.json rename lib/workload/{stateful => stateless}/filemanager/docs/ARCHITECTURE.md (100%) diff --git a/.secrets.baseline b/.secrets.baseline index eabe339a9..d8f9c2403 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -75,6 +75,10 @@ { "path": "detect_secrets.filters.allowlist.is_line_allowlisted" }, + { + "path": "detect_secrets.filters.common.is_baseline_file", + "filename": ".secrets.baseline" + }, { "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies", "min_level": 2 @@ -109,10 +113,10 @@ { "path": "detect_secrets.filters.regex.should_exclude_file", "pattern": [ - "^(yarn.lock|.yarn/|.local/|openapi/)" + "^(yarn.lock|.yarn/|.local/|openapi/)|.sqlx/" ] } ], "results": {}, - "generated_at": "2023-05-24T11:39:46Z" + "generated_at": "2024-02-18T03:27:13Z" } diff --git a/config/constants.ts b/config/constants.ts index d5137f110..c38d7a3eb 100644 --- a/config/constants.ts +++ b/config/constants.ts @@ -36,6 +36,7 @@ const orcaBusStatefulConfig = { cloudwatchLogsExports: ['orcabus-postgresql'], }, databaseSecurityGroupName: 'database-security-group', + inboundSecurityGroupName: 'inbound-database-security-group', }, securityGroupProps: { securityGroupName: lambdaSecurityGroupName, @@ -82,7 +83,7 @@ const filemanagerDependencies: FilemanagerDependencies = { eventSourceBuckets: ['umccr-temp-dev'], eventSourceQueueName: eventSourceConfig.queueName, databaseSecretName: orcaBusStatefulConfig.databaseProps.masterSecretName, - databaseSecurityGroupName: orcaBusStatefulConfig.databaseProps.databaseSecurityGroupName, + databaseSecurityGroupName: orcaBusStatefulConfig.databaseProps.inboundSecurityGroupName, }; interface EnvironmentConfig { diff --git a/lib/workload/orcabus-stateful-stack.ts b/lib/workload/orcabus-stateful-stack.ts index 990f41753..532b6bbb4 100644 --- a/lib/workload/orcabus-stateful-stack.ts +++ b/lib/workload/orcabus-stateful-stack.ts @@ -2,7 +2,7 @@ import * as cdk from 'aws-cdk-lib'; import { Construct } from 'constructs'; import { getVpc } from './stateful/vpc/component'; import { EventBusConstruct, EventBusProps } from './stateful/eventbridge/component'; -import { Database, DatabasePropsNoVPC } from './stateful/database/component'; +import { Database, ConfigurableDatabaseProps } from './stateful/database/component'; import { SecurityGroupConstruct, SecurityGroupProps } from './stateful/securitygroup/component'; import { SchemaRegistryConstruct, SchemaRegistryProps } from './stateful/schemaregistry/component'; import { EventSource, EventSourceProps } from './stateful/event_source/component'; @@ -10,7 +10,7 @@ import { EventSource, EventSourceProps } from './stateful/event_source/component export interface OrcaBusStatefulConfig { schemaRegistryProps: SchemaRegistryProps; eventBusProps: EventBusProps; - databaseProps: DatabasePropsNoVPC; + databaseProps: ConfigurableDatabaseProps; securityGroupProps: SecurityGroupProps; eventSourceProps?: EventSourceProps; } diff --git a/lib/workload/orcabus-stateless-stack.ts b/lib/workload/orcabus-stateless-stack.ts index 59f920e36..61df62e02 100644 --- a/lib/workload/orcabus-stateless-stack.ts +++ b/lib/workload/orcabus-stateless-stack.ts @@ -78,7 +78,7 @@ export class OrcaBusStatelessStack extends cdk.Stack { // Opting to reconstruct the dependencies here, and pass them into the service as constructs. const queue = Queue.fromQueueArn( this, - 'Filemanager Queue', + 'FilemanagerQueue', Arn.format( { resource: dependencies.eventSourceQueueName, @@ -87,23 +87,23 @@ export class OrcaBusStatelessStack extends cdk.Stack { this ) ); - const securityGroup = SecurityGroup.fromLookupByName( + const databaseSecurityGroup = SecurityGroup.fromLookupByName( this, - 'Filemanager Database Security Group', + 'FilemanagerDatabaseSecurityGroup', dependencies.databaseSecurityGroupName, vpc ); const databaseSecret = Secret.fromSecretNameV2( this, - 'Filemanager Database Secret', + 'FilemanagerDatabaseSecret', dependencies.databaseSecretName ); new Filemanager(this, 'Filemanager', { buckets: dependencies.eventSourceBuckets, buildEnvironment: {}, - databaseSecret: databaseSecret, - databaseSecurityGroup: securityGroup, + databaseSecret, + databaseSecurityGroup, eventSources: [queue], migrateDatabase: true, vpc: vpc, diff --git a/lib/workload/stateful/database/component.ts b/lib/workload/stateful/database/component.ts index abc3f6290..624bfee0b 100644 --- a/lib/workload/stateful/database/component.ts +++ b/lib/workload/stateful/database/component.ts @@ -2,6 +2,8 @@ import { Construct } from 'constructs'; import { RemovalPolicy, Duration } from 'aws-cdk-lib'; import * as rds from 'aws-cdk-lib/aws-rds'; import * as ec2 from 'aws-cdk-lib/aws-ec2'; +import { SecurityGroup } from 'aws-cdk-lib/aws-ec2'; +import { DatabaseCluster } from 'aws-cdk-lib/aws-rds'; /** * Props for enabling enhanced monitoring. @@ -28,7 +30,7 @@ type MonitoringProps = { /** * Database props without a VPC. */ -export type DatabasePropsNoVPC = MonitoringProps & { +export type ConfigurableDatabaseProps = MonitoringProps & { /** * The cluster identifier. */ @@ -78,15 +80,16 @@ export type DatabasePropsNoVPC = MonitoringProps & { */ removalPolicy: RemovalPolicy; /** - * Inbound security groups that are allowed to connect to the database. + * Create an inbound security group that can connect to the database. Stateless resources can recreate + * this security group to access the database. */ - allowedInboundSG?: ec2.SecurityGroup; + inboundSecurityGroupName: string; }; /** * Database props with a vpc. */ -export type DatabaseProps = DatabasePropsNoVPC & { +export type DatabaseProps = ConfigurableDatabaseProps & { /** * The database VPC. */ @@ -94,8 +97,9 @@ export type DatabaseProps = DatabasePropsNoVPC & { }; export class Database extends Construct { - readonly securityGroup: ec2.SecurityGroup; - readonly cluster: rds.DatabaseCluster; + readonly securityGroup: SecurityGroup; + readonly inboundSecurityGroup: SecurityGroup; + readonly cluster: DatabaseCluster; constructor(scope: Construct, id: string, props: DatabaseProps) { super(scope, id); @@ -113,14 +117,19 @@ export class Database extends Construct { description: 'security group for OrcaBus RDS', }); - // give compute sg to access the rds - if (props.allowedInboundSG) { - this.securityGroup.addIngressRule( - props.allowedInboundSG, - ec2.Port.tcp(props.dbPort), - 'allow the OrcaBus compute sg to access db' - ); - } + this.inboundSecurityGroup = new ec2.SecurityGroup(this, 'DbInboundSecurityGroup', { + vpc: props.vpc, + allowAllOutbound: false, + allowAllIpv6Outbound: false, + securityGroupName: props.inboundSecurityGroupName, + description: 'an inbound security group to connect to the OrcaBus RDS', + }); + + this.securityGroup.addIngressRule( + this.inboundSecurityGroup, + ec2.Port.tcp(props.dbPort), + 'allow the OrcaBus security group to access db' + ); this.cluster = new rds.DatabaseCluster(this, id + 'Cluster', { engine: rds.DatabaseClusterEngine.auroraPostgres({ version: props.version }), diff --git a/lib/workload/stateless/filemanager/.sqlx/query-6515aa29536fb79eb777151f8650c8eec41a9bd4bcc72a06d36a5d0eed482115.json b/lib/workload/stateless/filemanager/.sqlx/query-6515aa29536fb79eb777151f8650c8eec41a9bd4bcc72a06d36a5d0eed482115.json new file mode 100644 index 000000000..ba27d4bf6 --- /dev/null +++ b/lib/workload/stateless/filemanager/.sqlx/query-6515aa29536fb79eb777151f8650c8eec41a9bd4bcc72a06d36a5d0eed482115.json @@ -0,0 +1,117 @@ +{ + "db_name": "PostgreSQL", + "query": "-- Update the matching s3_objects which should be re-ordered based on the deleted event. Returns the\n-- data associated with the event before the update, if an update occurred.\n\n-- First, unnest the input parameters into a query.\nwith input as (\n select\n *\n from unnest(\n $1::uuid[],\n $2::text[],\n $3::text[],\n $4::timestamptz[],\n $5::text[],\n $6::text[]\n ) as input (\n s3_object_id,\n bucket,\n key,\n deleted_date,\n version_id,\n deleted_sequencer\n )\n),\n-- Then, select the objects that match the bucket, key and version_id\ncurrent_objects as (\n select\n s3_object.*,\n input.s3_object_id as input_id,\n input.bucket as input_bucket,\n input.key as input_key,\n input.version_id as input_version_id,\n input.deleted_sequencer as input_deleted_sequencer,\n input.deleted_date as input_deleted_date\n from s3_object\n -- Grab the relevant values to update with.\n join input on\n input.bucket = s3_object.bucket and\n input.key = s3_object.key and\n input.version_id = s3_object.version_id\n -- Lock this pre-emptively for the update.\n for update\n),\n-- And filter them to the objects that need to be updated.\nobjects_to_update as (\n select\n *\n from current_objects\n where\n -- Check the sequencer condition. We only update if there is a deleted\n -- sequencer that is closer to the created sequencer.\n current_objects.created_sequencer < current_objects.input_deleted_sequencer and\n (\n -- Updating a null sequencer doesn't cause the event to be reprocessed.\n current_objects.deleted_sequencer is null or\n -- If a sequencer already exists this event should be reprocessed because this\n -- sequencer would belong to another object.\n current_objects.deleted_sequencer > current_objects.input_deleted_sequencer\n )\n -- And there should not be any objects with a deleted sequencer that is the same as the input deleted\n -- sequencer because this is a duplicate event that would cause a constraint error in the update.\n and current_objects.input_deleted_sequencer not in (\n select deleted_sequencer from current_objects where deleted_sequencer is not null\n )\n),\n-- Finally, update the required objects.\nupdate as (\n update s3_object\n set deleted_sequencer = objects_to_update.input_deleted_sequencer,\n deleted_date = objects_to_update.input_deleted_date,\n number_reordered = s3_object.number_reordered +\n case when objects_to_update.deleted_sequencer is null then 0 else 1 end\n from objects_to_update\n where s3_object.s3_object_id = objects_to_update.s3_object_id\n)\n-- Return the old values because these need to be reprocessed.\nselect\n -- Note, this is the passed through value from the input in order to identify this event later.\n input_id as \"s3_object_id!\",\n bucket,\n key,\n deleted_date as event_time,\n last_modified_date,\n e_tag,\n storage_class as \"storage_class?: StorageClass\",\n version_id,\n deleted_sequencer as sequencer,\n number_reordered,\n number_duplicate_events,\n size,\n -- This is used to simplify re-constructing the FlatS3EventMessages in the Lambda. I.e. this update detected an\n -- out of order deleted event, so return a deleted event back.\n 'Deleted' as \"event_type!: EventType\"\nfrom objects_to_update;\n", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "s3_object_id!", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "bucket", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "key", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "event_time", + "type_info": "Timestamptz" + }, + { + "ordinal": 4, + "name": "last_modified_date", + "type_info": "Timestamptz" + }, + { + "ordinal": 5, + "name": "e_tag", + "type_info": "Text" + }, + { + "ordinal": 6, + "name": "storage_class?: StorageClass", + "type_info": { + "Custom": { + "name": "storage_class", + "kind": { + "Enum": [ + "DeepArchive", + "Glacier", + "GlacierIr", + "IntelligentTiering", + "OnezoneIa", + "Outposts", + "ReducedRedundancy", + "Snow", + "Standard", + "StandardIa" + ] + } + } + } + }, + { + "ordinal": 7, + "name": "version_id", + "type_info": "Text" + }, + { + "ordinal": 8, + "name": "sequencer", + "type_info": "Text" + }, + { + "ordinal": 9, + "name": "number_reordered", + "type_info": "Int4" + }, + { + "ordinal": 10, + "name": "number_duplicate_events", + "type_info": "Int4" + }, + { + "ordinal": 11, + "name": "size", + "type_info": "Int4" + }, + { + "ordinal": 12, + "name": "event_type!: EventType", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "UuidArray", + "TextArray", + "TextArray", + "TimestamptzArray", + "TextArray", + "TextArray" + ] + }, + "nullable": [ + null, + false, + false, + true, + true, + true, + true, + true, + true, + false, + false, + true, + null + ] + }, + "hash": "6515aa29536fb79eb777151f8650c8eec41a9bd4bcc72a06d36a5d0eed482115" +} diff --git a/lib/workload/stateless/filemanager/.sqlx/query-a21baafb9a67774ec404c4ef80ff43b6f0d78fd3a31ae2517ba889ed50496dff.json b/lib/workload/stateless/filemanager/.sqlx/query-a21baafb9a67774ec404c4ef80ff43b6f0d78fd3a31ae2517ba889ed50496dff.json new file mode 100644 index 000000000..81955501e --- /dev/null +++ b/lib/workload/stateless/filemanager/.sqlx/query-a21baafb9a67774ec404c4ef80ff43b6f0d78fd3a31ae2517ba889ed50496dff.json @@ -0,0 +1,64 @@ +{ + "db_name": "PostgreSQL", + "query": "-- Bulk insert of s3 objects.\ninsert into s3_object (\n s3_object_id,\n object_id,\n bucket,\n key,\n created_date,\n size,\n checksum,\n last_modified_date,\n e_tag,\n storage_class,\n version_id,\n created_sequencer\n)\nvalues (\n unnest($1::uuid[]),\n unnest($2::uuid[]),\n unnest($3::text[]),\n unnest($4::text[]),\n unnest($5::timestamptz[]),\n unnest($6::integer[]),\n unnest($7::text[]),\n unnest($8::timestamptz[]),\n unnest($9::text[]),\n unnest($10::storage_class[]),\n unnest($11::text[]),\n unnest($12::text[])\n) on conflict on constraint created_sequencer_unique do update\n set number_duplicate_events = s3_object.number_duplicate_events + 1\n returning object_id, number_duplicate_events;\n", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "object_id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "number_duplicate_events", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "UuidArray", + "UuidArray", + "TextArray", + "TextArray", + "TimestamptzArray", + "Int4Array", + "TextArray", + "TimestamptzArray", + "TextArray", + { + "Custom": { + "name": "_storage_class", + "kind": { + "Array": { + "Custom": { + "name": "storage_class", + "kind": { + "Enum": [ + "DeepArchive", + "Glacier", + "GlacierIr", + "IntelligentTiering", + "OnezoneIa", + "Outposts", + "ReducedRedundancy", + "Snow", + "Standard", + "StandardIa" + ] + } + } + } + } + } + }, + "TextArray", + "TextArray" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "a21baafb9a67774ec404c4ef80ff43b6f0d78fd3a31ae2517ba889ed50496dff" +} diff --git a/lib/workload/stateless/filemanager/.sqlx/query-b93388efd27ae9253929e157bc8df247af518d4fafe42761f1488d1927d616c6.json b/lib/workload/stateless/filemanager/.sqlx/query-b93388efd27ae9253929e157bc8df247af518d4fafe42761f1488d1927d616c6.json new file mode 100644 index 000000000..ee9c54553 --- /dev/null +++ b/lib/workload/stateless/filemanager/.sqlx/query-b93388efd27ae9253929e157bc8df247af518d4fafe42761f1488d1927d616c6.json @@ -0,0 +1,65 @@ +{ + "db_name": "PostgreSQL", + "query": "-- Bulk insert of s3 objects.\ninsert into s3_object (\n s3_object_id,\n object_id,\n bucket,\n key,\n deleted_date,\n size,\n checksum,\n last_modified_date,\n e_tag,\n storage_class,\n version_id,\n deleted_sequencer,\n number_reordered\n)\nvalues (\n unnest($1::uuid[]),\n unnest($2::uuid[]),\n unnest($3::text[]),\n unnest($4::text[]),\n unnest($5::timestamptz[]),\n unnest($6::integer[]),\n unnest($7::text[]),\n unnest($8::timestamptz[]),\n unnest($9::text[]),\n unnest($10::storage_class[]),\n unnest($11::text[]),\n unnest($12::text[]),\n unnest($13::integer[])\n) on conflict on constraint deleted_sequencer_unique do update\n set number_duplicate_events = s3_object.number_duplicate_events + 1\n returning object_id, number_duplicate_events;\n", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "object_id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "number_duplicate_events", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "UuidArray", + "UuidArray", + "TextArray", + "TextArray", + "TimestamptzArray", + "Int4Array", + "TextArray", + "TimestamptzArray", + "TextArray", + { + "Custom": { + "name": "_storage_class", + "kind": { + "Array": { + "Custom": { + "name": "storage_class", + "kind": { + "Enum": [ + "DeepArchive", + "Glacier", + "GlacierIr", + "IntelligentTiering", + "OnezoneIa", + "Outposts", + "ReducedRedundancy", + "Snow", + "Standard", + "StandardIa" + ] + } + } + } + } + } + }, + "TextArray", + "TextArray", + "Int4Array" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "b93388efd27ae9253929e157bc8df247af518d4fafe42761f1488d1927d616c6" +} diff --git a/lib/workload/stateless/filemanager/.sqlx/query-bfa37a3a6325c7bf5449aba02be91338f4623a239c677301c9d13bcbd5e65e78.json b/lib/workload/stateless/filemanager/.sqlx/query-bfa37a3a6325c7bf5449aba02be91338f4623a239c677301c9d13bcbd5e65e78.json new file mode 100644 index 000000000..6a44fe521 --- /dev/null +++ b/lib/workload/stateless/filemanager/.sqlx/query-bfa37a3a6325c7bf5449aba02be91338f4623a239c677301c9d13bcbd5e65e78.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "-- Bulk insert of objects\ninsert into object (object_id)\nvalues (\n unnest($1::uuid[])\n);\n", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "UuidArray" + ] + }, + "nullable": [] + }, + "hash": "bfa37a3a6325c7bf5449aba02be91338f4623a239c677301c9d13bcbd5e65e78" +} diff --git a/lib/workload/stateless/filemanager/.sqlx/query-c0649cd0af246c20b2ee60a3b27e2130db9c1526f858159de63c3af383f4a3b6.json b/lib/workload/stateless/filemanager/.sqlx/query-c0649cd0af246c20b2ee60a3b27e2130db9c1526f858159de63c3af383f4a3b6.json new file mode 100644 index 000000000..34ab88e3f --- /dev/null +++ b/lib/workload/stateless/filemanager/.sqlx/query-c0649cd0af246c20b2ee60a3b27e2130db9c1526f858159de63c3af383f4a3b6.json @@ -0,0 +1,147 @@ +{ + "db_name": "PostgreSQL", + "query": "-- Update the matching s3_objects which should be re-ordered based on the created event. Returns the\n-- data associated with the event before the update, if an update occurred.\n\n-- First, unnest the input parameters into a query.\nwith input as (\n select\n *\n from unnest(\n $1::uuid[],\n $2::text[],\n $3::text[],\n $4::timestamptz[],\n $5::integer[],\n $6::text[],\n $7::timestamptz[],\n $8::text[],\n $9::storage_class[],\n $10::text[],\n $11::text[]\n ) as input (\n s3_object_id,\n bucket,\n key,\n created_date,\n size,\n checksum,\n last_modified_date,\n e_tag,\n storage_class,\n version_id,\n created_sequencer\n )\n),\n-- Then, select the objects that need to be updated.\ncurrent_objects as (\n select\n s3_object.*,\n input.s3_object_id as input_id,\n input.bucket as input_bucket,\n input.key as input_key,\n input.version_id as input_version_id,\n input.created_sequencer as input_created_sequencer,\n input.created_date as input_created_date,\n input.size as input_size,\n input.checksum as input_checksum,\n input.last_modified_date as input_last_modified_date,\n input.e_tag as input_e_tag,\n input.storage_class as input_storage_class\n from s3_object\n -- Grab the relevant values to update with.\n join input on\n input.bucket = s3_object.bucket and\n input.key = s3_object.key and\n input.version_id = s3_object.version_id\n -- Lock this pre-emptively for the update.\n for update\n),\n-- And filter them to the objects that need to be updated.\nobjects_to_update as (\n select\n *\n from current_objects\n where\n -- Check the sequencer condition. We only update if there is a created\n -- sequencer that is closer to the deleted sequencer.\n current_objects.deleted_sequencer > current_objects.input_created_sequencer and\n (\n -- Updating a null sequencer doesn't cause the event to be reprocessed.\n current_objects.created_sequencer is null or\n -- If a sequencer already exists this event should be reprocessed because this\n -- sequencer could belong to another object.\n current_objects.created_sequencer < current_objects.input_created_sequencer\n )\n -- And there should not be any objects with a created sequencer that is the same as the input created\n -- sequencer because this is a duplicate event that would cause a constraint error in the update.\n and current_objects.input_created_sequencer not in (\n select created_sequencer from current_objects where created_sequencer is not null\n )\n),\n-- Finally, update the required objects.\nupdate as (\n update s3_object\n set created_sequencer = objects_to_update.input_created_sequencer,\n created_date = objects_to_update.input_created_date,\n size = coalesce(objects_to_update.input_size, objects_to_update.size),\n checksum = coalesce(objects_to_update.input_checksum, objects_to_update.checksum),\n last_modified_date = coalesce(objects_to_update.input_last_modified_date, objects_to_update.last_modified_date),\n e_tag = coalesce(objects_to_update.e_tag, objects_to_update.e_tag),\n storage_class = objects_to_update.storage_class,\n number_reordered = s3_object.number_reordered +\n -- Note the asymmetry between this and the reorder for deleted query.\n case when objects_to_update.deleted_sequencer is not null or objects_to_update.created_sequencer is not null then\n 1\n else\n 0\n end\n from objects_to_update\n where s3_object.s3_object_id = objects_to_update.s3_object_id\n)\n-- Return the old values because these need to be reprocessed.\nselect\n -- Note, this is the passed through value from the input in order to identify this event later.\n input_id as \"s3_object_id!\",\n bucket,\n key,\n created_date as event_time,\n last_modified_date,\n e_tag,\n storage_class as \"storage_class?: StorageClass\",\n version_id,\n created_sequencer as sequencer,\n number_reordered,\n number_duplicate_events,\n size,\n -- This is used to simplify re-constructing the FlatS3EventMessages in the Lambda. I.e. this update detected an\n -- out of order created event, so return a created event back.\n 'Created' as \"event_type!: EventType\"\nfrom objects_to_update;\n", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "s3_object_id!", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "bucket", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "key", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "event_time", + "type_info": "Timestamptz" + }, + { + "ordinal": 4, + "name": "last_modified_date", + "type_info": "Timestamptz" + }, + { + "ordinal": 5, + "name": "e_tag", + "type_info": "Text" + }, + { + "ordinal": 6, + "name": "storage_class?: StorageClass", + "type_info": { + "Custom": { + "name": "storage_class", + "kind": { + "Enum": [ + "DeepArchive", + "Glacier", + "GlacierIr", + "IntelligentTiering", + "OnezoneIa", + "Outposts", + "ReducedRedundancy", + "Snow", + "Standard", + "StandardIa" + ] + } + } + } + }, + { + "ordinal": 7, + "name": "version_id", + "type_info": "Text" + }, + { + "ordinal": 8, + "name": "sequencer", + "type_info": "Text" + }, + { + "ordinal": 9, + "name": "number_reordered", + "type_info": "Int4" + }, + { + "ordinal": 10, + "name": "number_duplicate_events", + "type_info": "Int4" + }, + { + "ordinal": 11, + "name": "size", + "type_info": "Int4" + }, + { + "ordinal": 12, + "name": "event_type!: EventType", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "UuidArray", + "TextArray", + "TextArray", + "TimestamptzArray", + "Int4Array", + "TextArray", + "TimestamptzArray", + "TextArray", + { + "Custom": { + "name": "_storage_class", + "kind": { + "Array": { + "Custom": { + "name": "storage_class", + "kind": { + "Enum": [ + "DeepArchive", + "Glacier", + "GlacierIr", + "IntelligentTiering", + "OnezoneIa", + "Outposts", + "ReducedRedundancy", + "Snow", + "Standard", + "StandardIa" + ] + } + } + } + } + } + }, + "TextArray", + "TextArray" + ] + }, + "nullable": [ + null, + false, + false, + true, + true, + true, + true, + true, + true, + false, + false, + true, + null + ] + }, + "hash": "c0649cd0af246c20b2ee60a3b27e2130db9c1526f858159de63c3af383f4a3b6" +} diff --git a/lib/workload/stateless/filemanager/Makefile b/lib/workload/stateless/filemanager/Makefile index 483123c4c..ef946520a 100644 --- a/lib/workload/stateless/filemanager/Makefile +++ b/lib/workload/stateless/filemanager/Makefile @@ -36,14 +36,17 @@ clippy: build check: lint clippy check-fix: lint-fix clippy - ## Clean clean: docker-clean @cargo clean -## Connect to the database +## Database related targets psql: @docker exec -it filemanager_db psql filemanager -U filemanager +prepare: + cargo sqlx prepare --workspace +prepare-check: + cargo sqlx prepare --workspace --check ## Help text help: diff --git a/lib/workload/stateless/filemanager/README.md b/lib/workload/stateless/filemanager/README.md index 0a795176d..c4872b8d2 100644 --- a/lib/workload/stateless/filemanager/README.md +++ b/lib/workload/stateless/filemanager/README.md @@ -42,10 +42,16 @@ or cargo install sccache && export RUSTC_WRAPPER=`which sccache` ``` -Then, cargo-watch can be used to recompile files as they change: +The sqlx-cli is also required for compiling code without a database running: ```sh -cargo install cargo-watch sqlx-cli +cargo install sqlx-cli +``` + +cargo-watch can be used to recompile files as they change: + +```sh +cargo install cargo-watch make watch ``` @@ -82,6 +88,38 @@ Which runs `cargo test -- --ignored`. ## Database +### Offline compilation + +In order to compile code without a local database running, queries should be prepared using `sqlx-cli`: + +```sh +make prepare +``` + +which runs: + +```sh +cargo sqlx prepare --workspace +``` + +This creates a `.sqlx` file in the workspace directory which contains the pre-compiled query information. +This is useful for deploying the orcabus code without a dependency on the filemanager's Docker compose, and it also +ensures that cdk tests which synthesize the filemanager stack without additional modifications. + +In order to check if this file is out of date, run: + +```sh +make prepare-check +``` + +which runs: + +```sh +cargo sqlx prepare --workspace --check +``` + +### Database connection + To connect to the local postgres database, run: ```bash diff --git a/lib/workload/stateless/filemanager/deploy/constructs/functions/function.ts b/lib/workload/stateless/filemanager/deploy/constructs/functions/function.ts index 80c77bda0..67c9202fc 100644 --- a/lib/workload/stateless/filemanager/deploy/constructs/functions/function.ts +++ b/lib/workload/stateless/filemanager/deploy/constructs/functions/function.ts @@ -7,10 +7,24 @@ import { Architecture, Version } from 'aws-cdk-lib/aws-lambda'; import { ManagedPolicy, PolicyStatement, Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam'; import { ISecret } from 'aws-cdk-lib/aws-secretsmanager'; +/** + * Properties for the database. + */ +export type DatabaseProps = { + /** + * The database secret. + */ + readonly databaseSecret: ISecret; + /** + * The database security group. + */ + readonly databaseSecurityGroup: ISecurityGroup; +} + /** * Props for a Rust function without the package. */ -export type FunctionPropsNoPackage = { +export type FunctionPropsNoPackage = DatabaseProps & { /** * Additional build environment variables when building the Lambda function. */ @@ -23,14 +37,6 @@ export type FunctionPropsNoPackage = { * Vpc for the function. */ readonly vpc: IVpc; - /** - * The database secret. - */ - readonly databaseSecret: ISecret; - /** - * The database security group. - */ - readonly databaseSecurityGroup: ISecurityGroup; }; /** diff --git a/lib/workload/stateless/filemanager/deploy/lib/filemanager.ts b/lib/workload/stateless/filemanager/deploy/lib/filemanager.ts index c4a943586..fd051fc09 100644 --- a/lib/workload/stateless/filemanager/deploy/lib/filemanager.ts +++ b/lib/workload/stateless/filemanager/deploy/lib/filemanager.ts @@ -3,26 +3,18 @@ import { IngestFunction, IngestFunctionProps } from '../constructs/functions/ing import { CdkResourceInvoke } from '../../../functions/cdk_resource_invoke'; import { MigrateFunction } from '../constructs/functions/migrate'; import * as fn from '../constructs/functions/function'; -import { ISecurityGroup, IVpc } from 'aws-cdk-lib/aws-ec2'; +import { IVpc } from 'aws-cdk-lib/aws-ec2'; import { IQueue } from 'aws-cdk-lib/aws-sqs'; -import { ISecret } from 'aws-cdk-lib/aws-secretsmanager'; +import { DatabaseProps } from '../constructs/functions/function'; /** * Props for the filemanager stack. */ -type FilemanagerProps = IngestFunctionProps & { +type FilemanagerProps = IngestFunctionProps & DatabaseProps & { /** * VPC to use for filemanager. */ readonly vpc: IVpc, - /** - * The database secret. - */ - readonly databaseSecret: ISecret; - /** - * The database security group. - */ - readonly databaseSecurityGroup: ISecurityGroup; /** * Whether to initialize a database migration. */ @@ -42,21 +34,21 @@ type FilemanagerProps = IngestFunctionProps & { * Construct used to configure the filemanager. */ export class Filemanager extends Construct { - constructor(scope: Construct, id: string, settings: FilemanagerProps) { + constructor(scope: Construct, id: string, props: FilemanagerProps) { super(scope, id); - if (settings?.migrateDatabase) { + if (props?.migrateDatabase) { new CdkResourceInvoke(this, 'MigrateDatabase', { - vpc: settings.vpc, + vpc: props.vpc, createFunction: (scope: Construct, id: string, props: fn.FunctionPropsNoPackage) => { return new MigrateFunction(scope, id, props); }, functionProps: { - vpc: settings.vpc, - databaseSecret: settings.databaseSecret, - databaseSecurityGroup: settings.databaseSecurityGroup, - buildEnvironment: settings?.buildEnvironment, - rustLog: settings?.rustLog, + vpc: props.vpc, + databaseSecret: props.databaseSecret, + databaseSecurityGroup: props.databaseSecurityGroup, + buildEnvironment: props?.buildEnvironment, + rustLog: props?.rustLog, }, id: 'MigrateFunction', // Assuming no dependencies because the database will already exist. @@ -64,13 +56,13 @@ export class Filemanager extends Construct { } new IngestFunction(this, 'IngestLambda', { - vpc: settings.vpc, - databaseSecret: settings.databaseSecret, - databaseSecurityGroup: settings.databaseSecurityGroup, - eventSources: settings.eventSources, - buckets: settings.buckets, - buildEnvironment: settings?.buildEnvironment, - rustLog: settings?.rustLog, + vpc: props.vpc, + databaseSecret: props.databaseSecret, + databaseSecurityGroup: props.databaseSecurityGroup, + eventSources: props.eventSources, + buckets: props.buckets, + buildEnvironment: props?.buildEnvironment, + rustLog: props?.rustLog, }); } } diff --git a/lib/workload/stateful/filemanager/docs/ARCHITECTURE.md b/lib/workload/stateless/filemanager/docs/ARCHITECTURE.md similarity index 100% rename from lib/workload/stateful/filemanager/docs/ARCHITECTURE.md rename to lib/workload/stateless/filemanager/docs/ARCHITECTURE.md diff --git a/lib/workload/stateless/functions/cdk_resource_invoke.ts b/lib/workload/stateless/functions/cdk_resource_invoke.ts index a274530e2..a440d9556 100644 --- a/lib/workload/stateless/functions/cdk_resource_invoke.ts +++ b/lib/workload/stateless/functions/cdk_resource_invoke.ts @@ -9,6 +9,7 @@ import { CfnOutput, Stack, Token } from 'aws-cdk-lib'; import { IVpc, SubnetType } from 'aws-cdk-lib/aws-ec2'; import { ManagedPolicy, PolicyStatement, Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam'; import { Version } from 'aws-cdk-lib/aws-lambda'; +import { createHash } from 'node:crypto'; /** * The interface by which the generic type of `CdkResourceInvoke` is constrained by. @@ -66,6 +67,10 @@ export type CdkResourceInvokeProps = { * Dependencies for this resource. */ dependencies?: IDependable[]; + /** + * Any payload to pass to the Lambda function. + */ + payload?: string; }; /** @@ -82,9 +87,12 @@ export class CdkResourceInvoke extends Construct { super(scope, id); const stack = Stack.of(this); + + // It's necessary to hash this because stack names can exceed the 64 character limit of function names. + const stackHash = this.hashValue(stack.stackName); this._function = props.createFunction(this, props.id, { ...props.functionProps, - functionName: `${stack.stackName}-ResourceInvokeFunction-${props.id}`, + functionName: `${stackHash}-ResourceInvokeFunction-${props.id}`, }); // Call another lambda function with no arguments. @@ -93,9 +101,10 @@ export class CdkResourceInvoke extends Construct { action: 'invoke', parameters: { FunctionName: this.function.functionName, + ...(props.payload && { Payload: props.payload }) }, physicalResourceId: PhysicalResourceId.of( - `${id}-AwsSdkCall-${this.function.currentVersion}` + `${id}-AwsSdkCall-${this.function.currentVersion + this.hashValue(props.payload)}` ), }; @@ -107,7 +116,7 @@ export class CdkResourceInvoke extends Construct { resources: [ // This needs to have permissions to run any `ResourceInvokeFunction` because it is deployed as a // singleton Lambda function. - `arn:aws:lambda:${stack.region}:${stack.account}:function:${stack.stackName}-ResourceInvokeFunction-*`, + `arn:aws:lambda:${stack.region}:${stack.account}:function:${stackHash}-ResourceInvokeFunction-*`, ], actions: ['lambda:InvokeFunction'], }) @@ -138,6 +147,14 @@ export class CdkResourceInvoke extends Construct { }); } + private hashValue(value?: string): string { + if (!value) { + return ''; + } + + return createHash('md5').update(value).digest('hex').substring(0, 24); + } + /** * Add a dependency to this resource. */ diff --git a/test/stateful/databaseConstruct.test.ts b/test/stateful/databaseConstruct.test.ts index 31b1780d2..51a0c0eb5 100644 --- a/test/stateful/databaseConstruct.test.ts +++ b/test/stateful/databaseConstruct.test.ts @@ -39,18 +39,19 @@ test('Test DBCluster created props', () => { }); test('Test other SG Allow Ingress to DB SG', () => { - const allowedSG = new ec2.SecurityGroup(stack, 'AllowedSG', { - securityGroupName: 'Allowed DB Ingress', + const props = constructConfig.stackProps.orcaBusStatefulConfig.databaseProps; + props.inboundSecurityGroupName = 'test-security-group'; + const database = new Database(stack, 'TestDatabaseConstruct', { vpc, + ...props, }); - const sgLogicalId = stack.getLogicalId(allowedSG.node.defaultChild as ec2.CfnSecurityGroup); - new Database(stack, 'TestDatabaseConstruct', { - vpc, - ...constructConfig.stackProps.orcaBusStatefulConfig.databaseProps, - allowedInboundSG: allowedSG, - }); + const sgLogicalId = stack.getLogicalId( + database.inboundSecurityGroup.node.defaultChild as ec2.CfnSecurityGroup + ); + const template = Template.fromStack(stack); + console.log(JSON.stringify(template, undefined, 2)); template.hasResourceProperties('AWS::EC2::SecurityGroupIngress', { ToPort: dbProps.dbPort, @@ -59,4 +60,8 @@ test('Test other SG Allow Ingress to DB SG', () => { 'Fn::GetAtt': [sgLogicalId, 'GroupId'], }, }); + + template.hasResourceProperties('AWS::EC2::SecurityGroup', { + GroupName: props.inboundSecurityGroupName, + }); }); From 360b2b3efcd347e3297a04e2381a20b8cc06deee Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Mon, 19 Feb 2024 09:24:56 +1100 Subject: [PATCH 12/13] test: add cdk resource invoke test --- config/constants.ts | 1 - test/stateless/cdkResourceInvoke.test.ts | 72 ++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 test/stateless/cdkResourceInvoke.test.ts diff --git a/config/constants.ts b/config/constants.ts index c38d7a3eb..76b4c3602 100644 --- a/config/constants.ts +++ b/config/constants.ts @@ -35,7 +35,6 @@ const orcaBusStatefulConfig = { monitoring: { cloudwatchLogsExports: ['orcabus-postgresql'], }, - databaseSecurityGroupName: 'database-security-group', inboundSecurityGroupName: 'inbound-database-security-group', }, securityGroupProps: { diff --git a/test/stateless/cdkResourceInvoke.test.ts b/test/stateless/cdkResourceInvoke.test.ts new file mode 100644 index 000000000..b397acdf1 --- /dev/null +++ b/test/stateless/cdkResourceInvoke.test.ts @@ -0,0 +1,72 @@ +import * as cdk from 'aws-cdk-lib'; +import { Match, Template } from 'aws-cdk-lib/assertions'; +import { CdkResourceInvoke } from '../../lib/workload/stateless/functions/cdk_resource_invoke'; +import * as ec2 from 'aws-cdk-lib/aws-ec2'; +import * as lambda from 'aws-cdk-lib/aws-lambda'; + +let stack: cdk.Stack; +let vpc: ec2.Vpc; + +beforeEach(() => { + stack = new cdk.Stack(); + vpc = new ec2.Vpc(stack, 'MockExistingVPC', { + subnetConfiguration: [ + { + cidrMask: 24, + name: 'publicSubnet', + subnetType: ec2.SubnetType.PUBLIC, + }, + { + cidrMask: 24, + name: 'privateWithEgressSubnet', + subnetType: ec2.SubnetType.PRIVATE_WITH_EGRESS, + }, + ], + }); +}); + +test('Test CdkResourceInvoke', () => { + new CdkResourceInvoke(stack, 'TestCdkResourceInvoke', { + functionProps: { + code: new lambda.InlineCode('exports.handler = async (event) => console.log(event)'), + runtime: lambda.Runtime.NODEJS_20_X, + handler: 'index.handler', + }, + id: 'TestFunction', + vpc, + createFunction: (scope, id, props) => { + return new lambda.Function(scope, id, props); + }, + }); + const template = Template.fromStack(stack); + + const expectedHash = '7a1920d61156abc05a60135a'; // pragma: allowlist secret + + // Contains the Lambda function. + template.hasResourceProperties('AWS::Lambda::Function', { + Code: { + ZipFile: 'exports.handler = async (event) => console.log(event)', + }, + FunctionName: `${expectedHash}-ResourceInvokeFunction-TestFunction`, + Handler: 'index.handler', + Runtime: 'nodejs20.x', + }); + + // Has policy to invoke it. + template.hasResourceProperties('AWS::IAM::Policy', { + PolicyDocument: { + Statement: [ + { + Action: 'lambda:InvokeFunction', + Effect: 'Allow', + Resource: { + 'Fn::Join': [ + '', + Match.arrayWith([`:function:${expectedHash}-ResourceInvokeFunction-*`]), + ], + }, + }, + ], + }, + }); +}); From ab6189b00cba903499b298c3db00e358922957e2 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Mon, 19 Feb 2024 15:49:31 +1100 Subject: [PATCH 13/13] refactor: revert database construct security group and rearrange filemanager dependencies --- config/constants.ts | 3 -- lib/workload/orcabus-stateful-stack.ts | 1 + lib/workload/orcabus-stateless-stack.ts | 19 +++++----- lib/workload/stateful/database/component.ts | 38 +++++++------------ .../{docker-compose.yml => compose.yml} | 0 test/stateful/databaseConstruct.test.ts | 21 ++++------ 6 files changed, 32 insertions(+), 50 deletions(-) rename lib/workload/stateless/filemanager/{docker-compose.yml => compose.yml} (100%) diff --git a/config/constants.ts b/config/constants.ts index 76b4c3602..451fcecd9 100644 --- a/config/constants.ts +++ b/config/constants.ts @@ -31,11 +31,9 @@ const orcaBusStatefulConfig = { username: 'postgres', dbPort: 5432, masterSecretName: rdsMasterSecretName, - securityGroupName: 'orcabus-database-security-group', monitoring: { cloudwatchLogsExports: ['orcabus-postgresql'], }, - inboundSecurityGroupName: 'inbound-database-security-group', }, securityGroupProps: { securityGroupName: lambdaSecurityGroupName, @@ -82,7 +80,6 @@ const filemanagerDependencies: FilemanagerDependencies = { eventSourceBuckets: ['umccr-temp-dev'], eventSourceQueueName: eventSourceConfig.queueName, databaseSecretName: orcaBusStatefulConfig.databaseProps.masterSecretName, - databaseSecurityGroupName: orcaBusStatefulConfig.databaseProps.inboundSecurityGroupName, }; interface EnvironmentConfig { diff --git a/lib/workload/orcabus-stateful-stack.ts b/lib/workload/orcabus-stateful-stack.ts index 532b6bbb4..fb6ea1fc2 100644 --- a/lib/workload/orcabus-stateful-stack.ts +++ b/lib/workload/orcabus-stateful-stack.ts @@ -42,6 +42,7 @@ export class OrcaBusStatefulStack extends cdk.Stack { this.database = new Database(this, 'OrcaBusDatabaseConstruct', { vpc, + allowedInboundSG: this.securityGroup.computeSecurityGroup, ...props.databaseProps, }); diff --git a/lib/workload/orcabus-stateless-stack.ts b/lib/workload/orcabus-stateless-stack.ts index 61df62e02..b4ff767f0 100644 --- a/lib/workload/orcabus-stateless-stack.ts +++ b/lib/workload/orcabus-stateless-stack.ts @@ -31,10 +31,6 @@ export interface FilemanagerDependencies { * Database secret name for the filemanager. */ databaseSecretName: string; - /** - * Database security group name to allow the filemanager Lambda to connect. - */ - databaseSecurityGroupName: string; } export class OrcaBusStatelessStack extends cdk.Stack { @@ -65,7 +61,10 @@ export class OrcaBusStatelessStack extends cdk.Stack { this.createSequenceRunManager(); if (props.filemanagerDependencies) { - this.createFilemanager(props.filemanagerDependencies, this.vpc); + this.createFilemanager({ + ...props.filemanagerDependencies, + lambdaSecurityGroupName: props.lambdaSecurityGroupName, + }); } } @@ -74,7 +73,9 @@ export class OrcaBusStatelessStack extends cdk.Stack { // However, the implementation is still incomplete... } - private createFilemanager(dependencies: FilemanagerDependencies, vpc: IVpc) { + private createFilemanager( + dependencies: FilemanagerDependencies & { lambdaSecurityGroupName: string } + ) { // Opting to reconstruct the dependencies here, and pass them into the service as constructs. const queue = Queue.fromQueueArn( this, @@ -90,8 +91,8 @@ export class OrcaBusStatelessStack extends cdk.Stack { const databaseSecurityGroup = SecurityGroup.fromLookupByName( this, 'FilemanagerDatabaseSecurityGroup', - dependencies.databaseSecurityGroupName, - vpc + dependencies.lambdaSecurityGroupName, + this.vpc ); const databaseSecret = Secret.fromSecretNameV2( this, @@ -106,7 +107,7 @@ export class OrcaBusStatelessStack extends cdk.Stack { databaseSecurityGroup, eventSources: [queue], migrateDatabase: true, - vpc: vpc, + vpc: this.vpc, }); } } diff --git a/lib/workload/stateful/database/component.ts b/lib/workload/stateful/database/component.ts index 624bfee0b..2052590e7 100644 --- a/lib/workload/stateful/database/component.ts +++ b/lib/workload/stateful/database/component.ts @@ -47,10 +47,6 @@ export type ConfigurableDatabaseProps = MonitoringProps & { * Database username. */ username: string; - /** - * Database security group name. - */ - securityGroupName: string; /** * Database secret name. */ @@ -79,26 +75,24 @@ export type ConfigurableDatabaseProps = MonitoringProps & { * The database removal policy. */ removalPolicy: RemovalPolicy; - /** - * Create an inbound security group that can connect to the database. Stateless resources can recreate - * this security group to access the database. - */ - inboundSecurityGroupName: string; }; /** - * Database props with a vpc. + * Database props with vpc and inbound security group. */ export type DatabaseProps = ConfigurableDatabaseProps & { /** * The database VPC. */ vpc: ec2.IVpc; + /** + * Inbound security group for the database. + */ + allowedInboundSG?: ec2.SecurityGroup; }; export class Database extends Construct { readonly securityGroup: SecurityGroup; - readonly inboundSecurityGroup: SecurityGroup; readonly cluster: DatabaseCluster; constructor(scope: Construct, id: string, props: DatabaseProps) { @@ -113,23 +107,17 @@ export class Database extends Construct { vpc: props.vpc, allowAllOutbound: false, allowAllIpv6Outbound: false, - securityGroupName: props.securityGroupName, description: 'security group for OrcaBus RDS', }); - this.inboundSecurityGroup = new ec2.SecurityGroup(this, 'DbInboundSecurityGroup', { - vpc: props.vpc, - allowAllOutbound: false, - allowAllIpv6Outbound: false, - securityGroupName: props.inboundSecurityGroupName, - description: 'an inbound security group to connect to the OrcaBus RDS', - }); - - this.securityGroup.addIngressRule( - this.inboundSecurityGroup, - ec2.Port.tcp(props.dbPort), - 'allow the OrcaBus security group to access db' - ); + // give compute sg to access the rds + if (props.allowedInboundSG) { + this.securityGroup.addIngressRule( + props.allowedInboundSG, + ec2.Port.tcp(props.dbPort), + 'allow the OrcaBus compute sg to access db' + ); + } this.cluster = new rds.DatabaseCluster(this, id + 'Cluster', { engine: rds.DatabaseClusterEngine.auroraPostgres({ version: props.version }), diff --git a/lib/workload/stateless/filemanager/docker-compose.yml b/lib/workload/stateless/filemanager/compose.yml similarity index 100% rename from lib/workload/stateless/filemanager/docker-compose.yml rename to lib/workload/stateless/filemanager/compose.yml diff --git a/test/stateful/databaseConstruct.test.ts b/test/stateful/databaseConstruct.test.ts index 51a0c0eb5..31b1780d2 100644 --- a/test/stateful/databaseConstruct.test.ts +++ b/test/stateful/databaseConstruct.test.ts @@ -39,19 +39,18 @@ test('Test DBCluster created props', () => { }); test('Test other SG Allow Ingress to DB SG', () => { - const props = constructConfig.stackProps.orcaBusStatefulConfig.databaseProps; - props.inboundSecurityGroupName = 'test-security-group'; - const database = new Database(stack, 'TestDatabaseConstruct', { + const allowedSG = new ec2.SecurityGroup(stack, 'AllowedSG', { + securityGroupName: 'Allowed DB Ingress', vpc, - ...props, }); + const sgLogicalId = stack.getLogicalId(allowedSG.node.defaultChild as ec2.CfnSecurityGroup); - const sgLogicalId = stack.getLogicalId( - database.inboundSecurityGroup.node.defaultChild as ec2.CfnSecurityGroup - ); - + new Database(stack, 'TestDatabaseConstruct', { + vpc, + ...constructConfig.stackProps.orcaBusStatefulConfig.databaseProps, + allowedInboundSG: allowedSG, + }); const template = Template.fromStack(stack); - console.log(JSON.stringify(template, undefined, 2)); template.hasResourceProperties('AWS::EC2::SecurityGroupIngress', { ToPort: dbProps.dbPort, @@ -60,8 +59,4 @@ test('Test other SG Allow Ingress to DB SG', () => { 'Fn::GetAtt': [sgLogicalId, 'GroupId'], }, }); - - template.hasResourceProperties('AWS::EC2::SecurityGroup', { - GroupName: props.inboundSecurityGroupName, - }); });