diff --git a/lib/workload/stateful/filemanager/Cargo.lock b/lib/workload/stateful/filemanager/Cargo.lock index 69f596575..e7cf0b3e5 100644 --- a/lib/workload/stateful/filemanager/Cargo.lock +++ b/lib/workload/stateful/filemanager/Cargo.lock @@ -1073,9 +1073,11 @@ dependencies = [ "axum", "chrono", "dotenvy", + "filemanager", "futures", "hyper 1.0.1", "lambda_runtime", + "lazy_static", "mockall", "mockall_double", "serde", @@ -1122,6 +1124,22 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "filemanager-migrate-lambda" +version = "0.1.0" +dependencies = [ + "aws-config", + "aws-sdk-sts", + "aws_lambda_events 0.12.1", + "filemanager", + "lambda_runtime", + "serde", + "serde_json", + "tokio", + "tracing", + "tracing-subscriber", +] + [[package]] name = "finl_unicode" version = "1.2.0" @@ -1946,9 +1964,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "openssl-probe" diff --git a/lib/workload/stateful/filemanager/Cargo.toml b/lib/workload/stateful/filemanager/Cargo.toml index e498db2bf..40e4c7565 100644 --- a/lib/workload/stateful/filemanager/Cargo.toml +++ b/lib/workload/stateful/filemanager/Cargo.toml @@ -5,6 +5,7 @@ members = [ "filemanager", "filemanager-http-lambda", "filemanager-ingest-lambda", + "filemanager-migrate-lambda" ] [workspace.package] diff --git a/lib/workload/stateful/filemanager/README.md b/lib/workload/stateful/filemanager/README.md index 906324651..0cc85da94 100644 --- a/lib/workload/stateful/filemanager/README.md +++ b/lib/workload/stateful/filemanager/README.md @@ -28,10 +28,21 @@ this might interfere and complain about non-existing roles and users. ### Tooling prerequisites, testing and building the code -For development of the rust workspace, install a build cache (sccache) and build manually: +For development of the rust workspace, it's recommended to install a build cache (sccache) to improve compilation speeds: ```sh brew install sccache && export RUSTC_WRAPPER=`which sccache` +``` + +or + +```sh +cargo install sccache && export RUSTC_WRAPPER=`which sccache` +``` + +Then install build prerequisites to build: + +```sh cargo install cargo-watch sqlx-cli cargo build --all-targets --all-features ``` diff --git a/lib/workload/stateful/filemanager/build.rs b/lib/workload/stateful/filemanager/build.rs index d5068697c..a8998c8ce 100644 --- a/lib/workload/stateful/filemanager/build.rs +++ b/lib/workload/stateful/filemanager/build.rs @@ -1,5 +1,5 @@ // generated by `sqlx migrate build-script` fn main() { // trigger recompilation when a new migration is added - println!("cargo:rerun-if-changed=migrations"); -} + println!("cargo:rerun-if-changed=database/migrations"); +} \ No newline at end of file diff --git a/lib/workload/stateful/filemanager/database/migrations/0001_add_object_table.sql b/lib/workload/stateful/filemanager/database/migrations/0001_add_object_table.sql index 7973d2493..3d9245e0e 100644 --- a/lib/workload/stateful/filemanager/database/migrations/0001_add_object_table.sql +++ b/lib/workload/stateful/filemanager/database/migrations/0001_add_object_table.sql @@ -7,13 +7,13 @@ create table object ( -- The name of the object. key varchar(1024) not null, -- The size of the object. - size int not null, + size int default null, -- A unique identifier for the object, if it is present. hash varchar(255) default null, -- When this object was created. - created_date timestamptz not null, + created_date timestamptz not null default now(), -- When this object was last modified. - last_modified_date timestamptz not null, + last_modified_date timestamptz not null default now(), -- When this object was deleted, a null value means that the object has not yet been deleted. deleted_date timestamptz default null, -- The date of the object and its id combined. diff --git a/lib/workload/stateful/filemanager/database/migrations/0002_add_s3_object_table.sql b/lib/workload/stateful/filemanager/database/migrations/0002_add_s3_object_table.sql index f3ecce153..ba2fc4cce 100644 --- a/lib/workload/stateful/filemanager/database/migrations/0002_add_s3_object_table.sql +++ b/lib/workload/stateful/filemanager/database/migrations/0002_add_s3_object_table.sql @@ -5,5 +5,5 @@ create table s3_object( -- The object id. object_id uuid references object (object_id) primary key, -- The S3 storage class of the object. - storage_class storage_class default null + storage_class storage_class not null ); \ No newline at end of file diff --git a/lib/workload/stateful/filemanager/deploy/README.md b/lib/workload/stateful/filemanager/deploy/README.md index 33425be40..6c30a6215 100644 --- a/lib/workload/stateful/filemanager/deploy/README.md +++ b/lib/workload/stateful/filemanager/deploy/README.md @@ -2,9 +2,20 @@ This folder contains CDK deployment code for filemanager. The CDK code can be deployed using `cdk`: - ```sh npm install cdk bootstrap cdk deploy ``` + +By default, the stack does not perform database migration. To migrate the database, use the script inside `package.json`: + +```sh +npm run migrate -- cdk deploy +``` + +or set `FILEMANAGER_DEPLOY_DATABASE_MIGRATION`: + +```sh +export FILEMANAGER_DEPLOY_DATABASE_MIGRATION="true" +``` diff --git a/lib/workload/stateful/filemanager/deploy/bin/filemanager.ts b/lib/workload/stateful/filemanager/deploy/bin/filemanager.ts new file mode 100644 index 000000000..061692463 --- /dev/null +++ b/lib/workload/stateful/filemanager/deploy/bin/filemanager.ts @@ -0,0 +1,37 @@ +#!/usr/bin/env node + +import 'source-map-support/register'; +import * as cdk from 'aws-cdk-lib'; +import { FilemanagerStack } from '../lib/filemanager_stack'; +import { Tags } from 'aws-cdk-lib'; + +export const STACK_NAME = 'FilemanagerStack'; +const STACK_DESCRIPTION = 'A stack deploying filemanager to dev.'; + +const app = new cdk.App(); +new FilemanagerStack( + app, + STACK_NAME, + { + stackName: STACK_NAME, + description: STACK_DESCRIPTION, + tags: { + Stack: STACK_NAME, + }, + env: { + region: 'ap-southeast-2', + }, + }, + { + destroyOnRemove: true, + enableMonitoring: { + enablePerformanceInsights: true, + }, + public: [ + // Put your IP here if you want the database to be reachable. + ], + migrateDatabase: process.env.FILEMANAGER_DEPLOY_MIGRATE_DATABASE == 'true', + } +); + +Tags.of(app).add('Stack', STACK_NAME); diff --git a/lib/workload/stateful/filemanager/deploy/cdk.json b/lib/workload/stateful/filemanager/deploy/cdk.json index 7447948b9..a795fe39a 100644 --- a/lib/workload/stateful/filemanager/deploy/cdk.json +++ b/lib/workload/stateful/filemanager/deploy/cdk.json @@ -1,5 +1,5 @@ { - "app": "npx ts-node --prefer-ts-exts stack/stack.ts", + "app": "npx ts-node --prefer-ts-exts bin/filemanager.ts", "watch": { "include": ["**"], "exclude": [ diff --git a/lib/workload/stateful/filemanager/deploy/constructs/cdk_resource_invoke.ts b/lib/workload/stateful/filemanager/deploy/constructs/cdk_resource_invoke.ts new file mode 100644 index 000000000..78f5fa9ea --- /dev/null +++ b/lib/workload/stateful/filemanager/deploy/constructs/cdk_resource_invoke.ts @@ -0,0 +1,136 @@ +import { Construct, IDependable } from 'constructs'; +import { + AwsCustomResource, + AwsCustomResourcePolicy, + AwsSdkCall, + PhysicalResourceId, +} from 'aws-cdk-lib/custom-resources'; +import { IVpc, SubnetType } from 'aws-cdk-lib/aws-ec2'; +import * as fn from './functions/function'; +import { ManagedPolicy, PolicyStatement, Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam'; +import { CfnOutput, Stack, Token } from 'aws-cdk-lib'; + +/** + * Props for the resource invoke construct. + */ +export type CdkResourceInvokeProps = { + /** + * Vpc for the function. + */ + vpc: IVpc; + /** + * The function to create. This will override the function name to ensure that it remains + * callable using the singleton function created by `AwsCustomResource`. See + * https://github.com/aws-samples/amazon-rds-init-cdk/blob/239626632f399ebe4928410a49d5ac5d009a6502/lib/resource-initializer.ts#L69-L71. + */ + createFunction: (scope: Construct, id: string, props: fn.FunctionPropsNoPackage) => fn.Function; + /** + * Function props when creating the Lambda function. + */ + functionProps: fn.FunctionPropsNoPackage; + /** + * Name to use when creating the function. + */ + id: string; + /** + * Dependencies for this resource. + */ + dependencies?: IDependable[]; +}; + +/** + * A construct for invoking a Lambda function for resource initialization. + */ +export class CdkResourceInvoke extends Construct { + private readonly _response: string; + private readonly _customResource: AwsCustomResource; + private readonly _function: fn.Function; + + constructor(scope: Construct, id: string, props: CdkResourceInvokeProps) { + super(scope, id); + + const stack = Stack.of(this); + this._function = props.createFunction(this, props.id, { + ...props.functionProps, + functionName: `${stack.stackName}-ResourceInvokeFunction-${props.id}`, + }); + + // Call another lambda function with no arguments. + const sdkCall: AwsSdkCall = { + service: 'Lambda', + action: 'invoke', + parameters: { + FunctionName: this.function.functionName(), + }, + physicalResourceId: PhysicalResourceId.of( + `${id}-AwsSdkCall-${this.function.currentVersion()}` + ), + }; + + const role = new Role(this, 'AwsCustomResourceRole', { + assumedBy: new ServicePrincipal('lambda.amazonaws.com'), + }); + role.addToPolicy( + new PolicyStatement({ + resources: [ + // This needs to have permissions to run any `ResourceInvokeFunction` because it is deployed as a + // singleton Lambda function. + `arn:aws:lambda:${stack.region}:${stack.account}:function:${stack.stackName}-ResourceInvokeFunction-*`, + ], + actions: ['lambda:InvokeFunction'], + }) + ); + // Also require VPC access for a Lambda function within the VPC. + role.addManagedPolicy( + ManagedPolicy.fromAwsManagedPolicyName('service-role/AWSLambdaVPCAccessExecutionRole') + ); + + this._customResource = new AwsCustomResource(this, 'AwsCustomResource', { + policy: AwsCustomResourcePolicy.fromSdkCalls({ + resources: AwsCustomResourcePolicy.ANY_RESOURCE, + }), + onUpdate: sdkCall, + role: role, + vpc: props.vpc, + vpcSubnets: { subnetType: SubnetType.PRIVATE_WITH_EGRESS }, + }); + + this._response = this.customResource.getResponseField('Payload'); + + // Add any dependencies. + props.dependencies?.forEach((dependency) => this.addDependency(dependency)); + + // Output the result. + new CfnOutput(this, 'MigrateDatabaseResponse', { + value: Token.asString(this.response), + }); + } + + /** + * Add a dependency to this resource. + */ + addDependency(dependency: IDependable) { + this.customResource.node.addDependency(dependency); + } + + /** + * Get the function response. + */ + get response(): string { + return this._response; + } + + /** + * Get the custom resource. + */ + get customResource(): AwsCustomResource { + return this._customResource; + } + + /** + * Get the function. + */ + get function(): fn.Function { + return this._function; + } +} diff --git a/lib/workload/stateful/filemanager/deploy/constructs/database.ts b/lib/workload/stateful/filemanager/deploy/constructs/database.ts new file mode 100644 index 000000000..1b8530808 --- /dev/null +++ b/lib/workload/stateful/filemanager/deploy/constructs/database.ts @@ -0,0 +1,185 @@ +import { Construct } from 'constructs'; +import { IVpc, SecurityGroup, SubnetType } from 'aws-cdk-lib/aws-ec2'; +import { ISecret } from 'aws-cdk-lib/aws-secretsmanager'; +import { + AuroraPostgresEngineVersion, + ClusterInstance, + DatabaseCluster, + DatabaseClusterEngine, +} from 'aws-cdk-lib/aws-rds'; +import { aws_ec2 as ec2, aws_rds as rds, Duration, RemovalPolicy } from 'aws-cdk-lib'; +import { ManagedPolicy, Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam'; + +/** + * Props for enabling enhanced monitoring. + */ +export type EnableMonitoringProps = { + /** + * Add cloud watch exports. + */ + readonly cloudwatchLogsExports?: string[]; + /** + * Enable performance insights. + */ + readonly enablePerformanceInsights?: boolean; + /** + * The interval for monitoring, defaults to 60 seconds. + */ + readonly monitoringInterval?: Duration; +}; + +/** + * Settable database props that can be configured. + */ +export type DatabaseSettings = { + /** + * If present, specifies the database as public and adds additional inbound CIDRs to the security group. + */ + readonly public?: string[]; + /** + * Whether to destroy the database on stack removal. Defaults to keeping a snapshot. + */ + readonly destroyOnRemove?: boolean; + /** + * Enable enhanced monitoring. + */ + readonly enableMonitoring?: EnableMonitoringProps; + /** + * Minimum ACU capacity, defaults to 0.5. + */ + readonly minCapacity?: number; + /** + * Maximum ACU capacity, defaults to 4. + */ + readonly maxCapacity?: number; + /** + * Port to use for the database. The default for the engine is used if not specified. + */ + readonly port?: number; +}; + +/** + * Props for the database. + */ +export type DatabaseProps = DatabaseSettings & { + /** + * Vpc for the database. + */ + readonly vpc: IVpc; + /** + * Secret for database credentials + */ + readonly secret: ISecret; + /** + * The name of the database initially created. + */ + readonly databaseName: string; +}; + +/** + * A construct for the postgres database used with filemanager. + */ +export class Database extends Construct { + private readonly _securityGroup: SecurityGroup; + private readonly _cluster: DatabaseCluster; + private readonly _unsafeConnection: string; + + constructor(scope: Construct, id: string, props: DatabaseProps) { + super(scope, id); + + // Create security group with no outbound connections, because outbound connections + // shouldn't be very useful for a database anyway. + this._securityGroup = new SecurityGroup(this, 'SecurityGroup', { + vpc: props.vpc, + allowAllOutbound: false, + allowAllIpv6Outbound: false, + description: 'Security group for communicating with the filemanager RDS instance', + }); + + // Creates roles for enhanced RDS monitoring, if enabled. + let enableMonitoring; + if (props.enableMonitoring) { + const monitoringRole = new Role(this, 'DatabaseMonitoringRole', { + assumedBy: new ServicePrincipal('monitoring.rds.amazonaws.com'), + }); + monitoringRole.addManagedPolicy( + ManagedPolicy.fromAwsManagedPolicyName('service-role/AmazonRDSEnhancedMonitoringRole') + ); + + enableMonitoring = { + enablePerformanceInsights: props.enableMonitoring.enablePerformanceInsights, + cloudwatchLogsExports: props.enableMonitoring.cloudwatchLogsExports, + monitoringInterval: props.enableMonitoring.monitoringInterval?.toSeconds() ?? 60, + monitoringRoleArn: monitoringRole.roleArn, + }; + } + + // Serverless V2 Cluster. + this._cluster = new DatabaseCluster(this, 'Cluster', { + vpc: props.vpc, + vpcSubnets: { + subnetType: props.public ? SubnetType.PUBLIC : SubnetType.PRIVATE_ISOLATED, + }, + securityGroups: [this._securityGroup], + credentials: rds.Credentials.fromSecret(props.secret), + removalPolicy: props.destroyOnRemove ? RemovalPolicy.DESTROY : RemovalPolicy.SNAPSHOT, + defaultDatabaseName: props.databaseName, + port: props.port, + engine: DatabaseClusterEngine.auroraPostgres({ + version: AuroraPostgresEngineVersion.VER_15_4, + }), + serverlessV2MinCapacity: props.minCapacity ?? 0.5, + serverlessV2MaxCapacity: props.maxCapacity ?? 4, + writer: ClusterInstance.serverlessV2('Writer', { + ...(enableMonitoring && { ...enableMonitoring }), + }), + }); + + if (props.public) { + // If it's public, set the CIDRs from the config. + props.public.forEach((cidr) => { + this._securityGroup.addIngressRule( + ec2.Peer.ipv4(cidr), + ec2.Port.tcp(this._cluster.clusterEndpoint.port) + ); + }); + } else { + // Any inbound connections within the same security group are allowed access to the database port. + this._securityGroup.addIngressRule( + this._securityGroup, + ec2.Port.tcp(this._cluster.clusterEndpoint.port) + ); + } + + this._unsafeConnection = + `postgres://` + + `${props.secret.secretValueFromJson('username').unsafeUnwrap()}` + + `:` + + `${props.secret.secretValueFromJson('password').unsafeUnwrap()}` + + `@` + + `${this._cluster.clusterEndpoint.socketAddress}` + + `/` + + `${props.databaseName}`; + } + + /** + * Get the serverless cluster. + */ + get cluster(): DatabaseCluster { + return this._cluster; + } + + /** + * Get the security group for the database. + */ + get securityGroup(): SecurityGroup { + return this._securityGroup; + } + + /** + * Get the connection string. Unsafe because it contains the username and password. + */ + get unsafeConnection(): string { + return this._unsafeConnection; + } +} diff --git a/lib/workload/stateful/filemanager/deploy/constructs/functions/function.ts b/lib/workload/stateful/filemanager/deploy/constructs/functions/function.ts new file mode 100644 index 000000000..06346026b --- /dev/null +++ b/lib/workload/stateful/filemanager/deploy/constructs/functions/function.ts @@ -0,0 +1,154 @@ +import { IVpc, SecurityGroup, SubnetType } from 'aws-cdk-lib/aws-ec2'; +import { Database } from '../database'; +import { Architecture, IDestination, Version } from 'aws-cdk-lib/aws-lambda'; +import { ManagedPolicy, PolicyStatement, Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam'; +import { Construct } from 'constructs'; +import { RustFunction } from 'rust.aws-cdk-lambda'; +import { Duration } from 'aws-cdk-lib'; +import { Settings as CargoSettings } from 'rust.aws-cdk-lambda/dist/settings'; + +/** + * Settable values for a Rust function. + */ +export type FunctionSettings = { + /** + * Additional build environment variables when building the Lambda function. + */ + readonly buildEnvironment?: { [key: string]: string | undefined }; + /** + * RUST_LOG string, defaults to trace on local crates and info everywhere else. + */ + readonly rustLog?: string; +}; + +/** + * Props for a Rust function without the package. + */ +export type FunctionPropsNoPackage = FunctionSettings & { + /** + * Vpc for the function. + */ + readonly vpc: IVpc; + /** + * Database that the function uses. + */ + readonly database: Database; + /** + * The destination to post failed invocations to. + */ + readonly onFailure?: IDestination; + /** + * Additional policies to add to the Lambda role. + */ + readonly policies?: PolicyStatement[]; + /** + * Name of the Lambda function resource. + */ + readonly functionName?: string; +}; + +/** + * Props for the Rust function. + */ +export type FunctionProps = FunctionPropsNoPackage & { + /** + * The package to build for this function. + */ + readonly package: string; +}; + +/** + * A construct for a Rust Lambda function. + */ +export class Function extends Construct { + private readonly _function: RustFunction; + private readonly _role: Role; + + constructor(scope: Construct, id: string, props: FunctionProps) { + super(scope, id); + + // Lambda role needs SQS execution role. + this._role = new Role(this, 'Role', { + assumedBy: new ServicePrincipal('lambda.amazonaws.com'), + description: 'Lambda execution role for ' + id, + }); + // Lambda needs VPC access if it is created in a VPC. + this.addManagedPolicy('service-role/AWSLambdaVPCAccessExecutionRole'); + props.policies?.forEach((policy) => { + this._role.addToPolicy(policy); + }); + + // Lambda needs to be able to reach out to access S3, security manager (eventually), etc. + // Could this use an endpoint instead? + const securityGroup = new SecurityGroup(this, 'SecurityGroup', { + vpc: props.vpc, + allowAllOutbound: true, + description: 'Security group that allows a filemanager Lambda function to egress out.', + }); + + CargoSettings.BUILD_INDIVIDUALLY = true; + + this._function = new RustFunction(this, 'RustFunction', { + package: props.package, + target: 'aarch64-unknown-linux-gnu', + memorySize: 128, + timeout: Duration.seconds(28), + environment: { + // Todo use security manager to get connection string rather than passing it in as an environment variable + DATABASE_URL: props.database.unsafeConnection, + RUST_LOG: + props.rustLog ?? `info,${props.package.replace('-', '_')}=trace,filemanager=trace`, + }, + buildEnvironment: props.buildEnvironment, + extraBuildArgs: ['--manifest-path', `../${props.package}/Cargo.toml`], + architecture: Architecture.ARM_64, + role: this._role, + onFailure: props.onFailure, + vpc: props.vpc, + vpcSubnets: { subnetType: SubnetType.PRIVATE_WITH_EGRESS }, + securityGroups: [ + securityGroup, + // Allow access to database. + props.database.securityGroup, + ], + functionName: props.functionName, + }); + + // Todo this should probably connect to an RDS proxy rather than directly to the database. + } + + /** + * Add a managed policy to the function's role. + */ + addManagedPolicy(policyName: string) { + this._role.addManagedPolicy(ManagedPolicy.fromAwsManagedPolicyName(policyName)); + } + + /** + * Get the function name. + */ + functionName(): string { + return this.function.functionName; + } + + /** + * Get the function version. + */ + currentVersion(): Version { + return this.function.currentVersion; + } + + /** + * Get the function IAM role. + */ + get role(): Role { + return this._role; + } + + /** + * Get the Lambda function. + */ + get function(): RustFunction { + return this._function; + } +} diff --git a/lib/workload/stateful/filemanager/deploy/constructs/functions/ingest.ts b/lib/workload/stateful/filemanager/deploy/constructs/functions/ingest.ts new file mode 100644 index 000000000..4f4426495 --- /dev/null +++ b/lib/workload/stateful/filemanager/deploy/constructs/functions/ingest.ts @@ -0,0 +1,34 @@ +import { Construct } from 'constructs'; +import { IQueue } from 'aws-cdk-lib/aws-sqs'; +import * as lambdaEventSources from 'aws-cdk-lib/aws-lambda-event-sources'; +import * as fn from './function'; + +/** + * Settable values for the ingest function. + */ +export type IngestFunctionSettings = fn.FunctionSettings; + +/** + * Props for the ingest function. + */ +export type IngestFunctionProps = IngestFunctionSettings & + fn.FunctionPropsNoPackage & { + /** + * The SQS queue URL to receive events from. + */ + readonly queue: IQueue; + }; + +/** + * A construct for the Lambda ingest function. + */ +export class IngestFunction extends fn.Function { + constructor(scope: Construct, id: string, props: IngestFunctionProps) { + super(scope, id, { package: 'filemanager-ingest-lambda', ...props }); + + this.addManagedPolicy('service-role/AWSLambdaSQSQueueExecutionRole'); + + const eventSource = new lambdaEventSources.SqsEventSource(props.queue); + this.function.addEventSource(eventSource); + } +} diff --git a/lib/workload/stateful/filemanager/deploy/constructs/functions/migrate.ts b/lib/workload/stateful/filemanager/deploy/constructs/functions/migrate.ts new file mode 100644 index 000000000..45982ca2a --- /dev/null +++ b/lib/workload/stateful/filemanager/deploy/constructs/functions/migrate.ts @@ -0,0 +1,21 @@ +import { Construct } from 'constructs'; +import * as fn from './function'; + +/** + * Settable values for the migrate function. + */ +export type MigrateFunctionSettings = fn.FunctionSettings; + +/** + * Props for the migrate function. + */ +export type MigrateFunctionProps = MigrateFunctionSettings & fn.FunctionPropsNoPackage; + +/** + * A construct for the Lambda migrate function. + */ +export class MigrateFunction extends fn.Function { + constructor(scope: Construct, id: string, props: MigrateFunctionProps) { + super(scope, id, { package: 'filemanager-migrate-lambda', ...props }); + } +} diff --git a/lib/workload/stateful/filemanager/deploy/lib/filemanager_stack.ts b/lib/workload/stateful/filemanager/deploy/lib/filemanager_stack.ts new file mode 100644 index 000000000..3f91c0816 --- /dev/null +++ b/lib/workload/stateful/filemanager/deploy/lib/filemanager_stack.ts @@ -0,0 +1,126 @@ +import { RemovalPolicy, Stack, StackProps } from 'aws-cdk-lib'; +import { Construct } from 'constructs'; +import * as s3 from 'aws-cdk-lib/aws-s3'; +import * as lambdaDestinations from 'aws-cdk-lib/aws-lambda-destinations'; +import * as ec2 from 'aws-cdk-lib/aws-ec2'; +import { IngestFunction, IngestFunctionSettings } from '../constructs/functions/ingest'; +import { Database, DatabaseSettings } from '../constructs/database'; +import { SubnetType } from 'aws-cdk-lib/aws-ec2'; +import { SqsDestination } from 'aws-cdk-lib/aws-s3-notifications'; +import { Bucket, EventType } from 'aws-cdk-lib/aws-s3'; +import { PolicyStatement } from 'aws-cdk-lib/aws-iam'; +import { Queue } from 'aws-cdk-lib/aws-sqs'; +import { Secret } from 'aws-cdk-lib/aws-secretsmanager'; +import { CdkResourceInvoke } from '../constructs/cdk_resource_invoke'; +import { MigrateFunction } from '../constructs/functions/migrate'; +import * as fn from '../constructs/functions/function'; + +/** + * Common settings for the filemanager stack. + */ +type Settings = DatabaseSettings & + IngestFunctionSettings & { + /** + * The name of the database. Defaults to `filemanager`. + */ + databaseName?: string; + /** + * Whether to initialize a database migration. + */ + migrateDatabase?: boolean; + }; + +/** + * Stack used to deploy filemanager. + */ +export class FilemanagerStack extends Stack { + constructor(scope: Construct, id: string, props: StackProps, settings?: Settings) { + super(scope, id, props); + + const queue = new Queue(this, 'Queue'); + + const testBucket = new Bucket(this, 'Bucket', { + bucketName: 'filemanager-test-ingest', + blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL, + encryption: s3.BucketEncryption.S3_MANAGED, + enforceSSL: true, + removalPolicy: RemovalPolicy.DESTROY, + }); + const testBucketPolicy = new PolicyStatement({ + actions: ['s3:List*', 's3:Get*'], + resources: ['arn:aws:s3:::*'], + }); + testBucket.addEventNotification(EventType.OBJECT_CREATED, new SqsDestination(queue)); + testBucket.addEventNotification(EventType.OBJECT_REMOVED, new SqsDestination(queue)); + + const deadLetterQueue = new Queue(this, 'DeadLetterQueue'); + const deadLetterQueueDestination = new lambdaDestinations.SqsDestination(deadLetterQueue); + + const vpc = new ec2.Vpc(this, 'Vpc', { + maxAzs: 99, // As many as there are available in the region + natGateways: 1, + subnetConfiguration: [ + { + name: 'ingress', + subnetType: SubnetType.PUBLIC, + }, + { + name: 'application', + subnetType: SubnetType.PRIVATE_WITH_EGRESS, + }, + { + name: 'database', + subnetType: SubnetType.PRIVATE_ISOLATED, + }, + ], + }); + + const secret = new Secret(this, 'FilemanagerDatabaseSecret', { + secretName: 'FilemanagerDatabaseSecret', // pragma: allowlist secret + generateSecretString: { + secretStringTemplate: JSON.stringify({ username: 'filemanager' }), + excludePunctuation: true, + generateStringKey: 'password', + }, + }); + + const database = new Database(this, 'Database', { + vpc, + databaseName: settings?.databaseName ?? 'filemanager', + secret, + destroyOnRemove: settings?.destroyOnRemove, + enableMonitoring: settings?.enableMonitoring, + minCapacity: settings?.minCapacity, + maxCapacity: settings?.maxCapacity, + port: settings?.port, + public: settings?.public, + }); + + if (settings?.migrateDatabase) { + new CdkResourceInvoke(this, 'MigrateDatabase', { + vpc, + createFunction: (scope: Construct, id: string, props: fn.FunctionPropsNoPackage) => { + return new MigrateFunction(scope, id, props); + }, + functionProps: { + vpc, + database, + buildEnvironment: settings?.buildEnvironment, + rustLog: settings?.rustLog, + }, + id: 'MigrateFunction', + dependencies: [database.cluster], + }); + } + + new IngestFunction(this, 'IngestLambda', { + vpc, + database, + queue, + onFailure: deadLetterQueueDestination, + policies: [testBucketPolicy], + buildEnvironment: settings?.buildEnvironment, + rustLog: settings?.rustLog, + }); + } +} diff --git a/lib/workload/stateful/filemanager/deploy/package-lock.json b/lib/workload/stateful/filemanager/deploy/package-lock.json index 29ffd9696..9f8b1741f 100644 --- a/lib/workload/stateful/filemanager/deploy/package-lock.json +++ b/lib/workload/stateful/filemanager/deploy/package-lock.json @@ -18,13 +18,25 @@ "source-map-support": "^0.5.21" }, "bin": { - "filemanager-stack": "stack/filemanager_stack.js" + "filemanager": "bin/filemanager.js" }, "devDependencies": { "@types/node": "^20.5.9", + "@typescript-eslint/eslint-plugin": "^6.14.0", "aws-cdk": "^2.114.1", + "cross-env": "^7.0.3", "prettier": "3.0.3", - "typescript": "^5.2.2" + "typescript": "^5.3.3" + } + }, + "node_modules/@aashutoshrathi/word-wrap": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@aashutoshrathi/word-wrap/-/word-wrap-1.2.6.tgz", + "integrity": "sha512-1Yjs2SvM8TflER/OD3cOjhWWOZb58A2t7wpE2S9XfBYTiIl+XFhQG2bjy4Pu1I+EAlCNUzRDYDdFwFYUKvXcIA==", + "dev": true, + "peer": true, + "engines": { + "node": ">=0.10.0" } }, "node_modules/@aws-cdk/asset-awscli-v1": { @@ -80,11 +92,105 @@ "constructs": "^10.0.0" } }, + "node_modules/@eslint-community/eslint-utils": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz", + "integrity": "sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA==", + "dev": true, + "dependencies": { + "eslint-visitor-keys": "^3.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "peerDependencies": { + "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" + } + }, + "node_modules/@eslint-community/regexpp": { + "version": "4.10.0", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.10.0.tgz", + "integrity": "sha512-Cu96Sd2By9mCNTx2iyKOmq10v22jUVQv0lQnlGNy16oE9589yE+QADPbrMGCkA51cKZSg3Pu/aTJVTGfL/qjUA==", + "dev": true, + "engines": { + "node": "^12.0.0 || ^14.0.0 || >=16.0.0" + } + }, + "node_modules/@eslint/eslintrc": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.1.4.tgz", + "integrity": "sha512-269Z39MS6wVJtsoUl10L60WdkhJVdPG24Q4eZTH3nnF6lpvSShEK3wQjDX9JRWAUPvPh7COouPpU9IrqaZFvtQ==", + "dev": true, + "peer": true, + "dependencies": { + "ajv": "^6.12.4", + "debug": "^4.3.2", + "espree": "^9.6.0", + "globals": "^13.19.0", + "ignore": "^5.2.0", + "import-fresh": "^3.2.1", + "js-yaml": "^4.1.0", + "minimatch": "^3.1.2", + "strip-json-comments": "^3.1.1" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@eslint/js": { + "version": "8.55.0", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.55.0.tgz", + "integrity": "sha512-qQfo2mxH5yVom1kacMtZZJFVdW+E70mqHMJvVg6WTLo+VBuQJ4TojZlfWBjK0ve5BdEeNAVxOsl/nvNMpJOaJA==", + "dev": true, + "peer": true, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, + "node_modules/@humanwhocodes/config-array": { + "version": "0.11.13", + "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.13.tgz", + "integrity": "sha512-JSBDMiDKSzQVngfRjOdFXgFfklaXI4K9nLF49Auh21lmBWRLIK3+xTErTWD4KU54pb6coM6ESE7Awz/FNU3zgQ==", + "dev": true, + "peer": true, + "dependencies": { + "@humanwhocodes/object-schema": "^2.0.1", + "debug": "^4.1.1", + "minimatch": "^3.0.5" + }, + "engines": { + "node": ">=10.10.0" + } + }, + "node_modules/@humanwhocodes/module-importer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", + "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", + "dev": true, + "peer": true, + "engines": { + "node": ">=12.22" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@humanwhocodes/object-schema": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-2.0.1.tgz", + "integrity": "sha512-dvuCeX5fC9dXgJn9t+X5atfmgQAzUOWqS1254Gh0m6i8wKd10ebXkfNKiRK+1GWi/yTvvLDHpoxLr0xxxeslWw==", + "dev": true, + "peer": true + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", - "optional": true, + "devOptional": true, "dependencies": { "@nodelib/fs.stat": "2.0.5", "run-parallel": "^1.1.9" @@ -97,7 +203,7 @@ "version": "2.0.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", - "optional": true, + "devOptional": true, "engines": { "node": ">= 8" } @@ -106,7 +212,7 @@ "version": "1.2.8", "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", - "optional": true, + "devOptional": true, "dependencies": { "@nodelib/fs.scandir": "2.1.5", "fastq": "^1.6.0" @@ -124,6 +230,12 @@ "@types/node": "*" } }, + "node_modules/@types/json-schema": { + "version": "7.0.15", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", + "dev": true + }, "node_modules/@types/minimist": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.5.tgz", @@ -145,12 +257,244 @@ "integrity": "sha512-PtrlVaOaI44/3pl3cvnlK+GxOM3re2526TJvPvh7W+keHIXdV4TE0ylpPBAcvFQCbGitaTXwL9u+RF7qtVeazQ==", "optional": true }, + "node_modules/@types/semver": { + "version": "7.5.6", + "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.5.6.tgz", + "integrity": "sha512-dn1l8LaMea/IjDoHNd9J52uBbInB796CDffS6VdIxvqYCPSG0V0DzHp76GpaWnlhg88uYyPbXCDIowa86ybd5A==", + "dev": true + }, "node_modules/@types/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/@types/which/-/which-2.0.2.tgz", "integrity": "sha512-113D3mDkZDjo+EeUEHCFy0qniNc1ZpecGiAU7WSo7YDoSzolZIQKpYFHrPpjkB2nuyahcKfrmLXeQlh7gqJYdw==", "optional": true }, + "node_modules/@typescript-eslint/eslint-plugin": { + "version": "6.14.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-6.14.0.tgz", + "integrity": "sha512-1ZJBykBCXaSHG94vMMKmiHoL0MhNHKSVlcHVYZNw+BKxufhqQVTOawNpwwI1P5nIFZ/4jLVop0mcY6mJJDFNaw==", + "dev": true, + "dependencies": { + "@eslint-community/regexpp": "^4.5.1", + "@typescript-eslint/scope-manager": "6.14.0", + "@typescript-eslint/type-utils": "6.14.0", + "@typescript-eslint/utils": "6.14.0", + "@typescript-eslint/visitor-keys": "6.14.0", + "debug": "^4.3.4", + "graphemer": "^1.4.0", + "ignore": "^5.2.4", + "natural-compare": "^1.4.0", + "semver": "^7.5.4", + "ts-api-utils": "^1.0.1" + }, + "engines": { + "node": "^16.0.0 || >=18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "@typescript-eslint/parser": "^6.0.0 || ^6.0.0-alpha", + "eslint": "^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/@typescript-eslint/parser": { + "version": "6.14.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-6.14.0.tgz", + "integrity": "sha512-QjToC14CKacd4Pa7JK4GeB/vHmWFJckec49FR4hmIRf97+KXole0T97xxu9IFiPxVQ1DBWrQ5wreLwAGwWAVQA==", + "dev": true, + "peer": true, + "dependencies": { + "@typescript-eslint/scope-manager": "6.14.0", + "@typescript-eslint/types": "6.14.0", + "@typescript-eslint/typescript-estree": "6.14.0", + "@typescript-eslint/visitor-keys": "6.14.0", + "debug": "^4.3.4" + }, + "engines": { + "node": "^16.0.0 || >=18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/@typescript-eslint/scope-manager": { + "version": "6.14.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-6.14.0.tgz", + "integrity": "sha512-VT7CFWHbZipPncAZtuALr9y3EuzY1b1t1AEkIq2bTXUPKw+pHoXflGNG5L+Gv6nKul1cz1VH8fz16IThIU0tdg==", + "dev": true, + "dependencies": { + "@typescript-eslint/types": "6.14.0", + "@typescript-eslint/visitor-keys": "6.14.0" + }, + "engines": { + "node": "^16.0.0 || >=18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@typescript-eslint/type-utils": { + "version": "6.14.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-6.14.0.tgz", + "integrity": "sha512-x6OC9Q7HfYKqjnuNu5a7kffIYs3No30isapRBJl1iCHLitD8O0lFbRcVGiOcuyN837fqXzPZ1NS10maQzZMKqw==", + "dev": true, + "dependencies": { + "@typescript-eslint/typescript-estree": "6.14.0", + "@typescript-eslint/utils": "6.14.0", + "debug": "^4.3.4", + "ts-api-utils": "^1.0.1" + }, + "engines": { + "node": "^16.0.0 || >=18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/@typescript-eslint/types": { + "version": "6.14.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-6.14.0.tgz", + "integrity": "sha512-uty9H2K4Xs8E47z3SnXEPRNDfsis8JO27amp2GNCnzGETEW3yTqEIVg5+AI7U276oGF/tw6ZA+UesxeQ104ceA==", + "dev": true, + "engines": { + "node": "^16.0.0 || >=18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@typescript-eslint/typescript-estree": { + "version": "6.14.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-6.14.0.tgz", + "integrity": "sha512-yPkaLwK0yH2mZKFE/bXkPAkkFgOv15GJAUzgUVonAbv0Hr4PK/N2yaA/4XQbTZQdygiDkpt5DkxPELqHguNvyw==", + "dev": true, + "dependencies": { + "@typescript-eslint/types": "6.14.0", + "@typescript-eslint/visitor-keys": "6.14.0", + "debug": "^4.3.4", + "globby": "^11.1.0", + "is-glob": "^4.0.3", + "semver": "^7.5.4", + "ts-api-utils": "^1.0.1" + }, + "engines": { + "node": "^16.0.0 || >=18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/globby": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/globby/-/globby-11.1.0.tgz", + "integrity": "sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==", + "dev": true, + "dependencies": { + "array-union": "^2.1.0", + "dir-glob": "^3.0.1", + "fast-glob": "^3.2.9", + "ignore": "^5.2.0", + "merge2": "^1.4.1", + "slash": "^3.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/slash": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", + "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/@typescript-eslint/utils": { + "version": "6.14.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-6.14.0.tgz", + "integrity": "sha512-XwRTnbvRr7Ey9a1NT6jqdKX8y/atWG+8fAIu3z73HSP8h06i3r/ClMhmaF/RGWGW1tHJEwij1uEg2GbEmPYvYg==", + "dev": true, + "dependencies": { + "@eslint-community/eslint-utils": "^4.4.0", + "@types/json-schema": "^7.0.12", + "@types/semver": "^7.5.0", + "@typescript-eslint/scope-manager": "6.14.0", + "@typescript-eslint/types": "6.14.0", + "@typescript-eslint/typescript-estree": "6.14.0", + "semver": "^7.5.4" + }, + "engines": { + "node": "^16.0.0 || >=18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^7.0.0 || ^8.0.0" + } + }, + "node_modules/@typescript-eslint/visitor-keys": { + "version": "6.14.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-6.14.0.tgz", + "integrity": "sha512-fB5cw6GRhJUz03MrROVuj5Zm/Q+XWlVdIsFj+Zb1Hvqouc8t+XP2H5y53QYU/MGtd2dPg6/vJJlhoX3xc2ehfw==", + "dev": true, + "dependencies": { + "@typescript-eslint/types": "6.14.0", + "eslint-visitor-keys": "^3.4.1" + }, + "engines": { + "node": "^16.0.0 || >=18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@ungap/structured-clone": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz", + "integrity": "sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==", + "dev": true, + "peer": true + }, "node_modules/@ziglang/cli": { "version": "0.0.11", "resolved": "https://registry.npmjs.org/@ziglang/cli/-/cli-0.0.11.tgz", @@ -169,6 +513,88 @@ "zig-uninstall": "uninstall.js" } }, + "node_modules/acorn": { + "version": "8.11.2", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.2.tgz", + "integrity": "sha512-nc0Axzp/0FILLEVsm4fNwLCwMttvhEI263QtVPQcbpfZZ3ts0hLsZGOpE6czNlid7CJ9MlyH8reXkpsf3YUY4w==", + "dev": true, + "peer": true, + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-jsx": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", + "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", + "dev": true, + "peer": true, + "peerDependencies": { + "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" + } + }, + "node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "peer": true, + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "peer": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "peer": true, + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true, + "peer": true + }, + "node_modules/array-union": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", + "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==", + "dev": true, + "engines": { + "node": ">=8" + } + }, "node_modules/aws-cdk": { "version": "2.114.1", "resolved": "https://registry.npmjs.org/aws-cdk/-/aws-cdk-2.114.1.tgz", @@ -524,11 +950,29 @@ "node": ">= 6" } }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true, + "peer": true + }, + "node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "peer": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, "node_modules/braces": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", - "optional": true, + "devOptional": true, "dependencies": { "fill-range": "^7.0.1" }, @@ -541,6 +985,16 @@ "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==" }, + "node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "dev": true, + "peer": true, + "engines": { + "node": ">=6" + } + }, "node_modules/chalk": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.3.0.tgz", @@ -553,6 +1007,33 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "peer": true, + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true, + "peer": true + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true, + "peer": true + }, "node_modules/constructs": { "version": "10.3.0", "resolved": "https://registry.npmjs.org/constructs/-/constructs-10.3.0.tgz", @@ -561,20 +1042,76 @@ "node": ">= 16.14.0" } }, - "node_modules/data-uri-to-buffer": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", - "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", - "optional": true, + "node_modules/cross-env": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-7.0.3.tgz", + "integrity": "sha512-+/HKd6EgcQCJGh2PSjZuUitQBQynKor4wrFbRg4DtAgS1aWO+gU52xpH7M9ScGgXSYmAVS9bIJ8EzuaGw0oNAw==", + "dev": true, + "dependencies": { + "cross-spawn": "^7.0.1" + }, + "bin": { + "cross-env": "src/bin/cross-env.js", + "cross-env-shell": "src/bin/cross-env-shell.js" + }, "engines": { - "node": ">= 12" + "node": ">=10.14", + "npm": ">=6", + "yarn": ">=1" } }, - "node_modules/dir-glob": { + "node_modules/cross-spawn": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", + "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "dev": true, + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "optional": true, + "engines": { + "node": ">= 12" + } + }, + "node_modules/debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dev": true, + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/deep-is": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", + "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", + "dev": true, + "peer": true + }, + "node_modules/dir-glob": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==", - "optional": true, + "devOptional": true, "dependencies": { "path-type": "^4.0.0" }, @@ -582,6 +1119,19 @@ "node": ">=8" } }, + "node_modules/doctrine": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", + "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==", + "dev": true, + "peer": true, + "dependencies": { + "esutils": "^2.0.2" + }, + "engines": { + "node": ">=6.0.0" + } + }, "node_modules/dotenv": { "version": "16.3.1", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.3.1.tgz", @@ -599,6 +1149,198 @@ "integrity": "sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==", "optional": true }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "dev": true, + "peer": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/eslint": { + "version": "8.55.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.55.0.tgz", + "integrity": "sha512-iyUUAM0PCKj5QpwGfmCAG9XXbZCWsqP/eWAWrG/W0umvjuLRBECwSFdt+rCntju0xEH7teIABPwXpahftIaTdA==", + "dev": true, + "peer": true, + "dependencies": { + "@eslint-community/eslint-utils": "^4.2.0", + "@eslint-community/regexpp": "^4.6.1", + "@eslint/eslintrc": "^2.1.4", + "@eslint/js": "8.55.0", + "@humanwhocodes/config-array": "^0.11.13", + "@humanwhocodes/module-importer": "^1.0.1", + "@nodelib/fs.walk": "^1.2.8", + "@ungap/structured-clone": "^1.2.0", + "ajv": "^6.12.4", + "chalk": "^4.0.0", + "cross-spawn": "^7.0.2", + "debug": "^4.3.2", + "doctrine": "^3.0.0", + "escape-string-regexp": "^4.0.0", + "eslint-scope": "^7.2.2", + "eslint-visitor-keys": "^3.4.3", + "espree": "^9.6.1", + "esquery": "^1.4.2", + "esutils": "^2.0.2", + "fast-deep-equal": "^3.1.3", + "file-entry-cache": "^6.0.1", + "find-up": "^5.0.0", + "glob-parent": "^6.0.2", + "globals": "^13.19.0", + "graphemer": "^1.4.0", + "ignore": "^5.2.0", + "imurmurhash": "^0.1.4", + "is-glob": "^4.0.0", + "is-path-inside": "^3.0.3", + "js-yaml": "^4.1.0", + "json-stable-stringify-without-jsonify": "^1.0.1", + "levn": "^0.4.1", + "lodash.merge": "^4.6.2", + "minimatch": "^3.1.2", + "natural-compare": "^1.4.0", + "optionator": "^0.9.3", + "strip-ansi": "^6.0.1", + "text-table": "^0.2.0" + }, + "bin": { + "eslint": "bin/eslint.js" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint-scope": { + "version": "7.2.2", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.2.2.tgz", + "integrity": "sha512-dOt21O7lTMhDM+X9mB4GX+DZrZtCUJPL/wlcTqxyrx5IvO0IYtILdtrQGQp+8n5S0gwSVmOf9NQrjMOgfQZlIg==", + "dev": true, + "peer": true, + "dependencies": { + "esrecurse": "^4.3.0", + "estraverse": "^5.2.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint-visitor-keys": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", + "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", + "dev": true, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint/node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "peer": true, + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/eslint/node_modules/glob-parent": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", + "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", + "dev": true, + "peer": true, + "dependencies": { + "is-glob": "^4.0.3" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/espree": { + "version": "9.6.1", + "resolved": "https://registry.npmjs.org/espree/-/espree-9.6.1.tgz", + "integrity": "sha512-oruZaFkjorTpF32kDSI5/75ViwGeZginGGy2NoOSg3Q9bnwlnmDm4HLnkl0RE3n+njDXR037aY1+x58Z/zFdwQ==", + "dev": true, + "peer": true, + "dependencies": { + "acorn": "^8.9.0", + "acorn-jsx": "^5.3.2", + "eslint-visitor-keys": "^3.4.1" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/esquery": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.5.0.tgz", + "integrity": "sha512-YQLXUplAwJgCydQ78IMJywZCceoqk1oH01OERdSAJc/7U2AylwjhSCLDEtqwg811idIS/9fIU5GjG73IgjKMVg==", + "dev": true, + "peer": true, + "dependencies": { + "estraverse": "^5.1.0" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "dev": true, + "peer": true, + "dependencies": { + "estraverse": "^5.2.0" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "dev": true, + "peer": true, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "dev": true, + "peer": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/event-stream": { "version": "3.3.4", "resolved": "https://registry.npmjs.org/event-stream/-/event-stream-3.3.4.tgz", @@ -614,11 +1356,18 @@ "through": "~2.3.1" } }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true, + "peer": true + }, "node_modules/fast-glob": { "version": "3.3.2", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.2.tgz", "integrity": "sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow==", - "optional": true, + "devOptional": true, "dependencies": { "@nodelib/fs.stat": "^2.0.2", "@nodelib/fs.walk": "^1.2.3", @@ -630,11 +1379,25 @@ "node": ">=8.6.0" } }, + "node_modules/fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "dev": true, + "peer": true + }, + "node_modules/fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", + "dev": true, + "peer": true + }, "node_modules/fastq": { "version": "1.15.0", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz", "integrity": "sha512-wBrocU2LCXXa+lWBt8RoIRD89Fi8OdABODa/kEnyeyjS5aZO5/GNvI5sEINADqP/h8M29UHTHUb53sUu5Ihqdw==", - "optional": true, + "devOptional": true, "dependencies": { "reusify": "^1.0.4" } @@ -662,11 +1425,24 @@ "node": "^12.20 || >= 14.13" } }, + "node_modules/file-entry-cache": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", + "integrity": "sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==", + "dev": true, + "peer": true, + "dependencies": { + "flat-cache": "^3.0.4" + }, + "engines": { + "node": "^10.12.0 || >=12.0.0" + } + }, "node_modules/fill-range": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", - "optional": true, + "devOptional": true, "dependencies": { "to-regex-range": "^5.0.1" }, @@ -674,6 +1450,45 @@ "node": ">=8" } }, + "node_modules/find-up": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", + "dev": true, + "peer": true, + "dependencies": { + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/flat-cache": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.2.0.tgz", + "integrity": "sha512-CYcENa+FtcUKLmhhqyctpclsq7QF38pKjZHsGNiSQF5r4FtoKDWabFDl3hzaEQMvT1LHEysw5twgLvpYYb4vbw==", + "dev": true, + "peer": true, + "dependencies": { + "flatted": "^3.2.9", + "keyv": "^4.5.3", + "rimraf": "^3.0.2" + }, + "engines": { + "node": "^10.12.0 || >=12.0.0" + } + }, + "node_modules/flatted": { + "version": "3.2.9", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.2.9.tgz", + "integrity": "sha512-36yxDn5H7OFZQla0/jFJmbIKTdZAQHngCedGxiMmpNfEZM0sdEeT+WczLQrjK6D7o2aiyLYDnkw0R3JK0Qv1RQ==", + "dev": true, + "peer": true + }, "node_modules/formdata-polyfill": { "version": "4.0.10", "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", @@ -706,6 +1521,13 @@ "node": ">=12" } }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "dev": true, + "peer": true + }, "node_modules/fsevents": { "version": "2.3.2", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", @@ -720,11 +1542,32 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, + "peer": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/glob-parent": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", - "optional": true, + "devOptional": true, "dependencies": { "is-glob": "^4.0.1" }, @@ -732,6 +1575,22 @@ "node": ">= 6" } }, + "node_modules/globals": { + "version": "13.24.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-13.24.0.tgz", + "integrity": "sha512-AhO5QUcj8llrbG09iWhPU2B204J1xnPeL8kQmVorSsy+Sjj1sk8gIyh6cUocGmH4L0UuhAJy+hJMRA4mgA4mFQ==", + "dev": true, + "peer": true, + "dependencies": { + "type-fest": "^0.20.2" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/globby": { "version": "13.2.2", "resolved": "https://registry.npmjs.org/globby/-/globby-13.2.2.tgz", @@ -757,20 +1616,81 @@ "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", "optional": true }, + "node_modules/graphemer": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/graphemer/-/graphemer-1.4.0.tgz", + "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==", + "dev": true + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "peer": true, + "engines": { + "node": ">=8" + } + }, "node_modules/ignore": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.0.tgz", "integrity": "sha512-g7dmpshy+gD7mh88OC9NwSGTKoc3kyLAZQRU1mt53Aw/vnvfXnbC+F/7F7QoYVKbV+KNvJx8wArewKy1vXMtlg==", - "optional": true, + "devOptional": true, "engines": { "node": ">= 4" } }, + "node_modules/import-fresh": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz", + "integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==", + "dev": true, + "peer": true, + "dependencies": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "dev": true, + "peer": true, + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "dev": true, + "peer": true, + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true, + "peer": true + }, "node_modules/is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", - "optional": true, + "devOptional": true, "engines": { "node": ">=0.10.0" } @@ -779,7 +1699,7 @@ "version": "4.0.3", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", - "optional": true, + "devOptional": true, "dependencies": { "is-extglob": "^2.1.1" }, @@ -791,16 +1711,60 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", - "optional": true, + "devOptional": true, "engines": { "node": ">=0.12.0" } }, + "node_modules/is-path-inside": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz", + "integrity": "sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==", + "dev": true, + "peer": true, + "engines": { + "node": ">=8" + } + }, "node_modules/isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", - "optional": true + "devOptional": true + }, + "node_modules/js-yaml": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "dev": true, + "peer": true, + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/json-buffer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", + "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", + "dev": true, + "peer": true + }, + "node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true, + "peer": true + }, + "node_modules/json-stable-stringify-without-jsonify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", + "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", + "dev": true, + "peer": true }, "node_modules/jsonfile": { "version": "6.1.0", @@ -814,6 +1778,65 @@ "graceful-fs": "^4.1.6" } }, + "node_modules/keyv": { + "version": "4.5.4", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", + "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", + "dev": true, + "peer": true, + "dependencies": { + "json-buffer": "3.0.1" + } + }, + "node_modules/levn": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", + "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", + "dev": true, + "peer": true, + "dependencies": { + "prelude-ls": "^1.2.1", + "type-check": "~0.4.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "dev": true, + "peer": true, + "dependencies": { + "p-locate": "^5.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/lodash.merge": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", + "dev": true, + "peer": true + }, + "node_modules/lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dev": true, + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/map-stream": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/map-stream/-/map-stream-0.1.0.tgz", @@ -824,7 +1847,7 @@ "version": "1.4.1", "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", - "optional": true, + "devOptional": true, "engines": { "node": ">= 8" } @@ -833,7 +1856,7 @@ "version": "4.0.5", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==", - "optional": true, + "devOptional": true, "dependencies": { "braces": "^3.0.2", "picomatch": "^2.3.1" @@ -842,6 +1865,19 @@ "node": ">=8.6" } }, + "node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "peer": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, "node_modules/minimist": { "version": "1.2.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", @@ -851,6 +1887,18 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, + "node_modules/natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", + "dev": true + }, "node_modules/node-domexception": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", @@ -888,11 +1936,113 @@ "url": "https://opencollective.com/node-fetch" } }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, + "peer": true, + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/optionator": { + "version": "0.9.3", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.3.tgz", + "integrity": "sha512-JjCoypp+jKn1ttEFExxhetCKeJt9zhAgAve5FXHixTvFDW/5aEktX9bufBKLRRMdU7bNtpLfcGu94B3cdEJgjg==", + "dev": true, + "peer": true, + "dependencies": { + "@aashutoshrathi/word-wrap": "^1.2.3", + "deep-is": "^0.1.3", + "fast-levenshtein": "^2.0.6", + "levn": "^0.4.1", + "prelude-ls": "^1.2.1", + "type-check": "^0.4.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "peer": true, + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "dev": true, + "peer": true, + "dependencies": { + "p-limit": "^3.0.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "dev": true, + "peer": true, + "dependencies": { + "callsites": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true, + "peer": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "dev": true, + "peer": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "engines": { + "node": ">=8" + } + }, "node_modules/path-type": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", - "optional": true, + "devOptional": true, "engines": { "node": ">=8" } @@ -910,7 +2060,7 @@ "version": "2.3.1", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", - "optional": true, + "devOptional": true, "engines": { "node": ">=8.6" }, @@ -918,6 +2068,16 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/prelude-ls": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", + "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", + "dev": true, + "peer": true, + "engines": { + "node": ">= 0.8.0" + } + }, "node_modules/prettier": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.0.3.tgz", @@ -948,10 +2108,21 @@ "node": ">= 0.10" } }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "dev": true, + "peer": true, + "engines": { + "node": ">=6" + } + }, "node_modules/queue-microtask": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", + "devOptional": true, "funding": [ { "type": "github", @@ -965,23 +2136,49 @@ "type": "consulting", "url": "https://feross.org/support" } - ], - "optional": true + ] + }, + "node_modules/resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "dev": true, + "peer": true, + "engines": { + "node": ">=4" + } }, "node_modules/reusify": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", - "optional": true, + "devOptional": true, "engines": { "iojs": ">=1.0.0", "node": ">=0.10.0" } }, + "node_modules/rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "dev": true, + "peer": true, + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/run-parallel": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", + "devOptional": true, "funding": [ { "type": "github", @@ -996,7 +2193,6 @@ "url": "https://feross.org/support" } ], - "optional": true, "dependencies": { "queue-microtask": "^1.2.2" } @@ -1021,6 +2217,42 @@ "constructs": "^10.*" } }, + "node_modules/semver": { + "version": "7.5.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", + "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", + "dev": true, + "dependencies": { + "lru-cache": "^6.0.0" + }, + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "engines": { + "node": ">=8" + } + }, "node_modules/slash": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/slash/-/slash-4.0.0.tgz", @@ -1071,6 +2303,52 @@ "duplexer": "~0.1.1" } }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "peer": true, + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "dev": true, + "peer": true, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "peer": true, + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/text-table": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", + "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==", + "dev": true, + "peer": true + }, "node_modules/through": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", @@ -1081,7 +2359,7 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "optional": true, + "devOptional": true, "dependencies": { "is-number": "^7.0.0" }, @@ -1094,6 +2372,44 @@ "resolved": "https://registry.npmjs.org/toml/-/toml-3.0.0.tgz", "integrity": "sha512-y/mWCZinnvxjTKYhJ+pYxwD0mRLVvOtdS2Awbgxln6iEnt4rk0yBxeSBHkGJcPucRiG0e55mwWp+g/05rsrd6w==" }, + "node_modules/ts-api-utils": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-1.0.3.tgz", + "integrity": "sha512-wNMeqtMz5NtwpT/UZGY5alT+VoKdSsOOP/kqHFcUW1P/VRhH2wJ48+DN2WwUliNbQ976ETwDL0Ifd2VVvgonvg==", + "dev": true, + "engines": { + "node": ">=16.13.0" + }, + "peerDependencies": { + "typescript": ">=4.2.0" + } + }, + "node_modules/type-check": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", + "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", + "dev": true, + "peer": true, + "dependencies": { + "prelude-ls": "^1.2.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/type-fest": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", + "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==", + "dev": true, + "peer": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/typescript": { "version": "5.3.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.3.tgz", @@ -1122,6 +2438,16 @@ "node": ">= 10.0.0" } }, + "node_modules/uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "dev": true, + "peer": true, + "dependencies": { + "punycode": "^2.1.0" + } + }, "node_modules/web-streams-polyfill": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz", @@ -1135,7 +2461,7 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "optional": true, + "devOptional": true, "dependencies": { "isexe": "^2.0.0" }, @@ -1146,6 +2472,19 @@ "node": ">= 8" } }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true, + "peer": true + }, + "node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + }, "node_modules/yaml": { "version": "2.3.4", "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.3.4.tgz", @@ -1155,6 +2494,19 @@ "node": ">= 14" } }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "peer": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/zx": { "version": "6.2.5", "resolved": "https://registry.npmjs.org/zx/-/zx-6.2.5.tgz", diff --git a/lib/workload/stateful/filemanager/deploy/package.json b/lib/workload/stateful/filemanager/deploy/package.json index 6a596b27a..f92fb23fb 100644 --- a/lib/workload/stateful/filemanager/deploy/package.json +++ b/lib/workload/stateful/filemanager/deploy/package.json @@ -2,18 +2,21 @@ "name": "filemanager", "version": "0.1", "bin": { - "filemanager-stack": "stack/filemanager_stack.js" + "filemanager": "bin/filemanager.js" }, "scripts": { "build": "tsc", "watch": "tsc -w", - "cdk": "cdk" + "cdk": "cdk", + "migrate": "cross-env FILEMANAGER_DEPLOY_MIGRATE_DATABASE=true" }, "devDependencies": { "@types/node": "^20.5.9", + "@typescript-eslint/eslint-plugin": "^6.14.0", "aws-cdk": "^2.114.1", + "cross-env": "^7.0.3", "prettier": "3.0.3", - "typescript": "^5.2.2" + "typescript": "^5.3.3" }, "dependencies": { "@aws-cdk/aws-apigatewayv2-alpha": "^2.114.1-alpha.0", diff --git a/lib/workload/stateful/filemanager/deploy/stack/filemanager_stack.ts b/lib/workload/stateful/filemanager/deploy/stack/filemanager_stack.ts deleted file mode 100644 index d62721802..000000000 --- a/lib/workload/stateful/filemanager/deploy/stack/filemanager_stack.ts +++ /dev/null @@ -1,132 +0,0 @@ -import { Duration, RemovalPolicy, Stack, StackProps, Tags } from 'aws-cdk-lib'; -import { Construct } from 'constructs'; -import * as iam from 'aws-cdk-lib/aws-iam'; -import { RustFunction, Settings as CargoSettings } from 'rust.aws-cdk-lambda'; -import { Architecture } from 'aws-cdk-lib/aws-lambda'; -import * as s3 from 'aws-cdk-lib/aws-s3'; -import * as rds from 'aws-cdk-lib/aws-rds'; -import * as sqs from 'aws-cdk-lib/aws-sqs'; -import * as lambdaDestinations from 'aws-cdk-lib/aws-lambda-destinations'; -import * as lambdaEventSources from 'aws-cdk-lib/aws-lambda-event-sources'; -import * as ec2 from 'aws-cdk-lib/aws-ec2'; -import * as secretsmanager from 'aws-cdk-lib/aws-secretsmanager'; -import * as s3n from 'aws-cdk-lib/aws-s3-notifications'; - -/** - * Common settings for the filemanager stack. - */ -interface Settings { - database_url: string; - endpoint_url?: string; - stack_name: string; - buildEnvironment?: NodeJS.ProcessEnv; -} - -/** - * Stack used to deploy filemanager. - */ -export class FilemanagerStack extends Stack { - constructor(scope: Construct, id: string, settings: Settings, props?: StackProps) { - super(scope, id, props); - - Tags.of(this).add('Stack', settings.stack_name); - - const lambdaRole = new iam.Role(this, id + 'Role', { - assumedBy: new iam.ServicePrincipal('lambda.amazonaws.com'), - description: 'Lambda execution role for ' + id, - }); - - const queue = new sqs.Queue(this, id + 'Queue'); - - const testBucket = new s3.Bucket(this, id + 'Bucket', { - bucketName: 'filemanager-test-ingest', - blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL, - encryption: s3.BucketEncryption.S3_MANAGED, - enforceSSL: true, - removalPolicy: RemovalPolicy.DESTROY, - }); - - testBucket.addEventNotification(s3.EventType.OBJECT_CREATED, new s3n.SqsDestination(queue)); - testBucket.addEventNotification(s3.EventType.OBJECT_REMOVED, new s3n.SqsDestination(queue)); - - lambdaRole.addManagedPolicy( - iam.ManagedPolicy.fromAwsManagedPolicyName('service-role/AWSLambdaSQSQueueExecutionRole') - ); - - const s3BucketPolicy = new iam.PolicyStatement({ - actions: ['s3:List*', 's3:Get*'], - resources: ['arn:aws:s3:::*'], - }); - lambdaRole.addToPolicy(s3BucketPolicy); - - const deadLetterQueue = new sqs.Queue(this, id + 'DeadLetterQueue'); - const deadLetterQueueDestination = new lambdaDestinations.SqsDestination(deadLetterQueue); - - CargoSettings.WORKSPACE_DIR = '../'; - CargoSettings.BUILD_INDIVIDUALLY = true; - - const filemanagerLambda = new RustFunction(this, id + 'IngestLambdaFunction', { - package: 'filemanager-ingest-lambda', - target: 'aarch64-unknown-linux-gnu', - - memorySize: 128, - timeout: Duration.seconds(28), - environment: { - DATABASE_URL: settings.database_url, - ...(settings.endpoint_url && { ENDPOINT_URL: settings.endpoint_url }), - SQS_QUEUE_URL: queue.queueUrl, - RUST_LOG: 'info,filemanager_ingest_lambda=trace,filemanager=trace', - }, - buildEnvironment: settings.buildEnvironment, - architecture: Architecture.ARM_64, - role: lambdaRole, - onFailure: deadLetterQueueDestination, - }); - - const eventSource = new lambdaEventSources.SqsEventSource(queue); - filemanagerLambda.addEventSource(eventSource); - - // VPC - //const vpc = ec2.Vpc.fromLookup(this, 'main-vpc', { isDefault: false }); - const vpc = new ec2.Vpc(this, 'vpc', { - ipAddresses: ec2.IpAddresses.cidr('10.0.0.0/24'), - maxAzs: 99, // As many as there are available in the region - natGateways: 1, - subnetConfiguration: [ - { - name: 'ingress', - subnetType: ec2.SubnetType.PUBLIC, - }, - { - name: 'application', - subnetType: ec2.SubnetType.PRIVATE_WITH_EGRESS, - }, - { - name: 'database', - subnetType: ec2.SubnetType.PRIVATE_ISOLATED, - }, - ], - }); - - // Secret - new secretsmanager.Secret(this, 'filemanager_db_secret', { - secretName: 'filemanager_db_secret', // pragma: allowlist secret - generateSecretString: { - secretStringTemplate: JSON.stringify({ username: 'filemanager' }), - excludePunctuation: true, - generateStringKey: 'password', - }, - }); - - // RDS - new rds.ServerlessCluster(this, 'Database', { - engine: rds.DatabaseClusterEngine.auroraPostgres({ - version: rds.AuroraPostgresEngineVersion.VER_13_12, - }), - defaultDatabaseName: 'filemanager', - credentials: rds.Credentials.fromGeneratedSecret('filemanager_db_secret'), - removalPolicy: RemovalPolicy.DESTROY, - vpc, - }); - } -} diff --git a/lib/workload/stateful/filemanager/deploy/stack/stack.ts b/lib/workload/stateful/filemanager/deploy/stack/stack.ts deleted file mode 100644 index 3cc4f175e..000000000 --- a/lib/workload/stateful/filemanager/deploy/stack/stack.ts +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env node - -import 'source-map-support/register'; -import * as cdk from 'aws-cdk-lib'; -import { FilemanagerStack } from './filemanager_stack'; - -export const STACK_NAME = 'FilemanagerStack'; -const STACK_DESCRIPTION = 'A stack deploying filemanager to dev.'; - -const app = new cdk.App(); -new FilemanagerStack( - app, - STACK_NAME, - { - database_url: 'postgresql://filemanager:filemanager@db:5432/filemanager', // pragma: allowlist secret - stack_name: STACK_NAME, - buildEnvironment: { - // Override release profile to match defaults for dev builds. - CARGO_PROFILE_RELEASE_OPT_LEVEL: '0', - CARGO_PROFILE_RELEASE_DEBUG_ASSERTIONS: 'true', - CARGO_PROFILE_RELEASE_OVERFLOW_CHECKS: 'true', - CARGO_PROFILE_RELEASE_PANIC: 'unwind', - CARGO_PROFILE_RELEASE_INCREMENTAL: 'true', - CARGO_PROFILE_RELEASE_CODEGEN_UNITS: '256', - - // Additionally speed up builds by removing debug info. Please enable this if required. - CARGO_PROFILE_RELEASE_DEBUG: 'false', - // Add SCCACHE to speed up compilation. FIXME: Not cross-platform right now as it's defined here :/ - RUSTC_WRAPPER: '/opt/homebrew/bin/sccache', - }, - }, - { - stackName: STACK_NAME, - description: STACK_DESCRIPTION, - tags: { - Stack: STACK_NAME, - }, - } -); diff --git a/lib/workload/stateful/filemanager/filemanager-migrate-lambda/Cargo.toml b/lib/workload/stateful/filemanager/filemanager-migrate-lambda/Cargo.toml new file mode 100644 index 000000000..13ee67f2b --- /dev/null +++ b/lib/workload/stateful/filemanager/filemanager-migrate-lambda/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "filemanager-migrate-lambda" +version = "0.1.0" +license.workspace = true +edition.workspace = true +authors.workspace = true + +[dependencies] +aws_lambda_events = "0.12" +lambda_runtime = "0.8" +serde = "1" +tokio = { version = "1", features = ["macros"] } +tracing = { version = "0.1", features = ["log"] } +tracing-subscriber = { version = "0.3", default-features = false, features = ["fmt"] } + +filemanager = { path = "../filemanager", features = ["migrate"] } +serde_json = "1" + +aws-config = "1" +aws-sdk-sts = "1" \ No newline at end of file diff --git a/lib/workload/stateful/filemanager/filemanager-migrate-lambda/src/main.rs b/lib/workload/stateful/filemanager/filemanager-migrate-lambda/src/main.rs new file mode 100644 index 000000000..3cfb27936 --- /dev/null +++ b/lib/workload/stateful/filemanager/filemanager-migrate-lambda/src/main.rs @@ -0,0 +1,24 @@ +use filemanager::database::aws::migration::Migration; +use filemanager::database::Migrate; +use lambda_runtime::{run, service_fn, Error, LambdaEvent}; +use std::collections::HashMap; +use tracing_subscriber::layer::SubscriberExt; +use tracing_subscriber::util::SubscriberInitExt; +use tracing_subscriber::{fmt, EnvFilter}; + +#[tokio::main] +async fn main() -> Result<(), Error> { + let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")); + + tracing_subscriber::registry() + .with(fmt::layer().json().without_time()) + .with(env_filter) + .init(); + + run(service_fn( + |_: LambdaEvent>| async move { + Migration::with_defaults().await?.migrate().await + }, + )) + .await +} diff --git a/lib/workload/stateful/filemanager/filemanager/Cargo.toml b/lib/workload/stateful/filemanager/filemanager/Cargo.toml index 161f5c492..5262ca838 100644 --- a/lib/workload/stateful/filemanager/filemanager/Cargo.toml +++ b/lib/workload/stateful/filemanager/filemanager/Cargo.toml @@ -6,6 +6,10 @@ authors.workspace = true license.workspace = true edition.workspace = true +[features] + +migrate = ["sqlx/migrate"] + [dependencies] axum = "0.6" hyper = { version = "1", features = ["full"] } @@ -39,3 +43,7 @@ futures = "0.3" [dev-dependencies] aws-smithy-runtime-api = "1" + +# The migrate feature is required to run sqlx tests +filemanager = { path = ".", features = ["migrate"] } +lazy_static = "1.4" diff --git a/lib/workload/stateful/filemanager/filemanager/src/database/aws/ingester.rs b/lib/workload/stateful/filemanager/filemanager/src/database/aws/ingester.rs index 2ab8697d7..c9b9e727c 100644 --- a/lib/workload/stateful/filemanager/filemanager/src/database/aws/ingester.rs +++ b/lib/workload/stateful/filemanager/filemanager/src/database/aws/ingester.rs @@ -7,7 +7,7 @@ use crate::database::{Client, Ingest}; use crate::error::Result; use crate::events::aws::StorageClass; use crate::events::aws::{Events, TransposedS3EventMessages}; -use crate::events::EventType; +use crate::events::EventSourceType; /// An ingester for S3 events. #[derive(Debug)] @@ -29,7 +29,7 @@ impl Ingester { } /// Ingest the events into the database by calling the insert and update queries. - pub async fn ingest_events(&mut self, events: Events) -> Result<()> { + pub async fn ingest_events(&self, events: Events) -> Result<()> { let Events { object_created, object_removed, @@ -55,8 +55,8 @@ impl Ingester { &object_ids, &buckets, &keys, - &sizes, - &e_tags, + &sizes as &[Option], + &e_tags as &[Option], &event_times, &last_modified_dates as &[Option>], &portal_run_ids @@ -100,9 +100,9 @@ impl Ingester { #[async_trait] impl Ingest for Ingester { - async fn ingest(&mut self, events: EventType) -> Result<()> { + async fn ingest(&self, events: EventSourceType) -> Result<()> { match events { - EventType::S3(events) => self.ingest_events(events).await, + EventSourceType::S3(events) => self.ingest_events(events).await, } } } @@ -110,20 +110,21 @@ impl Ingest for Ingester { #[cfg(test)] pub(crate) mod tests { use crate::database::aws::ingester::Ingester; + use crate::database::aws::migration::tests::MIGRATOR; use crate::database::{Client, Ingest}; use crate::events::aws::tests::{expected_events, EXPECTED_E_TAG}; use crate::events::aws::{Events, StorageClass}; - use crate::events::EventType; + use crate::events::EventSourceType; use chrono::{DateTime, Utc}; use sqlx::postgres::PgRow; use sqlx::{PgPool, Row}; - #[sqlx::test(migrations = "../database/migrations")] + #[sqlx::test(migrator = "MIGRATOR")] async fn ingest_object_created(pool: PgPool) { let mut events = test_events(); events.object_removed = Default::default(); - let mut ingester = test_ingester(pool); + let ingester = test_ingester(pool); ingester.ingest_events(events).await.unwrap(); let result = sqlx::query("select * from object") @@ -134,11 +135,11 @@ pub(crate) mod tests { assert_created(result); } - #[sqlx::test(migrations = "../database/migrations")] + #[sqlx::test(migrator = "MIGRATOR")] async fn ingest_object_removed(pool: PgPool) { let events = test_events(); - let mut ingester = test_ingester(pool); + let ingester = test_ingester(pool); ingester.ingest_events(events).await.unwrap(); let result = sqlx::query("select * from object") @@ -149,12 +150,12 @@ pub(crate) mod tests { assert_deleted(result); } - #[sqlx::test(migrations = "../database/migrations")] + #[sqlx::test(migrator = "MIGRATOR")] async fn ingest(pool: PgPool) { let events = test_events(); - let mut ingester = test_ingester(pool); - ingester.ingest(EventType::S3(events)).await.unwrap(); + let ingester = test_ingester(pool); + ingester.ingest(EventSourceType::S3(events)).await.unwrap(); let result = sqlx::query("select * from object") .fetch_one(ingester.client.pool()) @@ -211,7 +212,7 @@ pub(crate) mod tests { events.object_created.storage_classes[0] = Some(StorageClass::Standard); events.object_removed.last_modified_dates[0] = Some(DateTime::default()); - events.object_removed.storage_classes[0] = Some(StorageClass::Standard); + events.object_removed.storage_classes[0] = None; events } diff --git a/lib/workload/stateful/filemanager/filemanager/src/database/aws/migration.rs b/lib/workload/stateful/filemanager/filemanager/src/database/aws/migration.rs new file mode 100644 index 000000000..04c4923de --- /dev/null +++ b/lib/workload/stateful/filemanager/filemanager/src/database/aws/migration.rs @@ -0,0 +1,85 @@ +use crate::database::{Client, Migrate}; +use crate::error::Error::MigrateError; +use crate::error::Result; +use async_trait::async_trait; +use sqlx::migrate; +use sqlx::migrate::Migrator; +use tracing::trace; + +/// A struct to perform database migrations. +#[derive(Debug)] +pub struct Migration { + client: Client, +} + +impl Migration { + /// Create a new migration. + pub fn new(client: Client) -> Self { + Self { client } + } + + /// Create a new migration with a default database client. + pub async fn with_defaults() -> Result { + Ok(Self { + client: Client::default().await?, + }) + } + + /// Get the underlying sqlx migrator for the migrations. + pub fn migrator() -> Migrator { + migrate!("../database/migrations") + } + + /// Get a reference to the database client. + pub fn client(&self) -> &Client { + &self.client + } +} + +#[async_trait] +impl Migrate for Migration { + async fn migrate(&self) -> Result<()> { + trace!("applying migrations"); + Self::migrator() + .run(self.client().pool()) + .await + .map_err(|err| MigrateError(err.to_string())) + } +} + +#[cfg(test)] +pub(crate) mod tests { + use lazy_static::lazy_static; + use sqlx::PgPool; + + use super::*; + + lazy_static! { + pub(crate) static ref MIGRATOR: Migrator = Migration::migrator(); + } + + #[sqlx::test(migrations = false)] + async fn test_migrate(pool: PgPool) { + let migrate = Migration::new(Client::new(pool)); + + let not_exists = sqlx::query!( + "select exists (select from information_schema.tables where table_name = 'object')" + ) + .fetch_one(migrate.client.pool()) + .await + .unwrap(); + + assert!(!not_exists.exists.unwrap()); + + migrate.migrate().await.unwrap(); + + let exists = sqlx::query!( + "select exists (select from information_schema.tables where table_name = 'object')" + ) + .fetch_one(migrate.client.pool()) + .await + .unwrap(); + + assert!(exists.exists.unwrap()); + } +} diff --git a/lib/workload/stateful/filemanager/filemanager/src/database/aws/mod.rs b/lib/workload/stateful/filemanager/filemanager/src/database/aws/mod.rs index 55e301256..2a3edc691 100644 --- a/lib/workload/stateful/filemanager/filemanager/src/database/aws/mod.rs +++ b/lib/workload/stateful/filemanager/filemanager/src/database/aws/mod.rs @@ -5,6 +5,9 @@ use aws_sdk_s3::types::StorageClass; pub mod ingester; +#[cfg(feature = "migrate")] +pub mod migration; + /// An S3 object which matches the s3 object schema. #[derive(Debug, Clone)] pub struct CloudObject { diff --git a/lib/workload/stateful/filemanager/filemanager/src/database/mod.rs b/lib/workload/stateful/filemanager/filemanager/src/database/mod.rs index 0cec9a595..aacebdcde 100644 --- a/lib/workload/stateful/filemanager/filemanager/src/database/mod.rs +++ b/lib/workload/stateful/filemanager/filemanager/src/database/mod.rs @@ -6,7 +6,7 @@ use async_trait::async_trait; use sqlx::PgPool; use crate::error::Result; -use crate::events::EventType; +use crate::events::EventSourceType; pub mod aws; @@ -39,5 +39,12 @@ impl Client { #[async_trait] pub trait Ingest { /// Ingest the events. - async fn ingest(&mut self, events: EventType) -> Result<()>; + async fn ingest(&self, events: EventSourceType) -> Result<()>; +} + +/// Trait representing database migrations. +#[async_trait] +pub trait Migrate { + /// Migrate the database. + async fn migrate(&self) -> Result<()>; } diff --git a/lib/workload/stateful/filemanager/filemanager/src/events/aws/collecter.rs b/lib/workload/stateful/filemanager/filemanager/src/events/aws/collecter.rs index 3ad52587a..e4fbdb41c 100644 --- a/lib/workload/stateful/filemanager/filemanager/src/events/aws/collecter.rs +++ b/lib/workload/stateful/filemanager/filemanager/src/events/aws/collecter.rs @@ -4,14 +4,17 @@ use crate::error::Error::S3Error; use async_trait::async_trait; use aws_sdk_s3::operation::head_object::{HeadObjectError, HeadObjectOutput}; use aws_sdk_s3::primitives; +use aws_sdk_s3::types::StorageClass::Standard; use chrono::{DateTime, NaiveDateTime, Utc}; use futures::future::join_all; use mockall_double::double; use tracing::trace; use crate::error::Result; -use crate::events::aws::{Events, FlatS3EventMessage, FlatS3EventMessages, StorageClass}; -use crate::events::{Collect, EventType}; +use crate::events::aws::{ + EventType, Events, FlatS3EventMessage, FlatS3EventMessages, StorageClass, +}; +use crate::events::{Collect, EventSourceType}; /// Collect raw events into the processed form which the database module accepts. #[derive(Debug)] @@ -71,18 +74,37 @@ impl Collecter { ) -> Result { Ok(FlatS3EventMessages( join_all(events.into_inner().into_iter().map(|mut event| async move { + // No need to run this unnecessarily on removed events. + match event.event_type { + EventType::Removed | EventType::Other => return Ok(event), + _ => {} + }; + trace!(key = ?event.key, bucket = ?event.bucket, "updating event"); + // Race condition: it's possible that an object gets deleted so quickly that it + // occurs before calling head. This means that there may be cases where the storage + // class and other fields are not known. if let Some(head) = Self::head(client, &event.key, &event.bucket).await? { + trace!(head = ?head, "received head object output"); + let HeadObjectOutput { storage_class, last_modified, + content_length, + e_tag, .. } = head; - event = - event.with_storage_class(storage_class.and_then(StorageClass::from_aws)); - event = event.with_last_modified_date(Self::convert_datetime(last_modified)); + // S3 does not return a storage class for standard, which means this is the + // default. See https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadObject.html#API_HeadObject_ResponseSyntax + event = event + .update_storage_class(StorageClass::from_aws( + storage_class.unwrap_or(Standard), + )) + .update_last_modified_date(Self::convert_datetime(last_modified)) + .update_size(content_length) + .update_e_tag(e_tag); } Ok(event) @@ -96,13 +118,13 @@ impl Collecter { #[async_trait] impl Collect for Collecter { - async fn collect(self) -> Result { + async fn collect(self) -> Result { let (client, raw_events) = self.into_inner(); let raw_events = raw_events.sort_and_dedup(); let events = Self::update_events(&client, raw_events).await?; - Ok(EventType::S3(Events::from(events))) + Ok(EventSourceType::S3(Events::from(events))) } } @@ -110,7 +132,7 @@ impl Collect for Collecter { pub(crate) mod tests { use crate::events::aws::collecter::Collecter; use crate::events::aws::tests::expected_flat_events; - use crate::events::aws::StorageClass::Standard; + use crate::events::aws::StorageClass::IntelligentTiering; use aws_sdk_s3::error::SdkError; use aws_sdk_s3::primitives::{DateTimeFormat, SdkBody}; use aws_sdk_s3::types; @@ -119,6 +141,7 @@ pub(crate) mod tests { use aws_smithy_runtime_api::client::result::ServiceError; use chrono::{DateTime, Utc}; use mockall::predicate::eq; + use std::result; use super::*; @@ -136,7 +159,7 @@ pub(crate) mod tests { async fn head() { let mut collecter = test_collecter().await; - set_s3_client_expectations(&mut collecter.client, 1); + set_s3_client_expectations(&mut collecter.client, vec![|| Ok(expected_head_object())]); let result = Collecter::head(&collecter.client, "key", "bucket") .await @@ -148,19 +171,10 @@ pub(crate) mod tests { async fn head_not_found() { let mut collecter = test_collecter().await; - collecter - .client - .expect_head_object() - .with(eq("key"), eq("bucket")) - .times(1) - .returning(move |_, _| { - Err(SdkError::ServiceError( - ServiceError::builder() - .source(HeadObjectError::NotFound(NotFound::builder().build())) - .raw(HttpResponse::new(404.try_into().unwrap(), SdkBody::empty())) - .build(), - )) - }); + set_s3_client_expectations( + &mut collecter.client, + vec![|| Err(expected_head_object_not_found())], + ); let result = Collecter::head(&collecter.client, "key", "bucket") .await @@ -174,7 +188,7 @@ pub(crate) mod tests { let events = expected_flat_events().sort_and_dedup(); - set_s3_client_expectations(&mut collecter.client, 2); + set_s3_client_expectations(&mut collecter.client, vec![|| Ok(expected_head_object())]); let mut result = Collecter::update_events(&collecter.client, events) .await @@ -183,63 +197,78 @@ pub(crate) mod tests { .into_iter(); let first = result.next().unwrap(); - assert_eq!(first.storage_class, Some(Standard)); + assert_eq!(first.storage_class, Some(IntelligentTiering)); assert_eq!(first.last_modified_date, Some(Default::default())); let second = result.next().unwrap(); - assert_eq!(second.storage_class, Some(Standard)); - assert_eq!(second.last_modified_date, Some(Default::default())); + assert_eq!(second.storage_class, None); + assert_eq!(second.last_modified_date, None); } #[tokio::test] async fn collect() { let mut collecter = test_collecter().await; - set_s3_client_expectations(&mut collecter.client, 2); + set_s3_client_expectations(&mut collecter.client, vec![|| Ok(expected_head_object())]); let result = collecter.collect().await.unwrap(); assert_collected_events(result); } - pub(crate) fn assert_collected_events(result: EventType) { - assert!(matches!(result, EventType::S3(_))); + pub(crate) fn assert_collected_events(result: EventSourceType) { + assert!(matches!(result, EventSourceType::S3(_))); match result { - EventType::S3(events) => { - assert_eq!(events.object_created.storage_classes[0], Some(Standard)); + EventSourceType::S3(events) => { assert_eq!( - events.object_created.last_modified_dates[0], - Some(Default::default()) + events.object_created.storage_classes[0], + Some(IntelligentTiering) ); - - assert_eq!(events.object_removed.storage_classes[0], Some(Standard)); assert_eq!( - events.object_removed.last_modified_dates[0], + events.object_created.last_modified_dates[0], Some(Default::default()) ); + + assert_eq!(events.object_removed.storage_classes[0], None); + assert_eq!(events.object_removed.last_modified_dates[0], None); } } } - pub(crate) fn set_s3_client_expectations(client: &mut Client, times: usize) { - client + pub(crate) fn set_s3_client_expectations(client: &mut Client, expectations: Vec) + where + F: Fn() -> result::Result> + Send + 'static, + { + let client = client .expect_head_object() .with(eq("key"), eq("bucket")) - .times(times) - .returning(move |_, _| Ok(expected_head_object())); + .times(expectations.len()); + + for expectation in expectations { + client.returning(move |_, _| expectation()); + } } - fn expected_head_object() -> HeadObjectOutput { + pub(crate) fn expected_head_object() -> HeadObjectOutput { HeadObjectOutput::builder() .last_modified( primitives::DateTime::from_str("1970-01-01T00:00:00Z", DateTimeFormat::DateTime) .unwrap(), ) - .storage_class(types::StorageClass::Standard) + .storage_class(types::StorageClass::IntelligentTiering) .build() } + pub(crate) fn expected_head_object_not_found() -> SdkError { + SdkError::ServiceError( + ServiceError::builder() + .source(HeadObjectError::NotFound(NotFound::builder().build())) + .raw(HttpResponse::new(404.try_into().unwrap(), SdkBody::empty())) + .build(), + ) + } + async fn test_collecter() -> Collecter { Collecter::new(Client::default(), expected_flat_events()) } diff --git a/lib/workload/stateful/filemanager/filemanager/src/events/aws/collector_builder.rs b/lib/workload/stateful/filemanager/filemanager/src/events/aws/collector_builder.rs index 672e15ec2..02c3d09ff 100644 --- a/lib/workload/stateful/filemanager/filemanager/src/events/aws/collector_builder.rs +++ b/lib/workload/stateful/filemanager/filemanager/src/events/aws/collector_builder.rs @@ -66,7 +66,9 @@ impl CollecterBuilder { trace!(message = ?message, "got the message"); if let Some(body) = message.body() { - serde_json::from_str(body).map_err(|err| DeserializeError(err.to_string())) + let events: Option = serde_json::from_str(body) + .map_err(|err| DeserializeError(err.to_string()))?; + Ok(events.unwrap_or_default()) } else { Err(SQSReceiveError("No body in SQS message".to_string())) } @@ -100,7 +102,7 @@ impl CollecterBuilder { #[cfg(test)] pub(crate) mod tests { use crate::events::aws::collecter::tests::{ - assert_collected_events, set_s3_client_expectations, + assert_collected_events, expected_head_object, set_s3_client_expectations, }; use crate::events::aws::collector_builder::CollecterBuilder; use crate::events::aws::tests::{expected_event_record, expected_flat_events}; @@ -128,7 +130,7 @@ pub(crate) mod tests { let mut s3_client = S3Client::default(); set_sqs_client_expectations(&mut sqs_client); - set_s3_client_expectations(&mut s3_client, 2); + set_s3_client_expectations(&mut s3_client, vec![|| Ok(expected_head_object())]); let events = CollecterBuilder::default() .with_sqs_client(sqs_client) diff --git a/lib/workload/stateful/filemanager/filemanager/src/events/aws/mod.rs b/lib/workload/stateful/filemanager/filemanager/src/events/aws/mod.rs index 2e85e0be1..854aa6bed 100644 --- a/lib/workload/stateful/filemanager/filemanager/src/events/aws/mod.rs +++ b/lib/workload/stateful/filemanager/filemanager/src/events/aws/mod.rs @@ -12,6 +12,7 @@ use uuid::Uuid; use crate::error::Error; use crate::error::Error::DeserializeError; use crate::error::Result; +use crate::events::aws::EventType::{Created, Other, Removed}; pub mod collecter; pub mod collector_builder; @@ -67,8 +68,8 @@ pub struct TransposedS3EventMessages { pub event_names: Vec, pub buckets: Vec, pub keys: Vec, - pub sizes: Vec, - pub e_tags: Vec, + pub sizes: Vec>, + pub e_tags: Vec>, pub sequencers: Vec>, pub portal_run_ids: Vec, pub storage_classes: Vec>, @@ -108,6 +109,7 @@ impl TransposedS3EventMessages { portal_run_id, storage_class, last_modified_date, + .. } = message; self.object_ids.push(object_id); @@ -153,15 +155,20 @@ impl From for Events { let mut object_removed = FlatS3EventMessages::default(); let mut other = FlatS3EventMessages::default(); - messages.into_inner().into_iter().for_each(|message| { - if message.event_name.contains("ObjectCreated") { - object_created.0.push(message); - } else if message.event_name.contains("ObjectRemoved") { - object_removed.0.push(message); - } else { - other.0.push(message); - } - }); + messages + .into_inner() + .into_iter() + .for_each(|message| match message.event_type { + Created => { + object_created.0.push(message); + } + Removed => { + object_removed.0.push(message); + } + Other => { + other.0.push(message); + } + }); Self { object_created: TransposedS3EventMessages::from(object_created), @@ -281,6 +288,13 @@ impl PartialEq for FlatS3EventMessage { } } +#[derive(Debug, Eq, PartialEq)] +pub enum EventType { + Created, + Removed, + Other, +} + /// A flattened AWS S3 record #[derive(Debug, Eq)] pub struct FlatS3EventMessage { @@ -289,24 +303,41 @@ pub struct FlatS3EventMessage { pub event_name: String, pub bucket: String, pub key: String, - pub size: i32, - pub e_tag: String, + pub size: Option, + pub e_tag: Option, pub sequencer: Option, pub portal_run_id: String, pub storage_class: Option, pub last_modified_date: Option>, + pub event_type: EventType, } impl FlatS3EventMessage { - /// Update the storage class. - pub fn with_storage_class(mut self, storage_class: Option) -> Self { - self.storage_class = storage_class; + /// Update the storage class if not None.` + pub fn update_storage_class(mut self, storage_class: Option) -> Self { + storage_class + .into_iter() + .for_each(|storage_class| self.storage_class = Some(storage_class)); + self + } + + /// Update the last modified date if not None. + pub fn update_last_modified_date(mut self, last_modified_date: Option>) -> Self { + last_modified_date + .into_iter() + .for_each(|last_modified_date| self.last_modified_date = Some(last_modified_date)); + self + } + + /// Update the size if not None. + pub fn update_size(mut self, size: Option) -> Self { + size.into_iter().for_each(|size| self.size = Some(size)); self } - /// Update the last modified date. - pub fn with_last_modified_date(mut self, last_modified_date: Option>) -> Self { - self.last_modified_date = last_modified_date; + /// Update the e_tag if not None. + pub fn update_e_tag(mut self, e_tag: Option) -> Self { + e_tag.into_iter().for_each(|e_tag| self.e_tag = Some(e_tag)); self } } @@ -315,7 +346,7 @@ impl FlatS3EventMessage { #[derive(Debug, Serialize, Deserialize, Eq, PartialEq)] #[serde(rename_all = "camelCase")] pub struct S3EventMessage { - #[serde(alias = "Records")] + #[serde(rename = "Records")] pub records: Vec, } @@ -344,8 +375,8 @@ pub struct BucketRecord { #[serde(rename_all = "camelCase")] pub struct ObjectRecord { pub key: String, - pub size: i32, - pub e_tag: String, + pub size: Option, + pub e_tag: Option, pub sequencer: Option, } @@ -383,6 +414,14 @@ impl TryFrom for FlatS3EventMessages { let portal_run_id = event_time.format("%Y%m%d").to_string() + &object_id.to_string()[..8]; + let event_type = if event_name.contains("ObjectCreated") { + Created + } else if event_name.contains("ObjectRemoved") { + Removed + } else { + Other + }; + Ok(FlatS3EventMessage { object_id, event_time, @@ -393,9 +432,10 @@ impl TryFrom for FlatS3EventMessages { e_tag, sequencer, portal_run_id, - // Head field are optionally fetched later. + // Head field are fetched later. storage_class: None, last_modified_date: None, + event_type, }) }) .collect::>>()?, @@ -425,13 +465,28 @@ pub(crate) mod tests { let mut result = result.into_inner().into_iter(); let first = result.next().unwrap(); - assert_flat_s3_event(first, "ObjectRemoved:Delete", EXPECTED_SEQUENCER_DELETED); + assert_flat_s3_event( + first, + "ObjectRemoved:Delete", + EXPECTED_SEQUENCER_DELETED, + None, + ); let second = result.next().unwrap(); - assert_flat_s3_event(second, "ObjectCreated:Put", EXPECTED_SEQUENCER_CREATED); + assert_flat_s3_event( + second, + "ObjectCreated:Put", + EXPECTED_SEQUENCER_CREATED, + Some(0), + ); let third = result.next().unwrap(); - assert_flat_s3_event(third, "ObjectCreated:Put", EXPECTED_SEQUENCER_CREATED); + assert_flat_s3_event( + third, + "ObjectCreated:Put", + EXPECTED_SEQUENCER_CREATED, + Some(0), + ); } #[test] @@ -440,19 +495,34 @@ pub(crate) mod tests { let mut result = result.into_inner().into_iter(); let first = result.next().unwrap(); - assert_flat_s3_event(first, "ObjectCreated:Put", EXPECTED_SEQUENCER_CREATED); + assert_flat_s3_event( + first, + "ObjectCreated:Put", + EXPECTED_SEQUENCER_CREATED, + Some(0), + ); let second = result.next().unwrap(); - assert_flat_s3_event(second, "ObjectRemoved:Delete", EXPECTED_SEQUENCER_DELETED); + assert_flat_s3_event( + second, + "ObjectRemoved:Delete", + EXPECTED_SEQUENCER_DELETED, + None, + ); } - fn assert_flat_s3_event(event: FlatS3EventMessage, event_name: &str, sequencer: &str) { + fn assert_flat_s3_event( + event: FlatS3EventMessage, + event_name: &str, + sequencer: &str, + size: Option, + ) { assert_eq!(event.event_time, DateTime::::default()); assert_eq!(event.event_name, event_name); assert_eq!(event.bucket, "bucket"); assert_eq!(event.key, "key"); - assert_eq!(event.size, 0); - assert_eq!(event.e_tag, EXPECTED_E_TAG); // pragma: allowlist secret + assert_eq!(event.size, size); + assert_eq!(event.e_tag, Some(EXPECTED_E_TAG.to_string())); // pragma: allowlist secret assert_eq!(event.sequencer, Some(sequencer.to_string())); assert!(event.portal_run_id.starts_with("19700101")); assert_eq!(event.storage_class, None); @@ -470,8 +540,11 @@ pub(crate) mod tests { assert_eq!(result.object_created.event_names[0], "ObjectCreated:Put"); assert_eq!(result.object_created.buckets[0], "bucket"); assert_eq!(result.object_created.keys[0], "key"); - assert_eq!(result.object_created.sizes[0], 0); - assert_eq!(result.object_created.e_tags[0], EXPECTED_E_TAG); + assert_eq!(result.object_created.sizes[0], Some(0)); + assert_eq!( + result.object_created.e_tags[0], + Some(EXPECTED_E_TAG.to_string()) + ); assert_eq!( result.object_created.sequencers[0], Some(EXPECTED_SEQUENCER_CREATED.to_string()) @@ -487,8 +560,11 @@ pub(crate) mod tests { assert_eq!(result.object_removed.event_names[0], "ObjectRemoved:Delete"); assert_eq!(result.object_removed.buckets[0], "bucket"); assert_eq!(result.object_removed.keys[0], "key"); - assert_eq!(result.object_removed.sizes[0], 0); - assert_eq!(result.object_removed.e_tags[0], EXPECTED_E_TAG); + assert_eq!(result.object_removed.sizes[0], None); + assert_eq!( + result.object_removed.e_tags[0], + Some(EXPECTED_E_TAG.to_string()) + ); assert_eq!( result.object_removed.sequencers[0], Some(EXPECTED_SEQUENCER_DELETED.to_string()) @@ -588,7 +664,8 @@ pub(crate) mod tests { }, "object": { "key": "key", - "size": 0, + // ObjectRemoved::Delete does not have a size, even though this isn't documented + // anywhere. "eTag": EXPECTED_E_TAG, "versionId": "096fKKXTRTtl3on89fVO.nfljtsv6qko", "sequencer": EXPECTED_SEQUENCER_DELETED diff --git a/lib/workload/stateful/filemanager/filemanager/src/events/mod.rs b/lib/workload/stateful/filemanager/filemanager/src/events/mod.rs index fad126ba0..f1165b1cf 100644 --- a/lib/workload/stateful/filemanager/filemanager/src/events/mod.rs +++ b/lib/workload/stateful/filemanager/filemanager/src/events/mod.rs @@ -12,12 +12,12 @@ pub mod aws; #[async_trait] pub trait Collect { /// Collect into events. - async fn collect(self) -> Result; + async fn collect(self) -> Result; } /// The type of event. #[derive(Debug)] #[non_exhaustive] -pub enum EventType { +pub enum EventSourceType { S3(Events), } diff --git a/lib/workload/stateful/filemanager/filemanager/src/handlers/aws.rs b/lib/workload/stateful/filemanager/filemanager/src/handlers/aws.rs index f3331acf9..d75e2e2b3 100644 --- a/lib/workload/stateful/filemanager/filemanager/src/handlers/aws.rs +++ b/lib/workload/stateful/filemanager/filemanager/src/handlers/aws.rs @@ -32,7 +32,7 @@ pub async fn receive_and_ingest( .collect() .await?; - let mut ingester = if let Some(database_client) = database_client { + let ingester = if let Some(database_client) = database_client { Ingester::new(database_client) } else { Ingester::with_defaults().await? @@ -56,8 +56,8 @@ pub async fn ingest_event( .into_iter() .filter_map(|event| { event.body.map(|body| { - let body: FlatS3EventMessages = serde_json::from_str(&body)?; - Ok(body) + let body: Option = serde_json::from_str(&body)?; + Ok(body.unwrap_or_default()) }) }) .collect::, Error>>()? @@ -74,7 +74,7 @@ pub async fn ingest_event( trace!("ingesting events: {:?}", events); - let mut ingester = if let Some(database_client) = database_client { + let ingester = if let Some(database_client) = database_client { Ingester::new(database_client) } else { Ingester::with_defaults().await? @@ -90,19 +90,20 @@ pub async fn ingest_event( mod tests { use super::*; use crate::database::aws::ingester::tests::assert_deleted; - use crate::events::aws::collecter::tests::set_s3_client_expectations; + use crate::database::aws::migration::tests::MIGRATOR; + use crate::events::aws::collecter::tests::{expected_head_object, set_s3_client_expectations}; use crate::events::aws::collector_builder::tests::set_sqs_client_expectations; use crate::events::aws::tests::expected_event_record; use aws_lambda_events::sqs::SqsMessage; use sqlx::PgPool; - #[sqlx::test(migrations = "../database/migrations")] + #[sqlx::test(migrator = "MIGRATOR")] async fn test_receive_and_ingest(pool: PgPool) { let mut sqs_client = SQSClient::default(); let mut s3_client = S3Client::default(); set_sqs_client_expectations(&mut sqs_client); - set_s3_client_expectations(&mut s3_client, 2); + set_s3_client_expectations(&mut s3_client, vec![|| Ok(expected_head_object())]); let ingester = receive_and_ingest(s3_client, sqs_client, Some("url"), Some(Client::new(pool))) @@ -117,11 +118,11 @@ mod tests { assert_deleted(result); } - #[sqlx::test(migrations = "../database/migrations")] + #[sqlx::test(migrator = "MIGRATOR")] async fn test_ingest_event(pool: PgPool) { let mut s3_client = S3Client::default(); - set_s3_client_expectations(&mut s3_client, 2); + set_s3_client_expectations(&mut s3_client, vec![|| Ok(expected_head_object())]); let event = SqsEvent { records: vec![SqsMessage { diff --git a/lib/workload/stateful/filemanager/scripts/deploy.sh b/lib/workload/stateful/filemanager/scripts/deploy.sh index 385fa3c87..35138da61 100755 --- a/lib/workload/stateful/filemanager/scripts/deploy.sh +++ b/lib/workload/stateful/filemanager/scripts/deploy.sh @@ -1,15 +1,16 @@ #!/bin/sh -x # TODO: Takes too long for re-deploy, find further shortcuts -export AWS_ENDPOINT_URL=http://localhost:4566 export FM_BUCKET=filemanager-test-ingest docker compose down docker compose up --wait --wait-timeout 20 -d -cd deploy + +cd deploy || exit npm install yes | npx cdklocal destroy yes | npx cdklocal bootstrap + cd ../database && sqlx migrate run && cd .. cd deploy && yes | npx cdklocal deploy --require-approval never && cd .. diff --git a/lib/workload/stateful/filemanager/scripts/logs.sh b/lib/workload/stateful/filemanager/scripts/logs.sh index b018b5882..bdd068b33 100755 --- a/lib/workload/stateful/filemanager/scripts/logs.sh +++ b/lib/workload/stateful/filemanager/scripts/logs.sh @@ -1,7 +1,5 @@ #!/bin/sh -x -export AWS_ENDPOINT_URL=http://localhost:4566 - # Don't die when the whole stack re-deploys group_name=$(aws logs describe-log-groups --query 'logGroups[*].logGroupName' --output text) aws logs tail "$group_name" --follow