Merge branch 'main' into fix/workflowrun-state-handling

compose.yml

@@ -70,7 +70,8 @@ services:
       # Container database address for running server inside a docker container.
       - DATABASE_URL=postgresql://orcabus:orcabus@db:5432/filemanager
       - RUST_LOG=debug
-      - FILEMANAGER_API_CORS_ALLOW_ORIGINS=${FILEMANAGER_API_CORS_ALLOW_ORIGINS:-http://localhost:8400}
+      - FILEMANAGER_API_CORS_ALLOW_ORIGINS=${FILEMANAGER_API_CORS_ALLOW_ORIGINS:-http://localhost:3000}
+      - FILEMANAGER_API_CORS_ALLOW_HEADERS=${FILEMANAGER_API_CORS_ALLOW_HEADERS:-accept,authorization,content-type,user-agent,x-csrftoken,x-requested-with,x-amz-security-token,x-amz-date,content-disposition}
       - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-access_key_id}
       - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-secret_access_key}
       - AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION:-ap-southeast-2}

lib/workload/stateless/stacks/filemanager/.env.example

@@ -6,3 +6,4 @@ DATABASE_URL=postgresql://filemanager:filemanager@${FILEMANAGER_DATABASE_HOST}:$
 
 FILEMANAGER_LINKS_URL=localhost:8000
 FILEMANAGER_API_CORS_ALLOW_ORIGINS=http://localhost:8000
+FILEMANAGER_API_CORS_ALLOW_HEADERS=accept,authorization,content-type,user-agent,x-csrftoken,x-requested-with,x-amz-security-token,x-amz-date,content-disposition

lib/workload/stateless/stacks/filemanager/compose.yml

@@ -19,7 +19,8 @@ services:
       # Container database address for running server inside a docker container.
       - DATABASE_URL=postgresql://filemanager:filemanager@postgres:4321/filemanager
       - RUST_LOG=debug
-      - FILEMANAGER_API_CORS_ALLOW_ORIGINS=${FILEMANAGER_API_CORS_ALLOW_ORIGINS:-http://localhost:8000}
+      - FILEMANAGER_API_CORS_ALLOW_ORIGINS=${FILEMANAGER_API_CORS_ALLOW_ORIGINS:-http://localhost:3000}
+      - FILEMANAGER_API_CORS_ALLOW_HEADERS=${FILEMANAGER_API_CORS_ALLOW_HEADERS:-accept,authorization,content-type,user-agent,x-csrftoken,x-requested-with,x-amz-security-token,x-amz-date,content-disposition}
       - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-access_key_id}
       - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-secret_access_key}
       - AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION:-ap-southeast-2}

lib/workload/stateless/stacks/filemanager/deploy/constructs/functions/api.ts

@@ -6,7 +6,7 @@ import { BucketProps } from './ingest';
 /**
  * Props for the API function.
  */
-export type ApiFunctionProps = fn.FunctionPropsNoPackage & DatabaseProps & BucketProps;
+export type ApiFunctionProps = fn.FunctionPropsConfigurable & DatabaseProps & BucketProps;
 
 /**
  * A construct for the Lambda API function.

lib/workload/stateless/stacks/filemanager/deploy/constructs/functions/function.ts

@@ -31,9 +31,9 @@ export type DatabaseProps = {
 };
 
 /**
- * Props for a Rust function without the package.
+ * Props for a Rust function which can be configured from the top-level orcabus context.
  */
-export type FunctionPropsNoPackage = {
+export type FunctionPropsConfigurable = {
   /**
    * Additional build environment variables when building the Lambda function.
    */
@@ -57,9 +57,9 @@ export type FunctionPropsNoPackage = {
 };
 
 /**
- * Props for the Rust function.
+ * Props for the Rust function which can be configured from the top-level orcabus context.
  */
-export type FunctionProps = FunctionPropsNoPackage &
+export type FunctionProps = FunctionPropsConfigurable &
   DatabaseProps & {
     /**
      * The package to build for this function.
@@ -69,6 +69,10 @@ export type FunctionProps = FunctionPropsNoPackage &
      * Name of the Lambda function resource.
      */
     readonly functionName?: string;
+    /**
+     * The timeout for the Lambda function, defaults to 28 seconds.
+     */
+    readonly timeout?: Duration;
   };
 
 /**
@@ -121,7 +125,7 @@ export class Function extends Construct {
         },
       },
       memorySize: 128,
-      timeout: Duration.seconds(28),
+      timeout: props.timeout ?? Duration.seconds(28),
       environment: {
         // No password here, using RDS IAM to generate credentials.
         PGHOST: props.host,

lib/workload/stateless/stacks/filemanager/deploy/constructs/functions/ingest.ts

@@ -28,7 +28,7 @@ export type EventSourceProps = {
 /**
  * Props for the ingest function.
  */
-export type IngestFunctionProps = fn.FunctionPropsNoPackage & DatabaseProps & EventSourceProps;
+export type IngestFunctionProps = fn.FunctionPropsConfigurable & DatabaseProps & EventSourceProps;
 
 /**
  * A construct for the Lambda ingest function.

lib/workload/stateless/stacks/filemanager/deploy/constructs/functions/inventory.ts

@@ -19,7 +19,7 @@ export type InventoryFunctionConfig = {
 /**
  * Props for the inventory function.
  */
-export type InventoryFunctionProps = fn.FunctionPropsNoPackage &
+export type InventoryFunctionProps = fn.FunctionPropsConfigurable &
   DatabaseProps &
   InventoryFunctionConfig;
 

lib/workload/stateless/stacks/filemanager/deploy/constructs/functions/migrate.ts

@@ -1,17 +1,31 @@
 import { Construct } from 'constructs';
 import * as fn from './function';
 import { DatabaseProps } from './function';
+import { Duration, Stack } from 'aws-cdk-lib';
+import { PolicyStatement } from 'aws-cdk-lib/aws-iam';
 
 /**
  * Props for the migrate function.
  */
-export type MigrateFunctionProps = fn.FunctionPropsNoPackage & DatabaseProps;
+export type MigrateFunctionProps = fn.FunctionPropsConfigurable & DatabaseProps;
 
 /**
  * A construct for the Lambda migrate function.
  */
 export class MigrateFunction extends fn.Function {
   constructor(scope: Construct, id: string, props: MigrateFunctionProps) {
-    super(scope, id, { package: 'filemanager-migrate-lambda', ...props });
+    super(scope, id, {
+      package: 'filemanager-migrate-lambda',
+      timeout: Duration.minutes(2),
+      ...props,
+    });
+
+    // Need to be able to determine if the stack is in rollback state.
+    this.addToPolicy(
+      new PolicyStatement({
+        actions: ['cloudformation:DescribeStacks'],
+        resources: [Stack.of(this).stackId],
+      })
+    );
   }
 }

lib/workload/stateless/stacks/filemanager/docs/API_GUIDE.md

@@ -14,12 +14,14 @@ This serves Swagger OpenAPI docs at `http://localhost:8000/swagger-ui` when usin
 
 The API has some environment variables that can be used to configure behaviour (for the presigned url route):
 
-| Option                               | Description                                                                                                                    | Type                | Default                     |
-|--------------------------------------|--------------------------------------------------------------------------------------------------------------------------------|---------------------|-----------------------------|
-| `FILEMANAGER_API_LINKS_URL`          | Override the URL which is used to generate pagination links. By default the `HOST` header is used to created pagination links. | URL                 | Not set                     |
-| `FILEMANAGER_API_PRESIGN_LIMIT`      | The maximum file size in bytes which presigned URLs will be generated for.                                                     | Integer             | `"20971520"`                | 
-| `FILEMANAGER_API_PRESIGN_EXPIRY`     | The expiry time for presigned urls.                                                                                            | Duration in seconds | `"300"`                     |
-| `FILEMANAGER_API_CORS_ALLOW_ORIGINS` | The origins to allow for CORS.                                                                                                 | List of origins     | Not set, no origins allowed |
+| Option                               | Description                                                                                                                    | Type                | Default                         |
+|--------------------------------------|--------------------------------------------------------------------------------------------------------------------------------|---------------------|---------------------------------|
+| `FILEMANAGER_API_LINKS_URL`          | Override the URL which is used to generate pagination links. By default the `HOST` header is used to created pagination links. | URL                 | Not set                         |
+| `FILEMANAGER_API_PRESIGN_LIMIT`      | The maximum file size in bytes which presigned URLs will be generated for.                                                     | Integer             | `"20971520"`                    | 
+| `FILEMANAGER_API_PRESIGN_EXPIRY`     | The expiry time for presigned urls.                                                                                            | Duration in seconds | `"300"`                         |
+| `FILEMANAGER_API_CORS_ALLOW_ORIGINS` | The origins to allow for CORS.                                                                                                 | List of origins     | Not set, no origins allowed     |
+| `FILEMANAGER_API_CORS_ALLOW_METHODS` | The methods to allow for CORS.                                                                                                 | List of origins     | `"GET,HEAD,OPTIONS,POST,PATCH"` |
+| `FILEMANAGER_API_CORS_ALLOW_HEADERS` | The headers to allow for CORS.                                                                                                 | List of origins     | `"authorization"`               |
 
 The deployed instance of the filemanager API can be reached using the desired stage at `https://file.<stage>.umccr.org`
 using the orcabus API token. To retrieve the token, run:
@@ -102,6 +104,14 @@ curl --get -H "Authorization: Bearer $TOKEN" --data-urlencode "attributes[portal
 > Attributes on filemanager records start empty. They need to be added to the record to query on them later.
 > See [updating records](#updating-records)
 
+As a convience, the filemanager has an attributes route that can be used to query by top-level attribute properties.
+For example, the following is equivalent to the above query:
+
+```sh
+curl --get -H "Authorization: Bearer $TOKEN" --data-urlencode "portalRunId=202405212aecb782" \
+"https://file.dev.umccr.org/api/v1/s3/attributes" | jq
+```
+
 ### Wilcard matching
 
 The API supports using wildcards to match multiple characters in a value for most field. Use `*` to match multiple characters

lib/workload/stateless/stacks/filemanager/filemanager-api-lambda/src/main.rs

@@ -1,18 +1,19 @@
+use std::sync::Arc;
+
 use axum::extract::{Request, State};
 use axum::middleware::{from_fn_with_state, Next};
 use axum::response::{IntoResponse, Response};
-use filemanager::env::Config;
+use lambda_http::run;
 use lambda_http::Error;
-use std::sync::Arc;
+use tracing::debug;
 
 use filemanager::clients::aws::s3;
 use filemanager::database::Client;
+use filemanager::env::Config;
 use filemanager::handlers::aws::{create_database_pool, update_credentials};
 use filemanager::handlers::init_tracing;
 use filemanager::routes::error::{ErrorResponse, ErrorStatusCode};
 use filemanager::routes::{router, AppState};
-use lambda_http::run;
-use tracing::debug;
 
 #[tokio::main]
 async fn main() -> Result<(), Error> {

lib/workload/stateless/stacks/filemanager/filemanager-api-server/src/main.rs

@@ -1,5 +1,16 @@
+use std::io;
+use std::path::PathBuf;
+use std::sync::Arc;
+
 use axum::serve;
 use clap::{Parser, Subcommand};
+use http::Uri;
+use sea_orm::ConnectionTrait;
+use tokio::fs::File;
+use tokio::io::AsyncReadExt;
+use tokio::net::TcpListener;
+use tracing::{debug, info};
+
 use filemanager::clients::aws::s3;
 use filemanager::database::aws::migration::Migration;
 use filemanager::database::{Client, Migrate};
@@ -10,15 +21,6 @@ use filemanager::handlers::init_tracing_with_format;
 use filemanager::handlers::Format::Pretty;
 use filemanager::queries::EntriesBuilder;
 use filemanager::routes::{router, AppState};
-use http::Uri;
-use sea_orm::ConnectionTrait;
-use std::io;
-use std::path::PathBuf;
-use std::sync::Arc;
-use tokio::fs::File;
-use tokio::io::AsyncReadExt;
-use tokio::net::TcpListener;
-use tracing::{debug, info};
 
 /// Run the filemanager API server locally to explore the API.
 #[derive(Parser, Debug)]

lib/workload/stateless/stacks/filemanager/filemanager-inventory-lambda/src/main.rs

@@ -1,7 +1,7 @@
-use filemanager::clients::aws::s3::Client;
 use lambda_runtime::{run, service_fn, Error, LambdaEvent};
 use serde::Deserialize;
 
+use filemanager::clients::aws::s3::Client;
 use filemanager::database::Client as DbClient;
 use filemanager::env::Config;
 use filemanager::events::aws::inventory::Manifest;
@@ -59,11 +59,13 @@ async fn main() -> Result<(), Error> {
 
 #[cfg(test)]
 mod tests {
-    use super::*;
     use aws_sdk_s3::types::InventoryFormat;
-    use filemanager::events::aws::inventory::File;
     use serde_json::{from_str, json};
 
+    use filemanager::events::aws::inventory::File;
+
+    use super::*;
+
     #[test]
     fn deserialize_bucket_key() {
         let bucket_key = json!({

lib/workload/stateless/stacks/filemanager/filemanager-migrate-lambda/Cargo.toml

@@ -8,10 +8,13 @@ rust-version.workspace = true
 
 [dependencies]
 serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+
 tokio = { version = "1", features = ["macros"] }
 tracing = { version = "0.1" }
 
 aws_lambda_events = "0.15"
+aws-sdk-cloudformation = "1"
 lambda_runtime = "0.13"
 
 filemanager = { path = "../filemanager", features = ["migrate"] }