Merge pull request #529 from umccr/attributes/portal-run-id

filemanager: annotate portal run id

.github/workflows/prbuild.yml

@@ -36,6 +36,7 @@ env:
   #
   # And from pyenv docs setting this PYENV_VERSION as env-var would set the python version globally
   PYENV_VERSION: '3.12'
+  GOENV_VERSION: '1.22'
 
 jobs:
   pre-commit-lint-security:
@@ -82,21 +83,24 @@ jobs:
     runs-on: codebuild-orcabus-codebuild-gh-runner-${{ github.run_id }}-${{ github.run_attempt }}
     if: ${{ !github.event.pull_request.draft }}
     steps:
+      - name: Rust installation
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          components: rustfmt
+
       - name: Print toolchain versions
         run: |
           node -v
           python -V
           pip3 -V
+          rustc --version
+          go version
           make --version
+          docker version
 
       - name: Checkout code
         uses: actions/checkout@v4
 
-      - name: Rust installation
-        uses: actions-rust-lang/setup-rust-toolchain@v1
-        with:
-          components: rustfmt
-
       - name: Install dependencies
         run: |
           yarn install --immutable

Makefile

@@ -53,6 +53,7 @@ test-stateless-app-suite:
 	@(cd lib/workload/stateless/stacks/sequence-run-manager && $(MAKE) test)
 	@(cd lib/workload/stateless/stacks/metadata-manager && $(MAKE) test)
 	@(cd lib/workload/stateless/stacks/filemanager && $(MAKE) test)
+	@(cd lib/workload/stateless/stacks/fmannotator && $(MAKE) test)
 	@(cd lib/workload/stateless/stacks/bclconvert-manager && $(MAKE) test)
 	@(cd lib/workload/stateless/stacks/workflow-manager && $(MAKE) test)
 

config/config.ts

@@ -47,6 +47,7 @@ import {
   getRnasumIcav2PipelineManagerStackProps,
   getRnasumIcav2PipelineTableStackProps,
 } from './stacks/rnasumPipelineManager';
+import { getFmAnnotatorProps } from './stacks/fmAnnotator';
 
 interface EnvironmentConfig {
   name: string;
@@ -102,6 +103,7 @@ export const getEnvironmentConfig = (stage: AppStage): EnvironmentConfig | null
       bclConvertManagerStackProps: getBclConvertManagerStackProps(stage),
       workflowManagerStackProps: getWorkflowManagerStackProps(stage),
       stackyMcStackFaceProps: getGlueStackProps(stage),
+      fmAnnotatorProps: getFmAnnotatorProps(),
     },
   };
 

config/stacks/fmAnnotator.ts

@@ -0,0 +1,10 @@
+import { FMAnnotatorConfigurableProps } from '../../lib/workload/stateless/stacks/fmannotator/deploy/stack';
+import { eventBusName, jwtSecretName, vpcProps } from '../constants';
+
+export const getFmAnnotatorProps = (): FMAnnotatorConfigurableProps => {
+  return {
+    vpcProps,
+    eventBusName,
+    jwtSecretName,
+  };
+};

lib/pipeline/statelessPipelineStack.ts

@@ -91,6 +91,7 @@ export class StatelessPipelineStack extends cdk.Stack {
           install: {
             'runtime-versions': {
               python: '3.12',
+              golang: '1.22',
             },
           },
         },
@@ -126,6 +127,18 @@ export class StatelessPipelineStack extends cdk.Stack {
           resources: ['*'],
         }),
       ],
+      partialBuildSpec: codebuild.BuildSpec.fromObject({
+        phases: {
+          install: {
+            'runtime-versions': {
+              // Don't strictly need golang here as CDK can do docker bundling, but `local`
+              // bundling tends to be faster.
+              golang: '1.22',
+            },
+          },
+        },
+        version: '0.2',
+      }),
     });
 
     const pipeline = new pipelines.CodePipeline(this, 'Pipeline', {

lib/workload/components/api-gateway/index.ts

@@ -42,6 +42,7 @@ export interface ApiGatewayConstructProps {
 
 export class ApiGatewayConstruct extends Construct {
   private readonly _httpApi: HttpApi;
+  private readonly _domainName: string;
 
   constructor(scope: Construct, id: string, props: ApiGatewayConstructProps) {
     super(scope, id);
@@ -54,9 +55,9 @@ export class ApiGatewayConstruct extends Construct {
     );
     const hostedZoneId = StringParameter.valueForStringParameter(this, '/hosted_zone/umccr/id');
 
-    const domainName = `${props.customDomainNamePrefix}.${hostedDomainName}`;
+    this._domainName = `${props.customDomainNamePrefix}.${hostedDomainName}`;
     const apiGWDomainName = new DomainName(this, 'UmccrDomainName', {
-      domainName: `${props.customDomainNamePrefix}.${hostedDomainName}`,
+      domainName: this.domainName,
       certificate: Certificate.fromCertificateArn(this, 'cert', umccrAcmArn),
     });
 
@@ -85,7 +86,7 @@ export class ApiGatewayConstruct extends Construct {
         hostedZoneId,
         zoneName: hostedDomainName,
       }),
-      recordName: domainName,
+      recordName: this.domainName,
       target: RecordTarget.fromAlias(
         new ApiGatewayv2DomainProperties(
           apiGWDomainName.regionalDomainName,
@@ -177,4 +178,8 @@ export class ApiGatewayConstruct extends Construct {
   get httpApi(): HttpApi {
     return this._httpApi;
   }
+
+  get domainName(): string {
+    return this._domainName;
+  }
 }

lib/workload/stateless/stacks/filemanager/.env.example

@@ -3,6 +3,7 @@ FILEMANAGER_DATABASE_HOST=0.0.0.0
 FILEMANAGER_DATABASE_PORT=4321
 
 DATABASE_URL=postgresql://filemanager:filemanager@${FILEMANAGER_DATABASE_HOST}:${FILEMANAGER_DATABASE_PORT}/filemanager #pragma: allowlist secret
+API_PORT=8000
 
 FILEMANAGER_LINKS_URL=localhost:8000
 FILEMANAGER_API_CORS_ALLOW_ORIGINS=http://localhost:8000

lib/workload/stateless/stacks/filemanager/Makefile

@@ -40,7 +40,7 @@ docker-find:
 	@docker ps --filter name=$(DOCKER_PROJECT_NAME)-postgres --latest --format "{{.ID}}" | xargs -I {} docker port {} | tail -n 1 | awk '{print $$NF}'
 docker-api: docker-postgres
 	# Run the local API server in a docker container.
-	@docker compose -p "$(DOCKER_PROJECT_NAME)" up api
+	@docker compose -p "$(DOCKER_PROJECT_NAME)" -f compose-api.yml up api
 
 ## Build related commands
 build: docker-postgres

lib/workload/stateless/stacks/filemanager/compose-api.yml

@@ -0,0 +1,20 @@
+services:
+  api:
+    build:
+      context: .
+      args:
+        # The build itself needs access to the database.
+        - DATABASE_URL=${BUILD_DATABASE_URL:-postgresql://filemanager:[email protected]:4321/filemanager} # pragma: allowlist secret
+    environment:
+      # Container database address for running server inside a docker container.
+      - DATABASE_URL=${DATABASE_URL:-postgresql://filemanager:filemanager@postgres:4321/filemanager}
+      - RUST_LOG=debug
+      - FILEMANAGER_API_CORS_ALLOW_ORIGINS=${FILEMANAGER_API_CORS_ALLOW_ORIGINS:-http://localhost:3000}
+      - FILEMANAGER_API_CORS_ALLOW_HEADERS=${FILEMANAGER_API_CORS_ALLOW_HEADERS:-accept,authorization,content-type,user-agent,x-csrftoken,x-requested-with,x-amz-security-token,x-amz-date,content-disposition}
+      - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-access_key_id}
+      - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-secret_access_key}
+      - AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION:-ap-southeast-2}
+      - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN:-session_token}
+    ports:
+      - '${API_PORT}:8000'
+    restart: always

lib/workload/stateless/stacks/filemanager/compose.yml

@@ -3,30 +3,9 @@ services:
     build: database
     restart: always
     environment:
-      - POSTGRES_DATABASE=filemanager
+      - POSTGRES_DB=filemanager
       - POSTGRES_USER=filemanager
       - POSTGRES_PASSWORD=filemanager
       - PGPORT=4321
     ports:
       - "${FILEMANAGER_DATABASE_HOST}:${FILEMANAGER_DATABASE_PORT}:4321"
-  api:
-    build:
-      context: .
-      args:
-        # The build itself needs access to the database.
-        DATABASE_URL: postgresql://filemanager:[email protected]:4321/filemanager # pragma: allowlist secret
-    environment:
-      # Container database address for running server inside a docker container.
-      - DATABASE_URL=postgresql://filemanager:filemanager@postgres:4321/filemanager
-      - RUST_LOG=debug
-      - FILEMANAGER_API_CORS_ALLOW_ORIGINS=${FILEMANAGER_API_CORS_ALLOW_ORIGINS:-http://localhost:3000}
-      - FILEMANAGER_API_CORS_ALLOW_HEADERS=${FILEMANAGER_API_CORS_ALLOW_HEADERS:-accept,authorization,content-type,user-agent,x-csrftoken,x-requested-with,x-amz-security-token,x-amz-date,content-disposition}
-      - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-access_key_id}
-      - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-secret_access_key}
-      - AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION:-ap-southeast-2}
-      - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN:-session_token}
-    ports:
-      - "8000:8000"
-    depends_on:
-      - postgres
-    restart: always

lib/workload/stateless/stacks/filemanager/deploy/stack.ts

@@ -48,6 +48,7 @@ export class Filemanager extends Stack {
   private readonly host: string;
   private readonly securityGroup: ISecurityGroup;
   private readonly queue: IQueue;
+  readonly domainName: string;
 
   constructor(scope: Construct, id: string, props: FilemanagerProps) {
     super(scope, id, props);
@@ -93,8 +94,9 @@ export class Filemanager extends Stack {
     );
 
     this.createIngestFunction(props);
-    this.createApiFunction(props);
     this.createInventoryFunction(props);
+
+    this.domainName = this.createApiFunction(props);
   }
 
   private createIngestRole(name: string) {
@@ -130,9 +132,9 @@ export class Filemanager extends Stack {
   }
 
   /**
-   * Query function and API Gateway fronting the function.
+   * Query function and API Gateway fronting the function. Returns the configured domain name.
    */
-  private createApiFunction(props: FilemanagerProps) {
+  private createApiFunction(props: FilemanagerProps): string {
     let apiLambda = new ApiFunction(this, 'ApiFunction', {
       vpc: this.vpc,
       host: this.host,
@@ -141,13 +143,14 @@ export class Filemanager extends Stack {
       ...props,
     });
 
-    const ApiGateway = new ApiGatewayConstruct(this, 'ApiGateway', {
+    const apiGateway = new ApiGatewayConstruct(this, 'ApiGateway', {
       region: this.region,
       apiName: 'FileManager',
       customDomainNamePrefix: 'file',
       ...props,
     });
-    const httpApi = ApiGateway.httpApi;
+
+    const httpApi = apiGateway.httpApi;
 
     const apiIntegration = new HttpLambdaIntegration('ApiIntegration', apiLambda.function);
 
@@ -156,5 +159,7 @@ export class Filemanager extends Stack {
       integration: apiIntegration,
       routeKey: HttpRouteKey.with('/{proxy+}', HttpMethod.ANY),
     });
+
+    return apiGateway.domainName;
   }
 }

lib/workload/stateless/stacks/filemanager/filemanager-build/src/gen_entities.rs

@@ -17,6 +17,8 @@ use crate::Config;
 pub async fn generate_entities() -> Result<()> {
     let config = Config::load()?;
 
+    println!("{:#?}", config);
+
     let out_dir = config.out_dir;
     let command: &[&_] = &[
         "sea-orm-cli",

lib/workload/stateless/stacks/fmannotator/.gitignore

@@ -0,0 +1 @@
+build

lib/workload/stateless/stacks/fmannotator/Makefile

@@ -0,0 +1,28 @@
+## Default target
+default: help
+
+install: build
+all: build
+
+## Build related commands
+build:
+	@go build -o build
+test:
+	@go test
+lint:
+	@go fmt
+golangci-lint:
+	@golangci-lint run
+check: lint golangci-lint
+
+## Clean
+clean:
+	@go mod tidy
+
+## Help text
+help:
+	@printf "The attribute annotator Makefile.\n\n"
+	@printf "Usage: make <target>\n"
+
+# No targets are files, so they are all PHONY.
+.PHONY: *

lib/workload/stateless/stacks/fmannotator/README.md

@@ -0,0 +1,52 @@
+# FMAnnotator
+
+The FMAnnotator service annotates records using the FileManager API.
+
+## Development
+
+This service is written in Go, which should be [installed][golang]. Please also install [golangci-lint] to run lints on
+the codebase.
+
+This project is organised using the Go Lambda function CDK, and contains Lambda function handlers under [`cmd`][cmd]
+
+Makefile is used to simplify development. Tests can be run by using:
+
+```sh
+make test
+```
+
+Lints and checks can be run using:
+
+```sh
+make check
+```
+
+To update the [go.mod][go-mod] and download any new go modules:
+
+```sh
+make clean
+```
+
+## Project layout
+
+This service has the following structure:
+
+* [cmd]: The Lambda handler `main.go` functions.
+* [deploy]: CDK deployment code.
+* [fixtures]: Database fixtures and test data.
+* [internal]: Internal package containing common test code.
+* [schema]: Generated EventBridge code-bindings.
+
+Top-level `.go` files contain library related code which implements the functionality of the FMAnnotator.
+Tests are defined using the `_test.go` suffix.
+
+[golang]: https://go.dev/doc/install
+[golangci-lint]: https://golangci-lint.run/welcome/install/#local-installation
+[cmd]: cmd
+[api]: api.go
+[config]: config.go
+[handlers]: handlers.go
+[internal]: internal
+[fixtures]: fixtures
+[schema]: schema
+[go-mod]: go.mod