From f97ac1ab1b3631707955a16ff4066d39f9b3d868 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Thu, 29 Aug 2024 08:55:11 +1000 Subject: [PATCH 01/13] feat: setup attribute linker microservice --- config/config.ts | 2 + config/stacks/attributeLinker.ts | 9 ++ .../stacks/attribute-linker/deploy/stack.ts | 82 +++++++++++++++++++ .../stateless/stacks/attribute-linker/go.mod | 5 ++ .../stateless/stacks/attribute-linker/go.sum | 10 +++ .../stateless/stacks/attribute-linker/main.go | 19 +++++ .../stacks/attribute-linker/schema/README.md | 14 ++++ .../workflowrunstatechange/event.go | 54 ++++++++++++ .../workflowrunstatechange/library_record.go | 14 ++++ .../workflowrunstatechange/marshaller.go | 24 ++++++ .../workflowrunstatechange/payload.go | 19 +++++ .../workflow_run_state_change.go | 48 +++++++++++ .../statelessStackCollectionClass.ts | 8 ++ package.json | 1 + yarn.lock | 11 +++ 15 files changed, 320 insertions(+) create mode 100644 config/stacks/attributeLinker.ts create mode 100644 lib/workload/stateless/stacks/attribute-linker/deploy/stack.ts create mode 100644 lib/workload/stateless/stacks/attribute-linker/go.mod create mode 100644 lib/workload/stateless/stacks/attribute-linker/go.sum create mode 100644 lib/workload/stateless/stacks/attribute-linker/main.go create mode 100644 lib/workload/stateless/stacks/attribute-linker/schema/README.md create mode 100644 lib/workload/stateless/stacks/attribute-linker/schema/orcabus_workflowmanager/workflowrunstatechange/event.go create mode 100644 lib/workload/stateless/stacks/attribute-linker/schema/orcabus_workflowmanager/workflowrunstatechange/library_record.go create mode 100644 lib/workload/stateless/stacks/attribute-linker/schema/orcabus_workflowmanager/workflowrunstatechange/marshaller.go create mode 100644 lib/workload/stateless/stacks/attribute-linker/schema/orcabus_workflowmanager/workflowrunstatechange/payload.go create mode 100644 lib/workload/stateless/stacks/attribute-linker/schema/orcabus_workflowmanager/workflowrunstatechange/workflow_run_state_change.go diff --git a/config/config.ts b/config/config.ts index 383eb82e6..56bd1f910 100644 --- a/config/config.ts +++ b/config/config.ts @@ -43,6 +43,7 @@ import { getUmccriseIcav2PipelineManagerStackProps, getUmccriseIcav2PipelineTableStackProps, } from './stacks/umccrisePipelineManager'; +import { getAttributeLinkerProps } from './stacks/attributeLinker'; interface EnvironmentConfig { name: string; @@ -96,6 +97,7 @@ export const getEnvironmentConfig = (stage: AppStage): EnvironmentConfig | null bclConvertManagerStackProps: getBclConvertManagerStackProps(stage), workflowManagerStackProps: getWorkflowManagerStackProps(stage), stackyMcStackFaceProps: getGlueStackProps(stage), + attributeLinkerProps: getAttributeLinkerProps(), }, }; diff --git a/config/stacks/attributeLinker.ts b/config/stacks/attributeLinker.ts new file mode 100644 index 000000000..9dae0931b --- /dev/null +++ b/config/stacks/attributeLinker.ts @@ -0,0 +1,9 @@ +import { AttributeLinkerProps } from '../../lib/workload/stateless/stacks/attribute-linker/deploy/stack'; +import { eventBusName, vpcProps } from '../constants'; + +export const getAttributeLinkerProps = (): AttributeLinkerProps => { + return { + vpcProps, + eventBusName, + }; +}; diff --git a/lib/workload/stateless/stacks/attribute-linker/deploy/stack.ts b/lib/workload/stateless/stacks/attribute-linker/deploy/stack.ts new file mode 100644 index 000000000..9a32b7430 --- /dev/null +++ b/lib/workload/stateless/stacks/attribute-linker/deploy/stack.ts @@ -0,0 +1,82 @@ +import { aws_events_targets as targets, Duration, Stack, StackProps } from 'aws-cdk-lib'; +import { DatabaseProps } from '../../filemanager/deploy/constructs/functions/function'; +import { + ISecurityGroup, + IVpc, + SecurityGroup, + SubnetType, + Vpc, + VpcLookupOptions, +} from 'aws-cdk-lib/aws-ec2'; +import { ApiGwLogsConfig } from '../../../../components/api-gateway'; +import { Construct } from 'constructs'; +import { GoFunction } from '@aws-cdk/aws-lambda-go-alpha'; +import path from 'path'; +import { FILEMANAGER_SERVICE_NAME } from '../../filemanager/deploy/stack'; +import { Architecture } from 'aws-cdk-lib/aws-lambda'; +import { EventBus, IEventBus, Rule } from 'aws-cdk-lib/aws-events'; + +/** + * Config for the attribute linker. + */ +export type AttributeLinkerConfig = { + vpcProps: VpcLookupOptions; + eventBusName: string; +}; + +/** + * Props for the attribute linker stack. + */ +export type AttributeLinkerProps = StackProps & AttributeLinkerConfig; + +/** + * Construct used to configure the attribute linker. + */ +export class AttributeLinker extends Stack { + private readonly vpc: IVpc; + private readonly securityGroup: ISecurityGroup; + private readonly eventBus: IEventBus; + + constructor(scope: Construct, id: string, props: AttributeLinkerProps) { + super(scope, id, props); + + this.vpc = Vpc.fromLookup(this, 'MainVpc', props.vpcProps); + this.eventBus = EventBus.fromEventBusName(this, 'OrcaBusMain', props.eventBusName); + + this.securityGroup = new SecurityGroup(this, 'SecurityGroup', { + vpc: this.vpc, + allowAllOutbound: true, + description: 'Security group that allows the attribute linker Lambda to egress out.', + }); + + const entry = path.join(__dirname, '..'); + const fn = new GoFunction(this, 'handler', { + entry, + memorySize: 128, + timeout: Duration.seconds(28), + architecture: Architecture.ARM_64, + vpc: this.vpc, + vpcSubnets: { subnetType: SubnetType.PRIVATE_WITH_EGRESS }, + securityGroups: [this.securityGroup], + }); + + const eventRule = new Rule(this, 'EventRule', { + description: 'Send WorkflowRunStateChange events to the AttributeLinker Lambda', + eventBus: this.eventBus, + }); + + eventRule.addTarget(new targets.LambdaFunction(fn)); + eventRule.addEventPattern({ + source: ['orcabus.workflowmanager'], + detailType: ['WorkflowRunStateChange'], + detail: { + status: [ + { 'equals-ignore-case': 'SUCCEEDED' }, + { 'equals-ignore-case': 'FAILED' }, + { 'equals-ignore-case': 'ABORTED' }, + { 'equals-ignore-case': 'TEST' }, + ], + }, + }); + } +} diff --git a/lib/workload/stateless/stacks/attribute-linker/go.mod b/lib/workload/stateless/stacks/attribute-linker/go.mod new file mode 100644 index 000000000..3562d51ef --- /dev/null +++ b/lib/workload/stateless/stacks/attribute-linker/go.mod @@ -0,0 +1,5 @@ +module github.com/umccr/orcabus/lib/workload/stateless/stacks/attribute-linker + +go 1.23 + +require github.com/aws/aws-lambda-go v1.47.0 diff --git a/lib/workload/stateless/stacks/attribute-linker/go.sum b/lib/workload/stateless/stacks/attribute-linker/go.sum new file mode 100644 index 000000000..e803789f9 --- /dev/null +++ b/lib/workload/stateless/stacks/attribute-linker/go.sum @@ -0,0 +1,10 @@ +github.com/aws/aws-lambda-go v1.47.0 h1:0H8s0vumYx/YKs4sE7YM0ktwL2eWse+kfopsRI1sXVI= +github.com/aws/aws-lambda-go v1.47.0/go.mod h1:dpMpZgvWx5vuQJfBt0zqBha60q7Dd7RfgJv23DymV8A= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s= +github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/lib/workload/stateless/stacks/attribute-linker/main.go b/lib/workload/stateless/stacks/attribute-linker/main.go new file mode 100644 index 000000000..258f7c06c --- /dev/null +++ b/lib/workload/stateless/stacks/attribute-linker/main.go @@ -0,0 +1,19 @@ +package main + +import ( + "encoding/json" + "fmt" + "github.com/aws/aws-lambda-go/lambda" + "github.com/umccr/orcabus/lib/workload/stateless/stacks/attribute-linker/schema/orcabus_workflowmanager/workflowrunstatechange" +) + +func handler(event workflowrunstatechange.Event) (string, error) { + b, err := json.Marshal(event) + a := string(b) + fmt.Println(a) + return a, err +} + +func main() { + lambda.Start(handler) +} diff --git a/lib/workload/stateless/stacks/attribute-linker/schema/README.md b/lib/workload/stateless/stacks/attribute-linker/schema/README.md new file mode 100644 index 000000000..edea2adcf --- /dev/null +++ b/lib/workload/stateless/stacks/attribute-linker/schema/README.md @@ -0,0 +1,14 @@ +# AWSEvent + +*Automatically generated by the [Amazon Event Schemas](https://aws.amazon.com/)* + +## Requirements +1. Go 1.x + +## Installation +* Golang Installation - [Golang](https://golang.org) + +Put the package under your project folder and add the following to import: +```golang +import "schema/orcabus_workflowmanager/workflowrunstatechange" +``` diff --git a/lib/workload/stateless/stacks/attribute-linker/schema/orcabus_workflowmanager/workflowrunstatechange/event.go b/lib/workload/stateless/stacks/attribute-linker/schema/orcabus_workflowmanager/workflowrunstatechange/event.go new file mode 100644 index 000000000..cab41befa --- /dev/null +++ b/lib/workload/stateless/stacks/attribute-linker/schema/orcabus_workflowmanager/workflowrunstatechange/event.go @@ -0,0 +1,54 @@ +package workflowrunstatechange + +import ( + "time" +) + +// EventBridge custom event schema for orcabus.workflowmanager@WorkflowRunStateChange +type Event struct { + Account string `json:"account,omitempty"` + Detail WorkflowRunStateChange `json:"detail,omitempty"` + DetailType string `json:"detail-type,omitempty"` + Id string `json:"id,omitempty"` + Region string `json:"region,omitempty"` + Resources []string `json:"resources,omitempty"` + Source string `json:"source,omitempty"` + Time time.Time `json:"time,omitempty"` + Version string `json:"version,omitempty"` +} + +func (e *Event) SetAccount(account string) { + e.Account = account +} + +func (e *Event) SetDetail(detail WorkflowRunStateChange) { + e.Detail = detail +} + +func (e *Event) SetDetailType(detailType string) { + e.DetailType = detailType +} + +func (e *Event) SetId(id string) { + e.Id = id +} + +func (e *Event) SetRegion(region string) { + e.Region = region +} + +func (e *Event) SetResources(resources []string) { + e.Resources = resources +} + +func (e *Event) SetSource(source string) { + e.Source = source +} + +func (e *Event) SetTime(time time.Time) { + e.Time = time +} + +func (e *Event) SetVersion(version string) { + e.Version = version +} diff --git a/lib/workload/stateless/stacks/attribute-linker/schema/orcabus_workflowmanager/workflowrunstatechange/library_record.go b/lib/workload/stateless/stacks/attribute-linker/schema/orcabus_workflowmanager/workflowrunstatechange/library_record.go new file mode 100644 index 000000000..669307eb7 --- /dev/null +++ b/lib/workload/stateless/stacks/attribute-linker/schema/orcabus_workflowmanager/workflowrunstatechange/library_record.go @@ -0,0 +1,14 @@ +package workflowrunstatechange + +type LibraryRecord struct { + LibraryId string `json:"libraryId,omitempty"` + OrcabusId string `json:"orcabusId,omitempty"` +} + +func (l *LibraryRecord) SetLibraryId(libraryId string) { + l.LibraryId = libraryId +} + +func (l *LibraryRecord) SetOrcabusId(orcabusId string) { + l.OrcabusId = orcabusId +} diff --git a/lib/workload/stateless/stacks/attribute-linker/schema/orcabus_workflowmanager/workflowrunstatechange/marshaller.go b/lib/workload/stateless/stacks/attribute-linker/schema/orcabus_workflowmanager/workflowrunstatechange/marshaller.go new file mode 100644 index 000000000..c08cb15c3 --- /dev/null +++ b/lib/workload/stateless/stacks/attribute-linker/schema/orcabus_workflowmanager/workflowrunstatechange/marshaller.go @@ -0,0 +1,24 @@ +package workflowrunstatechange + +import ( + "encoding/json" +) + +func Marshal(inputEvent interface{}) ([]byte, error) { + outputStream, err := json.Marshal(inputEvent) + if err != nil { + return nil, err + } + + return outputStream, nil +} + +func Unmarshal(inputStream []byte) (map[string]interface{}, error) { + var outputEvent map[string]interface{} + err := json.Unmarshal(inputStream, &outputEvent) + if err != nil { + return nil, err + } + + return outputEvent, nil +} diff --git a/lib/workload/stateless/stacks/attribute-linker/schema/orcabus_workflowmanager/workflowrunstatechange/payload.go b/lib/workload/stateless/stacks/attribute-linker/schema/orcabus_workflowmanager/workflowrunstatechange/payload.go new file mode 100644 index 000000000..54fb1ccd8 --- /dev/null +++ b/lib/workload/stateless/stacks/attribute-linker/schema/orcabus_workflowmanager/workflowrunstatechange/payload.go @@ -0,0 +1,19 @@ +package workflowrunstatechange + +type Payload struct { + Data interface{} `json:"data,omitempty"` + RefId string `json:"refId,omitempty"` + Version string `json:"version,omitempty"` +} + +func (p *Payload) SetData(data interface{}) { + p.Data = data +} + +func (p *Payload) SetRefId(refId string) { + p.RefId = refId +} + +func (p *Payload) SetVersion(version string) { + p.Version = version +} diff --git a/lib/workload/stateless/stacks/attribute-linker/schema/orcabus_workflowmanager/workflowrunstatechange/workflow_run_state_change.go b/lib/workload/stateless/stacks/attribute-linker/schema/orcabus_workflowmanager/workflowrunstatechange/workflow_run_state_change.go new file mode 100644 index 000000000..cea378437 --- /dev/null +++ b/lib/workload/stateless/stacks/attribute-linker/schema/orcabus_workflowmanager/workflowrunstatechange/workflow_run_state_change.go @@ -0,0 +1,48 @@ +package workflowrunstatechange + +import ( + "time" +) + +type WorkflowRunStateChange struct { + LinkedLibraries []LibraryRecord `json:"linkedLibraries,omitempty"` + Payload Payload `json:"payload,omitempty"` + PortalRunId string `json:"portalRunId,omitempty"` + Status string `json:"status,omitempty"` + Timestamp time.Time `json:"timestamp,omitempty"` + WorkflowName string `json:"workflowName,omitempty"` + WorkflowRunName string `json:"workflowRunName,omitempty"` + WorkflowVersion string `json:"workflowVersion,omitempty"` +} + +func (w *WorkflowRunStateChange) SetLinkedLibraries(linkedLibraries []LibraryRecord) { + w.LinkedLibraries = linkedLibraries +} + +func (w *WorkflowRunStateChange) SetPayload(payload Payload) { + w.Payload = payload +} + +func (w *WorkflowRunStateChange) SetPortalRunId(portalRunId string) { + w.PortalRunId = portalRunId +} + +func (w *WorkflowRunStateChange) SetStatus(status string) { + w.Status = status +} + +func (w *WorkflowRunStateChange) SetTimestamp(timestamp time.Time) { + w.Timestamp = timestamp +} + +func (w *WorkflowRunStateChange) SetWorkflowName(workflowName string) { + w.WorkflowName = workflowName +} + +func (w *WorkflowRunStateChange) SetWorkflowRunName(workflowRunName string) { + w.WorkflowRunName = workflowRunName +} + +func (w *WorkflowRunStateChange) SetWorkflowVersion(workflowVersion string) { + w.WorkflowVersion = workflowVersion +} diff --git a/lib/workload/stateless/statelessStackCollectionClass.ts b/lib/workload/stateless/statelessStackCollectionClass.ts index 34258feac..cf7522726 100644 --- a/lib/workload/stateless/statelessStackCollectionClass.ts +++ b/lib/workload/stateless/statelessStackCollectionClass.ts @@ -53,6 +53,7 @@ import { UmccriseIcav2PipelineManagerStack, UmccriseIcav2PipelineManagerStackProps, } from './stacks/umccrise-pipeline-manager/deploy'; +import { AttributeLinker, AttributeLinkerProps } from './stacks/attribute-linker/deploy/stack'; export interface StatelessStackCollectionProps { metadataManagerStackProps: MetadataManagerStackProps; @@ -71,6 +72,7 @@ export interface StatelessStackCollectionProps { bclConvertManagerStackProps: BclConvertManagerStackProps; workflowManagerStackProps: WorkflowManagerStackProps; stackyMcStackFaceProps: GlueStackProps; + attributeLinkerProps: AttributeLinkerProps; } export class StatelessStackCollection { @@ -91,6 +93,7 @@ export class StatelessStackCollection { readonly bclConvertManagerStack: Stack; readonly workflowManagerStack: Stack; readonly stackyMcStackFaceStack: Stack; + readonly attributeLinkerStack: Stack; constructor( scope: Construct, @@ -208,6 +211,11 @@ export class StatelessStackCollection { ...this.createTemplateProps(env, 'StackyMcStackFaceStack'), ...statelessConfiguration.stackyMcStackFaceProps, }); + + this.attributeLinkerStack = new AttributeLinker(scope, 'AttributeLinkerStack', { + ...this.createTemplateProps(env, 'AttributeLinkerStack'), + ...statelessConfiguration.attributeLinkerProps, + }); } /** diff --git a/package.json b/package.json index 858719276..094bf9f0e 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "cdk-stateful": "cdk --app 'yarn run -B ts-node --prefer-ts-exts bin/statefulPipeline.ts'" }, "dependencies": { + "@aws-cdk/aws-lambda-go-alpha": "^2.154.1-alpha.0", "@aws-cdk/aws-lambda-python-alpha": "2.152.0-alpha.0", "@aws-cdk/aws-pipes-alpha": "^2.152.0-alpha.0", "@aws-cdk/aws-pipes-sources-alpha": "^2.152.0-alpha.0", diff --git a/yarn.lock b/yarn.lock index 491b46630..88bef37ff 100644 --- a/yarn.lock +++ b/yarn.lock @@ -36,6 +36,16 @@ __metadata: languageName: node linkType: hard +"@aws-cdk/aws-lambda-go-alpha@npm:^2.154.1-alpha.0": + version: 2.154.1-alpha.0 + resolution: "@aws-cdk/aws-lambda-go-alpha@npm:2.154.1-alpha.0" + peerDependencies: + aws-cdk-lib: ^2.154.1 + constructs: ^10.0.0 + checksum: 10/54ee0471035ac77dc9a9d3c1fbfce9301e04938c90ebd7b1543c369cdd7cc4f61d09ab6e93951da8f7ffab56cd8691b87618d604ecc0443d0e07404dd05d32e9 + languageName: node + linkType: hard + "@aws-cdk/aws-lambda-python-alpha@npm:2.152.0-alpha.0": version: 2.152.0-alpha.0 resolution: "@aws-cdk/aws-lambda-python-alpha@npm:2.152.0-alpha.0" @@ -4022,6 +4032,7 @@ __metadata: version: 0.0.0-use.local resolution: "orcabus@workspace:." dependencies: + "@aws-cdk/aws-lambda-go-alpha": "npm:^2.154.1-alpha.0" "@aws-cdk/aws-lambda-python-alpha": "npm:2.152.0-alpha.0" "@aws-cdk/aws-pipes-alpha": "npm:^2.152.0-alpha.0" "@aws-cdk/aws-pipes-sources-alpha": "npm:^2.152.0-alpha.0" From 87a3e629d8d93ce16fdf87d7c61d48380b637db1 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Thu, 29 Aug 2024 15:17:56 +1000 Subject: [PATCH 02/13] feat(attributes): add main handler function --- .../stacks/attribute-annotator/.gitignore | 1 + .../stacks/attribute-annotator/Makefile | 29 +++++++ .../deploy/stack.ts | 0 .../go.mod | 0 .../go.sum | 0 .../stacks/attribute-annotator/main.go | 75 +++++++++++++++++++ .../schema/README.md | 0 .../workflowrunstatechange/event.go | 0 .../workflowrunstatechange/library_record.go | 0 .../workflowrunstatechange/marshaller.go | 0 .../workflowrunstatechange/payload.go | 0 .../workflow_run_state_change.go | 0 .../stateless/stacks/attribute-linker/main.go | 19 ----- 13 files changed, 105 insertions(+), 19 deletions(-) create mode 100644 lib/workload/stateless/stacks/attribute-annotator/.gitignore create mode 100644 lib/workload/stateless/stacks/attribute-annotator/Makefile rename lib/workload/stateless/stacks/{attribute-linker => attribute-annotator}/deploy/stack.ts (100%) rename lib/workload/stateless/stacks/{attribute-linker => attribute-annotator}/go.mod (100%) rename lib/workload/stateless/stacks/{attribute-linker => attribute-annotator}/go.sum (100%) create mode 100644 lib/workload/stateless/stacks/attribute-annotator/main.go rename lib/workload/stateless/stacks/{attribute-linker => attribute-annotator}/schema/README.md (100%) rename lib/workload/stateless/stacks/{attribute-linker => attribute-annotator}/schema/orcabus_workflowmanager/workflowrunstatechange/event.go (100%) rename lib/workload/stateless/stacks/{attribute-linker => attribute-annotator}/schema/orcabus_workflowmanager/workflowrunstatechange/library_record.go (100%) rename lib/workload/stateless/stacks/{attribute-linker => attribute-annotator}/schema/orcabus_workflowmanager/workflowrunstatechange/marshaller.go (100%) rename lib/workload/stateless/stacks/{attribute-linker => attribute-annotator}/schema/orcabus_workflowmanager/workflowrunstatechange/payload.go (100%) rename lib/workload/stateless/stacks/{attribute-linker => attribute-annotator}/schema/orcabus_workflowmanager/workflowrunstatechange/workflow_run_state_change.go (100%) delete mode 100644 lib/workload/stateless/stacks/attribute-linker/main.go diff --git a/lib/workload/stateless/stacks/attribute-annotator/.gitignore b/lib/workload/stateless/stacks/attribute-annotator/.gitignore new file mode 100644 index 000000000..c795b054e --- /dev/null +++ b/lib/workload/stateless/stacks/attribute-annotator/.gitignore @@ -0,0 +1 @@ +build \ No newline at end of file diff --git a/lib/workload/stateless/stacks/attribute-annotator/Makefile b/lib/workload/stateless/stacks/attribute-annotator/Makefile new file mode 100644 index 000000000..2fe4e45d3 --- /dev/null +++ b/lib/workload/stateless/stacks/attribute-annotator/Makefile @@ -0,0 +1,29 @@ +## Default target +default: help + +install: build +all: build + +## Build related commands +build: + @go build -o build +lint: + @gofmt -l +lint-fix: + @gofmt +golangci-lint: + @golangci-lint run +check: lint golangci-lint +check-fix: lint-fix golangci-lint + +## Clean +clean: + @go mod tidy + +## Help text +help: + @printf "The attribute annotator Makefile.\n\n" + @printf "Usage: make \n" + +# No targets are files, so they are all PHONY. +.PHONY: * diff --git a/lib/workload/stateless/stacks/attribute-linker/deploy/stack.ts b/lib/workload/stateless/stacks/attribute-annotator/deploy/stack.ts similarity index 100% rename from lib/workload/stateless/stacks/attribute-linker/deploy/stack.ts rename to lib/workload/stateless/stacks/attribute-annotator/deploy/stack.ts diff --git a/lib/workload/stateless/stacks/attribute-linker/go.mod b/lib/workload/stateless/stacks/attribute-annotator/go.mod similarity index 100% rename from lib/workload/stateless/stacks/attribute-linker/go.mod rename to lib/workload/stateless/stacks/attribute-annotator/go.mod diff --git a/lib/workload/stateless/stacks/attribute-linker/go.sum b/lib/workload/stateless/stacks/attribute-annotator/go.sum similarity index 100% rename from lib/workload/stateless/stacks/attribute-linker/go.sum rename to lib/workload/stateless/stacks/attribute-annotator/go.sum diff --git a/lib/workload/stateless/stacks/attribute-annotator/main.go b/lib/workload/stateless/stacks/attribute-annotator/main.go new file mode 100644 index 000000000..9abf7b7ff --- /dev/null +++ b/lib/workload/stateless/stacks/attribute-annotator/main.go @@ -0,0 +1,75 @@ +package main + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "github.com/aws/aws-lambda-go/lambda" + "github.com/umccr/orcabus/lib/workload/stateless/stacks/attribute-linker/schema/orcabus_workflowmanager/workflowrunstatechange" + "io" + "log/slog" + "net/http" + "net/url" + "strings" +) + +type AttributePatch struct { + Op string `json:"op"` + Path string `json:"path"` + Value string `json:"value"` +} + +func handler(event workflowrunstatechange.Event) (err error) { + status := strings.ToUpper(event.Detail.Status) + if status != "SUCCEEDED" && status != "FAILED" && status != "ABORTED" { + return nil + } + + patch, err := json.Marshal(AttributePatch{ + "add", + "/portalRunId", + event.Detail.PortalRunId, + }) + if err != nil { + return err + } + + client := &http.Client{} + + req, err := http.NewRequest( + "PATCH", + "https://file.dev.umccr.org/api/v1/s3", + bytes.NewBuffer(patch), + ) + if err != nil { + return err + } + + req.URL.RawQuery = url.Values{ + "key": {fmt.Sprintf("*%v*", event.Detail.PortalRunId)}, + }.Encode() + + slog.Debug(fmt.Sprintf("sending attributes patch %v to %v", patch, req.URL.String())) + + resp, err := client.Do(req) + if err != nil { + return err + } + defer func() { + err = errors.Join(resp.Body.Close()) + }() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + + slog.Debug(fmt.Sprintf("received response %v with body: %v", resp.StatusCode, body)) + + return nil +} + +func main() { + lambda.Start(handler) +} diff --git a/lib/workload/stateless/stacks/attribute-linker/schema/README.md b/lib/workload/stateless/stacks/attribute-annotator/schema/README.md similarity index 100% rename from lib/workload/stateless/stacks/attribute-linker/schema/README.md rename to lib/workload/stateless/stacks/attribute-annotator/schema/README.md diff --git a/lib/workload/stateless/stacks/attribute-linker/schema/orcabus_workflowmanager/workflowrunstatechange/event.go b/lib/workload/stateless/stacks/attribute-annotator/schema/orcabus_workflowmanager/workflowrunstatechange/event.go similarity index 100% rename from lib/workload/stateless/stacks/attribute-linker/schema/orcabus_workflowmanager/workflowrunstatechange/event.go rename to lib/workload/stateless/stacks/attribute-annotator/schema/orcabus_workflowmanager/workflowrunstatechange/event.go diff --git a/lib/workload/stateless/stacks/attribute-linker/schema/orcabus_workflowmanager/workflowrunstatechange/library_record.go b/lib/workload/stateless/stacks/attribute-annotator/schema/orcabus_workflowmanager/workflowrunstatechange/library_record.go similarity index 100% rename from lib/workload/stateless/stacks/attribute-linker/schema/orcabus_workflowmanager/workflowrunstatechange/library_record.go rename to lib/workload/stateless/stacks/attribute-annotator/schema/orcabus_workflowmanager/workflowrunstatechange/library_record.go diff --git a/lib/workload/stateless/stacks/attribute-linker/schema/orcabus_workflowmanager/workflowrunstatechange/marshaller.go b/lib/workload/stateless/stacks/attribute-annotator/schema/orcabus_workflowmanager/workflowrunstatechange/marshaller.go similarity index 100% rename from lib/workload/stateless/stacks/attribute-linker/schema/orcabus_workflowmanager/workflowrunstatechange/marshaller.go rename to lib/workload/stateless/stacks/attribute-annotator/schema/orcabus_workflowmanager/workflowrunstatechange/marshaller.go diff --git a/lib/workload/stateless/stacks/attribute-linker/schema/orcabus_workflowmanager/workflowrunstatechange/payload.go b/lib/workload/stateless/stacks/attribute-annotator/schema/orcabus_workflowmanager/workflowrunstatechange/payload.go similarity index 100% rename from lib/workload/stateless/stacks/attribute-linker/schema/orcabus_workflowmanager/workflowrunstatechange/payload.go rename to lib/workload/stateless/stacks/attribute-annotator/schema/orcabus_workflowmanager/workflowrunstatechange/payload.go diff --git a/lib/workload/stateless/stacks/attribute-linker/schema/orcabus_workflowmanager/workflowrunstatechange/workflow_run_state_change.go b/lib/workload/stateless/stacks/attribute-annotator/schema/orcabus_workflowmanager/workflowrunstatechange/workflow_run_state_change.go similarity index 100% rename from lib/workload/stateless/stacks/attribute-linker/schema/orcabus_workflowmanager/workflowrunstatechange/workflow_run_state_change.go rename to lib/workload/stateless/stacks/attribute-annotator/schema/orcabus_workflowmanager/workflowrunstatechange/workflow_run_state_change.go diff --git a/lib/workload/stateless/stacks/attribute-linker/main.go b/lib/workload/stateless/stacks/attribute-linker/main.go deleted file mode 100644 index 258f7c06c..000000000 --- a/lib/workload/stateless/stacks/attribute-linker/main.go +++ /dev/null @@ -1,19 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "github.com/aws/aws-lambda-go/lambda" - "github.com/umccr/orcabus/lib/workload/stateless/stacks/attribute-linker/schema/orcabus_workflowmanager/workflowrunstatechange" -) - -func handler(event workflowrunstatechange.Event) (string, error) { - b, err := json.Marshal(event) - a := string(b) - fmt.Println(a) - return a, err -} - -func main() { - lambda.Start(handler) -} From abb5910f0c0e022a27fe7c522e8da00e66197c21 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Thu, 29 Aug 2024 15:54:45 +1000 Subject: [PATCH 03/13] feat(attributes): make filemanager endpoint configurable --- config/stacks/attributeLinker.ts | 4 ++-- lib/workload/components/api-gateway/index.ts | 11 ++++++++--- .../stacks/attribute-annotator/deploy/stack.ts | 17 ++++++++++++----- .../stateless/stacks/attribute-annotator/go.mod | 5 ++++- .../stateless/stacks/attribute-annotator/go.sum | 2 ++ .../stacks/attribute-annotator/main.go | 17 ++++++++++++++++- .../stacks/filemanager/deploy/stack.ts | 15 ++++++++++----- .../stateless/statelessStackCollectionClass.ts | 15 ++++++++++----- 8 files changed, 64 insertions(+), 22 deletions(-) diff --git a/config/stacks/attributeLinker.ts b/config/stacks/attributeLinker.ts index 9dae0931b..490716dfc 100644 --- a/config/stacks/attributeLinker.ts +++ b/config/stacks/attributeLinker.ts @@ -1,7 +1,7 @@ -import { AttributeLinkerProps } from '../../lib/workload/stateless/stacks/attribute-linker/deploy/stack'; +import { AttributeLinkerConfigurableProps } from '../../lib/workload/stateless/stacks/attribute-annotator/deploy/stack'; import { eventBusName, vpcProps } from '../constants'; -export const getAttributeLinkerProps = (): AttributeLinkerProps => { +export const getAttributeLinkerProps = (): AttributeLinkerConfigurableProps => { return { vpcProps, eventBusName, diff --git a/lib/workload/components/api-gateway/index.ts b/lib/workload/components/api-gateway/index.ts index a359c1bc3..2c3c6f3a7 100644 --- a/lib/workload/components/api-gateway/index.ts +++ b/lib/workload/components/api-gateway/index.ts @@ -42,6 +42,7 @@ export interface ApiGatewayConstructProps { export class ApiGatewayConstruct extends Construct { private readonly _httpApi: HttpApi; + private readonly _domainName: string; constructor(scope: Construct, id: string, props: ApiGatewayConstructProps) { super(scope, id); @@ -54,9 +55,9 @@ export class ApiGatewayConstruct extends Construct { ); const hostedZoneId = StringParameter.valueForStringParameter(this, '/hosted_zone/umccr/id'); - const domainName = `${props.customDomainNamePrefix}.${hostedDomainName}`; + this._domainName = `${props.customDomainNamePrefix}.${hostedDomainName}`; const apiGWDomainName = new DomainName(this, 'UmccrDomainName', { - domainName: `${props.customDomainNamePrefix}.${hostedDomainName}`, + domainName: this.domainName, certificate: Certificate.fromCertificateArn(this, 'cert', umccrAcmArn), }); @@ -85,7 +86,7 @@ export class ApiGatewayConstruct extends Construct { hostedZoneId, zoneName: hostedDomainName, }), - recordName: domainName, + recordName: this.domainName, target: RecordTarget.fromAlias( new ApiGatewayv2DomainProperties( apiGWDomainName.regionalDomainName, @@ -177,4 +178,8 @@ export class ApiGatewayConstruct extends Construct { get httpApi(): HttpApi { return this._httpApi; } + + get domainName(): string { + return this._domainName; + } } diff --git a/lib/workload/stateless/stacks/attribute-annotator/deploy/stack.ts b/lib/workload/stateless/stacks/attribute-annotator/deploy/stack.ts index 9a32b7430..75ba561a9 100644 --- a/lib/workload/stateless/stacks/attribute-annotator/deploy/stack.ts +++ b/lib/workload/stateless/stacks/attribute-annotator/deploy/stack.ts @@ -1,5 +1,4 @@ import { aws_events_targets as targets, Duration, Stack, StackProps } from 'aws-cdk-lib'; -import { DatabaseProps } from '../../filemanager/deploy/constructs/functions/function'; import { ISecurityGroup, IVpc, @@ -8,11 +7,9 @@ import { Vpc, VpcLookupOptions, } from 'aws-cdk-lib/aws-ec2'; -import { ApiGwLogsConfig } from '../../../../components/api-gateway'; import { Construct } from 'constructs'; import { GoFunction } from '@aws-cdk/aws-lambda-go-alpha'; import path from 'path'; -import { FILEMANAGER_SERVICE_NAME } from '../../filemanager/deploy/stack'; import { Architecture } from 'aws-cdk-lib/aws-lambda'; import { EventBus, IEventBus, Rule } from 'aws-cdk-lib/aws-events'; @@ -24,15 +21,22 @@ export type AttributeLinkerConfig = { eventBusName: string; }; +/** + * Props for the attribute linker stack which can be configured + */ +export type AttributeLinkerConfigurableProps = StackProps & AttributeLinkerConfig; + /** * Props for the attribute linker stack. */ -export type AttributeLinkerProps = StackProps & AttributeLinkerConfig; +export type AttributeLinkerProps = AttributeLinkerConfigurableProps & { + domainName: string; +}; /** * Construct used to configure the attribute linker. */ -export class AttributeLinker extends Stack { +export class AttributeAnnotator extends Stack { private readonly vpc: IVpc; private readonly securityGroup: ISecurityGroup; private readonly eventBus: IEventBus; @@ -52,6 +56,9 @@ export class AttributeLinker extends Stack { const entry = path.join(__dirname, '..'); const fn = new GoFunction(this, 'handler', { entry, + environment: { + ANNOTATOR_FILEMANAGER_ENDPOINT: props.domainName, + }, memorySize: 128, timeout: Duration.seconds(28), architecture: Architecture.ARM_64, diff --git a/lib/workload/stateless/stacks/attribute-annotator/go.mod b/lib/workload/stateless/stacks/attribute-annotator/go.mod index 3562d51ef..2aa27189e 100644 --- a/lib/workload/stateless/stacks/attribute-annotator/go.mod +++ b/lib/workload/stateless/stacks/attribute-annotator/go.mod @@ -2,4 +2,7 @@ module github.com/umccr/orcabus/lib/workload/stateless/stacks/attribute-linker go 1.23 -require github.com/aws/aws-lambda-go v1.47.0 +require ( + github.com/aws/aws-lambda-go v1.47.0 + github.com/kelseyhightower/envconfig v1.4.0 +) diff --git a/lib/workload/stateless/stacks/attribute-annotator/go.sum b/lib/workload/stateless/stacks/attribute-annotator/go.sum index e803789f9..9eb28bb78 100644 --- a/lib/workload/stateless/stacks/attribute-annotator/go.sum +++ b/lib/workload/stateless/stacks/attribute-annotator/go.sum @@ -2,6 +2,8 @@ github.com/aws/aws-lambda-go v1.47.0 h1:0H8s0vumYx/YKs4sE7YM0ktwL2eWse+kfopsRI1s github.com/aws/aws-lambda-go v1.47.0/go.mod h1:dpMpZgvWx5vuQJfBt0zqBha60q7Dd7RfgJv23DymV8A= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= +github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s= diff --git a/lib/workload/stateless/stacks/attribute-annotator/main.go b/lib/workload/stateless/stacks/attribute-annotator/main.go index 9abf7b7ff..0c0cfb7c9 100644 --- a/lib/workload/stateless/stacks/attribute-annotator/main.go +++ b/lib/workload/stateless/stacks/attribute-annotator/main.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "github.com/aws/aws-lambda-go/lambda" + "github.com/kelseyhightower/envconfig" "github.com/umccr/orcabus/lib/workload/stateless/stacks/attribute-linker/schema/orcabus_workflowmanager/workflowrunstatechange" "io" "log/slog" @@ -14,6 +15,10 @@ import ( "strings" ) +type Config struct { + FileManagerEndpoint string +} + type AttributePatch struct { Op string `json:"op"` Path string `json:"path"` @@ -35,11 +40,21 @@ func handler(event workflowrunstatechange.Event) (err error) { return err } + var config Config + err = envconfig.Process("annotator", &config) + if err != nil { + return err + } + endpoint, err := url.JoinPath(config.FileManagerEndpoint, "/api/v1/s3") + if err != nil { + return err + } + client := &http.Client{} req, err := http.NewRequest( "PATCH", - "https://file.dev.umccr.org/api/v1/s3", + endpoint, bytes.NewBuffer(patch), ) if err != nil { diff --git a/lib/workload/stateless/stacks/filemanager/deploy/stack.ts b/lib/workload/stateless/stacks/filemanager/deploy/stack.ts index bd338f50f..829d06b10 100644 --- a/lib/workload/stateless/stacks/filemanager/deploy/stack.ts +++ b/lib/workload/stateless/stacks/filemanager/deploy/stack.ts @@ -48,6 +48,7 @@ export class Filemanager extends Stack { private readonly host: string; private readonly securityGroup: ISecurityGroup; private readonly queue: IQueue; + readonly domainName: string; constructor(scope: Construct, id: string, props: FilemanagerProps) { super(scope, id, props); @@ -93,8 +94,9 @@ export class Filemanager extends Stack { ); this.createIngestFunction(props); - this.createApiFunction(props); this.createInventoryFunction(props); + + this.domainName = this.createApiFunction(props); } private createIngestRole(name: string) { @@ -130,9 +132,9 @@ export class Filemanager extends Stack { } /** - * Query function and API Gateway fronting the function. + * Query function and API Gateway fronting the function. Returns the configured domain name. */ - private createApiFunction(props: FilemanagerProps) { + private createApiFunction(props: FilemanagerProps): string { let apiLambda = new ApiFunction(this, 'ApiFunction', { vpc: this.vpc, host: this.host, @@ -141,13 +143,14 @@ export class Filemanager extends Stack { ...props, }); - const ApiGateway = new ApiGatewayConstruct(this, 'ApiGateway', { + const apiGateway = new ApiGatewayConstruct(this, 'ApiGateway', { region: this.region, apiName: 'FileManager', customDomainNamePrefix: 'file', ...props, }); - const httpApi = ApiGateway.httpApi; + + const httpApi = apiGateway.httpApi; const apiIntegration = new HttpLambdaIntegration('ApiIntegration', apiLambda.function); @@ -156,5 +159,7 @@ export class Filemanager extends Stack { integration: apiIntegration, routeKey: HttpRouteKey.with('/{proxy+}', HttpMethod.ANY), }); + + return apiGateway.domainName; } } diff --git a/lib/workload/stateless/statelessStackCollectionClass.ts b/lib/workload/stateless/statelessStackCollectionClass.ts index cf7522726..29ff331ef 100644 --- a/lib/workload/stateless/statelessStackCollectionClass.ts +++ b/lib/workload/stateless/statelessStackCollectionClass.ts @@ -53,7 +53,10 @@ import { UmccriseIcav2PipelineManagerStack, UmccriseIcav2PipelineManagerStackProps, } from './stacks/umccrise-pipeline-manager/deploy'; -import { AttributeLinker, AttributeLinkerProps } from './stacks/attribute-linker/deploy/stack'; +import { + AttributeAnnotator, + AttributeLinkerConfigurableProps, +} from './stacks/attribute-annotator/deploy/stack'; export interface StatelessStackCollectionProps { metadataManagerStackProps: MetadataManagerStackProps; @@ -72,7 +75,7 @@ export interface StatelessStackCollectionProps { bclConvertManagerStackProps: BclConvertManagerStackProps; workflowManagerStackProps: WorkflowManagerStackProps; stackyMcStackFaceProps: GlueStackProps; - attributeLinkerProps: AttributeLinkerProps; + attributeLinkerProps: AttributeLinkerConfigurableProps; } export class StatelessStackCollection { @@ -93,7 +96,7 @@ export class StatelessStackCollection { readonly bclConvertManagerStack: Stack; readonly workflowManagerStack: Stack; readonly stackyMcStackFaceStack: Stack; - readonly attributeLinkerStack: Stack; + readonly attributeAnnotatorStack: Stack; constructor( scope: Construct, @@ -110,10 +113,11 @@ export class StatelessStackCollection { ...statelessConfiguration.dataSchemaStackProps, }); - this.fileManagerStack = new Filemanager(scope, 'FileManagerStack', { + const fileManagerStack = new Filemanager(scope, 'FileManagerStack', { ...this.createTemplateProps(env, 'FileManagerStack'), ...statelessConfiguration.fileManagerStackProps, }); + this.fileManagerStack = fileManagerStack; this.metadataManagerStack = new MetadataManagerStack(scope, 'MetadataManagerStack', { ...this.createTemplateProps(env, 'MetadataManagerStack'), @@ -212,9 +216,10 @@ export class StatelessStackCollection { ...statelessConfiguration.stackyMcStackFaceProps, }); - this.attributeLinkerStack = new AttributeLinker(scope, 'AttributeLinkerStack', { + this.attributeAnnotatorStack = new AttributeAnnotator(scope, 'AttributeLinkerStack', { ...this.createTemplateProps(env, 'AttributeLinkerStack'), ...statelessConfiguration.attributeLinkerProps, + domainName: fileManagerStack.domainName, }); } From 7b845fd4e187f0751b3ca276a7c6159b29ef0ab9 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Mon, 2 Sep 2024 16:09:27 +1000 Subject: [PATCH 04/13] test: test annotator linking --- .../fixtures/event_failed.json | 18 ++ .../fixtures/event_succeeded.json | 18 ++ .../fixtures/s3_object.yml | 31 ++ .../stacks/attribute-annotator/go.mod | 70 +++++ .../stacks/attribute-annotator/go.sum | 270 +++++++++++++++++- .../attribute-annotator/handler_test.go | 231 +++++++++++++++ .../stacks/attribute-annotator/main.go | 20 +- .../stateless/stacks/filemanager/.env.example | 1 + .../stateless/stacks/filemanager/Makefile | 2 +- .../stacks/filemanager/compose-api.yml | 20 ++ .../stateless/stacks/filemanager/compose.yml | 23 +- 11 files changed, 672 insertions(+), 32 deletions(-) create mode 100644 lib/workload/stateless/stacks/attribute-annotator/fixtures/event_failed.json create mode 100644 lib/workload/stateless/stacks/attribute-annotator/fixtures/event_succeeded.json create mode 100644 lib/workload/stateless/stacks/attribute-annotator/fixtures/s3_object.yml create mode 100644 lib/workload/stateless/stacks/attribute-annotator/handler_test.go create mode 100644 lib/workload/stateless/stacks/filemanager/compose-api.yml diff --git a/lib/workload/stateless/stacks/attribute-annotator/fixtures/event_failed.json b/lib/workload/stateless/stacks/attribute-annotator/fixtures/event_failed.json new file mode 100644 index 000000000..bab2590d9 --- /dev/null +++ b/lib/workload/stateless/stacks/attribute-annotator/fixtures/event_failed.json @@ -0,0 +1,18 @@ +{ + "version": "0", + "id": "ae02404d-9965-e3c7-2343-89cb5d46fedf", + "detail-type": "WorkflowRunStateChange", + "source": "orcabus.workflowmanager", + "account": "123456789012", + "time": "2024-09-02T00:00:01Z", + "region": "ap-southeast-2", + "resources": [], + "detail": { + "portalRunId": "202409021221e6c6", + "workflowName": "wts", + "workflowRunName": "umccr--automated--wts--4-2-4--202409021221e6c6", + "workflowVersion": "4.2.4", + "timestamp": "2024-09-02T00:00:00Z", + "status": "FAILED" + } +} diff --git a/lib/workload/stateless/stacks/attribute-annotator/fixtures/event_succeeded.json b/lib/workload/stateless/stacks/attribute-annotator/fixtures/event_succeeded.json new file mode 100644 index 000000000..a945ab17b --- /dev/null +++ b/lib/workload/stateless/stacks/attribute-annotator/fixtures/event_succeeded.json @@ -0,0 +1,18 @@ +{ + "version": "0", + "id": "ae02404d-9965-e3c7-2343-89cb5d46fedf", + "detail-type": "WorkflowRunStateChange", + "source": "orcabus.workflowmanager", + "account": "123456789012", + "time": "2024-09-02T00:00:01Z", + "region": "ap-southeast-2", + "resources": [], + "detail": { + "portalRunId": "202409021221e6e6", + "workflowName": "wts", + "workflowRunName": "umccr--automated--wts--4-2-4--202409021221e6e6", + "workflowVersion": "4.2.4", + "timestamp": "2024-09-02T00:00:00Z", + "status": "SUCCEEDED" + } +} diff --git a/lib/workload/stateless/stacks/attribute-annotator/fixtures/s3_object.yml b/lib/workload/stateless/stacks/attribute-annotator/fixtures/s3_object.yml new file mode 100644 index 000000000..558e7ea6e --- /dev/null +++ b/lib/workload/stateless/stacks/attribute-annotator/fixtures/s3_object.yml @@ -0,0 +1,31 @@ +- s3_object_id: RAW=gen_random_uuid() + event_type: 'Created' + bucket: 'bucket' + key: 'byob-icav2/development/analysis/wts/202409021221e6e6/_manifest.json' + event_time: 2024-09-02 00:00:00 + size: 5 + storage_class: 'Standard' +- s3_object_id: RAW=gen_random_uuid() + event_type: 'Deleted' + bucket: 'bucket' + key: 'byob-icav2/development/analysis/wts/202409021221e6e6/_manifest.json' + event_time: 2024-09-03 00:00:00 +- s3_object_id: RAW=gen_random_uuid() + event_type: 'Created' + bucket: 'bucket' + key: 'byob-icav2/development/analysis/wts/202409021221e6e6/_tags.json' + event_time: 2024-09-04 00:00:00 + size: 10 + storage_class: 'Standard' +- s3_object_id: RAW=gen_random_uuid() + event_type: 'Created' + bucket: 'bucket' + key: 'byob-icav2/development/analysis/wts/202409021221e6c6/_manifest.json' + event_time: 2024-09-05 00:00:00 + size: 3 + storage_class: 'Standard' +- s3_object_id: RAW=gen_random_uuid() + event_type: 'Deleted' + bucket: 'bucket' + key: 'byob-icav2/development/analysis/wts/202409021221e6c6/_manifest.json' + event_time: 2024-09-06 00:00:00 diff --git a/lib/workload/stateless/stacks/attribute-annotator/go.mod b/lib/workload/stateless/stacks/attribute-annotator/go.mod index 2aa27189e..d58a6425c 100644 --- a/lib/workload/stateless/stacks/attribute-annotator/go.mod +++ b/lib/workload/stateless/stacks/attribute-annotator/go.mod @@ -4,5 +4,75 @@ go 1.23 require ( github.com/aws/aws-lambda-go v1.47.0 + github.com/docker/go-connections v0.5.0 + github.com/go-testfixtures/testfixtures/v3 v3.12.0 + github.com/google/uuid v1.6.0 + github.com/jmoiron/sqlx v1.4.0 github.com/kelseyhightower/envconfig v1.4.0 + github.com/lib/pq v1.10.9 + github.com/stretchr/testify v1.9.0 + github.com/testcontainers/testcontainers-go v0.33.0 +) + +require ( + dario.cat/mergo v1.0.0 // indirect + github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect + github.com/ClickHouse/ch-go v0.61.5 // indirect + github.com/ClickHouse/clickhouse-go/v2 v2.26.0 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/andybalholm/brotli v1.1.0 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect + github.com/containerd/containerd v1.7.18 // indirect + github.com/containerd/log v0.1.0 // indirect + github.com/containerd/platforms v0.2.1 // indirect + github.com/cpuguy83/dockercfg v0.3.1 // indirect + github.com/creack/pty v1.1.21 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/docker/docker v27.1.1+incompatible // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-faster/city v1.0.1 // indirect + github.com/go-faster/errors v0.7.1 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/klauspost/compress v1.17.7 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/magiconair/properties v1.8.7 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/patternmatcher v0.6.0 // indirect + github.com/moby/sys/sequential v0.5.0 // indirect + github.com/moby/sys/user v0.1.0 // indirect + github.com/moby/term v0.5.0 // indirect + github.com/morikuni/aec v1.0.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.0 // indirect + github.com/paulmach/orb v0.11.1 // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/segmentio/asm v1.2.0 // indirect + github.com/shirou/gopsutil/v3 v3.23.12 // indirect + github.com/shoenig/go-m1cpu v0.1.6 // indirect + github.com/shopspring/decimal v1.4.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect + github.com/yusufpapurcu/wmi v1.2.3 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect + go.opentelemetry.io/otel v1.26.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0 // indirect + go.opentelemetry.io/otel/metric v1.26.0 // indirect + go.opentelemetry.io/otel/trace v1.26.0 // indirect + golang.org/x/crypto v0.24.0 // indirect + golang.org/x/net v0.26.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.21.0 // indirect + golang.org/x/time v0.3.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/lib/workload/stateless/stacks/attribute-annotator/go.sum b/lib/workload/stateless/stacks/attribute-annotator/go.sum index 9eb28bb78..7e6290736 100644 --- a/lib/workload/stateless/stacks/attribute-annotator/go.sum +++ b/lib/workload/stateless/stacks/attribute-annotator/go.sum @@ -1,12 +1,278 @@ +dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/ClickHouse/ch-go v0.61.5 h1:zwR8QbYI0tsMiEcze/uIMK+Tz1D3XZXLdNrlaOpeEI4= +github.com/ClickHouse/ch-go v0.61.5/go.mod h1:s1LJW/F/LcFs5HJnuogFMta50kKDO0lf9zzfrbl0RQg= +github.com/ClickHouse/clickhouse-go/v2 v2.26.0 h1:j4/y6NYaCcFkJwN/TU700ebW+nmsIy34RmUAAcZKy9w= +github.com/ClickHouse/clickhouse-go/v2 v2.26.0/go.mod h1:iDTViXk2Fgvf1jn2dbJd1ys+fBkdD1UMRnXlwmhijhQ= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= +github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/aws/aws-lambda-go v1.47.0 h1:0H8s0vumYx/YKs4sE7YM0ktwL2eWse+kfopsRI1sXVI= github.com/aws/aws-lambda-go v1.47.0/go.mod h1:dpMpZgvWx5vuQJfBt0zqBha60q7Dd7RfgJv23DymV8A= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/containerd/containerd v1.7.18 h1:jqjZTQNfXGoEaZdW1WwPU0RqSn1Bm2Ay/KJPUuO8nao= +github.com/containerd/containerd v1.7.18/go.mod h1:IYEk9/IO6wAPUz2bCMVUbsfXjzw5UNP5fLz4PsUygQ4= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= +github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= +github.com/cpuguy83/dockercfg v0.3.1 h1:/FpZ+JaygUR/lZP2NlFI2DVfrOEMAIKP5wWEJdoYe9E= +github.com/cpuguy83/dockercfg v0.3.1/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= +github.com/creack/pty v1.1.21 h1:1/QdRyBaHHJP61QkWMXlOIBfsgdDeeKfK8SYVUWJKf0= +github.com/creack/pty v1.1.21/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/denisenkom/go-mssqldb v0.12.3 h1:pBSGx9Tq67pBOTLmxNuirNTeB8Vjmf886Kx+8Y+8shw= +github.com/denisenkom/go-mssqldb v0.12.3/go.mod h1:k0mtMFOnU+AihqFxPMiF05rtiDrorD1Vrm1KEz5hxDo= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/docker v27.1.1+incompatible h1:hO/M4MtV36kzKldqnA37IWhebRA+LnqqcqDja6kVaKY= +github.com/docker/docker v27.1.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= +github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw= +github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw= +github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg= +github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= +github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= +github.com/go-testfixtures/testfixtures/v3 v3.12.0 h1:Ew0+c2o1mXSUqMwjuNup3MK/vw1HkLS3ILljX5C6lVE= +github.com/go-testfixtures/testfixtures/v3 v3.12.0/go.mod h1:13F0m6/DtqqSDso9IAVuhbZ4I7AiRAHrolmDMu9v5vY= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= +github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A= +github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EOqtpKwwwHI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= +github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v1.14.3 h1:bVoTr12EGANZz66nZPkMInAV/KHD2TxH9npjXXgiB3w= +github.com/jackc/pgconn v1.14.3/go.mod h1:RZbme4uasqzybK2RK5c65VsHxoyaml09lx3tXOcO/VM= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3/v2 v2.3.3 h1:1HLSx5H+tXR9pW3in3zaztoEwQYRC9SQaYUHjTSUOag= +github.com/jackc/pgproto3/v2 v2.3.3/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgtype v1.14.0 h1:y+xUdabmyMkJLyApYuPj38mW+aAIqCe5uuBB51rH3Vw= +github.com/jackc/pgtype v1.14.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgx/v4 v4.18.3 h1:dE2/TrEsGX3RBprb3qryqSV9Y60iZN1C6i8IrmW9/BA= +github.com/jackc/pgx/v4 v4.18.3/go.mod h1:Ey4Oru5tH5sB6tV7hDmfWFahwF15Eb7DNXlRKx2CkVw= +github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= +github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= +github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= +github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= +github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= +github.com/moby/sys/user v0.1.0 h1:WmZ93f5Ux6het5iituh9x2zAG7NFY9Aqi49jjE1PaQg= +github.com/moby/sys/user v0.1.0/go.mod h1:fKJhFOnsCN6xZ5gSfbM6zaHGgDJMrqt9/reuj4T7MmU= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= +github.com/paulmach/orb v0.11.1 h1:3koVegMC4X/WeiXYz9iswopaTwMem53NzTJuTF20JzU= +github.com/paulmach/orb v0.11.1/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= +github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= +github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= +github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s= -github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= +github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= +github.com/shirou/gopsutil/v3 v3.23.12 h1:z90NtUkp3bMtmICZKpC4+WaknU1eXtp5vtbQ11DgpE4= +github.com/shirou/gopsutil/v3 v3.23.12/go.mod h1:1FrWgea594Jp7qmjHUUPlJDTPgcsb9mGnXDxavtikzM= +github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= +github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/testcontainers/testcontainers-go v0.33.0 h1:zJS9PfXYT5O0ZFXM2xxXfk4J5UMw/kRiISng037Gxdw= +github.com/testcontainers/testcontainers-go v0.33.0/go.mod h1:W80YpTa8D5C3Yy16icheD01UTDu+LmXIA2Keo+jWtT8= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= +github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= +github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= +go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs= +go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 h1:cl5P5/GIfFh4t6xyruOgJP5QiA1pw4fYYdv6nc6CBWw= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0/go.mod h1:zgBdWWAu7oEEMC06MMKc5NLbA/1YDXV1sMpSqEeLQLg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0 h1:digkEZCJWobwBqMwC0cwCq8/wkkRy/OowZg5OArWZrM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0/go.mod h1:/OpE/y70qVkndM0TrxT4KBoN3RsFZP0QaofcfYrj76I= +go.opentelemetry.io/otel/metric v1.26.0 h1:7S39CLuY5Jgg9CrnA9HHiEjGMF/X2VHvoXGgSllRz30= +go.opentelemetry.io/otel/metric v1.26.0/go.mod h1:SY+rHOI4cEawI9a7N1A4nIg/nTQXe1ccCNWYOJUrpX4= +go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= +go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= +go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2LP5sQA= +go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0= +go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= +go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= +golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 h1:RFiFrvy37/mpSpdySBDrUdipW/dHwsRwh3J3+A9VgT4= +google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237/go.mod h1:Z5Iiy3jtmioajWHDGFk7CeugTyHtPvMHA4UTmUkyalE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/grpc v1.64.1 h1:LKtvyfbX3UGVPFcGqJ9ItpVWW6oN/2XqTxfAnwRRXiA= +google.golang.org/grpc v1.64.1/go.mod h1:hiQF4LFZelK2WKaP6W0L92zGHtiQdZxk8CrSdvyjeP0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= +gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= diff --git a/lib/workload/stateless/stacks/attribute-annotator/handler_test.go b/lib/workload/stateless/stacks/attribute-annotator/handler_test.go new file mode 100644 index 000000000..2da3929e4 --- /dev/null +++ b/lib/workload/stateless/stacks/attribute-annotator/handler_test.go @@ -0,0 +1,231 @@ +package main + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "github.com/docker/go-connections/nat" + "github.com/go-testfixtures/testfixtures/v3" + "github.com/google/uuid" + "github.com/jmoiron/sqlx" + _ "github.com/lib/pq" + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" + "github.com/umccr/orcabus/lib/workload/stateless/stacks/attribute-linker/schema/orcabus_workflowmanager/workflowrunstatechange" + "os" + "strconv" + "strings" + "testing" + "time" +) + +var ( + fmEndpoint string + databaseEndpoint string + db *sql.DB +) + +func setupService(t *testing.T, buildContext string, port nat.Port, wait wait.Strategy, env map[string]string) (string, nat.Port) { + ctx := context.Background() + + args := make(map[string]*string) + for k, v := range env { + args[k] = &v + } + + containerName := strings.ReplaceAll(strings.Trim(buildContext, "./"), "/", "_") + req := testcontainers.ContainerRequest{ + FromDockerfile: testcontainers.FromDockerfile{ + Context: buildContext, + Repo: containerName, + Tag: containerName, + BuildArgs: args, + KeepImage: true, + }, + ExposedPorts: []string{port.Port()}, + Env: env, + WaitingFor: wait, + Name: containerName, + } + container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + Reuse: true, + }) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, container.Terminate(ctx)) + }) + + ip, err := container.Host(ctx) + require.NoError(t, err) + port, err = container.MappedPort(ctx, port) + require.NoError(t, err) + + fmt.Printf("setup `%v` at `%v:%v`\n", containerName, ip, port.Port()) + + return ip, port +} + +func setupFileManager(t *testing.T) { + // This has to be such a large timeout because FM needs to compile/build with access to a Postgres database. If + // the timeout is too low, before the FM compilation connects to the database, testcontainers will kill the postgres + // container as there are no connections to it. + // This is another good reason to remove the requirement to compile with a database, as most software doesn't + // expect this to be the case. + t.Setenv("TESTCONTAINERS_RYUK_CONNECTION_TIMEOUT", "10m") + t.Setenv("TESTCONTAINERS_RYUK_RECONNECTION_TIMEOUT", "5m") + + // Database + testDatabaseName := fmt.Sprintf("filemanager_test_%v", strings.ReplaceAll(uuid.New().String(), "-", "_")) + databaseIp, port := setupService(t, "../filemanager/database", "4321", wait.ForLog("database system is ready to accept connections"). + WithOccurrence(2), + map[string]string{ + "POSTGRES_DB": testDatabaseName, + "POSTGRES_USER": "filemanager", + "POSTGRES_PASSWORD": "filemanager", // pragma: allowlist secret + "PGPORT": "4321", + }) + + // API + intPort, err := strconv.Atoi(port.Port()) + require.NoError(t, err) + + databaseFmt := "postgresql://filemanager:filemanager@%v:%v/%v?sslmode=disable" // pragma: allowlist secret + databaseUrl := fmt.Sprintf(databaseFmt, "host.docker.internal", intPort, testDatabaseName) + ip, port := setupService(t, "../filemanager", "8000", wait.ForHTTP("/api/v1/s3/count"), + map[string]string{ + "DATABASE_URL": databaseUrl, + }) + + fmEndpoint = fmt.Sprintf("http://%v:%v", ip, port.Port()) + databaseEndpoint = fmt.Sprintf(databaseFmt, databaseIp, intPort, testDatabaseName) + + t.Setenv("ANNOTATOR_FILE_MANAGER_ENDPOINT", fmEndpoint) + loadFixtures(t, databaseEndpoint) +} + +func loadFixtures(t *testing.T, databaseUrl string) { + var err error + db, err = sql.Open("postgres", databaseUrl) + require.NoError(t, err) + + fixtures, err := testfixtures.New( + testfixtures.Database(db), + testfixtures.Dialect("postgres"), + testfixtures.Directory("fixtures"), + ) + require.NoError(t, err) + + fmt.Printf("loading data into test database: %v\n", databaseUrl) + + err = fixtures.Load() + require.NoError(t, err) +} + +type S3Object struct { + EventType string `db:"event_type"` + Bucket string `db:"bucket"` + Key string `db:"key"` + EventTime sql.NullTime `db:"event_time"` + Size sql.NullInt64 `db:"size"` + StorageClass sql.NullString `db:"storage_class"` +} + +func testEvent(t *testing.T, path string) workflowrunstatechange.Event { + b, err := os.ReadFile(path) + require.NoError(t, err) + + var event workflowrunstatechange.Event + err = json.Unmarshal(b, &event) + require.NoError(t, err) + + return event +} + +func queryObjects(t *testing.T, db *sql.DB, query string) []S3Object { + var s3Objects []S3Object + err := sqlx.NewDb(db, "postgres").Unsafe().Select(&s3Objects, query) + require.NoError(t, err) + + return s3Objects +} + +type TestCase struct { + event string + portalRunId string + expected []S3Object +} + +func successCase(location *time.Location) TestCase { + return TestCase{"fixtures/event_succeeded.json", "202409021221e6e6", []S3Object{ + { + "Created", + "bucket", + "byob-icav2/development/analysis/wts/202409021221e6e6/_manifest.json", + sql.NullTime{Time: time.Date(2024, 9, 2, 0, 0, 0, 0, location), Valid: true}, + sql.NullInt64{Int64: 5, Valid: true}, + sql.NullString{String: "Standard", Valid: true}, + }, + { + "Deleted", + "bucket", + "byob-icav2/development/analysis/wts/202409021221e6e6/_manifest.json", + sql.NullTime{Time: time.Date(2024, 9, 3, 0, 0, 0, 0, location), Valid: true}, + sql.NullInt64{}, + sql.NullString{}, + }, + { + "Created", + "bucket", + "byob-icav2/development/analysis/wts/202409021221e6e6/_tags.json", + sql.NullTime{Time: time.Date(2024, 9, 4, 0, 0, 0, 0, location), Valid: true}, + sql.NullInt64{Int64: 10, Valid: true}, + sql.NullString{String: "Standard", Valid: true}, + }, + }} +} + +func failCase(location *time.Location) TestCase { + return TestCase{"fixtures/event_failed.json", "202409021221e6c6", []S3Object{ + { + "Created", + "bucket", + "byob-icav2/development/analysis/wts/202409021221e6c6/_manifest.json", + sql.NullTime{Time: time.Date(2024, 9, 5, 0, 0, 0, 0, location), Valid: true}, + sql.NullInt64{Int64: 3, Valid: true}, + sql.NullString{String: "Standard", Valid: true}, + }, + { + "Deleted", + "bucket", + "byob-icav2/development/analysis/wts/202409021221e6c6/_manifest.json", + sql.NullTime{Time: time.Date(2024, 9, 6, 0, 0, 0, 0, location), Valid: true}, + sql.NullInt64{}, + sql.NullString{}, + }, + }} +} + +func TestHandler(t *testing.T) { + setupFileManager(t) + + location, err := time.LoadLocation("Etc/UTC") + require.NoError(t, err) + + testCases := []TestCase{successCase(location), failCase(location)} + + for _, tc := range testCases { + t.Run(fmt.Sprintf("%v", tc.event), func(t *testing.T) { + event := testEvent(t, tc.event) + + err := Handler(event) + require.NoError(t, err) + + s3Objects := queryObjects(t, db, fmt.Sprintf("select * from s3_object where key like '%%%v%%'", tc.portalRunId)) + require.Equal(t, tc.expected, s3Objects) + }) + } +} diff --git a/lib/workload/stateless/stacks/attribute-annotator/main.go b/lib/workload/stateless/stacks/attribute-annotator/main.go index 0c0cfb7c9..0ef31bfee 100644 --- a/lib/workload/stateless/stacks/attribute-annotator/main.go +++ b/lib/workload/stateless/stacks/attribute-annotator/main.go @@ -16,26 +16,28 @@ import ( ) type Config struct { - FileManagerEndpoint string + FileManagerEndpoint string `required:"true" split_words:"true"` } -type AttributePatch struct { +type JsonPatch struct { Op string `json:"op"` Path string `json:"path"` - Value string `json:"value"` + Value string `json:"value,omitempty"` } -func handler(event workflowrunstatechange.Event) (err error) { +type PatchList []JsonPatch + +func Handler(event workflowrunstatechange.Event) (err error) { status := strings.ToUpper(event.Detail.Status) if status != "SUCCEEDED" && status != "FAILED" && status != "ABORTED" { return nil } - patch, err := json.Marshal(AttributePatch{ + patch, err := json.Marshal(PatchList{JsonPatch{ "add", "/portalRunId", event.Detail.PortalRunId, - }) + }}) if err != nil { return err } @@ -64,6 +66,7 @@ func handler(event workflowrunstatechange.Event) (err error) { req.URL.RawQuery = url.Values{ "key": {fmt.Sprintf("*%v*", event.Detail.PortalRunId)}, }.Encode() + req.Header.Add("Content-Type", "application/json") slog.Debug(fmt.Sprintf("sending attributes patch %v to %v", patch, req.URL.String())) @@ -79,6 +82,9 @@ func handler(event workflowrunstatechange.Event) (err error) { if err != nil { return err } + if resp.StatusCode != 200 { + return errors.New(fmt.Sprintf("error annotating attributes with status %v: %v", resp.Status, string(body))) + } slog.Debug(fmt.Sprintf("received response %v with body: %v", resp.StatusCode, body)) @@ -86,5 +92,5 @@ func handler(event workflowrunstatechange.Event) (err error) { } func main() { - lambda.Start(handler) + lambda.Start(Handler) } diff --git a/lib/workload/stateless/stacks/filemanager/.env.example b/lib/workload/stateless/stacks/filemanager/.env.example index 88a97b5da..781acc1e4 100644 --- a/lib/workload/stateless/stacks/filemanager/.env.example +++ b/lib/workload/stateless/stacks/filemanager/.env.example @@ -3,6 +3,7 @@ FILEMANAGER_DATABASE_HOST=0.0.0.0 FILEMANAGER_DATABASE_PORT=4321 DATABASE_URL=postgresql://filemanager:filemanager@${FILEMANAGER_DATABASE_HOST}:${FILEMANAGER_DATABASE_PORT}/filemanager #pragma: allowlist secret +API_PORT=8000 FILEMANAGER_LINKS_URL=localhost:8000 FILEMANAGER_API_CORS_ALLOW_ORIGINS=http://localhost:8000 diff --git a/lib/workload/stateless/stacks/filemanager/Makefile b/lib/workload/stateless/stacks/filemanager/Makefile index 8d34e8509..a020bb3b0 100644 --- a/lib/workload/stateless/stacks/filemanager/Makefile +++ b/lib/workload/stateless/stacks/filemanager/Makefile @@ -40,7 +40,7 @@ docker-find: @docker ps --filter name=$(DOCKER_PROJECT_NAME)-postgres --latest --format "{{.ID}}" | xargs -I {} docker port {} | tail -n 1 | awk '{print $$NF}' docker-api: docker-postgres # Run the local API server in a docker container. - @docker compose -p "$(DOCKER_PROJECT_NAME)" up api + @docker compose -p "$(DOCKER_PROJECT_NAME)" -f compose-api.yml up api ## Build related commands build: docker-postgres diff --git a/lib/workload/stateless/stacks/filemanager/compose-api.yml b/lib/workload/stateless/stacks/filemanager/compose-api.yml new file mode 100644 index 000000000..360ab5769 --- /dev/null +++ b/lib/workload/stateless/stacks/filemanager/compose-api.yml @@ -0,0 +1,20 @@ +services: + api: + build: + context: . + args: + # The build itself needs access to the database. + - DATABASE_URL=${BUILD_DATABASE_URL:-postgresql://filemanager:filemanager@host.docker.internal:4321/filemanager} # pragma: allowlist secret + environment: + # Container database address for running server inside a docker container. + - DATABASE_URL=${DATABASE_URL:-postgresql://filemanager:filemanager@postgres:4321/filemanager} + - RUST_LOG=debug + - FILEMANAGER_API_CORS_ALLOW_ORIGINS=${FILEMANAGER_API_CORS_ALLOW_ORIGINS:-http://localhost:3000} + - FILEMANAGER_API_CORS_ALLOW_HEADERS=${FILEMANAGER_API_CORS_ALLOW_HEADERS:-accept,authorization,content-type,user-agent,x-csrftoken,x-requested-with,x-amz-security-token,x-amz-date,content-disposition} + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-access_key_id} + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-secret_access_key} + - AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION:-ap-southeast-2} + - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN:-session_token} + ports: + - '${API_PORT}:8000' + restart: always diff --git a/lib/workload/stateless/stacks/filemanager/compose.yml b/lib/workload/stateless/stacks/filemanager/compose.yml index 234e01807..f9bd6761b 100644 --- a/lib/workload/stateless/stacks/filemanager/compose.yml +++ b/lib/workload/stateless/stacks/filemanager/compose.yml @@ -3,30 +3,9 @@ services: build: database restart: always environment: - - POSTGRES_DATABASE=filemanager + - POSTGRES_DB=filemanager - POSTGRES_USER=filemanager - POSTGRES_PASSWORD=filemanager - PGPORT=4321 ports: - "${FILEMANAGER_DATABASE_HOST}:${FILEMANAGER_DATABASE_PORT}:4321" - api: - build: - context: . - args: - # The build itself needs access to the database. - DATABASE_URL: postgresql://filemanager:filemanager@host.docker.internal:4321/filemanager # pragma: allowlist secret - environment: - # Container database address for running server inside a docker container. - - DATABASE_URL=postgresql://filemanager:filemanager@postgres:4321/filemanager - - RUST_LOG=debug - - FILEMANAGER_API_CORS_ALLOW_ORIGINS=${FILEMANAGER_API_CORS_ALLOW_ORIGINS:-http://localhost:3000} - - FILEMANAGER_API_CORS_ALLOW_HEADERS=${FILEMANAGER_API_CORS_ALLOW_HEADERS:-accept,authorization,content-type,user-agent,x-csrftoken,x-requested-with,x-amz-security-token,x-amz-date,content-disposition} - - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-access_key_id} - - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-secret_access_key} - - AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION:-ap-southeast-2} - - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN:-session_token} - ports: - - "8000:8000" - depends_on: - - postgres - restart: always From fb43a916d560492f0eb21b6b6d3d39112759232f Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Tue, 3 Sep 2024 09:16:42 +1000 Subject: [PATCH 05/13] refactor: separate into modules, add some tests --- Makefile | 1 + config/stacks/attributeLinker.ts | 2 +- lib/pipeline/statelessPipelineStack.ts | 13 ++ .../stacks/attribute-annotator/main.go | 96 ------------- .../.gitignore | 0 .../Makefile | 4 +- .../stateless/stacks/fmannotator/api.go | 86 ++++++++++++ .../stateless/stacks/fmannotator/api_test.go | 43 ++++++ .../stateless/stacks/fmannotator/cmd/main.go | 10 ++ .../stateless/stacks/fmannotator/config.go | 15 ++ .../deploy/stack.ts | 0 .../fixtures/event_failed.json | 0 .../fixtures/event_succeeded.json | 0 .../fixtures/s3_object.yml | 0 .../go.mod | 2 +- .../go.sum | 0 .../stateless/stacks/fmannotator/handler.go | 48 +++++++ .../stacks/fmannotator/handler_test.go | 94 +++++++++++++ .../internal/test/test.go} | 128 ++++-------------- .../schema/README.md | 0 .../workflowrunstatechange/event.go | 0 .../workflowrunstatechange/library_record.go | 0 .../workflowrunstatechange/marshaller.go | 0 .../workflowrunstatechange/payload.go | 0 .../workflow_run_state_change.go | 0 .../statelessStackCollectionClass.ts | 2 +- 26 files changed, 340 insertions(+), 204 deletions(-) delete mode 100644 lib/workload/stateless/stacks/attribute-annotator/main.go rename lib/workload/stateless/stacks/{attribute-annotator => fmannotator}/.gitignore (100%) rename lib/workload/stateless/stacks/{attribute-annotator => fmannotator}/Makefile (90%) create mode 100644 lib/workload/stateless/stacks/fmannotator/api.go create mode 100644 lib/workload/stateless/stacks/fmannotator/api_test.go create mode 100644 lib/workload/stateless/stacks/fmannotator/cmd/main.go create mode 100644 lib/workload/stateless/stacks/fmannotator/config.go rename lib/workload/stateless/stacks/{attribute-annotator => fmannotator}/deploy/stack.ts (100%) rename lib/workload/stateless/stacks/{attribute-annotator => fmannotator}/fixtures/event_failed.json (100%) rename lib/workload/stateless/stacks/{attribute-annotator => fmannotator}/fixtures/event_succeeded.json (100%) rename lib/workload/stateless/stacks/{attribute-annotator => fmannotator}/fixtures/s3_object.yml (100%) rename lib/workload/stateless/stacks/{attribute-annotator => fmannotator}/go.mod (97%) rename lib/workload/stateless/stacks/{attribute-annotator => fmannotator}/go.sum (100%) create mode 100644 lib/workload/stateless/stacks/fmannotator/handler.go create mode 100644 lib/workload/stateless/stacks/fmannotator/handler_test.go rename lib/workload/stateless/stacks/{attribute-annotator/handler_test.go => fmannotator/internal/test/test.go} (56%) rename lib/workload/stateless/stacks/{attribute-annotator => fmannotator}/schema/README.md (100%) rename lib/workload/stateless/stacks/{attribute-annotator => fmannotator}/schema/orcabus_workflowmanager/workflowrunstatechange/event.go (100%) rename lib/workload/stateless/stacks/{attribute-annotator => fmannotator}/schema/orcabus_workflowmanager/workflowrunstatechange/library_record.go (100%) rename lib/workload/stateless/stacks/{attribute-annotator => fmannotator}/schema/orcabus_workflowmanager/workflowrunstatechange/marshaller.go (100%) rename lib/workload/stateless/stacks/{attribute-annotator => fmannotator}/schema/orcabus_workflowmanager/workflowrunstatechange/payload.go (100%) rename lib/workload/stateless/stacks/{attribute-annotator => fmannotator}/schema/orcabus_workflowmanager/workflowrunstatechange/workflow_run_state_change.go (100%) diff --git a/Makefile b/Makefile index 607dd112b..c41b99e54 100644 --- a/Makefile +++ b/Makefile @@ -53,6 +53,7 @@ test-stateless-app-suite: @(cd lib/workload/stateless/stacks/sequence-run-manager && $(MAKE) test) @(cd lib/workload/stateless/stacks/metadata-manager && $(MAKE) test) @(cd lib/workload/stateless/stacks/filemanager && $(MAKE) test) + @(cd lib/workload/stateless/stacks/fmannotator && $(MAKE) test) @(cd lib/workload/stateless/stacks/bclconvert-manager && $(MAKE) test) @(cd lib/workload/stateless/stacks/workflow-manager && $(MAKE) test) diff --git a/config/stacks/attributeLinker.ts b/config/stacks/attributeLinker.ts index 490716dfc..9ea5e1807 100644 --- a/config/stacks/attributeLinker.ts +++ b/config/stacks/attributeLinker.ts @@ -1,4 +1,4 @@ -import { AttributeLinkerConfigurableProps } from '../../lib/workload/stateless/stacks/attribute-annotator/deploy/stack'; +import { AttributeLinkerConfigurableProps } from '../../lib/workload/stateless/stacks/fmannotator/deploy/stack'; import { eventBusName, vpcProps } from '../constants'; export const getAttributeLinkerProps = (): AttributeLinkerConfigurableProps => { diff --git a/lib/pipeline/statelessPipelineStack.ts b/lib/pipeline/statelessPipelineStack.ts index f3b0acaad..6d28c3bd7 100644 --- a/lib/pipeline/statelessPipelineStack.ts +++ b/lib/pipeline/statelessPipelineStack.ts @@ -91,6 +91,7 @@ export class StatelessPipelineStack extends cdk.Stack { install: { 'runtime-versions': { python: '3.12', + golang: '1.22', }, }, }, @@ -126,6 +127,18 @@ export class StatelessPipelineStack extends cdk.Stack { resources: ['*'], }), ], + partialBuildSpec: codebuild.BuildSpec.fromObject({ + phases: { + install: { + 'runtime-versions': { + // Don't strictly need golang here as CDK can do docker bundling, but `local` + // bundling tends to be faster. + golang: '1.22', + }, + }, + }, + version: '0.2', + }), }); const pipeline = new pipelines.CodePipeline(this, 'Pipeline', { diff --git a/lib/workload/stateless/stacks/attribute-annotator/main.go b/lib/workload/stateless/stacks/attribute-annotator/main.go deleted file mode 100644 index 0ef31bfee..000000000 --- a/lib/workload/stateless/stacks/attribute-annotator/main.go +++ /dev/null @@ -1,96 +0,0 @@ -package main - -import ( - "bytes" - "encoding/json" - "errors" - "fmt" - "github.com/aws/aws-lambda-go/lambda" - "github.com/kelseyhightower/envconfig" - "github.com/umccr/orcabus/lib/workload/stateless/stacks/attribute-linker/schema/orcabus_workflowmanager/workflowrunstatechange" - "io" - "log/slog" - "net/http" - "net/url" - "strings" -) - -type Config struct { - FileManagerEndpoint string `required:"true" split_words:"true"` -} - -type JsonPatch struct { - Op string `json:"op"` - Path string `json:"path"` - Value string `json:"value,omitempty"` -} - -type PatchList []JsonPatch - -func Handler(event workflowrunstatechange.Event) (err error) { - status := strings.ToUpper(event.Detail.Status) - if status != "SUCCEEDED" && status != "FAILED" && status != "ABORTED" { - return nil - } - - patch, err := json.Marshal(PatchList{JsonPatch{ - "add", - "/portalRunId", - event.Detail.PortalRunId, - }}) - if err != nil { - return err - } - - var config Config - err = envconfig.Process("annotator", &config) - if err != nil { - return err - } - endpoint, err := url.JoinPath(config.FileManagerEndpoint, "/api/v1/s3") - if err != nil { - return err - } - - client := &http.Client{} - - req, err := http.NewRequest( - "PATCH", - endpoint, - bytes.NewBuffer(patch), - ) - if err != nil { - return err - } - - req.URL.RawQuery = url.Values{ - "key": {fmt.Sprintf("*%v*", event.Detail.PortalRunId)}, - }.Encode() - req.Header.Add("Content-Type", "application/json") - - slog.Debug(fmt.Sprintf("sending attributes patch %v to %v", patch, req.URL.String())) - - resp, err := client.Do(req) - if err != nil { - return err - } - defer func() { - err = errors.Join(resp.Body.Close()) - }() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return err - } - if resp.StatusCode != 200 { - return errors.New(fmt.Sprintf("error annotating attributes with status %v: %v", resp.Status, string(body))) - } - - slog.Debug(fmt.Sprintf("received response %v with body: %v", resp.StatusCode, body)) - - return nil -} - -func main() { - lambda.Start(Handler) -} diff --git a/lib/workload/stateless/stacks/attribute-annotator/.gitignore b/lib/workload/stateless/stacks/fmannotator/.gitignore similarity index 100% rename from lib/workload/stateless/stacks/attribute-annotator/.gitignore rename to lib/workload/stateless/stacks/fmannotator/.gitignore diff --git a/lib/workload/stateless/stacks/attribute-annotator/Makefile b/lib/workload/stateless/stacks/fmannotator/Makefile similarity index 90% rename from lib/workload/stateless/stacks/attribute-annotator/Makefile rename to lib/workload/stateless/stacks/fmannotator/Makefile index 2fe4e45d3..459b0edcc 100644 --- a/lib/workload/stateless/stacks/attribute-annotator/Makefile +++ b/lib/workload/stateless/stacks/fmannotator/Makefile @@ -7,12 +7,14 @@ all: build ## Build related commands build: @go build -o build +test: + @go test lint: @gofmt -l lint-fix: @gofmt golangci-lint: - @golangci-lint run + @golangci-lint run --verbose check: lint golangci-lint check-fix: lint-fix golangci-lint diff --git a/lib/workload/stateless/stacks/fmannotator/api.go b/lib/workload/stateless/stacks/fmannotator/api.go new file mode 100644 index 000000000..b236cc4b2 --- /dev/null +++ b/lib/workload/stateless/stacks/fmannotator/api.go @@ -0,0 +1,86 @@ +package fmannotator + +import ( + "encoding/json" + "errors" + "fmt" + "github.com/umccr/orcabus/lib/workload/stateless/stacks/fmannotator/schema/orcabus_workflowmanager/workflowrunstatechange" + "io" + "log/slog" + "net/http" + "net/url" +) + +// A JsonPatch used to update FM records. +type JsonPatch struct { + Op string `json:"op"` + Path string `json:"path"` + Value string `json:"value,omitempty"` +} + +// The PatchList for updating records. +type PatchList []JsonPatch + +// MarshallPortalRunId MarshallPatch Convert an event into a JSON patch using the portalRunId. +func MarshallPortalRunId(event *workflowrunstatechange.Event) ([]byte, error) { + return json.Marshal(PatchList{JsonPatch{ + "add", + "/portalRunId", + event.Detail.PortalRunId, + }}) +} + +// The ApiClient which is used to send requests to filemanager. +type ApiClient struct { + Client *http.Client + Request *http.Request +} + +// NewApiClient Create a new ApiClient. +func NewApiClient(config *Config, body io.Reader) (*ApiClient, error) { + req, err := http.NewRequest("GET", config.FileManagerEndpoint, body) + if err != nil { + return nil, err + } + req.Header.Add("Content-Type", "application/json") + + return &ApiClient{ + Client: &http.Client{}, + Request: req, + }, nil +} + +// WithMethod Set the method for the request. +func (c *ApiClient) WithMethod(method string) *ApiClient { + c.Request.Method = method + return c +} + +// WithS3Endpoint Set the endpoint to query S3 records. +func (c *ApiClient) WithS3Endpoint() *ApiClient { + c.Request.URL.Path = "/api/v1/s3" + return c +} + +// WithQuery Set the query parameters. +func (c *ApiClient) WithQuery(query url.Values) *ApiClient { + c.Request.URL.RawQuery = query.Encode() + return c +} + +// Do the request returning the response body and status code. +func (c *ApiClient) Do() ([]byte, int, error) { + slog.Debug(fmt.Sprintf("sending request to %v", c.Request.URL.String())) + + resp, err := c.Client.Do(c.Request) + if err != nil { + return []byte{}, 0, err + } + defer func() { + err = errors.Join(resp.Body.Close()) + }() + + status := resp.StatusCode + body, err := io.ReadAll(resp.Body) + return body, status, err +} diff --git a/lib/workload/stateless/stacks/fmannotator/api_test.go b/lib/workload/stateless/stacks/fmannotator/api_test.go new file mode 100644 index 000000000..137d0d713 --- /dev/null +++ b/lib/workload/stateless/stacks/fmannotator/api_test.go @@ -0,0 +1,43 @@ +package fmannotator + +import ( + "bytes" + "encoding/json" + "github.com/stretchr/testify/require" + "github.com/umccr/orcabus/lib/workload/stateless/stacks/fmannotator/internal/test" + "net/url" + "testing" +) + +func TestMarshallPortalRunId(t *testing.T) { + event := test.CreateEvent(t, "event_succeeded.json") + body, err := MarshallPortalRunId(&event) + require.NoError(t, err) + + var patch PatchList + err = json.Unmarshal(body, &patch) + require.NoError(t, err) + + require.Equal(t, PatchList{JsonPatch{ + "add", + "/portalRunId", + "202409021221e6e6", + }}, patch) +} + +func TestApiBuild(t *testing.T) { + config := Config{"http://localhost:8000"} + + api, err := NewApiClient(&config, bytes.NewBuffer([]byte{})) + require.NoError(t, err) + + api = api.WithMethod("PATCH").WithS3Endpoint().WithQuery(url.Values{ + "key": {"value"}, + }) + + require.Equal(t, []string{"application/json"}, api.Request.Header.Values("Content-Type")) + require.Equal(t, "PATCH", api.Request.Method) + require.Equal(t, "localhost:8000", api.Request.URL.Host) + require.Equal(t, "/api/v1/s3", api.Request.URL.Path) + require.Equal(t, "key=value", api.Request.URL.RawQuery) +} diff --git a/lib/workload/stateless/stacks/fmannotator/cmd/main.go b/lib/workload/stateless/stacks/fmannotator/cmd/main.go new file mode 100644 index 000000000..1ce7b59c1 --- /dev/null +++ b/lib/workload/stateless/stacks/fmannotator/cmd/main.go @@ -0,0 +1,10 @@ +package main + +import ( + "github.com/aws/aws-lambda-go/lambda" + "github.com/umccr/orcabus/lib/workload/stateless/stacks/fmannotator" +) + +func main() { + lambda.Start(fmannotator.Handler) +} diff --git a/lib/workload/stateless/stacks/fmannotator/config.go b/lib/workload/stateless/stacks/fmannotator/config.go new file mode 100644 index 000000000..36a598f96 --- /dev/null +++ b/lib/workload/stateless/stacks/fmannotator/config.go @@ -0,0 +1,15 @@ +package fmannotator + +import "github.com/kelseyhightower/envconfig" + +// Config Configuration for the fmannotator +type Config struct { + FileManagerEndpoint string `required:"true" split_words:"true"` +} + +// LoadConfig Load config from the environment. +func LoadConfig() (Config, error) { + var config Config + err := envconfig.Process("annotator", &config) + return config, err +} diff --git a/lib/workload/stateless/stacks/attribute-annotator/deploy/stack.ts b/lib/workload/stateless/stacks/fmannotator/deploy/stack.ts similarity index 100% rename from lib/workload/stateless/stacks/attribute-annotator/deploy/stack.ts rename to lib/workload/stateless/stacks/fmannotator/deploy/stack.ts diff --git a/lib/workload/stateless/stacks/attribute-annotator/fixtures/event_failed.json b/lib/workload/stateless/stacks/fmannotator/fixtures/event_failed.json similarity index 100% rename from lib/workload/stateless/stacks/attribute-annotator/fixtures/event_failed.json rename to lib/workload/stateless/stacks/fmannotator/fixtures/event_failed.json diff --git a/lib/workload/stateless/stacks/attribute-annotator/fixtures/event_succeeded.json b/lib/workload/stateless/stacks/fmannotator/fixtures/event_succeeded.json similarity index 100% rename from lib/workload/stateless/stacks/attribute-annotator/fixtures/event_succeeded.json rename to lib/workload/stateless/stacks/fmannotator/fixtures/event_succeeded.json diff --git a/lib/workload/stateless/stacks/attribute-annotator/fixtures/s3_object.yml b/lib/workload/stateless/stacks/fmannotator/fixtures/s3_object.yml similarity index 100% rename from lib/workload/stateless/stacks/attribute-annotator/fixtures/s3_object.yml rename to lib/workload/stateless/stacks/fmannotator/fixtures/s3_object.yml diff --git a/lib/workload/stateless/stacks/attribute-annotator/go.mod b/lib/workload/stateless/stacks/fmannotator/go.mod similarity index 97% rename from lib/workload/stateless/stacks/attribute-annotator/go.mod rename to lib/workload/stateless/stacks/fmannotator/go.mod index d58a6425c..75dee0530 100644 --- a/lib/workload/stateless/stacks/attribute-annotator/go.mod +++ b/lib/workload/stateless/stacks/fmannotator/go.mod @@ -1,4 +1,4 @@ -module github.com/umccr/orcabus/lib/workload/stateless/stacks/attribute-linker +module github.com/umccr/orcabus/lib/workload/stateless/stacks/fmannotator go 1.23 diff --git a/lib/workload/stateless/stacks/attribute-annotator/go.sum b/lib/workload/stateless/stacks/fmannotator/go.sum similarity index 100% rename from lib/workload/stateless/stacks/attribute-annotator/go.sum rename to lib/workload/stateless/stacks/fmannotator/go.sum diff --git a/lib/workload/stateless/stacks/fmannotator/handler.go b/lib/workload/stateless/stacks/fmannotator/handler.go new file mode 100644 index 000000000..0cb9449a8 --- /dev/null +++ b/lib/workload/stateless/stacks/fmannotator/handler.go @@ -0,0 +1,48 @@ +package fmannotator + +import ( + "bytes" + "errors" + "fmt" + "github.com/umccr/orcabus/lib/workload/stateless/stacks/fmannotator/schema/orcabus_workflowmanager/workflowrunstatechange" + "log/slog" + "net/url" + "strings" +) + +func Handler(event workflowrunstatechange.Event) (err error) { + eventStatus := strings.ToUpper(event.Detail.Status) + if eventStatus != "SUCCEEDED" && eventStatus != "FAILED" && eventStatus != "ABORTED" { + return nil + } + + config, err := LoadConfig() + if err != nil { + return err + } + + patch, err := MarshallPortalRunId(&event) + if err != nil { + return err + } + + resp, err := NewApiClient(&config, bytes.NewBuffer(patch)) + if err != nil { + return err + } + + resp = resp.WithMethod("PATCH").WithS3Endpoint().WithQuery(url.Values{ + "key": {fmt.Sprintf("*%v*", event.Detail.PortalRunId)}, + }) + body, status, err := resp.Do() + if err != nil { + return err + } + if status != 200 { + return errors.New(fmt.Sprintf("error annotating attributes with status %v: %v", status, string(body))) + } + + slog.Debug(fmt.Sprintf("received response %v with body: %v", status, body)) + + return nil +} diff --git a/lib/workload/stateless/stacks/fmannotator/handler_test.go b/lib/workload/stateless/stacks/fmannotator/handler_test.go new file mode 100644 index 000000000..be4ef68eb --- /dev/null +++ b/lib/workload/stateless/stacks/fmannotator/handler_test.go @@ -0,0 +1,94 @@ +package fmannotator + +import ( + "database/sql" + "fmt" + _ "github.com/lib/pq" + "github.com/stretchr/testify/require" + "github.com/umccr/orcabus/lib/workload/stateless/stacks/fmannotator/internal/test" + "testing" + "time" +) + +var ( + fmEndpoint string + databaseEndpoint string + db *sql.DB +) + +type TestCase struct { + event string + portalRunId string + expected []test.S3Object +} + +func successCase(location *time.Location) TestCase { + return TestCase{"event_succeeded.json", "202409021221e6e6", []test.S3Object{ + { + "Created", + "bucket", + "byob-icav2/development/analysis/wts/202409021221e6e6/_manifest.json", + sql.NullTime{Time: time.Date(2024, 9, 2, 0, 0, 0, 0, location), Valid: true}, + sql.NullInt64{Int64: 5, Valid: true}, + sql.NullString{String: "Standard", Valid: true}, + }, + { + "Deleted", + "bucket", + "byob-icav2/development/analysis/wts/202409021221e6e6/_manifest.json", + sql.NullTime{Time: time.Date(2024, 9, 3, 0, 0, 0, 0, location), Valid: true}, + sql.NullInt64{}, + sql.NullString{}, + }, + { + "Created", + "bucket", + "byob-icav2/development/analysis/wts/202409021221e6e6/_tags.json", + sql.NullTime{Time: time.Date(2024, 9, 4, 0, 0, 0, 0, location), Valid: true}, + sql.NullInt64{Int64: 10, Valid: true}, + sql.NullString{String: "Standard", Valid: true}, + }, + }} +} + +func failCase(location *time.Location) TestCase { + return TestCase{"event_failed.json", "202409021221e6c6", []test.S3Object{ + { + "Created", + "bucket", + "byob-icav2/development/analysis/wts/202409021221e6c6/_manifest.json", + sql.NullTime{Time: time.Date(2024, 9, 5, 0, 0, 0, 0, location), Valid: true}, + sql.NullInt64{Int64: 3, Valid: true}, + sql.NullString{String: "Standard", Valid: true}, + }, + { + "Deleted", + "bucket", + "byob-icav2/development/analysis/wts/202409021221e6c6/_manifest.json", + sql.NullTime{Time: time.Date(2024, 9, 6, 0, 0, 0, 0, location), Valid: true}, + sql.NullInt64{}, + sql.NullString{}, + }, + }} +} + +func TestHandler(t *testing.T) { + test.SetupFileManager(t, &fmEndpoint, &databaseEndpoint, db) + + location, err := time.LoadLocation("Etc/UTC") + require.NoError(t, err) + + testCases := []TestCase{successCase(location), failCase(location)} + + for _, tc := range testCases { + t.Run(fmt.Sprintf("%v", tc.event), func(t *testing.T) { + event := test.CreateEvent(t, tc.event) + + err := Handler(event) + require.NoError(t, err) + + s3Objects := test.QueryObjects(t, db, fmt.Sprintf("select * from s3_object where key like '%%%v%%'", tc.portalRunId)) + require.Equal(t, tc.expected, s3Objects) + }) + } +} diff --git a/lib/workload/stateless/stacks/attribute-annotator/handler_test.go b/lib/workload/stateless/stacks/fmannotator/internal/test/test.go similarity index 56% rename from lib/workload/stateless/stacks/attribute-annotator/handler_test.go rename to lib/workload/stateless/stacks/fmannotator/internal/test/test.go index 2da3929e4..ca991fc68 100644 --- a/lib/workload/stateless/stacks/attribute-annotator/handler_test.go +++ b/lib/workload/stateless/stacks/fmannotator/internal/test/test.go @@ -1,4 +1,4 @@ -package main +package test import ( "context" @@ -9,23 +9,26 @@ import ( "github.com/go-testfixtures/testfixtures/v3" "github.com/google/uuid" "github.com/jmoiron/sqlx" - _ "github.com/lib/pq" "github.com/stretchr/testify/require" "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/wait" - "github.com/umccr/orcabus/lib/workload/stateless/stacks/attribute-linker/schema/orcabus_workflowmanager/workflowrunstatechange" + "github.com/umccr/orcabus/lib/workload/stateless/stacks/fmannotator/schema/orcabus_workflowmanager/workflowrunstatechange" "os" + "path/filepath" + "runtime" "strconv" "strings" "testing" - "time" ) -var ( - fmEndpoint string - databaseEndpoint string - db *sql.DB -) +type S3Object struct { + EventType string `db:"event_type"` + Bucket string `db:"bucket"` + Key string `db:"key"` + EventTime sql.NullTime `db:"event_time"` + Size sql.NullInt64 `db:"size"` + StorageClass sql.NullString `db:"storage_class"` +} func setupService(t *testing.T, buildContext string, port nat.Port, wait wait.Strategy, env map[string]string) (string, nat.Port) { ctx := context.Background() @@ -69,7 +72,7 @@ func setupService(t *testing.T, buildContext string, port nat.Port, wait wait.St return ip, port } -func setupFileManager(t *testing.T) { +func SetupFileManager(t *testing.T, fmEndpoint *string, databaseEndpoint *string, db *sql.DB) { // This has to be such a large timeout because FM needs to compile/build with access to a Postgres database. If // the timeout is too low, before the FM compilation connects to the database, testcontainers will kill the postgres // container as there are no connections to it. @@ -100,16 +103,16 @@ func setupFileManager(t *testing.T) { "DATABASE_URL": databaseUrl, }) - fmEndpoint = fmt.Sprintf("http://%v:%v", ip, port.Port()) - databaseEndpoint = fmt.Sprintf(databaseFmt, databaseIp, intPort, testDatabaseName) + *fmEndpoint = fmt.Sprintf("http://%v:%v", ip, port.Port()) + *databaseEndpoint = fmt.Sprintf(databaseFmt, databaseIp, intPort, testDatabaseName) - t.Setenv("ANNOTATOR_FILE_MANAGER_ENDPOINT", fmEndpoint) - loadFixtures(t, databaseEndpoint) + t.Setenv("ANNOTATOR_FILE_MANAGER_ENDPOINT", *fmEndpoint) + loadFixtures(t, databaseEndpoint, db) } -func loadFixtures(t *testing.T, databaseUrl string) { +func loadFixtures(t *testing.T, databaseUrl *string, db *sql.DB) { var err error - db, err = sql.Open("postgres", databaseUrl) + db, err = sql.Open("postgres", *databaseUrl) require.NoError(t, err) fixtures, err := testfixtures.New( @@ -125,17 +128,11 @@ func loadFixtures(t *testing.T, databaseUrl string) { require.NoError(t, err) } -type S3Object struct { - EventType string `db:"event_type"` - Bucket string `db:"bucket"` - Key string `db:"key"` - EventTime sql.NullTime `db:"event_time"` - Size sql.NullInt64 `db:"size"` - StorageClass sql.NullString `db:"storage_class"` -} +func CreateEvent(t *testing.T, path string) workflowrunstatechange.Event { + _, file, _, _ := runtime.Caller(0) + root := filepath.Join(filepath.Dir(file), "../..") -func testEvent(t *testing.T, path string) workflowrunstatechange.Event { - b, err := os.ReadFile(path) + b, err := os.ReadFile(filepath.Join(root, "fixtures", path)) require.NoError(t, err) var event workflowrunstatechange.Event @@ -145,87 +142,10 @@ func testEvent(t *testing.T, path string) workflowrunstatechange.Event { return event } -func queryObjects(t *testing.T, db *sql.DB, query string) []S3Object { +func QueryObjects(t *testing.T, db *sql.DB, query string) []S3Object { var s3Objects []S3Object err := sqlx.NewDb(db, "postgres").Unsafe().Select(&s3Objects, query) require.NoError(t, err) return s3Objects } - -type TestCase struct { - event string - portalRunId string - expected []S3Object -} - -func successCase(location *time.Location) TestCase { - return TestCase{"fixtures/event_succeeded.json", "202409021221e6e6", []S3Object{ - { - "Created", - "bucket", - "byob-icav2/development/analysis/wts/202409021221e6e6/_manifest.json", - sql.NullTime{Time: time.Date(2024, 9, 2, 0, 0, 0, 0, location), Valid: true}, - sql.NullInt64{Int64: 5, Valid: true}, - sql.NullString{String: "Standard", Valid: true}, - }, - { - "Deleted", - "bucket", - "byob-icav2/development/analysis/wts/202409021221e6e6/_manifest.json", - sql.NullTime{Time: time.Date(2024, 9, 3, 0, 0, 0, 0, location), Valid: true}, - sql.NullInt64{}, - sql.NullString{}, - }, - { - "Created", - "bucket", - "byob-icav2/development/analysis/wts/202409021221e6e6/_tags.json", - sql.NullTime{Time: time.Date(2024, 9, 4, 0, 0, 0, 0, location), Valid: true}, - sql.NullInt64{Int64: 10, Valid: true}, - sql.NullString{String: "Standard", Valid: true}, - }, - }} -} - -func failCase(location *time.Location) TestCase { - return TestCase{"fixtures/event_failed.json", "202409021221e6c6", []S3Object{ - { - "Created", - "bucket", - "byob-icav2/development/analysis/wts/202409021221e6c6/_manifest.json", - sql.NullTime{Time: time.Date(2024, 9, 5, 0, 0, 0, 0, location), Valid: true}, - sql.NullInt64{Int64: 3, Valid: true}, - sql.NullString{String: "Standard", Valid: true}, - }, - { - "Deleted", - "bucket", - "byob-icav2/development/analysis/wts/202409021221e6c6/_manifest.json", - sql.NullTime{Time: time.Date(2024, 9, 6, 0, 0, 0, 0, location), Valid: true}, - sql.NullInt64{}, - sql.NullString{}, - }, - }} -} - -func TestHandler(t *testing.T) { - setupFileManager(t) - - location, err := time.LoadLocation("Etc/UTC") - require.NoError(t, err) - - testCases := []TestCase{successCase(location), failCase(location)} - - for _, tc := range testCases { - t.Run(fmt.Sprintf("%v", tc.event), func(t *testing.T) { - event := testEvent(t, tc.event) - - err := Handler(event) - require.NoError(t, err) - - s3Objects := queryObjects(t, db, fmt.Sprintf("select * from s3_object where key like '%%%v%%'", tc.portalRunId)) - require.Equal(t, tc.expected, s3Objects) - }) - } -} diff --git a/lib/workload/stateless/stacks/attribute-annotator/schema/README.md b/lib/workload/stateless/stacks/fmannotator/schema/README.md similarity index 100% rename from lib/workload/stateless/stacks/attribute-annotator/schema/README.md rename to lib/workload/stateless/stacks/fmannotator/schema/README.md diff --git a/lib/workload/stateless/stacks/attribute-annotator/schema/orcabus_workflowmanager/workflowrunstatechange/event.go b/lib/workload/stateless/stacks/fmannotator/schema/orcabus_workflowmanager/workflowrunstatechange/event.go similarity index 100% rename from lib/workload/stateless/stacks/attribute-annotator/schema/orcabus_workflowmanager/workflowrunstatechange/event.go rename to lib/workload/stateless/stacks/fmannotator/schema/orcabus_workflowmanager/workflowrunstatechange/event.go diff --git a/lib/workload/stateless/stacks/attribute-annotator/schema/orcabus_workflowmanager/workflowrunstatechange/library_record.go b/lib/workload/stateless/stacks/fmannotator/schema/orcabus_workflowmanager/workflowrunstatechange/library_record.go similarity index 100% rename from lib/workload/stateless/stacks/attribute-annotator/schema/orcabus_workflowmanager/workflowrunstatechange/library_record.go rename to lib/workload/stateless/stacks/fmannotator/schema/orcabus_workflowmanager/workflowrunstatechange/library_record.go diff --git a/lib/workload/stateless/stacks/attribute-annotator/schema/orcabus_workflowmanager/workflowrunstatechange/marshaller.go b/lib/workload/stateless/stacks/fmannotator/schema/orcabus_workflowmanager/workflowrunstatechange/marshaller.go similarity index 100% rename from lib/workload/stateless/stacks/attribute-annotator/schema/orcabus_workflowmanager/workflowrunstatechange/marshaller.go rename to lib/workload/stateless/stacks/fmannotator/schema/orcabus_workflowmanager/workflowrunstatechange/marshaller.go diff --git a/lib/workload/stateless/stacks/attribute-annotator/schema/orcabus_workflowmanager/workflowrunstatechange/payload.go b/lib/workload/stateless/stacks/fmannotator/schema/orcabus_workflowmanager/workflowrunstatechange/payload.go similarity index 100% rename from lib/workload/stateless/stacks/attribute-annotator/schema/orcabus_workflowmanager/workflowrunstatechange/payload.go rename to lib/workload/stateless/stacks/fmannotator/schema/orcabus_workflowmanager/workflowrunstatechange/payload.go diff --git a/lib/workload/stateless/stacks/attribute-annotator/schema/orcabus_workflowmanager/workflowrunstatechange/workflow_run_state_change.go b/lib/workload/stateless/stacks/fmannotator/schema/orcabus_workflowmanager/workflowrunstatechange/workflow_run_state_change.go similarity index 100% rename from lib/workload/stateless/stacks/attribute-annotator/schema/orcabus_workflowmanager/workflowrunstatechange/workflow_run_state_change.go rename to lib/workload/stateless/stacks/fmannotator/schema/orcabus_workflowmanager/workflowrunstatechange/workflow_run_state_change.go diff --git a/lib/workload/stateless/statelessStackCollectionClass.ts b/lib/workload/stateless/statelessStackCollectionClass.ts index 29ff331ef..07d5b64da 100644 --- a/lib/workload/stateless/statelessStackCollectionClass.ts +++ b/lib/workload/stateless/statelessStackCollectionClass.ts @@ -56,7 +56,7 @@ import { import { AttributeAnnotator, AttributeLinkerConfigurableProps, -} from './stacks/attribute-annotator/deploy/stack'; +} from './stacks/fmannotator/deploy/stack'; export interface StatelessStackCollectionProps { metadataManagerStackProps: MetadataManagerStackProps; From d5bc557bfb1e6003c1f41cd37cfe77599ed55089 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Tue, 3 Sep 2024 13:37:15 +1000 Subject: [PATCH 06/13] fix(fmannotator): set secret for FM JWT token --- config/stacks/attributeLinker.ts | 4 +- .../stateless/stacks/fmannotator/api.go | 7 ++- .../stateless/stacks/fmannotator/api_test.go | 4 +- .../stateless/stacks/fmannotator/cmd/main.go | 26 +++++++- .../stateless/stacks/fmannotator/config.go | 3 +- .../stacks/fmannotator/deploy/stack.ts | 59 ++++++++++++++----- .../stateless/stacks/fmannotator/go.mod | 3 + .../stateless/stacks/fmannotator/go.sum | 45 ++++++++++++++ .../stateless/stacks/fmannotator/handler.go | 12 ++-- .../stacks/fmannotator/handler_test.go | 13 ++-- .../stacks/fmannotator/internal/test/test.go | 28 +++++---- .../statelessStackCollectionClass.ts | 15 ++--- 12 files changed, 162 insertions(+), 57 deletions(-) diff --git a/config/stacks/attributeLinker.ts b/config/stacks/attributeLinker.ts index 9ea5e1807..069310223 100644 --- a/config/stacks/attributeLinker.ts +++ b/config/stacks/attributeLinker.ts @@ -1,7 +1,7 @@ -import { AttributeLinkerConfigurableProps } from '../../lib/workload/stateless/stacks/fmannotator/deploy/stack'; +import { FMAnnotatorConfigurableProps } from '../../lib/workload/stateless/stacks/fmannotator/deploy/stack'; import { eventBusName, vpcProps } from '../constants'; -export const getAttributeLinkerProps = (): AttributeLinkerConfigurableProps => { +export const getAttributeLinkerProps = (): FMAnnotatorConfigurableProps => { return { vpcProps, eventBusName, diff --git a/lib/workload/stateless/stacks/fmannotator/api.go b/lib/workload/stateless/stacks/fmannotator/api.go index b236cc4b2..2382b979d 100644 --- a/lib/workload/stateless/stacks/fmannotator/api.go +++ b/lib/workload/stateless/stacks/fmannotator/api.go @@ -42,7 +42,6 @@ func NewApiClient(config *Config, body io.Reader) (*ApiClient, error) { if err != nil { return nil, err } - req.Header.Add("Content-Type", "application/json") return &ApiClient{ Client: &http.Client{}, @@ -56,6 +55,12 @@ func (c *ApiClient) WithMethod(method string) *ApiClient { return c } +// WithHeader Add a header to the request +func (c *ApiClient) WithHeader(key string, value string) *ApiClient { + c.Request.Header.Add(key, value) + return c +} + // WithS3Endpoint Set the endpoint to query S3 records. func (c *ApiClient) WithS3Endpoint() *ApiClient { c.Request.URL.Path = "/api/v1/s3" diff --git a/lib/workload/stateless/stacks/fmannotator/api_test.go b/lib/workload/stateless/stacks/fmannotator/api_test.go index 137d0d713..347f5c9e1 100644 --- a/lib/workload/stateless/stacks/fmannotator/api_test.go +++ b/lib/workload/stateless/stacks/fmannotator/api_test.go @@ -26,14 +26,14 @@ func TestMarshallPortalRunId(t *testing.T) { } func TestApiBuild(t *testing.T) { - config := Config{"http://localhost:8000"} + config := Config{"http://localhost:8000", "token"} api, err := NewApiClient(&config, bytes.NewBuffer([]byte{})) require.NoError(t, err) api = api.WithMethod("PATCH").WithS3Endpoint().WithQuery(url.Values{ "key": {"value"}, - }) + }).WithHeader("Content-Type", "application/json") require.Equal(t, []string{"application/json"}, api.Request.Header.Values("Content-Type")) require.Equal(t, "PATCH", api.Request.Method) diff --git a/lib/workload/stateless/stacks/fmannotator/cmd/main.go b/lib/workload/stateless/stacks/fmannotator/cmd/main.go index 1ce7b59c1..c9c30d63a 100644 --- a/lib/workload/stateless/stacks/fmannotator/cmd/main.go +++ b/lib/workload/stateless/stacks/fmannotator/cmd/main.go @@ -1,10 +1,34 @@ package main import ( + "fmt" "github.com/aws/aws-lambda-go/lambda" + "github.com/aws/aws-secretsmanager-caching-go/secretcache" "github.com/umccr/orcabus/lib/workload/stateless/stacks/fmannotator" + "github.com/umccr/orcabus/lib/workload/stateless/stacks/fmannotator/schema/orcabus_workflowmanager/workflowrunstatechange" + "log/slog" ) +var ( + secretCache, _ = secretcache.New() +) + +func Handler(event workflowrunstatechange.Event) error { + config, err := fmannotator.LoadConfig() + if err != nil { + return err + } + + token, err := secretCache.GetSecretString(config.FileManagerSecret) + if err != nil { + return err + } + + slog.Info(fmt.Sprintf("token is: %v", token)) + + return fmannotator.Handler(event, &config, token) +} + func main() { - lambda.Start(fmannotator.Handler) + lambda.Start(Handler) } diff --git a/lib/workload/stateless/stacks/fmannotator/config.go b/lib/workload/stateless/stacks/fmannotator/config.go index 36a598f96..f638b3f2d 100644 --- a/lib/workload/stateless/stacks/fmannotator/config.go +++ b/lib/workload/stateless/stacks/fmannotator/config.go @@ -5,11 +5,12 @@ import "github.com/kelseyhightower/envconfig" // Config Configuration for the fmannotator type Config struct { FileManagerEndpoint string `required:"true" split_words:"true"` + FileManagerSecret string `required:"true" split_words:"true"` } // LoadConfig Load config from the environment. func LoadConfig() (Config, error) { var config Config - err := envconfig.Process("annotator", &config) + err := envconfig.Process("fmannotator", &config) return config, err } diff --git a/lib/workload/stateless/stacks/fmannotator/deploy/stack.ts b/lib/workload/stateless/stacks/fmannotator/deploy/stack.ts index 75ba561a9..31f7d3b66 100644 --- a/lib/workload/stateless/stacks/fmannotator/deploy/stack.ts +++ b/lib/workload/stateless/stacks/fmannotator/deploy/stack.ts @@ -12,36 +12,41 @@ import { GoFunction } from '@aws-cdk/aws-lambda-go-alpha'; import path from 'path'; import { Architecture } from 'aws-cdk-lib/aws-lambda'; import { EventBus, IEventBus, Rule } from 'aws-cdk-lib/aws-events'; +import { ISecret } from 'aws-cdk-lib/aws-secretsmanager'; +import { NamedLambdaRole } from '../../../../components/named-lambda-role'; +import { ManagedPolicy, PolicyStatement, Role } from 'aws-cdk-lib/aws-iam'; /** - * Config for the attribute linker. + * Config for the FM annotator. */ -export type AttributeLinkerConfig = { +export type FMAnnotatorConfig = { vpcProps: VpcLookupOptions; eventBusName: string; }; /** - * Props for the attribute linker stack which can be configured + * Props for the FM annotator stack which can be configured */ -export type AttributeLinkerConfigurableProps = StackProps & AttributeLinkerConfig; +export type FMAnnotatorConfigurableProps = StackProps & FMAnnotatorConfig; /** - * Props for the attribute linker stack. + * Props for the FM annotator stack. */ -export type AttributeLinkerProps = AttributeLinkerConfigurableProps & { +export type FMAnnotatorProps = FMAnnotatorConfigurableProps & { domainName: string; + secret: ISecret; }; /** - * Construct used to configure the attribute linker. + * Construct used to configure the FM annotator. */ -export class AttributeAnnotator extends Stack { +export class FMAnnotator extends Stack { private readonly vpc: IVpc; private readonly securityGroup: ISecurityGroup; private readonly eventBus: IEventBus; + private readonly role: Role; - constructor(scope: Construct, id: string, props: AttributeLinkerProps) { + constructor(scope: Construct, id: string, props: FMAnnotatorProps) { super(scope, id, props); this.vpc = Vpc.fromLookup(this, 'MainVpc', props.vpcProps); @@ -50,40 +55,66 @@ export class AttributeAnnotator extends Stack { this.securityGroup = new SecurityGroup(this, 'SecurityGroup', { vpc: this.vpc, allowAllOutbound: true, - description: 'Security group that allows the attribute linker Lambda to egress out.', + description: 'Security group that allows the annotator Lambda to egress out.', }); + this.role = new NamedLambdaRole(this, 'Role'); + this.addAwsManagedPolicy('service-role/AWSLambdaVPCAccessExecutionRole'); + // Need access to secrets to fetch FM JWT token. + this.addToPolicy( + new PolicyStatement({ + actions: ['secretsmanager:DescribeSecret', 'secretsmanager:GetSecretValue'], + resources: [props.secret.secretArn], + }) + ); + const entry = path.join(__dirname, '..'); const fn = new GoFunction(this, 'handler', { entry, environment: { - ANNOTATOR_FILEMANAGER_ENDPOINT: props.domainName, + FMANNOTATOR_FILE_MANAGER_ENDPOINT: props.domainName, + FMANNOTATOR_FILE_MANAGER_SECRET: props.secret.secretName, }, memorySize: 128, timeout: Duration.seconds(28), architecture: Architecture.ARM_64, + role: this.role, vpc: this.vpc, vpcSubnets: { subnetType: SubnetType.PRIVATE_WITH_EGRESS }, securityGroups: [this.securityGroup], }); const eventRule = new Rule(this, 'EventRule', { - description: 'Send WorkflowRunStateChange events to the AttributeLinker Lambda', + description: 'Send WorkflowRunStateChange events to the annotator Lambda', eventBus: this.eventBus, }); eventRule.addTarget(new targets.LambdaFunction(fn)); eventRule.addEventPattern({ - source: ['orcabus.workflowmanager'], + // Allow accepting a self-made event used for testing. + source: ['orcabus.workflowmanager', 'orcabus.fmannotator'], detailType: ['WorkflowRunStateChange'], detail: { status: [ { 'equals-ignore-case': 'SUCCEEDED' }, { 'equals-ignore-case': 'FAILED' }, { 'equals-ignore-case': 'ABORTED' }, - { 'equals-ignore-case': 'TEST' }, ], }, }); } + + /** + * Add an AWS managed policy to the function's role. + */ + addAwsManagedPolicy(policyName: string) { + this.role.addManagedPolicy(ManagedPolicy.fromAwsManagedPolicyName(policyName)); + } + + /** + * Add a policy statement to this function's role. + */ + addToPolicy(policyStatement: PolicyStatement) { + this.role.addToPolicy(policyStatement); + } } diff --git a/lib/workload/stateless/stacks/fmannotator/go.mod b/lib/workload/stateless/stacks/fmannotator/go.mod index 75dee0530..495148464 100644 --- a/lib/workload/stateless/stacks/fmannotator/go.mod +++ b/lib/workload/stateless/stacks/fmannotator/go.mod @@ -4,6 +4,7 @@ go 1.23 require ( github.com/aws/aws-lambda-go v1.47.0 + github.com/aws/aws-secretsmanager-caching-go v1.2.0 github.com/docker/go-connections v0.5.0 github.com/go-testfixtures/testfixtures/v3 v3.12.0 github.com/google/uuid v1.6.0 @@ -21,6 +22,7 @@ require ( github.com/ClickHouse/clickhouse-go/v2 v2.26.0 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/andybalholm/brotli v1.1.0 // indirect + github.com/aws/aws-sdk-go v1.47.10 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/containerd/containerd v1.7.18 // indirect github.com/containerd/log v0.1.0 // indirect @@ -38,6 +40,7 @@ require ( github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/klauspost/compress v1.17.7 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/magiconair/properties v1.8.7 // indirect diff --git a/lib/workload/stateless/stacks/fmannotator/go.sum b/lib/workload/stateless/stacks/fmannotator/go.sum index 7e6290736..179fbe3c6 100644 --- a/lib/workload/stateless/stacks/fmannotator/go.sum +++ b/lib/workload/stateless/stacks/fmannotator/go.sum @@ -16,6 +16,10 @@ github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1 github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/aws/aws-lambda-go v1.47.0 h1:0H8s0vumYx/YKs4sE7YM0ktwL2eWse+kfopsRI1sXVI= github.com/aws/aws-lambda-go v1.47.0/go.mod h1:dpMpZgvWx5vuQJfBt0zqBha60q7Dd7RfgJv23DymV8A= +github.com/aws/aws-sdk-go v1.47.10 h1:cvufN7WkD1nlOgpRopsmxKQlFp5X1MfyAw4r7BBORQc= +github.com/aws/aws-sdk-go v1.47.10/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= +github.com/aws/aws-secretsmanager-caching-go v1.2.0 h1:gUA+CVKvFLj4OUSknhIrnt4dF7Y37+JrChKqfaehJME= +github.com/aws/aws-secretsmanager-caching-go v1.2.0/go.mod h1:6t2/zQIsigFMlnpOdGj503Dgaz24tMqIRhass9uoTBo= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/containerd/containerd v1.7.18 h1:jqjZTQNfXGoEaZdW1WwPU0RqSn1Bm2Ay/KJPUuO8nao= @@ -92,6 +96,10 @@ github.com/jackc/pgtype v1.14.0 h1:y+xUdabmyMkJLyApYuPj38mW+aAIqCe5uuBB51rH3Vw= github.com/jackc/pgtype v1.14.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= github.com/jackc/pgx/v4 v4.18.3 h1:dE2/TrEsGX3RBprb3qryqSV9Y60iZN1C6i8IrmW9/BA= github.com/jackc/pgx/v4 v4.18.3/go.mod h1:Ey4Oru5tH5sB6tV7hDmfWFahwF15Eb7DNXlRKx2CkVw= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= @@ -183,6 +191,7 @@ github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgk github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= @@ -205,22 +214,37 @@ go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v8 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -232,19 +256,35 @@ golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= @@ -253,6 +293,8 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -271,6 +313,9 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/lib/workload/stateless/stacks/fmannotator/handler.go b/lib/workload/stateless/stacks/fmannotator/handler.go index 0cb9449a8..b1b3473d2 100644 --- a/lib/workload/stateless/stacks/fmannotator/handler.go +++ b/lib/workload/stateless/stacks/fmannotator/handler.go @@ -10,30 +10,26 @@ import ( "strings" ) -func Handler(event workflowrunstatechange.Event) (err error) { +// Handler Handle an incoming WorkflowRunStateChange event using the config and FM endpoint token. +func Handler(event workflowrunstatechange.Event, config *Config, token string) (err error) { eventStatus := strings.ToUpper(event.Detail.Status) if eventStatus != "SUCCEEDED" && eventStatus != "FAILED" && eventStatus != "ABORTED" { return nil } - config, err := LoadConfig() - if err != nil { - return err - } - patch, err := MarshallPortalRunId(&event) if err != nil { return err } - resp, err := NewApiClient(&config, bytes.NewBuffer(patch)) + resp, err := NewApiClient(config, bytes.NewBuffer(patch)) if err != nil { return err } resp = resp.WithMethod("PATCH").WithS3Endpoint().WithQuery(url.Values{ "key": {fmt.Sprintf("*%v*", event.Detail.PortalRunId)}, - }) + }).WithHeader("Content-Type", "application/json").WithHeader("Authorization", fmt.Sprintf("Bearer %v", token)) body, status, err := resp.Do() if err != nil { return err diff --git a/lib/workload/stateless/stacks/fmannotator/handler_test.go b/lib/workload/stateless/stacks/fmannotator/handler_test.go index be4ef68eb..448472f96 100644 --- a/lib/workload/stateless/stacks/fmannotator/handler_test.go +++ b/lib/workload/stateless/stacks/fmannotator/handler_test.go @@ -10,12 +10,6 @@ import ( "time" ) -var ( - fmEndpoint string - databaseEndpoint string - db *sql.DB -) - type TestCase struct { event string portalRunId string @@ -73,7 +67,7 @@ func failCase(location *time.Location) TestCase { } func TestHandler(t *testing.T) { - test.SetupFileManager(t, &fmEndpoint, &databaseEndpoint, db) + db := test.SetupFileManager(t) location, err := time.LoadLocation("Etc/UTC") require.NoError(t, err) @@ -84,7 +78,10 @@ func TestHandler(t *testing.T) { t.Run(fmt.Sprintf("%v", tc.event), func(t *testing.T) { event := test.CreateEvent(t, tc.event) - err := Handler(event) + config, err := LoadConfig() + require.NoError(t, err) + + err = Handler(event, &config, "token") require.NoError(t, err) s3Objects := test.QueryObjects(t, db, fmt.Sprintf("select * from s3_object where key like '%%%v%%'", tc.portalRunId)) diff --git a/lib/workload/stateless/stacks/fmannotator/internal/test/test.go b/lib/workload/stateless/stacks/fmannotator/internal/test/test.go index ca991fc68..600ee63d7 100644 --- a/lib/workload/stateless/stacks/fmannotator/internal/test/test.go +++ b/lib/workload/stateless/stacks/fmannotator/internal/test/test.go @@ -72,7 +72,7 @@ func setupService(t *testing.T, buildContext string, port nat.Port, wait wait.St return ip, port } -func SetupFileManager(t *testing.T, fmEndpoint *string, databaseEndpoint *string, db *sql.DB) { +func SetupFileManager(t *testing.T) *sql.DB { // This has to be such a large timeout because FM needs to compile/build with access to a Postgres database. If // the timeout is too low, before the FM compilation connects to the database, testcontainers will kill the postgres // container as there are no connections to it. @@ -103,22 +103,24 @@ func SetupFileManager(t *testing.T, fmEndpoint *string, databaseEndpoint *string "DATABASE_URL": databaseUrl, }) - *fmEndpoint = fmt.Sprintf("http://%v:%v", ip, port.Port()) - *databaseEndpoint = fmt.Sprintf(databaseFmt, databaseIp, intPort, testDatabaseName) + fmEndpoint := fmt.Sprintf("http://%v:%v", ip, port.Port()) + databaseEndpoint := fmt.Sprintf(databaseFmt, databaseIp, intPort, testDatabaseName) - t.Setenv("ANNOTATOR_FILE_MANAGER_ENDPOINT", *fmEndpoint) - loadFixtures(t, databaseEndpoint, db) + t.Setenv("FMANNOTATOR_FILE_MANAGER_ENDPOINT", fmEndpoint) + t.Setenv("FMANNOTATOR_FILE_MANAGER_SECRET", "secret") + + return loadFixtures(t, databaseEndpoint) } -func loadFixtures(t *testing.T, databaseUrl *string, db *sql.DB) { +func loadFixtures(t *testing.T, databaseUrl string) *sql.DB { var err error - db, err = sql.Open("postgres", *databaseUrl) + db, err := sql.Open("postgres", databaseUrl) require.NoError(t, err) fixtures, err := testfixtures.New( testfixtures.Database(db), testfixtures.Dialect("postgres"), - testfixtures.Directory("fixtures"), + testfixtures.Directory(fixturesPath()), ) require.NoError(t, err) @@ -126,13 +128,17 @@ func loadFixtures(t *testing.T, databaseUrl *string, db *sql.DB) { err = fixtures.Load() require.NoError(t, err) + + return db } -func CreateEvent(t *testing.T, path string) workflowrunstatechange.Event { +func fixturesPath() string { _, file, _, _ := runtime.Caller(0) - root := filepath.Join(filepath.Dir(file), "../..") + return filepath.Join(filepath.Dir(file), "../../fixtures") +} - b, err := os.ReadFile(filepath.Join(root, "fixtures", path)) +func CreateEvent(t *testing.T, path string) workflowrunstatechange.Event { + b, err := os.ReadFile(filepath.Join(fixturesPath(), path)) require.NoError(t, err) var event workflowrunstatechange.Event diff --git a/lib/workload/stateless/statelessStackCollectionClass.ts b/lib/workload/stateless/statelessStackCollectionClass.ts index 07d5b64da..6b564ff23 100644 --- a/lib/workload/stateless/statelessStackCollectionClass.ts +++ b/lib/workload/stateless/statelessStackCollectionClass.ts @@ -53,10 +53,7 @@ import { UmccriseIcav2PipelineManagerStack, UmccriseIcav2PipelineManagerStackProps, } from './stacks/umccrise-pipeline-manager/deploy'; -import { - AttributeAnnotator, - AttributeLinkerConfigurableProps, -} from './stacks/fmannotator/deploy/stack'; +import { FMAnnotator, FMAnnotatorConfigurableProps } from './stacks/fmannotator/deploy/stack'; export interface StatelessStackCollectionProps { metadataManagerStackProps: MetadataManagerStackProps; @@ -75,7 +72,7 @@ export interface StatelessStackCollectionProps { bclConvertManagerStackProps: BclConvertManagerStackProps; workflowManagerStackProps: WorkflowManagerStackProps; stackyMcStackFaceProps: GlueStackProps; - attributeLinkerProps: AttributeLinkerConfigurableProps; + fmAnnotatorProps: FMAnnotatorConfigurableProps; } export class StatelessStackCollection { @@ -96,7 +93,7 @@ export class StatelessStackCollection { readonly bclConvertManagerStack: Stack; readonly workflowManagerStack: Stack; readonly stackyMcStackFaceStack: Stack; - readonly attributeAnnotatorStack: Stack; + readonly fmAnnotator: Stack; constructor( scope: Construct, @@ -216,9 +213,9 @@ export class StatelessStackCollection { ...statelessConfiguration.stackyMcStackFaceProps, }); - this.attributeAnnotatorStack = new AttributeAnnotator(scope, 'AttributeLinkerStack', { - ...this.createTemplateProps(env, 'AttributeLinkerStack'), - ...statelessConfiguration.attributeLinkerProps, + this.fmAnnotator = new FMAnnotator(scope, 'FMAnnotatorStack', { + ...this.createTemplateProps(env, 'FMAnnotatorStack'), + ...statelessConfiguration.fmAnnotatorProps, domainName: fileManagerStack.domainName, }); } From a5ec456eb5ef5f459d395c376a5c37ab6623083c Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Tue, 3 Sep 2024 15:48:16 +1000 Subject: [PATCH 07/13] refactor(fmannotator): add README.md, fix deployment and tests --- config/config.ts | 4 +- config/stacks/attributeLinker.ts | 9 -- config/stacks/fmAnnotator.ts | 10 ++ .../stateless/stacks/fmannotator/Makefile | 7 +- .../stateless/stacks/fmannotator/README.md | 52 +++++++++++ .../stateless/stacks/fmannotator/api.go | 3 + .../fmannotator/cmd/{ => portalrunid}/main.go | 23 ++++- .../stateless/stacks/fmannotator/config.go | 23 ++++- .../stacks/fmannotator/deploy/stack.ts | 20 ++-- .../stacks/fmannotator/handler_test.go | 91 ------------------- .../fmannotator/{handler.go => handlers.go} | 17 ++-- .../stacks/fmannotator/handlers_test.go | 91 +++++++++++++++++++ .../stacks/fmannotator/internal/test/test.go | 2 +- 13 files changed, 217 insertions(+), 135 deletions(-) delete mode 100644 config/stacks/attributeLinker.ts create mode 100644 config/stacks/fmAnnotator.ts create mode 100644 lib/workload/stateless/stacks/fmannotator/README.md rename lib/workload/stateless/stacks/fmannotator/cmd/{ => portalrunid}/main.go (54%) delete mode 100644 lib/workload/stateless/stacks/fmannotator/handler_test.go rename lib/workload/stateless/stacks/fmannotator/{handler.go => handlers.go} (56%) create mode 100644 lib/workload/stateless/stacks/fmannotator/handlers_test.go diff --git a/config/config.ts b/config/config.ts index 56bd1f910..5f71085d0 100644 --- a/config/config.ts +++ b/config/config.ts @@ -43,7 +43,7 @@ import { getUmccriseIcav2PipelineManagerStackProps, getUmccriseIcav2PipelineTableStackProps, } from './stacks/umccrisePipelineManager'; -import { getAttributeLinkerProps } from './stacks/attributeLinker'; +import { getFmAnnotatorProps } from './stacks/fmAnnotator'; interface EnvironmentConfig { name: string; @@ -97,7 +97,7 @@ export const getEnvironmentConfig = (stage: AppStage): EnvironmentConfig | null bclConvertManagerStackProps: getBclConvertManagerStackProps(stage), workflowManagerStackProps: getWorkflowManagerStackProps(stage), stackyMcStackFaceProps: getGlueStackProps(stage), - attributeLinkerProps: getAttributeLinkerProps(), + fmAnnotatorProps: getFmAnnotatorProps(), }, }; diff --git a/config/stacks/attributeLinker.ts b/config/stacks/attributeLinker.ts deleted file mode 100644 index 069310223..000000000 --- a/config/stacks/attributeLinker.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { FMAnnotatorConfigurableProps } from '../../lib/workload/stateless/stacks/fmannotator/deploy/stack'; -import { eventBusName, vpcProps } from '../constants'; - -export const getAttributeLinkerProps = (): FMAnnotatorConfigurableProps => { - return { - vpcProps, - eventBusName, - }; -}; diff --git a/config/stacks/fmAnnotator.ts b/config/stacks/fmAnnotator.ts new file mode 100644 index 000000000..11112ea7e --- /dev/null +++ b/config/stacks/fmAnnotator.ts @@ -0,0 +1,10 @@ +import { FMAnnotatorConfigurableProps } from '../../lib/workload/stateless/stacks/fmannotator/deploy/stack'; +import { eventBusName, jwtSecretName, vpcProps } from '../constants'; + +export const getFmAnnotatorProps = (): FMAnnotatorConfigurableProps => { + return { + vpcProps, + eventBusName, + jwtSecretName, + }; +}; diff --git a/lib/workload/stateless/stacks/fmannotator/Makefile b/lib/workload/stateless/stacks/fmannotator/Makefile index 459b0edcc..ecd547b28 100644 --- a/lib/workload/stateless/stacks/fmannotator/Makefile +++ b/lib/workload/stateless/stacks/fmannotator/Makefile @@ -10,13 +10,10 @@ build: test: @go test lint: - @gofmt -l -lint-fix: - @gofmt + @go fmt golangci-lint: - @golangci-lint run --verbose + @golangci-lint run check: lint golangci-lint -check-fix: lint-fix golangci-lint ## Clean clean: diff --git a/lib/workload/stateless/stacks/fmannotator/README.md b/lib/workload/stateless/stacks/fmannotator/README.md new file mode 100644 index 000000000..3ee6603df --- /dev/null +++ b/lib/workload/stateless/stacks/fmannotator/README.md @@ -0,0 +1,52 @@ +# FMAnnotator + +The FMAnnotator service annotates records using the FileManager API. + +## Development + +This service is written in Go, which should be [installed][golang]. Please also install [golangci-lint] to run lints on +the codebase. + +This project is organised using the Go Lambda function CDK, and contains Lambda function handlers under [`cmd`][cmd] + +Makefile is used to simplify development. Tests can be run by using: + +```sh +make test +``` + +Lints and checks can be run using: + +```sh +make check +``` + +To update the [go.mod][go-mod] and download any new go modules: + +```sh +make clean +``` + +## Project layout + +This service has the following structure: + +* [cmd]: The Lambda handler `main.go` functions. +* [deploy]: CDK deployment code. +* [fixtures]: Database fixtures and test data. +* [internal]: Internal package containing common test code. +* [schema]: Generated EventBridge code-bindings. + +Top-level `.go` files contain library related code which implements the functionality of the FMAnnotator. +Tests are defined using the `_test.go` suffix. + +[golang]: https://go.dev/doc/install +[golangci-lint]: https://golangci-lint.run/welcome/install/#local-installation +[cmd]: cmd +[api]: api.go +[config]: config.go +[handlers]: handlers.go +[internal]: internal +[fixtures]: fixtures +[schema]: schema +[go-mod]: go.mod diff --git a/lib/workload/stateless/stacks/fmannotator/api.go b/lib/workload/stateless/stacks/fmannotator/api.go index 2382b979d..483db7587 100644 --- a/lib/workload/stateless/stacks/fmannotator/api.go +++ b/lib/workload/stateless/stacks/fmannotator/api.go @@ -1,3 +1,4 @@ +// Package fmannotator This file contains definitions for calling the FileManager API. package fmannotator import ( @@ -43,6 +44,8 @@ func NewApiClient(config *Config, body io.Reader) (*ApiClient, error) { return nil, err } + slog.Debug(fmt.Sprintf("host set to: %s", req.Host)) + return &ApiClient{ Client: &http.Client{}, Request: req, diff --git a/lib/workload/stateless/stacks/fmannotator/cmd/main.go b/lib/workload/stateless/stacks/fmannotator/cmd/portalrunid/main.go similarity index 54% rename from lib/workload/stateless/stacks/fmannotator/cmd/main.go rename to lib/workload/stateless/stacks/fmannotator/cmd/portalrunid/main.go index c9c30d63a..11267ed4e 100644 --- a/lib/workload/stateless/stacks/fmannotator/cmd/main.go +++ b/lib/workload/stateless/stacks/fmannotator/cmd/portalrunid/main.go @@ -1,7 +1,7 @@ package main import ( - "fmt" + "encoding/json" "github.com/aws/aws-lambda-go/lambda" "github.com/aws/aws-secretsmanager-caching-go/secretcache" "github.com/umccr/orcabus/lib/workload/stateless/stacks/fmannotator" @@ -9,24 +9,39 @@ import ( "log/slog" ) +const ( + TokenIdField = "id_token" +) + var ( secretCache, _ = secretcache.New() ) +// Handler for the portalRunId annotator function. func Handler(event workflowrunstatechange.Event) error { + level, err := fmannotator.GetLogLevel() + if err != nil { + return err + } + slog.SetLogLoggerLevel(level) + config, err := fmannotator.LoadConfig() if err != nil { return err } - token, err := secretCache.GetSecretString(config.FileManagerSecret) + secret, err := secretCache.GetSecretString(config.FileManagerSecretName) if err != nil { return err } - slog.Info(fmt.Sprintf("token is: %v", token)) + secretKeys := make(map[string]string) + err = json.Unmarshal([]byte(secret), &secretKeys) + if err != nil { + return err + } - return fmannotator.Handler(event, &config, token) + return fmannotator.PortalRunId(event, &config, secretKeys[TokenIdField]) } func main() { diff --git a/lib/workload/stateless/stacks/fmannotator/config.go b/lib/workload/stateless/stacks/fmannotator/config.go index f638b3f2d..4a51a2d95 100644 --- a/lib/workload/stateless/stacks/fmannotator/config.go +++ b/lib/workload/stateless/stacks/fmannotator/config.go @@ -1,16 +1,31 @@ +// Package fmannotator config using environment variables. package fmannotator -import "github.com/kelseyhightower/envconfig" +import ( + "github.com/kelseyhightower/envconfig" + "log/slog" + "os" +) + +const ( + EnvPrefix = "fmannotator" +) // Config Configuration for the fmannotator type Config struct { - FileManagerEndpoint string `required:"true" split_words:"true"` - FileManagerSecret string `required:"true" split_words:"true"` + FileManagerEndpoint string `required:"true" split_words:"true"` + FileManagerSecretName string `required:"true" split_words:"true"` } // LoadConfig Load config from the environment. func LoadConfig() (Config, error) { var config Config - err := envconfig.Process("fmannotator", &config) + err := envconfig.Process(EnvPrefix, &config) return config, err } + +func GetLogLevel() (slog.Level, error) { + var level slog.Level + err := level.UnmarshalText([]byte(os.Getenv("GO_LOG"))) + return level, err +} diff --git a/lib/workload/stateless/stacks/fmannotator/deploy/stack.ts b/lib/workload/stateless/stacks/fmannotator/deploy/stack.ts index 31f7d3b66..a7dae456c 100644 --- a/lib/workload/stateless/stacks/fmannotator/deploy/stack.ts +++ b/lib/workload/stateless/stacks/fmannotator/deploy/stack.ts @@ -12,7 +12,7 @@ import { GoFunction } from '@aws-cdk/aws-lambda-go-alpha'; import path from 'path'; import { Architecture } from 'aws-cdk-lib/aws-lambda'; import { EventBus, IEventBus, Rule } from 'aws-cdk-lib/aws-events'; -import { ISecret } from 'aws-cdk-lib/aws-secretsmanager'; +import { Secret } from 'aws-cdk-lib/aws-secretsmanager'; import { NamedLambdaRole } from '../../../../components/named-lambda-role'; import { ManagedPolicy, PolicyStatement, Role } from 'aws-cdk-lib/aws-iam'; @@ -22,6 +22,7 @@ import { ManagedPolicy, PolicyStatement, Role } from 'aws-cdk-lib/aws-iam'; export type FMAnnotatorConfig = { vpcProps: VpcLookupOptions; eventBusName: string; + jwtSecretName: string; }; /** @@ -34,7 +35,6 @@ export type FMAnnotatorConfigurableProps = StackProps & FMAnnotatorConfig; */ export type FMAnnotatorProps = FMAnnotatorConfigurableProps & { domainName: string; - secret: ISecret; }; /** @@ -58,22 +58,20 @@ export class FMAnnotator extends Stack { description: 'Security group that allows the annotator Lambda to egress out.', }); + const tokenSecret = Secret.fromSecretNameV2(this, 'JwtSecret', props.jwtSecretName); + this.role = new NamedLambdaRole(this, 'Role'); this.addAwsManagedPolicy('service-role/AWSLambdaVPCAccessExecutionRole'); // Need access to secrets to fetch FM JWT token. - this.addToPolicy( - new PolicyStatement({ - actions: ['secretsmanager:DescribeSecret', 'secretsmanager:GetSecretValue'], - resources: [props.secret.secretArn], - }) - ); + tokenSecret.grantRead(this.role); - const entry = path.join(__dirname, '..'); + const entry = path.join(__dirname, '..', 'cmd', 'portalrunid'); const fn = new GoFunction(this, 'handler', { entry, environment: { - FMANNOTATOR_FILE_MANAGER_ENDPOINT: props.domainName, - FMANNOTATOR_FILE_MANAGER_SECRET: props.secret.secretName, + FMANNOTATOR_FILE_MANAGER_ENDPOINT: `https://${props.domainName}`, + FMANNOTATOR_FILE_MANAGER_SECRET_NAME: tokenSecret.secretName, + GO_LOG: 'debug', }, memorySize: 128, timeout: Duration.seconds(28), diff --git a/lib/workload/stateless/stacks/fmannotator/handler_test.go b/lib/workload/stateless/stacks/fmannotator/handler_test.go deleted file mode 100644 index 448472f96..000000000 --- a/lib/workload/stateless/stacks/fmannotator/handler_test.go +++ /dev/null @@ -1,91 +0,0 @@ -package fmannotator - -import ( - "database/sql" - "fmt" - _ "github.com/lib/pq" - "github.com/stretchr/testify/require" - "github.com/umccr/orcabus/lib/workload/stateless/stacks/fmannotator/internal/test" - "testing" - "time" -) - -type TestCase struct { - event string - portalRunId string - expected []test.S3Object -} - -func successCase(location *time.Location) TestCase { - return TestCase{"event_succeeded.json", "202409021221e6e6", []test.S3Object{ - { - "Created", - "bucket", - "byob-icav2/development/analysis/wts/202409021221e6e6/_manifest.json", - sql.NullTime{Time: time.Date(2024, 9, 2, 0, 0, 0, 0, location), Valid: true}, - sql.NullInt64{Int64: 5, Valid: true}, - sql.NullString{String: "Standard", Valid: true}, - }, - { - "Deleted", - "bucket", - "byob-icav2/development/analysis/wts/202409021221e6e6/_manifest.json", - sql.NullTime{Time: time.Date(2024, 9, 3, 0, 0, 0, 0, location), Valid: true}, - sql.NullInt64{}, - sql.NullString{}, - }, - { - "Created", - "bucket", - "byob-icav2/development/analysis/wts/202409021221e6e6/_tags.json", - sql.NullTime{Time: time.Date(2024, 9, 4, 0, 0, 0, 0, location), Valid: true}, - sql.NullInt64{Int64: 10, Valid: true}, - sql.NullString{String: "Standard", Valid: true}, - }, - }} -} - -func failCase(location *time.Location) TestCase { - return TestCase{"event_failed.json", "202409021221e6c6", []test.S3Object{ - { - "Created", - "bucket", - "byob-icav2/development/analysis/wts/202409021221e6c6/_manifest.json", - sql.NullTime{Time: time.Date(2024, 9, 5, 0, 0, 0, 0, location), Valid: true}, - sql.NullInt64{Int64: 3, Valid: true}, - sql.NullString{String: "Standard", Valid: true}, - }, - { - "Deleted", - "bucket", - "byob-icav2/development/analysis/wts/202409021221e6c6/_manifest.json", - sql.NullTime{Time: time.Date(2024, 9, 6, 0, 0, 0, 0, location), Valid: true}, - sql.NullInt64{}, - sql.NullString{}, - }, - }} -} - -func TestHandler(t *testing.T) { - db := test.SetupFileManager(t) - - location, err := time.LoadLocation("Etc/UTC") - require.NoError(t, err) - - testCases := []TestCase{successCase(location), failCase(location)} - - for _, tc := range testCases { - t.Run(fmt.Sprintf("%v", tc.event), func(t *testing.T) { - event := test.CreateEvent(t, tc.event) - - config, err := LoadConfig() - require.NoError(t, err) - - err = Handler(event, &config, "token") - require.NoError(t, err) - - s3Objects := test.QueryObjects(t, db, fmt.Sprintf("select * from s3_object where key like '%%%v%%'", tc.portalRunId)) - require.Equal(t, tc.expected, s3Objects) - }) - } -} diff --git a/lib/workload/stateless/stacks/fmannotator/handler.go b/lib/workload/stateless/stacks/fmannotator/handlers.go similarity index 56% rename from lib/workload/stateless/stacks/fmannotator/handler.go rename to lib/workload/stateless/stacks/fmannotator/handlers.go index b1b3473d2..9542ae1e0 100644 --- a/lib/workload/stateless/stacks/fmannotator/handler.go +++ b/lib/workload/stateless/stacks/fmannotator/handlers.go @@ -1,8 +1,8 @@ +// Package fmannotator Lambda handler implementations. package fmannotator import ( "bytes" - "errors" "fmt" "github.com/umccr/orcabus/lib/workload/stateless/stacks/fmannotator/schema/orcabus_workflowmanager/workflowrunstatechange" "log/slog" @@ -10,8 +10,8 @@ import ( "strings" ) -// Handler Handle an incoming WorkflowRunStateChange event using the config and FM endpoint token. -func Handler(event workflowrunstatechange.Event, config *Config, token string) (err error) { +// PortalRunId Annotate the portalRunId from an incoming WorkflowRunStateChange event using the config and FM endpoint token. +func PortalRunId(event workflowrunstatechange.Event, config *Config, token string) (err error) { eventStatus := strings.ToUpper(event.Detail.Status) if eventStatus != "SUCCEEDED" && eventStatus != "FAILED" && eventStatus != "ABORTED" { return nil @@ -22,20 +22,21 @@ func Handler(event workflowrunstatechange.Event, config *Config, token string) ( return err } - resp, err := NewApiClient(config, bytes.NewBuffer(patch)) + req, err := NewApiClient(config, bytes.NewBuffer(patch)) if err != nil { return err } - resp = resp.WithMethod("PATCH").WithS3Endpoint().WithQuery(url.Values{ + req = req.WithMethod("PATCH").WithS3Endpoint().WithQuery(url.Values{ "key": {fmt.Sprintf("*%v*", event.Detail.PortalRunId)}, - }).WithHeader("Content-Type", "application/json").WithHeader("Authorization", fmt.Sprintf("Bearer %v", token)) - body, status, err := resp.Do() + }).WithHeader("Content-Type", "application/json").WithHeader("Authorization", fmt.Sprintf("Bearer %s", token)) + + body, status, err := req.Do() if err != nil { return err } if status != 200 { - return errors.New(fmt.Sprintf("error annotating attributes with status %v: %v", status, string(body))) + return fmt.Errorf("error annotating attributes with status %v: %v", status, string(body)) } slog.Debug(fmt.Sprintf("received response %v with body: %v", status, body)) diff --git a/lib/workload/stateless/stacks/fmannotator/handlers_test.go b/lib/workload/stateless/stacks/fmannotator/handlers_test.go new file mode 100644 index 000000000..23e11f864 --- /dev/null +++ b/lib/workload/stateless/stacks/fmannotator/handlers_test.go @@ -0,0 +1,91 @@ +package fmannotator + +import ( + "database/sql" + "fmt" + _ "github.com/lib/pq" + "github.com/stretchr/testify/require" + "github.com/umccr/orcabus/lib/workload/stateless/stacks/fmannotator/internal/test" + "testing" + "time" +) + +type TestCase struct { + event string + portalRunId string + expected []test.S3Object +} + +func successCase(location *time.Location) TestCase { + return TestCase{"event_succeeded.json", "202409021221e6e6", []test.S3Object{ + { + EventType: "Created", + Bucket: "bucket", + Key: "byob-icav2/development/analysis/wts/202409021221e6e6/_manifest.json", + EventTime: sql.NullTime{Time: time.Date(2024, 9, 2, 0, 0, 0, 0, location), Valid: true}, + Size: sql.NullInt64{Int64: 5, Valid: true}, + StorageClass: sql.NullString{String: "Standard", Valid: true}, + }, + { + EventType: "Deleted", + Bucket: "bucket", + Key: "byob-icav2/development/analysis/wts/202409021221e6e6/_manifest.json", + EventTime: sql.NullTime{Time: time.Date(2024, 9, 3, 0, 0, 0, 0, location), Valid: true}, + Size: sql.NullInt64{}, + StorageClass: sql.NullString{}, + }, + { + EventType: "Created", + Bucket: "bucket", + Key: "byob-icav2/development/analysis/wts/202409021221e6e6/_tags.json", + EventTime: sql.NullTime{Time: time.Date(2024, 9, 4, 0, 0, 0, 0, location), Valid: true}, + Size: sql.NullInt64{Int64: 10, Valid: true}, + StorageClass: sql.NullString{String: "Standard", Valid: true}, + }, + }} +} + +func failCase(location *time.Location) TestCase { + return TestCase{"event_failed.json", "202409021221e6c6", []test.S3Object{ + { + EventType: "Created", + Bucket: "bucket", + Key: "byob-icav2/development/analysis/wts/202409021221e6c6/_manifest.json", + EventTime: sql.NullTime{Time: time.Date(2024, 9, 5, 0, 0, 0, 0, location), Valid: true}, + Size: sql.NullInt64{Int64: 3, Valid: true}, + StorageClass: sql.NullString{String: "Standard", Valid: true}, + }, + { + EventType: "Deleted", + Bucket: "bucket", + Key: "byob-icav2/development/analysis/wts/202409021221e6c6/_manifest.json", + EventTime: sql.NullTime{Time: time.Date(2024, 9, 6, 0, 0, 0, 0, location), Valid: true}, + Size: sql.NullInt64{}, + StorageClass: sql.NullString{}, + }, + }} +} + +func TestHandler(t *testing.T) { + db := test.SetupFileManager(t) + + location, err := time.LoadLocation("Etc/UTC") + require.NoError(t, err) + + testCases := []TestCase{successCase(location), failCase(location)} + + for _, tc := range testCases { + t.Run(fmt.Sprintf("%v", tc.event), func(t *testing.T) { + event := test.CreateEvent(t, tc.event) + + config, err := LoadConfig() + require.NoError(t, err) + + err = PortalRunId(event, &config, "token") + require.NoError(t, err) + + s3Objects := test.QueryObjects(t, db, fmt.Sprintf("select * from s3_object where key like '%%%v%%'", tc.portalRunId)) + require.Equal(t, tc.expected, s3Objects) + }) + } +} diff --git a/lib/workload/stateless/stacks/fmannotator/internal/test/test.go b/lib/workload/stateless/stacks/fmannotator/internal/test/test.go index 600ee63d7..4b2e65d43 100644 --- a/lib/workload/stateless/stacks/fmannotator/internal/test/test.go +++ b/lib/workload/stateless/stacks/fmannotator/internal/test/test.go @@ -107,7 +107,7 @@ func SetupFileManager(t *testing.T) *sql.DB { databaseEndpoint := fmt.Sprintf(databaseFmt, databaseIp, intPort, testDatabaseName) t.Setenv("FMANNOTATOR_FILE_MANAGER_ENDPOINT", fmEndpoint) - t.Setenv("FMANNOTATOR_FILE_MANAGER_SECRET", "secret") + t.Setenv("FMANNOTATOR_FILE_MANAGER_SECRET_NAME", "secret") return loadFixtures(t, databaseEndpoint) } From 69f1a2becc848259f3a2635b63330c5998183acf Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Tue, 3 Sep 2024 16:00:14 +1000 Subject: [PATCH 08/13] build: update yarn.lock --- yarn.lock | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/yarn.lock b/yarn.lock index 9e180db17..ebae8b2a9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -36,6 +36,16 @@ __metadata: languageName: node linkType: hard +"@aws-cdk/aws-lambda-go-alpha@npm:^2.155.0-alpha.0": + version: 2.155.0-alpha.0 + resolution: "@aws-cdk/aws-lambda-go-alpha@npm:2.155.0-alpha.0" + peerDependencies: + aws-cdk-lib: ^2.155.0 + constructs: ^10.0.0 + checksum: 10/2dbea0a36bed606fdb31b3e1b7454d5d9dffc0ddfc0bc21f103a5b2845280657e23c22e5cea72511fb903253aba1409bc00493fa044c6de73cabe85cb4b517d6 + languageName: node + linkType: hard + "@aws-cdk/aws-lambda-python-alpha@npm:2.155.0-alpha.0": version: 2.155.0-alpha.0 resolution: "@aws-cdk/aws-lambda-python-alpha@npm:2.155.0-alpha.0" @@ -4003,6 +4013,7 @@ __metadata: version: 0.0.0-use.local resolution: "orcabus@workspace:." dependencies: + "@aws-cdk/aws-lambda-go-alpha": "npm:^2.155.0-alpha.0" "@aws-cdk/aws-lambda-python-alpha": "npm:2.155.0-alpha.0" "@aws-cdk/aws-pipes-alpha": "npm:^2.155.0-alpha.0" "@aws-cdk/aws-pipes-sources-alpha": "npm:^2.155.0-alpha.0" From d9a26b2390d1c356c4f8fc81e46136f45a757538 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Tue, 3 Sep 2024 16:07:05 +1000 Subject: [PATCH 09/13] style(fmannotator): .gitignore spacing --- lib/workload/stateless/stacks/fmannotator/.gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/workload/stateless/stacks/fmannotator/.gitignore b/lib/workload/stateless/stacks/fmannotator/.gitignore index c795b054e..378eac25d 100644 --- a/lib/workload/stateless/stacks/fmannotator/.gitignore +++ b/lib/workload/stateless/stacks/fmannotator/.gitignore @@ -1 +1 @@ -build \ No newline at end of file +build From 4894d9cc42b981da75199d6d9a13e7b3271fbadd Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Tue, 3 Sep 2024 16:19:25 +1000 Subject: [PATCH 10/13] fix(fmannotator): go version --- lib/workload/stateless/stacks/fmannotator/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/workload/stateless/stacks/fmannotator/go.mod b/lib/workload/stateless/stacks/fmannotator/go.mod index 495148464..00b7413ad 100644 --- a/lib/workload/stateless/stacks/fmannotator/go.mod +++ b/lib/workload/stateless/stacks/fmannotator/go.mod @@ -1,6 +1,6 @@ module github.com/umccr/orcabus/lib/workload/stateless/stacks/fmannotator -go 1.23 +go 1.22 require ( github.com/aws/aws-lambda-go v1.47.0 From b1d532604c30801387be73e57bf6a2871ddc0b2d Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Tue, 3 Sep 2024 17:07:33 +1000 Subject: [PATCH 11/13] ci: update go version inside github action runner --- .github/workflows/prbuild.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/prbuild.yml b/.github/workflows/prbuild.yml index 0dda8cb0f..458f04681 100644 --- a/.github/workflows/prbuild.yml +++ b/.github/workflows/prbuild.yml @@ -36,6 +36,7 @@ env: # # And from pyenv docs setting this PYENV_VERSION as env-var would set the python version globally PYENV_VERSION: '3.12' + GOENV_VERSION: '1.22' jobs: pre-commit-lint-security: @@ -82,21 +83,23 @@ jobs: runs-on: codebuild-orcabus-codebuild-gh-runner-${{ github.run_id }}-${{ github.run_attempt }} if: ${{ !github.event.pull_request.draft }} steps: + - name: Rust installation + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + components: rustfmt + - name: Print toolchain versions run: | node -v python -V pip3 -V + rustc --version + go version make --version - name: Checkout code uses: actions/checkout@v4 - - name: Rust installation - uses: actions-rust-lang/setup-rust-toolchain@v1 - with: - components: rustfmt - - name: Install dependencies run: | yarn install --immutable From 3e1bee3c581331b26ee2d417ac7f3da008e7e351 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Tue, 3 Sep 2024 17:15:28 +1000 Subject: [PATCH 12/13] fix(fmannotator): use '/' when querying the portal run id --- lib/workload/stateless/stacks/fmannotator/handlers.go | 2 +- lib/workload/stateless/stacks/fmannotator/handlers_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/workload/stateless/stacks/fmannotator/handlers.go b/lib/workload/stateless/stacks/fmannotator/handlers.go index 9542ae1e0..18ec65fcf 100644 --- a/lib/workload/stateless/stacks/fmannotator/handlers.go +++ b/lib/workload/stateless/stacks/fmannotator/handlers.go @@ -28,7 +28,7 @@ func PortalRunId(event workflowrunstatechange.Event, config *Config, token strin } req = req.WithMethod("PATCH").WithS3Endpoint().WithQuery(url.Values{ - "key": {fmt.Sprintf("*%v*", event.Detail.PortalRunId)}, + "key": {fmt.Sprintf("*/%v/*", event.Detail.PortalRunId)}, }).WithHeader("Content-Type", "application/json").WithHeader("Authorization", fmt.Sprintf("Bearer %s", token)) body, status, err := req.Do() diff --git a/lib/workload/stateless/stacks/fmannotator/handlers_test.go b/lib/workload/stateless/stacks/fmannotator/handlers_test.go index 23e11f864..c498ceba2 100644 --- a/lib/workload/stateless/stacks/fmannotator/handlers_test.go +++ b/lib/workload/stateless/stacks/fmannotator/handlers_test.go @@ -84,7 +84,7 @@ func TestHandler(t *testing.T) { err = PortalRunId(event, &config, "token") require.NoError(t, err) - s3Objects := test.QueryObjects(t, db, fmt.Sprintf("select * from s3_object where key like '%%%v%%'", tc.portalRunId)) + s3Objects := test.QueryObjects(t, db, fmt.Sprintf("select * from s3_object where key like '%%/%v/%%'", tc.portalRunId)) require.Equal(t, tc.expected, s3Objects) }) } From f14bcc2309657f0ca7a8762fd9d72bbcb0815823 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Tue, 3 Sep 2024 20:15:45 +1000 Subject: [PATCH 13/13] test(fmannotator): workaround for issue which expects a valid config.json --- .github/workflows/prbuild.yml | 1 + .../filemanager-build/src/gen_entities.rs | 2 ++ .../stateless/stacks/fmannotator/go.mod | 2 +- .../stacks/fmannotator/internal/test/test.go | 26 +++++++++++++++---- 4 files changed, 25 insertions(+), 6 deletions(-) diff --git a/.github/workflows/prbuild.yml b/.github/workflows/prbuild.yml index 458f04681..979706173 100644 --- a/.github/workflows/prbuild.yml +++ b/.github/workflows/prbuild.yml @@ -96,6 +96,7 @@ jobs: rustc --version go version make --version + docker version - name: Checkout code uses: actions/checkout@v4 diff --git a/lib/workload/stateless/stacks/filemanager/filemanager-build/src/gen_entities.rs b/lib/workload/stateless/stacks/filemanager/filemanager-build/src/gen_entities.rs index a3d186a25..50511cfa3 100644 --- a/lib/workload/stateless/stacks/filemanager/filemanager-build/src/gen_entities.rs +++ b/lib/workload/stateless/stacks/filemanager/filemanager-build/src/gen_entities.rs @@ -17,6 +17,8 @@ use crate::Config; pub async fn generate_entities() -> Result<()> { let config = Config::load()?; + println!("{:#?}", config); + let out_dir = config.out_dir; let command: &[&_] = &[ "sea-orm-cli", diff --git a/lib/workload/stateless/stacks/fmannotator/go.mod b/lib/workload/stateless/stacks/fmannotator/go.mod index 00b7413ad..b4f499f47 100644 --- a/lib/workload/stateless/stacks/fmannotator/go.mod +++ b/lib/workload/stateless/stacks/fmannotator/go.mod @@ -5,6 +5,7 @@ go 1.22 require ( github.com/aws/aws-lambda-go v1.47.0 github.com/aws/aws-secretsmanager-caching-go v1.2.0 + github.com/docker/docker v27.1.1+incompatible github.com/docker/go-connections v0.5.0 github.com/go-testfixtures/testfixtures/v3 v3.12.0 github.com/google/uuid v1.6.0 @@ -31,7 +32,6 @@ require ( github.com/creack/pty v1.1.21 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/distribution/reference v0.6.0 // indirect - github.com/docker/docker v27.1.1+incompatible // indirect github.com/docker/go-units v0.5.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-faster/city v1.0.1 // indirect diff --git a/lib/workload/stateless/stacks/fmannotator/internal/test/test.go b/lib/workload/stateless/stacks/fmannotator/internal/test/test.go index 4b2e65d43..93a7a85ec 100644 --- a/lib/workload/stateless/stacks/fmannotator/internal/test/test.go +++ b/lib/workload/stateless/stacks/fmannotator/internal/test/test.go @@ -5,6 +5,8 @@ import ( "database/sql" "encoding/json" "fmt" + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" "github.com/docker/go-connections/nat" "github.com/go-testfixtures/testfixtures/v3" "github.com/google/uuid" @@ -14,6 +16,7 @@ import ( "github.com/testcontainers/testcontainers-go/wait" "github.com/umccr/orcabus/lib/workload/stateless/stacks/fmannotator/schema/orcabus_workflowmanager/workflowrunstatechange" "os" + "path" "path/filepath" "runtime" "strconv" @@ -41,16 +44,23 @@ func setupService(t *testing.T, buildContext string, port nat.Port, wait wait.St containerName := strings.ReplaceAll(strings.Trim(buildContext, "./"), "/", "_") req := testcontainers.ContainerRequest{ FromDockerfile: testcontainers.FromDockerfile{ - Context: buildContext, - Repo: containerName, - Tag: containerName, - BuildArgs: args, - KeepImage: true, + Context: buildContext, + Repo: containerName, + Tag: containerName, + BuildArgs: args, + KeepImage: true, + PrintBuildLog: true, + BuildOptionsModifier: func(options *types.ImageBuildOptions) { + options.ExtraHosts = []string{"host.docker.internal:host-gateway"} + }, }, ExposedPorts: []string{port.Port()}, Env: env, WaitingFor: wait, Name: containerName, + HostConfigModifier: func(config *container.HostConfig) { + config.ExtraHosts = []string{"host.docker.internal:host-gateway"} + }, } container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ ContainerRequest: req, @@ -81,6 +91,12 @@ func SetupFileManager(t *testing.T) *sql.DB { t.Setenv("TESTCONTAINERS_RYUK_CONNECTION_TIMEOUT", "10m") t.Setenv("TESTCONTAINERS_RYUK_RECONNECTION_TIMEOUT", "5m") + // This works around an issue in test containers which requires the presence of a config.json file. + dir := t.TempDir() + err := os.WriteFile(path.Join(dir, "config.json"), []byte("{}"), 0666) + require.NoError(t, err) + t.Setenv("DOCKER_CONFIG", dir) + // Database testDatabaseName := fmt.Sprintf("filemanager_test_%v", strings.ReplaceAll(uuid.New().String(), "-", "_")) databaseIp, port := setupService(t, "../filemanager/database", "4321", wait.ForLog("database system is ready to accept connections").