diff --git a/.github/workflows/prbuild.yml b/.github/workflows/prbuild.yml index 0dda8cb0f..979706173 100644 --- a/.github/workflows/prbuild.yml +++ b/.github/workflows/prbuild.yml @@ -36,6 +36,7 @@ env: # # And from pyenv docs setting this PYENV_VERSION as env-var would set the python version globally PYENV_VERSION: '3.12' + GOENV_VERSION: '1.22' jobs: pre-commit-lint-security: @@ -82,21 +83,24 @@ jobs: runs-on: codebuild-orcabus-codebuild-gh-runner-${{ github.run_id }}-${{ github.run_attempt }} if: ${{ !github.event.pull_request.draft }} steps: + - name: Rust installation + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + components: rustfmt + - name: Print toolchain versions run: | node -v python -V pip3 -V + rustc --version + go version make --version + docker version - name: Checkout code uses: actions/checkout@v4 - - name: Rust installation - uses: actions-rust-lang/setup-rust-toolchain@v1 - with: - components: rustfmt - - name: Install dependencies run: | yarn install --immutable diff --git a/Makefile b/Makefile index 607dd112b..c41b99e54 100644 --- a/Makefile +++ b/Makefile @@ -53,6 +53,7 @@ test-stateless-app-suite: @(cd lib/workload/stateless/stacks/sequence-run-manager && $(MAKE) test) @(cd lib/workload/stateless/stacks/metadata-manager && $(MAKE) test) @(cd lib/workload/stateless/stacks/filemanager && $(MAKE) test) + @(cd lib/workload/stateless/stacks/fmannotator && $(MAKE) test) @(cd lib/workload/stateless/stacks/bclconvert-manager && $(MAKE) test) @(cd lib/workload/stateless/stacks/workflow-manager && $(MAKE) test) diff --git a/config/config.ts b/config/config.ts index 8621742ac..f7dd0a45a 100644 --- a/config/config.ts +++ b/config/config.ts @@ -47,6 +47,7 @@ import { getRnasumIcav2PipelineManagerStackProps, getRnasumIcav2PipelineTableStackProps, } from './stacks/rnasumPipelineManager'; +import { getFmAnnotatorProps } from './stacks/fmAnnotator'; interface EnvironmentConfig { name: string; @@ -102,6 +103,7 @@ export const getEnvironmentConfig = (stage: AppStage): EnvironmentConfig | null bclConvertManagerStackProps: getBclConvertManagerStackProps(stage), workflowManagerStackProps: getWorkflowManagerStackProps(stage), stackyMcStackFaceProps: getGlueStackProps(stage), + fmAnnotatorProps: getFmAnnotatorProps(), }, }; diff --git a/config/stacks/fmAnnotator.ts b/config/stacks/fmAnnotator.ts new file mode 100644 index 000000000..11112ea7e --- /dev/null +++ b/config/stacks/fmAnnotator.ts @@ -0,0 +1,10 @@ +import { FMAnnotatorConfigurableProps } from '../../lib/workload/stateless/stacks/fmannotator/deploy/stack'; +import { eventBusName, jwtSecretName, vpcProps } from '../constants'; + +export const getFmAnnotatorProps = (): FMAnnotatorConfigurableProps => { + return { + vpcProps, + eventBusName, + jwtSecretName, + }; +}; diff --git a/lib/pipeline/statelessPipelineStack.ts b/lib/pipeline/statelessPipelineStack.ts index f3b0acaad..6d28c3bd7 100644 --- a/lib/pipeline/statelessPipelineStack.ts +++ b/lib/pipeline/statelessPipelineStack.ts @@ -91,6 +91,7 @@ export class StatelessPipelineStack extends cdk.Stack { install: { 'runtime-versions': { python: '3.12', + golang: '1.22', }, }, }, @@ -126,6 +127,18 @@ export class StatelessPipelineStack extends cdk.Stack { resources: ['*'], }), ], + partialBuildSpec: codebuild.BuildSpec.fromObject({ + phases: { + install: { + 'runtime-versions': { + // Don't strictly need golang here as CDK can do docker bundling, but `local` + // bundling tends to be faster. + golang: '1.22', + }, + }, + }, + version: '0.2', + }), }); const pipeline = new pipelines.CodePipeline(this, 'Pipeline', { diff --git a/lib/workload/components/api-gateway/index.ts b/lib/workload/components/api-gateway/index.ts index a359c1bc3..2c3c6f3a7 100644 --- a/lib/workload/components/api-gateway/index.ts +++ b/lib/workload/components/api-gateway/index.ts @@ -42,6 +42,7 @@ export interface ApiGatewayConstructProps { export class ApiGatewayConstruct extends Construct { private readonly _httpApi: HttpApi; + private readonly _domainName: string; constructor(scope: Construct, id: string, props: ApiGatewayConstructProps) { super(scope, id); @@ -54,9 +55,9 @@ export class ApiGatewayConstruct extends Construct { ); const hostedZoneId = StringParameter.valueForStringParameter(this, '/hosted_zone/umccr/id'); - const domainName = `${props.customDomainNamePrefix}.${hostedDomainName}`; + this._domainName = `${props.customDomainNamePrefix}.${hostedDomainName}`; const apiGWDomainName = new DomainName(this, 'UmccrDomainName', { - domainName: `${props.customDomainNamePrefix}.${hostedDomainName}`, + domainName: this.domainName, certificate: Certificate.fromCertificateArn(this, 'cert', umccrAcmArn), }); @@ -85,7 +86,7 @@ export class ApiGatewayConstruct extends Construct { hostedZoneId, zoneName: hostedDomainName, }), - recordName: domainName, + recordName: this.domainName, target: RecordTarget.fromAlias( new ApiGatewayv2DomainProperties( apiGWDomainName.regionalDomainName, @@ -177,4 +178,8 @@ export class ApiGatewayConstruct extends Construct { get httpApi(): HttpApi { return this._httpApi; } + + get domainName(): string { + return this._domainName; + } } diff --git a/lib/workload/stateless/stacks/filemanager/.env.example b/lib/workload/stateless/stacks/filemanager/.env.example index 88a97b5da..781acc1e4 100644 --- a/lib/workload/stateless/stacks/filemanager/.env.example +++ b/lib/workload/stateless/stacks/filemanager/.env.example @@ -3,6 +3,7 @@ FILEMANAGER_DATABASE_HOST=0.0.0.0 FILEMANAGER_DATABASE_PORT=4321 DATABASE_URL=postgresql://filemanager:filemanager@${FILEMANAGER_DATABASE_HOST}:${FILEMANAGER_DATABASE_PORT}/filemanager #pragma: allowlist secret +API_PORT=8000 FILEMANAGER_LINKS_URL=localhost:8000 FILEMANAGER_API_CORS_ALLOW_ORIGINS=http://localhost:8000 diff --git a/lib/workload/stateless/stacks/filemanager/Makefile b/lib/workload/stateless/stacks/filemanager/Makefile index 8d34e8509..a020bb3b0 100644 --- a/lib/workload/stateless/stacks/filemanager/Makefile +++ b/lib/workload/stateless/stacks/filemanager/Makefile @@ -40,7 +40,7 @@ docker-find: @docker ps --filter name=$(DOCKER_PROJECT_NAME)-postgres --latest --format "{{.ID}}" | xargs -I {} docker port {} | tail -n 1 | awk '{print $$NF}' docker-api: docker-postgres # Run the local API server in a docker container. - @docker compose -p "$(DOCKER_PROJECT_NAME)" up api + @docker compose -p "$(DOCKER_PROJECT_NAME)" -f compose-api.yml up api ## Build related commands build: docker-postgres diff --git a/lib/workload/stateless/stacks/filemanager/compose-api.yml b/lib/workload/stateless/stacks/filemanager/compose-api.yml new file mode 100644 index 000000000..360ab5769 --- /dev/null +++ b/lib/workload/stateless/stacks/filemanager/compose-api.yml @@ -0,0 +1,20 @@ +services: + api: + build: + context: . + args: + # The build itself needs access to the database. + - DATABASE_URL=${BUILD_DATABASE_URL:-postgresql://filemanager:filemanager@host.docker.internal:4321/filemanager} # pragma: allowlist secret + environment: + # Container database address for running server inside a docker container. + - DATABASE_URL=${DATABASE_URL:-postgresql://filemanager:filemanager@postgres:4321/filemanager} + - RUST_LOG=debug + - FILEMANAGER_API_CORS_ALLOW_ORIGINS=${FILEMANAGER_API_CORS_ALLOW_ORIGINS:-http://localhost:3000} + - FILEMANAGER_API_CORS_ALLOW_HEADERS=${FILEMANAGER_API_CORS_ALLOW_HEADERS:-accept,authorization,content-type,user-agent,x-csrftoken,x-requested-with,x-amz-security-token,x-amz-date,content-disposition} + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-access_key_id} + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-secret_access_key} + - AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION:-ap-southeast-2} + - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN:-session_token} + ports: + - '${API_PORT}:8000' + restart: always diff --git a/lib/workload/stateless/stacks/filemanager/compose.yml b/lib/workload/stateless/stacks/filemanager/compose.yml index 234e01807..f9bd6761b 100644 --- a/lib/workload/stateless/stacks/filemanager/compose.yml +++ b/lib/workload/stateless/stacks/filemanager/compose.yml @@ -3,30 +3,9 @@ services: build: database restart: always environment: - - POSTGRES_DATABASE=filemanager + - POSTGRES_DB=filemanager - POSTGRES_USER=filemanager - POSTGRES_PASSWORD=filemanager - PGPORT=4321 ports: - "${FILEMANAGER_DATABASE_HOST}:${FILEMANAGER_DATABASE_PORT}:4321" - api: - build: - context: . - args: - # The build itself needs access to the database. - DATABASE_URL: postgresql://filemanager:filemanager@host.docker.internal:4321/filemanager # pragma: allowlist secret - environment: - # Container database address for running server inside a docker container. - - DATABASE_URL=postgresql://filemanager:filemanager@postgres:4321/filemanager - - RUST_LOG=debug - - FILEMANAGER_API_CORS_ALLOW_ORIGINS=${FILEMANAGER_API_CORS_ALLOW_ORIGINS:-http://localhost:3000} - - FILEMANAGER_API_CORS_ALLOW_HEADERS=${FILEMANAGER_API_CORS_ALLOW_HEADERS:-accept,authorization,content-type,user-agent,x-csrftoken,x-requested-with,x-amz-security-token,x-amz-date,content-disposition} - - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-access_key_id} - - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-secret_access_key} - - AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION:-ap-southeast-2} - - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN:-session_token} - ports: - - "8000:8000" - depends_on: - - postgres - restart: always diff --git a/lib/workload/stateless/stacks/filemanager/deploy/stack.ts b/lib/workload/stateless/stacks/filemanager/deploy/stack.ts index bd338f50f..829d06b10 100644 --- a/lib/workload/stateless/stacks/filemanager/deploy/stack.ts +++ b/lib/workload/stateless/stacks/filemanager/deploy/stack.ts @@ -48,6 +48,7 @@ export class Filemanager extends Stack { private readonly host: string; private readonly securityGroup: ISecurityGroup; private readonly queue: IQueue; + readonly domainName: string; constructor(scope: Construct, id: string, props: FilemanagerProps) { super(scope, id, props); @@ -93,8 +94,9 @@ export class Filemanager extends Stack { ); this.createIngestFunction(props); - this.createApiFunction(props); this.createInventoryFunction(props); + + this.domainName = this.createApiFunction(props); } private createIngestRole(name: string) { @@ -130,9 +132,9 @@ export class Filemanager extends Stack { } /** - * Query function and API Gateway fronting the function. + * Query function and API Gateway fronting the function. Returns the configured domain name. */ - private createApiFunction(props: FilemanagerProps) { + private createApiFunction(props: FilemanagerProps): string { let apiLambda = new ApiFunction(this, 'ApiFunction', { vpc: this.vpc, host: this.host, @@ -141,13 +143,14 @@ export class Filemanager extends Stack { ...props, }); - const ApiGateway = new ApiGatewayConstruct(this, 'ApiGateway', { + const apiGateway = new ApiGatewayConstruct(this, 'ApiGateway', { region: this.region, apiName: 'FileManager', customDomainNamePrefix: 'file', ...props, }); - const httpApi = ApiGateway.httpApi; + + const httpApi = apiGateway.httpApi; const apiIntegration = new HttpLambdaIntegration('ApiIntegration', apiLambda.function); @@ -156,5 +159,7 @@ export class Filemanager extends Stack { integration: apiIntegration, routeKey: HttpRouteKey.with('/{proxy+}', HttpMethod.ANY), }); + + return apiGateway.domainName; } } diff --git a/lib/workload/stateless/stacks/filemanager/filemanager-build/src/gen_entities.rs b/lib/workload/stateless/stacks/filemanager/filemanager-build/src/gen_entities.rs index a3d186a25..50511cfa3 100644 --- a/lib/workload/stateless/stacks/filemanager/filemanager-build/src/gen_entities.rs +++ b/lib/workload/stateless/stacks/filemanager/filemanager-build/src/gen_entities.rs @@ -17,6 +17,8 @@ use crate::Config; pub async fn generate_entities() -> Result<()> { let config = Config::load()?; + println!("{:#?}", config); + let out_dir = config.out_dir; let command: &[&_] = &[ "sea-orm-cli", diff --git a/lib/workload/stateless/stacks/fmannotator/.gitignore b/lib/workload/stateless/stacks/fmannotator/.gitignore new file mode 100644 index 000000000..378eac25d --- /dev/null +++ b/lib/workload/stateless/stacks/fmannotator/.gitignore @@ -0,0 +1 @@ +build diff --git a/lib/workload/stateless/stacks/fmannotator/Makefile b/lib/workload/stateless/stacks/fmannotator/Makefile new file mode 100644 index 000000000..ecd547b28 --- /dev/null +++ b/lib/workload/stateless/stacks/fmannotator/Makefile @@ -0,0 +1,28 @@ +## Default target +default: help + +install: build +all: build + +## Build related commands +build: + @go build -o build +test: + @go test +lint: + @go fmt +golangci-lint: + @golangci-lint run +check: lint golangci-lint + +## Clean +clean: + @go mod tidy + +## Help text +help: + @printf "The attribute annotator Makefile.\n\n" + @printf "Usage: make \n" + +# No targets are files, so they are all PHONY. +.PHONY: * diff --git a/lib/workload/stateless/stacks/fmannotator/README.md b/lib/workload/stateless/stacks/fmannotator/README.md new file mode 100644 index 000000000..3ee6603df --- /dev/null +++ b/lib/workload/stateless/stacks/fmannotator/README.md @@ -0,0 +1,52 @@ +# FMAnnotator + +The FMAnnotator service annotates records using the FileManager API. + +## Development + +This service is written in Go, which should be [installed][golang]. Please also install [golangci-lint] to run lints on +the codebase. + +This project is organised using the Go Lambda function CDK, and contains Lambda function handlers under [`cmd`][cmd] + +Makefile is used to simplify development. Tests can be run by using: + +```sh +make test +``` + +Lints and checks can be run using: + +```sh +make check +``` + +To update the [go.mod][go-mod] and download any new go modules: + +```sh +make clean +``` + +## Project layout + +This service has the following structure: + +* [cmd]: The Lambda handler `main.go` functions. +* [deploy]: CDK deployment code. +* [fixtures]: Database fixtures and test data. +* [internal]: Internal package containing common test code. +* [schema]: Generated EventBridge code-bindings. + +Top-level `.go` files contain library related code which implements the functionality of the FMAnnotator. +Tests are defined using the `_test.go` suffix. + +[golang]: https://go.dev/doc/install +[golangci-lint]: https://golangci-lint.run/welcome/install/#local-installation +[cmd]: cmd +[api]: api.go +[config]: config.go +[handlers]: handlers.go +[internal]: internal +[fixtures]: fixtures +[schema]: schema +[go-mod]: go.mod diff --git a/lib/workload/stateless/stacks/fmannotator/api.go b/lib/workload/stateless/stacks/fmannotator/api.go new file mode 100644 index 000000000..483db7587 --- /dev/null +++ b/lib/workload/stateless/stacks/fmannotator/api.go @@ -0,0 +1,94 @@ +// Package fmannotator This file contains definitions for calling the FileManager API. +package fmannotator + +import ( + "encoding/json" + "errors" + "fmt" + "github.com/umccr/orcabus/lib/workload/stateless/stacks/fmannotator/schema/orcabus_workflowmanager/workflowrunstatechange" + "io" + "log/slog" + "net/http" + "net/url" +) + +// A JsonPatch used to update FM records. +type JsonPatch struct { + Op string `json:"op"` + Path string `json:"path"` + Value string `json:"value,omitempty"` +} + +// The PatchList for updating records. +type PatchList []JsonPatch + +// MarshallPortalRunId MarshallPatch Convert an event into a JSON patch using the portalRunId. +func MarshallPortalRunId(event *workflowrunstatechange.Event) ([]byte, error) { + return json.Marshal(PatchList{JsonPatch{ + "add", + "/portalRunId", + event.Detail.PortalRunId, + }}) +} + +// The ApiClient which is used to send requests to filemanager. +type ApiClient struct { + Client *http.Client + Request *http.Request +} + +// NewApiClient Create a new ApiClient. +func NewApiClient(config *Config, body io.Reader) (*ApiClient, error) { + req, err := http.NewRequest("GET", config.FileManagerEndpoint, body) + if err != nil { + return nil, err + } + + slog.Debug(fmt.Sprintf("host set to: %s", req.Host)) + + return &ApiClient{ + Client: &http.Client{}, + Request: req, + }, nil +} + +// WithMethod Set the method for the request. +func (c *ApiClient) WithMethod(method string) *ApiClient { + c.Request.Method = method + return c +} + +// WithHeader Add a header to the request +func (c *ApiClient) WithHeader(key string, value string) *ApiClient { + c.Request.Header.Add(key, value) + return c +} + +// WithS3Endpoint Set the endpoint to query S3 records. +func (c *ApiClient) WithS3Endpoint() *ApiClient { + c.Request.URL.Path = "/api/v1/s3" + return c +} + +// WithQuery Set the query parameters. +func (c *ApiClient) WithQuery(query url.Values) *ApiClient { + c.Request.URL.RawQuery = query.Encode() + return c +} + +// Do the request returning the response body and status code. +func (c *ApiClient) Do() ([]byte, int, error) { + slog.Debug(fmt.Sprintf("sending request to %v", c.Request.URL.String())) + + resp, err := c.Client.Do(c.Request) + if err != nil { + return []byte{}, 0, err + } + defer func() { + err = errors.Join(resp.Body.Close()) + }() + + status := resp.StatusCode + body, err := io.ReadAll(resp.Body) + return body, status, err +} diff --git a/lib/workload/stateless/stacks/fmannotator/api_test.go b/lib/workload/stateless/stacks/fmannotator/api_test.go new file mode 100644 index 000000000..347f5c9e1 --- /dev/null +++ b/lib/workload/stateless/stacks/fmannotator/api_test.go @@ -0,0 +1,43 @@ +package fmannotator + +import ( + "bytes" + "encoding/json" + "github.com/stretchr/testify/require" + "github.com/umccr/orcabus/lib/workload/stateless/stacks/fmannotator/internal/test" + "net/url" + "testing" +) + +func TestMarshallPortalRunId(t *testing.T) { + event := test.CreateEvent(t, "event_succeeded.json") + body, err := MarshallPortalRunId(&event) + require.NoError(t, err) + + var patch PatchList + err = json.Unmarshal(body, &patch) + require.NoError(t, err) + + require.Equal(t, PatchList{JsonPatch{ + "add", + "/portalRunId", + "202409021221e6e6", + }}, patch) +} + +func TestApiBuild(t *testing.T) { + config := Config{"http://localhost:8000", "token"} + + api, err := NewApiClient(&config, bytes.NewBuffer([]byte{})) + require.NoError(t, err) + + api = api.WithMethod("PATCH").WithS3Endpoint().WithQuery(url.Values{ + "key": {"value"}, + }).WithHeader("Content-Type", "application/json") + + require.Equal(t, []string{"application/json"}, api.Request.Header.Values("Content-Type")) + require.Equal(t, "PATCH", api.Request.Method) + require.Equal(t, "localhost:8000", api.Request.URL.Host) + require.Equal(t, "/api/v1/s3", api.Request.URL.Path) + require.Equal(t, "key=value", api.Request.URL.RawQuery) +} diff --git a/lib/workload/stateless/stacks/fmannotator/cmd/portalrunid/main.go b/lib/workload/stateless/stacks/fmannotator/cmd/portalrunid/main.go new file mode 100644 index 000000000..11267ed4e --- /dev/null +++ b/lib/workload/stateless/stacks/fmannotator/cmd/portalrunid/main.go @@ -0,0 +1,49 @@ +package main + +import ( + "encoding/json" + "github.com/aws/aws-lambda-go/lambda" + "github.com/aws/aws-secretsmanager-caching-go/secretcache" + "github.com/umccr/orcabus/lib/workload/stateless/stacks/fmannotator" + "github.com/umccr/orcabus/lib/workload/stateless/stacks/fmannotator/schema/orcabus_workflowmanager/workflowrunstatechange" + "log/slog" +) + +const ( + TokenIdField = "id_token" +) + +var ( + secretCache, _ = secretcache.New() +) + +// Handler for the portalRunId annotator function. +func Handler(event workflowrunstatechange.Event) error { + level, err := fmannotator.GetLogLevel() + if err != nil { + return err + } + slog.SetLogLoggerLevel(level) + + config, err := fmannotator.LoadConfig() + if err != nil { + return err + } + + secret, err := secretCache.GetSecretString(config.FileManagerSecretName) + if err != nil { + return err + } + + secretKeys := make(map[string]string) + err = json.Unmarshal([]byte(secret), &secretKeys) + if err != nil { + return err + } + + return fmannotator.PortalRunId(event, &config, secretKeys[TokenIdField]) +} + +func main() { + lambda.Start(Handler) +} diff --git a/lib/workload/stateless/stacks/fmannotator/config.go b/lib/workload/stateless/stacks/fmannotator/config.go new file mode 100644 index 000000000..4a51a2d95 --- /dev/null +++ b/lib/workload/stateless/stacks/fmannotator/config.go @@ -0,0 +1,31 @@ +// Package fmannotator config using environment variables. +package fmannotator + +import ( + "github.com/kelseyhightower/envconfig" + "log/slog" + "os" +) + +const ( + EnvPrefix = "fmannotator" +) + +// Config Configuration for the fmannotator +type Config struct { + FileManagerEndpoint string `required:"true" split_words:"true"` + FileManagerSecretName string `required:"true" split_words:"true"` +} + +// LoadConfig Load config from the environment. +func LoadConfig() (Config, error) { + var config Config + err := envconfig.Process(EnvPrefix, &config) + return config, err +} + +func GetLogLevel() (slog.Level, error) { + var level slog.Level + err := level.UnmarshalText([]byte(os.Getenv("GO_LOG"))) + return level, err +} diff --git a/lib/workload/stateless/stacks/fmannotator/deploy/stack.ts b/lib/workload/stateless/stacks/fmannotator/deploy/stack.ts new file mode 100644 index 000000000..a7dae456c --- /dev/null +++ b/lib/workload/stateless/stacks/fmannotator/deploy/stack.ts @@ -0,0 +1,118 @@ +import { aws_events_targets as targets, Duration, Stack, StackProps } from 'aws-cdk-lib'; +import { + ISecurityGroup, + IVpc, + SecurityGroup, + SubnetType, + Vpc, + VpcLookupOptions, +} from 'aws-cdk-lib/aws-ec2'; +import { Construct } from 'constructs'; +import { GoFunction } from '@aws-cdk/aws-lambda-go-alpha'; +import path from 'path'; +import { Architecture } from 'aws-cdk-lib/aws-lambda'; +import { EventBus, IEventBus, Rule } from 'aws-cdk-lib/aws-events'; +import { Secret } from 'aws-cdk-lib/aws-secretsmanager'; +import { NamedLambdaRole } from '../../../../components/named-lambda-role'; +import { ManagedPolicy, PolicyStatement, Role } from 'aws-cdk-lib/aws-iam'; + +/** + * Config for the FM annotator. + */ +export type FMAnnotatorConfig = { + vpcProps: VpcLookupOptions; + eventBusName: string; + jwtSecretName: string; +}; + +/** + * Props for the FM annotator stack which can be configured + */ +export type FMAnnotatorConfigurableProps = StackProps & FMAnnotatorConfig; + +/** + * Props for the FM annotator stack. + */ +export type FMAnnotatorProps = FMAnnotatorConfigurableProps & { + domainName: string; +}; + +/** + * Construct used to configure the FM annotator. + */ +export class FMAnnotator extends Stack { + private readonly vpc: IVpc; + private readonly securityGroup: ISecurityGroup; + private readonly eventBus: IEventBus; + private readonly role: Role; + + constructor(scope: Construct, id: string, props: FMAnnotatorProps) { + super(scope, id, props); + + this.vpc = Vpc.fromLookup(this, 'MainVpc', props.vpcProps); + this.eventBus = EventBus.fromEventBusName(this, 'OrcaBusMain', props.eventBusName); + + this.securityGroup = new SecurityGroup(this, 'SecurityGroup', { + vpc: this.vpc, + allowAllOutbound: true, + description: 'Security group that allows the annotator Lambda to egress out.', + }); + + const tokenSecret = Secret.fromSecretNameV2(this, 'JwtSecret', props.jwtSecretName); + + this.role = new NamedLambdaRole(this, 'Role'); + this.addAwsManagedPolicy('service-role/AWSLambdaVPCAccessExecutionRole'); + // Need access to secrets to fetch FM JWT token. + tokenSecret.grantRead(this.role); + + const entry = path.join(__dirname, '..', 'cmd', 'portalrunid'); + const fn = new GoFunction(this, 'handler', { + entry, + environment: { + FMANNOTATOR_FILE_MANAGER_ENDPOINT: `https://${props.domainName}`, + FMANNOTATOR_FILE_MANAGER_SECRET_NAME: tokenSecret.secretName, + GO_LOG: 'debug', + }, + memorySize: 128, + timeout: Duration.seconds(28), + architecture: Architecture.ARM_64, + role: this.role, + vpc: this.vpc, + vpcSubnets: { subnetType: SubnetType.PRIVATE_WITH_EGRESS }, + securityGroups: [this.securityGroup], + }); + + const eventRule = new Rule(this, 'EventRule', { + description: 'Send WorkflowRunStateChange events to the annotator Lambda', + eventBus: this.eventBus, + }); + + eventRule.addTarget(new targets.LambdaFunction(fn)); + eventRule.addEventPattern({ + // Allow accepting a self-made event used for testing. + source: ['orcabus.workflowmanager', 'orcabus.fmannotator'], + detailType: ['WorkflowRunStateChange'], + detail: { + status: [ + { 'equals-ignore-case': 'SUCCEEDED' }, + { 'equals-ignore-case': 'FAILED' }, + { 'equals-ignore-case': 'ABORTED' }, + ], + }, + }); + } + + /** + * Add an AWS managed policy to the function's role. + */ + addAwsManagedPolicy(policyName: string) { + this.role.addManagedPolicy(ManagedPolicy.fromAwsManagedPolicyName(policyName)); + } + + /** + * Add a policy statement to this function's role. + */ + addToPolicy(policyStatement: PolicyStatement) { + this.role.addToPolicy(policyStatement); + } +} diff --git a/lib/workload/stateless/stacks/fmannotator/fixtures/event_failed.json b/lib/workload/stateless/stacks/fmannotator/fixtures/event_failed.json new file mode 100644 index 000000000..bab2590d9 --- /dev/null +++ b/lib/workload/stateless/stacks/fmannotator/fixtures/event_failed.json @@ -0,0 +1,18 @@ +{ + "version": "0", + "id": "ae02404d-9965-e3c7-2343-89cb5d46fedf", + "detail-type": "WorkflowRunStateChange", + "source": "orcabus.workflowmanager", + "account": "123456789012", + "time": "2024-09-02T00:00:01Z", + "region": "ap-southeast-2", + "resources": [], + "detail": { + "portalRunId": "202409021221e6c6", + "workflowName": "wts", + "workflowRunName": "umccr--automated--wts--4-2-4--202409021221e6c6", + "workflowVersion": "4.2.4", + "timestamp": "2024-09-02T00:00:00Z", + "status": "FAILED" + } +} diff --git a/lib/workload/stateless/stacks/fmannotator/fixtures/event_succeeded.json b/lib/workload/stateless/stacks/fmannotator/fixtures/event_succeeded.json new file mode 100644 index 000000000..a945ab17b --- /dev/null +++ b/lib/workload/stateless/stacks/fmannotator/fixtures/event_succeeded.json @@ -0,0 +1,18 @@ +{ + "version": "0", + "id": "ae02404d-9965-e3c7-2343-89cb5d46fedf", + "detail-type": "WorkflowRunStateChange", + "source": "orcabus.workflowmanager", + "account": "123456789012", + "time": "2024-09-02T00:00:01Z", + "region": "ap-southeast-2", + "resources": [], + "detail": { + "portalRunId": "202409021221e6e6", + "workflowName": "wts", + "workflowRunName": "umccr--automated--wts--4-2-4--202409021221e6e6", + "workflowVersion": "4.2.4", + "timestamp": "2024-09-02T00:00:00Z", + "status": "SUCCEEDED" + } +} diff --git a/lib/workload/stateless/stacks/fmannotator/fixtures/s3_object.yml b/lib/workload/stateless/stacks/fmannotator/fixtures/s3_object.yml new file mode 100644 index 000000000..558e7ea6e --- /dev/null +++ b/lib/workload/stateless/stacks/fmannotator/fixtures/s3_object.yml @@ -0,0 +1,31 @@ +- s3_object_id: RAW=gen_random_uuid() + event_type: 'Created' + bucket: 'bucket' + key: 'byob-icav2/development/analysis/wts/202409021221e6e6/_manifest.json' + event_time: 2024-09-02 00:00:00 + size: 5 + storage_class: 'Standard' +- s3_object_id: RAW=gen_random_uuid() + event_type: 'Deleted' + bucket: 'bucket' + key: 'byob-icav2/development/analysis/wts/202409021221e6e6/_manifest.json' + event_time: 2024-09-03 00:00:00 +- s3_object_id: RAW=gen_random_uuid() + event_type: 'Created' + bucket: 'bucket' + key: 'byob-icav2/development/analysis/wts/202409021221e6e6/_tags.json' + event_time: 2024-09-04 00:00:00 + size: 10 + storage_class: 'Standard' +- s3_object_id: RAW=gen_random_uuid() + event_type: 'Created' + bucket: 'bucket' + key: 'byob-icav2/development/analysis/wts/202409021221e6c6/_manifest.json' + event_time: 2024-09-05 00:00:00 + size: 3 + storage_class: 'Standard' +- s3_object_id: RAW=gen_random_uuid() + event_type: 'Deleted' + bucket: 'bucket' + key: 'byob-icav2/development/analysis/wts/202409021221e6c6/_manifest.json' + event_time: 2024-09-06 00:00:00 diff --git a/lib/workload/stateless/stacks/fmannotator/go.mod b/lib/workload/stateless/stacks/fmannotator/go.mod new file mode 100644 index 000000000..b4f499f47 --- /dev/null +++ b/lib/workload/stateless/stacks/fmannotator/go.mod @@ -0,0 +1,81 @@ +module github.com/umccr/orcabus/lib/workload/stateless/stacks/fmannotator + +go 1.22 + +require ( + github.com/aws/aws-lambda-go v1.47.0 + github.com/aws/aws-secretsmanager-caching-go v1.2.0 + github.com/docker/docker v27.1.1+incompatible + github.com/docker/go-connections v0.5.0 + github.com/go-testfixtures/testfixtures/v3 v3.12.0 + github.com/google/uuid v1.6.0 + github.com/jmoiron/sqlx v1.4.0 + github.com/kelseyhightower/envconfig v1.4.0 + github.com/lib/pq v1.10.9 + github.com/stretchr/testify v1.9.0 + github.com/testcontainers/testcontainers-go v0.33.0 +) + +require ( + dario.cat/mergo v1.0.0 // indirect + github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect + github.com/ClickHouse/ch-go v0.61.5 // indirect + github.com/ClickHouse/clickhouse-go/v2 v2.26.0 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/andybalholm/brotli v1.1.0 // indirect + github.com/aws/aws-sdk-go v1.47.10 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect + github.com/containerd/containerd v1.7.18 // indirect + github.com/containerd/log v0.1.0 // indirect + github.com/containerd/platforms v0.2.1 // indirect + github.com/cpuguy83/dockercfg v0.3.1 // indirect + github.com/creack/pty v1.1.21 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-faster/city v1.0.1 // indirect + github.com/go-faster/errors v0.7.1 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/klauspost/compress v1.17.7 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/magiconair/properties v1.8.7 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/patternmatcher v0.6.0 // indirect + github.com/moby/sys/sequential v0.5.0 // indirect + github.com/moby/sys/user v0.1.0 // indirect + github.com/moby/term v0.5.0 // indirect + github.com/morikuni/aec v1.0.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.0 // indirect + github.com/paulmach/orb v0.11.1 // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/segmentio/asm v1.2.0 // indirect + github.com/shirou/gopsutil/v3 v3.23.12 // indirect + github.com/shoenig/go-m1cpu v0.1.6 // indirect + github.com/shopspring/decimal v1.4.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect + github.com/yusufpapurcu/wmi v1.2.3 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect + go.opentelemetry.io/otel v1.26.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0 // indirect + go.opentelemetry.io/otel/metric v1.26.0 // indirect + go.opentelemetry.io/otel/trace v1.26.0 // indirect + golang.org/x/crypto v0.24.0 // indirect + golang.org/x/net v0.26.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.21.0 // indirect + golang.org/x/time v0.3.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/lib/workload/stateless/stacks/fmannotator/go.sum b/lib/workload/stateless/stacks/fmannotator/go.sum new file mode 100644 index 000000000..179fbe3c6 --- /dev/null +++ b/lib/workload/stateless/stacks/fmannotator/go.sum @@ -0,0 +1,323 @@ +dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/ClickHouse/ch-go v0.61.5 h1:zwR8QbYI0tsMiEcze/uIMK+Tz1D3XZXLdNrlaOpeEI4= +github.com/ClickHouse/ch-go v0.61.5/go.mod h1:s1LJW/F/LcFs5HJnuogFMta50kKDO0lf9zzfrbl0RQg= +github.com/ClickHouse/clickhouse-go/v2 v2.26.0 h1:j4/y6NYaCcFkJwN/TU700ebW+nmsIy34RmUAAcZKy9w= +github.com/ClickHouse/clickhouse-go/v2 v2.26.0/go.mod h1:iDTViXk2Fgvf1jn2dbJd1ys+fBkdD1UMRnXlwmhijhQ= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= +github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= +github.com/aws/aws-lambda-go v1.47.0 h1:0H8s0vumYx/YKs4sE7YM0ktwL2eWse+kfopsRI1sXVI= +github.com/aws/aws-lambda-go v1.47.0/go.mod h1:dpMpZgvWx5vuQJfBt0zqBha60q7Dd7RfgJv23DymV8A= +github.com/aws/aws-sdk-go v1.47.10 h1:cvufN7WkD1nlOgpRopsmxKQlFp5X1MfyAw4r7BBORQc= +github.com/aws/aws-sdk-go v1.47.10/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= +github.com/aws/aws-secretsmanager-caching-go v1.2.0 h1:gUA+CVKvFLj4OUSknhIrnt4dF7Y37+JrChKqfaehJME= +github.com/aws/aws-secretsmanager-caching-go v1.2.0/go.mod h1:6t2/zQIsigFMlnpOdGj503Dgaz24tMqIRhass9uoTBo= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/containerd/containerd v1.7.18 h1:jqjZTQNfXGoEaZdW1WwPU0RqSn1Bm2Ay/KJPUuO8nao= +github.com/containerd/containerd v1.7.18/go.mod h1:IYEk9/IO6wAPUz2bCMVUbsfXjzw5UNP5fLz4PsUygQ4= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= +github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= +github.com/cpuguy83/dockercfg v0.3.1 h1:/FpZ+JaygUR/lZP2NlFI2DVfrOEMAIKP5wWEJdoYe9E= +github.com/cpuguy83/dockercfg v0.3.1/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= +github.com/creack/pty v1.1.21 h1:1/QdRyBaHHJP61QkWMXlOIBfsgdDeeKfK8SYVUWJKf0= +github.com/creack/pty v1.1.21/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/denisenkom/go-mssqldb v0.12.3 h1:pBSGx9Tq67pBOTLmxNuirNTeB8Vjmf886Kx+8Y+8shw= +github.com/denisenkom/go-mssqldb v0.12.3/go.mod h1:k0mtMFOnU+AihqFxPMiF05rtiDrorD1Vrm1KEz5hxDo= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/docker v27.1.1+incompatible h1:hO/M4MtV36kzKldqnA37IWhebRA+LnqqcqDja6kVaKY= +github.com/docker/docker v27.1.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= +github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw= +github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw= +github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg= +github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= +github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= +github.com/go-testfixtures/testfixtures/v3 v3.12.0 h1:Ew0+c2o1mXSUqMwjuNup3MK/vw1HkLS3ILljX5C6lVE= +github.com/go-testfixtures/testfixtures/v3 v3.12.0/go.mod h1:13F0m6/DtqqSDso9IAVuhbZ4I7AiRAHrolmDMu9v5vY= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= +github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A= +github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EOqtpKwwwHI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= +github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v1.14.3 h1:bVoTr12EGANZz66nZPkMInAV/KHD2TxH9npjXXgiB3w= +github.com/jackc/pgconn v1.14.3/go.mod h1:RZbme4uasqzybK2RK5c65VsHxoyaml09lx3tXOcO/VM= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3/v2 v2.3.3 h1:1HLSx5H+tXR9pW3in3zaztoEwQYRC9SQaYUHjTSUOag= +github.com/jackc/pgproto3/v2 v2.3.3/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgtype v1.14.0 h1:y+xUdabmyMkJLyApYuPj38mW+aAIqCe5uuBB51rH3Vw= +github.com/jackc/pgtype v1.14.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgx/v4 v4.18.3 h1:dE2/TrEsGX3RBprb3qryqSV9Y60iZN1C6i8IrmW9/BA= +github.com/jackc/pgx/v4 v4.18.3/go.mod h1:Ey4Oru5tH5sB6tV7hDmfWFahwF15Eb7DNXlRKx2CkVw= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= +github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= +github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= +github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= +github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= +github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= +github.com/moby/sys/user v0.1.0 h1:WmZ93f5Ux6het5iituh9x2zAG7NFY9Aqi49jjE1PaQg= +github.com/moby/sys/user v0.1.0/go.mod h1:fKJhFOnsCN6xZ5gSfbM6zaHGgDJMrqt9/reuj4T7MmU= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= +github.com/paulmach/orb v0.11.1 h1:3koVegMC4X/WeiXYz9iswopaTwMem53NzTJuTF20JzU= +github.com/paulmach/orb v0.11.1/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= +github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= +github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= +github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= +github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= +github.com/shirou/gopsutil/v3 v3.23.12 h1:z90NtUkp3bMtmICZKpC4+WaknU1eXtp5vtbQ11DgpE4= +github.com/shirou/gopsutil/v3 v3.23.12/go.mod h1:1FrWgea594Jp7qmjHUUPlJDTPgcsb9mGnXDxavtikzM= +github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= +github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/testcontainers/testcontainers-go v0.33.0 h1:zJS9PfXYT5O0ZFXM2xxXfk4J5UMw/kRiISng037Gxdw= +github.com/testcontainers/testcontainers-go v0.33.0/go.mod h1:W80YpTa8D5C3Yy16icheD01UTDu+LmXIA2Keo+jWtT8= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= +github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= +github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= +go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs= +go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 h1:cl5P5/GIfFh4t6xyruOgJP5QiA1pw4fYYdv6nc6CBWw= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0/go.mod h1:zgBdWWAu7oEEMC06MMKc5NLbA/1YDXV1sMpSqEeLQLg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0 h1:digkEZCJWobwBqMwC0cwCq8/wkkRy/OowZg5OArWZrM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0/go.mod h1:/OpE/y70qVkndM0TrxT4KBoN3RsFZP0QaofcfYrj76I= +go.opentelemetry.io/otel/metric v1.26.0 h1:7S39CLuY5Jgg9CrnA9HHiEjGMF/X2VHvoXGgSllRz30= +go.opentelemetry.io/otel/metric v1.26.0/go.mod h1:SY+rHOI4cEawI9a7N1A4nIg/nTQXe1ccCNWYOJUrpX4= +go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= +go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= +go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2LP5sQA= +go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0= +go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= +go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= +golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 h1:RFiFrvy37/mpSpdySBDrUdipW/dHwsRwh3J3+A9VgT4= +google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237/go.mod h1:Z5Iiy3jtmioajWHDGFk7CeugTyHtPvMHA4UTmUkyalE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/grpc v1.64.1 h1:LKtvyfbX3UGVPFcGqJ9ItpVWW6oN/2XqTxfAnwRRXiA= +google.golang.org/grpc v1.64.1/go.mod h1:hiQF4LFZelK2WKaP6W0L92zGHtiQdZxk8CrSdvyjeP0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= +gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= diff --git a/lib/workload/stateless/stacks/fmannotator/handlers.go b/lib/workload/stateless/stacks/fmannotator/handlers.go new file mode 100644 index 000000000..18ec65fcf --- /dev/null +++ b/lib/workload/stateless/stacks/fmannotator/handlers.go @@ -0,0 +1,45 @@ +// Package fmannotator Lambda handler implementations. +package fmannotator + +import ( + "bytes" + "fmt" + "github.com/umccr/orcabus/lib/workload/stateless/stacks/fmannotator/schema/orcabus_workflowmanager/workflowrunstatechange" + "log/slog" + "net/url" + "strings" +) + +// PortalRunId Annotate the portalRunId from an incoming WorkflowRunStateChange event using the config and FM endpoint token. +func PortalRunId(event workflowrunstatechange.Event, config *Config, token string) (err error) { + eventStatus := strings.ToUpper(event.Detail.Status) + if eventStatus != "SUCCEEDED" && eventStatus != "FAILED" && eventStatus != "ABORTED" { + return nil + } + + patch, err := MarshallPortalRunId(&event) + if err != nil { + return err + } + + req, err := NewApiClient(config, bytes.NewBuffer(patch)) + if err != nil { + return err + } + + req = req.WithMethod("PATCH").WithS3Endpoint().WithQuery(url.Values{ + "key": {fmt.Sprintf("*/%v/*", event.Detail.PortalRunId)}, + }).WithHeader("Content-Type", "application/json").WithHeader("Authorization", fmt.Sprintf("Bearer %s", token)) + + body, status, err := req.Do() + if err != nil { + return err + } + if status != 200 { + return fmt.Errorf("error annotating attributes with status %v: %v", status, string(body)) + } + + slog.Debug(fmt.Sprintf("received response %v with body: %v", status, body)) + + return nil +} diff --git a/lib/workload/stateless/stacks/fmannotator/handlers_test.go b/lib/workload/stateless/stacks/fmannotator/handlers_test.go new file mode 100644 index 000000000..c498ceba2 --- /dev/null +++ b/lib/workload/stateless/stacks/fmannotator/handlers_test.go @@ -0,0 +1,91 @@ +package fmannotator + +import ( + "database/sql" + "fmt" + _ "github.com/lib/pq" + "github.com/stretchr/testify/require" + "github.com/umccr/orcabus/lib/workload/stateless/stacks/fmannotator/internal/test" + "testing" + "time" +) + +type TestCase struct { + event string + portalRunId string + expected []test.S3Object +} + +func successCase(location *time.Location) TestCase { + return TestCase{"event_succeeded.json", "202409021221e6e6", []test.S3Object{ + { + EventType: "Created", + Bucket: "bucket", + Key: "byob-icav2/development/analysis/wts/202409021221e6e6/_manifest.json", + EventTime: sql.NullTime{Time: time.Date(2024, 9, 2, 0, 0, 0, 0, location), Valid: true}, + Size: sql.NullInt64{Int64: 5, Valid: true}, + StorageClass: sql.NullString{String: "Standard", Valid: true}, + }, + { + EventType: "Deleted", + Bucket: "bucket", + Key: "byob-icav2/development/analysis/wts/202409021221e6e6/_manifest.json", + EventTime: sql.NullTime{Time: time.Date(2024, 9, 3, 0, 0, 0, 0, location), Valid: true}, + Size: sql.NullInt64{}, + StorageClass: sql.NullString{}, + }, + { + EventType: "Created", + Bucket: "bucket", + Key: "byob-icav2/development/analysis/wts/202409021221e6e6/_tags.json", + EventTime: sql.NullTime{Time: time.Date(2024, 9, 4, 0, 0, 0, 0, location), Valid: true}, + Size: sql.NullInt64{Int64: 10, Valid: true}, + StorageClass: sql.NullString{String: "Standard", Valid: true}, + }, + }} +} + +func failCase(location *time.Location) TestCase { + return TestCase{"event_failed.json", "202409021221e6c6", []test.S3Object{ + { + EventType: "Created", + Bucket: "bucket", + Key: "byob-icav2/development/analysis/wts/202409021221e6c6/_manifest.json", + EventTime: sql.NullTime{Time: time.Date(2024, 9, 5, 0, 0, 0, 0, location), Valid: true}, + Size: sql.NullInt64{Int64: 3, Valid: true}, + StorageClass: sql.NullString{String: "Standard", Valid: true}, + }, + { + EventType: "Deleted", + Bucket: "bucket", + Key: "byob-icav2/development/analysis/wts/202409021221e6c6/_manifest.json", + EventTime: sql.NullTime{Time: time.Date(2024, 9, 6, 0, 0, 0, 0, location), Valid: true}, + Size: sql.NullInt64{}, + StorageClass: sql.NullString{}, + }, + }} +} + +func TestHandler(t *testing.T) { + db := test.SetupFileManager(t) + + location, err := time.LoadLocation("Etc/UTC") + require.NoError(t, err) + + testCases := []TestCase{successCase(location), failCase(location)} + + for _, tc := range testCases { + t.Run(fmt.Sprintf("%v", tc.event), func(t *testing.T) { + event := test.CreateEvent(t, tc.event) + + config, err := LoadConfig() + require.NoError(t, err) + + err = PortalRunId(event, &config, "token") + require.NoError(t, err) + + s3Objects := test.QueryObjects(t, db, fmt.Sprintf("select * from s3_object where key like '%%/%v/%%'", tc.portalRunId)) + require.Equal(t, tc.expected, s3Objects) + }) + } +} diff --git a/lib/workload/stateless/stacks/fmannotator/internal/test/test.go b/lib/workload/stateless/stacks/fmannotator/internal/test/test.go new file mode 100644 index 000000000..93a7a85ec --- /dev/null +++ b/lib/workload/stateless/stacks/fmannotator/internal/test/test.go @@ -0,0 +1,173 @@ +package test + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + "github.com/docker/go-connections/nat" + "github.com/go-testfixtures/testfixtures/v3" + "github.com/google/uuid" + "github.com/jmoiron/sqlx" + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" + "github.com/umccr/orcabus/lib/workload/stateless/stacks/fmannotator/schema/orcabus_workflowmanager/workflowrunstatechange" + "os" + "path" + "path/filepath" + "runtime" + "strconv" + "strings" + "testing" +) + +type S3Object struct { + EventType string `db:"event_type"` + Bucket string `db:"bucket"` + Key string `db:"key"` + EventTime sql.NullTime `db:"event_time"` + Size sql.NullInt64 `db:"size"` + StorageClass sql.NullString `db:"storage_class"` +} + +func setupService(t *testing.T, buildContext string, port nat.Port, wait wait.Strategy, env map[string]string) (string, nat.Port) { + ctx := context.Background() + + args := make(map[string]*string) + for k, v := range env { + args[k] = &v + } + + containerName := strings.ReplaceAll(strings.Trim(buildContext, "./"), "/", "_") + req := testcontainers.ContainerRequest{ + FromDockerfile: testcontainers.FromDockerfile{ + Context: buildContext, + Repo: containerName, + Tag: containerName, + BuildArgs: args, + KeepImage: true, + PrintBuildLog: true, + BuildOptionsModifier: func(options *types.ImageBuildOptions) { + options.ExtraHosts = []string{"host.docker.internal:host-gateway"} + }, + }, + ExposedPorts: []string{port.Port()}, + Env: env, + WaitingFor: wait, + Name: containerName, + HostConfigModifier: func(config *container.HostConfig) { + config.ExtraHosts = []string{"host.docker.internal:host-gateway"} + }, + } + container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + Reuse: true, + }) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, container.Terminate(ctx)) + }) + + ip, err := container.Host(ctx) + require.NoError(t, err) + port, err = container.MappedPort(ctx, port) + require.NoError(t, err) + + fmt.Printf("setup `%v` at `%v:%v`\n", containerName, ip, port.Port()) + + return ip, port +} + +func SetupFileManager(t *testing.T) *sql.DB { + // This has to be such a large timeout because FM needs to compile/build with access to a Postgres database. If + // the timeout is too low, before the FM compilation connects to the database, testcontainers will kill the postgres + // container as there are no connections to it. + // This is another good reason to remove the requirement to compile with a database, as most software doesn't + // expect this to be the case. + t.Setenv("TESTCONTAINERS_RYUK_CONNECTION_TIMEOUT", "10m") + t.Setenv("TESTCONTAINERS_RYUK_RECONNECTION_TIMEOUT", "5m") + + // This works around an issue in test containers which requires the presence of a config.json file. + dir := t.TempDir() + err := os.WriteFile(path.Join(dir, "config.json"), []byte("{}"), 0666) + require.NoError(t, err) + t.Setenv("DOCKER_CONFIG", dir) + + // Database + testDatabaseName := fmt.Sprintf("filemanager_test_%v", strings.ReplaceAll(uuid.New().String(), "-", "_")) + databaseIp, port := setupService(t, "../filemanager/database", "4321", wait.ForLog("database system is ready to accept connections"). + WithOccurrence(2), + map[string]string{ + "POSTGRES_DB": testDatabaseName, + "POSTGRES_USER": "filemanager", + "POSTGRES_PASSWORD": "filemanager", // pragma: allowlist secret + "PGPORT": "4321", + }) + + // API + intPort, err := strconv.Atoi(port.Port()) + require.NoError(t, err) + + databaseFmt := "postgresql://filemanager:filemanager@%v:%v/%v?sslmode=disable" // pragma: allowlist secret + databaseUrl := fmt.Sprintf(databaseFmt, "host.docker.internal", intPort, testDatabaseName) + ip, port := setupService(t, "../filemanager", "8000", wait.ForHTTP("/api/v1/s3/count"), + map[string]string{ + "DATABASE_URL": databaseUrl, + }) + + fmEndpoint := fmt.Sprintf("http://%v:%v", ip, port.Port()) + databaseEndpoint := fmt.Sprintf(databaseFmt, databaseIp, intPort, testDatabaseName) + + t.Setenv("FMANNOTATOR_FILE_MANAGER_ENDPOINT", fmEndpoint) + t.Setenv("FMANNOTATOR_FILE_MANAGER_SECRET_NAME", "secret") + + return loadFixtures(t, databaseEndpoint) +} + +func loadFixtures(t *testing.T, databaseUrl string) *sql.DB { + var err error + db, err := sql.Open("postgres", databaseUrl) + require.NoError(t, err) + + fixtures, err := testfixtures.New( + testfixtures.Database(db), + testfixtures.Dialect("postgres"), + testfixtures.Directory(fixturesPath()), + ) + require.NoError(t, err) + + fmt.Printf("loading data into test database: %v\n", databaseUrl) + + err = fixtures.Load() + require.NoError(t, err) + + return db +} + +func fixturesPath() string { + _, file, _, _ := runtime.Caller(0) + return filepath.Join(filepath.Dir(file), "../../fixtures") +} + +func CreateEvent(t *testing.T, path string) workflowrunstatechange.Event { + b, err := os.ReadFile(filepath.Join(fixturesPath(), path)) + require.NoError(t, err) + + var event workflowrunstatechange.Event + err = json.Unmarshal(b, &event) + require.NoError(t, err) + + return event +} + +func QueryObjects(t *testing.T, db *sql.DB, query string) []S3Object { + var s3Objects []S3Object + err := sqlx.NewDb(db, "postgres").Unsafe().Select(&s3Objects, query) + require.NoError(t, err) + + return s3Objects +} diff --git a/lib/workload/stateless/stacks/fmannotator/schema/README.md b/lib/workload/stateless/stacks/fmannotator/schema/README.md new file mode 100644 index 000000000..edea2adcf --- /dev/null +++ b/lib/workload/stateless/stacks/fmannotator/schema/README.md @@ -0,0 +1,14 @@ +# AWSEvent + +*Automatically generated by the [Amazon Event Schemas](https://aws.amazon.com/)* + +## Requirements +1. Go 1.x + +## Installation +* Golang Installation - [Golang](https://golang.org) + +Put the package under your project folder and add the following to import: +```golang +import "schema/orcabus_workflowmanager/workflowrunstatechange" +``` diff --git a/lib/workload/stateless/stacks/fmannotator/schema/orcabus_workflowmanager/workflowrunstatechange/event.go b/lib/workload/stateless/stacks/fmannotator/schema/orcabus_workflowmanager/workflowrunstatechange/event.go new file mode 100644 index 000000000..cab41befa --- /dev/null +++ b/lib/workload/stateless/stacks/fmannotator/schema/orcabus_workflowmanager/workflowrunstatechange/event.go @@ -0,0 +1,54 @@ +package workflowrunstatechange + +import ( + "time" +) + +// EventBridge custom event schema for orcabus.workflowmanager@WorkflowRunStateChange +type Event struct { + Account string `json:"account,omitempty"` + Detail WorkflowRunStateChange `json:"detail,omitempty"` + DetailType string `json:"detail-type,omitempty"` + Id string `json:"id,omitempty"` + Region string `json:"region,omitempty"` + Resources []string `json:"resources,omitempty"` + Source string `json:"source,omitempty"` + Time time.Time `json:"time,omitempty"` + Version string `json:"version,omitempty"` +} + +func (e *Event) SetAccount(account string) { + e.Account = account +} + +func (e *Event) SetDetail(detail WorkflowRunStateChange) { + e.Detail = detail +} + +func (e *Event) SetDetailType(detailType string) { + e.DetailType = detailType +} + +func (e *Event) SetId(id string) { + e.Id = id +} + +func (e *Event) SetRegion(region string) { + e.Region = region +} + +func (e *Event) SetResources(resources []string) { + e.Resources = resources +} + +func (e *Event) SetSource(source string) { + e.Source = source +} + +func (e *Event) SetTime(time time.Time) { + e.Time = time +} + +func (e *Event) SetVersion(version string) { + e.Version = version +} diff --git a/lib/workload/stateless/stacks/fmannotator/schema/orcabus_workflowmanager/workflowrunstatechange/library_record.go b/lib/workload/stateless/stacks/fmannotator/schema/orcabus_workflowmanager/workflowrunstatechange/library_record.go new file mode 100644 index 000000000..669307eb7 --- /dev/null +++ b/lib/workload/stateless/stacks/fmannotator/schema/orcabus_workflowmanager/workflowrunstatechange/library_record.go @@ -0,0 +1,14 @@ +package workflowrunstatechange + +type LibraryRecord struct { + LibraryId string `json:"libraryId,omitempty"` + OrcabusId string `json:"orcabusId,omitempty"` +} + +func (l *LibraryRecord) SetLibraryId(libraryId string) { + l.LibraryId = libraryId +} + +func (l *LibraryRecord) SetOrcabusId(orcabusId string) { + l.OrcabusId = orcabusId +} diff --git a/lib/workload/stateless/stacks/fmannotator/schema/orcabus_workflowmanager/workflowrunstatechange/marshaller.go b/lib/workload/stateless/stacks/fmannotator/schema/orcabus_workflowmanager/workflowrunstatechange/marshaller.go new file mode 100644 index 000000000..c08cb15c3 --- /dev/null +++ b/lib/workload/stateless/stacks/fmannotator/schema/orcabus_workflowmanager/workflowrunstatechange/marshaller.go @@ -0,0 +1,24 @@ +package workflowrunstatechange + +import ( + "encoding/json" +) + +func Marshal(inputEvent interface{}) ([]byte, error) { + outputStream, err := json.Marshal(inputEvent) + if err != nil { + return nil, err + } + + return outputStream, nil +} + +func Unmarshal(inputStream []byte) (map[string]interface{}, error) { + var outputEvent map[string]interface{} + err := json.Unmarshal(inputStream, &outputEvent) + if err != nil { + return nil, err + } + + return outputEvent, nil +} diff --git a/lib/workload/stateless/stacks/fmannotator/schema/orcabus_workflowmanager/workflowrunstatechange/payload.go b/lib/workload/stateless/stacks/fmannotator/schema/orcabus_workflowmanager/workflowrunstatechange/payload.go new file mode 100644 index 000000000..54fb1ccd8 --- /dev/null +++ b/lib/workload/stateless/stacks/fmannotator/schema/orcabus_workflowmanager/workflowrunstatechange/payload.go @@ -0,0 +1,19 @@ +package workflowrunstatechange + +type Payload struct { + Data interface{} `json:"data,omitempty"` + RefId string `json:"refId,omitempty"` + Version string `json:"version,omitempty"` +} + +func (p *Payload) SetData(data interface{}) { + p.Data = data +} + +func (p *Payload) SetRefId(refId string) { + p.RefId = refId +} + +func (p *Payload) SetVersion(version string) { + p.Version = version +} diff --git a/lib/workload/stateless/stacks/fmannotator/schema/orcabus_workflowmanager/workflowrunstatechange/workflow_run_state_change.go b/lib/workload/stateless/stacks/fmannotator/schema/orcabus_workflowmanager/workflowrunstatechange/workflow_run_state_change.go new file mode 100644 index 000000000..cea378437 --- /dev/null +++ b/lib/workload/stateless/stacks/fmannotator/schema/orcabus_workflowmanager/workflowrunstatechange/workflow_run_state_change.go @@ -0,0 +1,48 @@ +package workflowrunstatechange + +import ( + "time" +) + +type WorkflowRunStateChange struct { + LinkedLibraries []LibraryRecord `json:"linkedLibraries,omitempty"` + Payload Payload `json:"payload,omitempty"` + PortalRunId string `json:"portalRunId,omitempty"` + Status string `json:"status,omitempty"` + Timestamp time.Time `json:"timestamp,omitempty"` + WorkflowName string `json:"workflowName,omitempty"` + WorkflowRunName string `json:"workflowRunName,omitempty"` + WorkflowVersion string `json:"workflowVersion,omitempty"` +} + +func (w *WorkflowRunStateChange) SetLinkedLibraries(linkedLibraries []LibraryRecord) { + w.LinkedLibraries = linkedLibraries +} + +func (w *WorkflowRunStateChange) SetPayload(payload Payload) { + w.Payload = payload +} + +func (w *WorkflowRunStateChange) SetPortalRunId(portalRunId string) { + w.PortalRunId = portalRunId +} + +func (w *WorkflowRunStateChange) SetStatus(status string) { + w.Status = status +} + +func (w *WorkflowRunStateChange) SetTimestamp(timestamp time.Time) { + w.Timestamp = timestamp +} + +func (w *WorkflowRunStateChange) SetWorkflowName(workflowName string) { + w.WorkflowName = workflowName +} + +func (w *WorkflowRunStateChange) SetWorkflowRunName(workflowRunName string) { + w.WorkflowRunName = workflowRunName +} + +func (w *WorkflowRunStateChange) SetWorkflowVersion(workflowVersion string) { + w.WorkflowVersion = workflowVersion +} diff --git a/lib/workload/stateless/statelessStackCollectionClass.ts b/lib/workload/stateless/statelessStackCollectionClass.ts index a53f3e927..e60a40749 100644 --- a/lib/workload/stateless/statelessStackCollectionClass.ts +++ b/lib/workload/stateless/statelessStackCollectionClass.ts @@ -57,6 +57,7 @@ import { RnasumIcav2PipelineManagerStack, RnasumIcav2PipelineManagerStackProps, } from './stacks/rnasum-pipeline-manager/deploy'; +import { FMAnnotator, FMAnnotatorConfigurableProps } from './stacks/fmannotator/deploy/stack'; export interface StatelessStackCollectionProps { metadataManagerStackProps: MetadataManagerStackProps; @@ -76,6 +77,7 @@ export interface StatelessStackCollectionProps { bclConvertManagerStackProps: BclConvertManagerStackProps; workflowManagerStackProps: WorkflowManagerStackProps; stackyMcStackFaceProps: GlueStackProps; + fmAnnotatorProps: FMAnnotatorConfigurableProps; } export class StatelessStackCollection { @@ -97,6 +99,7 @@ export class StatelessStackCollection { readonly bclConvertManagerStack: Stack; readonly workflowManagerStack: Stack; readonly stackyMcStackFaceStack: Stack; + readonly fmAnnotator: Stack; constructor( scope: Construct, @@ -113,10 +116,11 @@ export class StatelessStackCollection { ...statelessConfiguration.dataSchemaStackProps, }); - this.fileManagerStack = new Filemanager(scope, 'FileManagerStack', { + const fileManagerStack = new Filemanager(scope, 'FileManagerStack', { ...this.createTemplateProps(env, 'FileManagerStack'), ...statelessConfiguration.fileManagerStackProps, }); + this.fileManagerStack = fileManagerStack; this.metadataManagerStack = new MetadataManagerStack(scope, 'MetadataManagerStack', { ...this.createTemplateProps(env, 'MetadataManagerStack'), @@ -223,6 +227,12 @@ export class StatelessStackCollection { ...this.createTemplateProps(env, 'StackyMcStackFaceStack'), ...statelessConfiguration.stackyMcStackFaceProps, }); + + this.fmAnnotator = new FMAnnotator(scope, 'FMAnnotatorStack', { + ...this.createTemplateProps(env, 'FMAnnotatorStack'), + ...statelessConfiguration.fmAnnotatorProps, + domainName: fileManagerStack.domainName, + }); } /** diff --git a/package.json b/package.json index e1e6e4a2f..e3b07fa0a 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "cdk-stateful": "cdk --app 'yarn run -B ts-node --prefer-ts-exts bin/statefulPipeline.ts'" }, "dependencies": { + "@aws-cdk/aws-lambda-go-alpha": "^2.155.0-alpha.0", "@aws-cdk/aws-lambda-python-alpha": "2.155.0-alpha.0", "@aws-cdk/aws-pipes-alpha": "^2.155.0-alpha.0", "@aws-cdk/aws-pipes-sources-alpha": "^2.155.0-alpha.0", diff --git a/yarn.lock b/yarn.lock index 9e180db17..ebae8b2a9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -36,6 +36,16 @@ __metadata: languageName: node linkType: hard +"@aws-cdk/aws-lambda-go-alpha@npm:^2.155.0-alpha.0": + version: 2.155.0-alpha.0 + resolution: "@aws-cdk/aws-lambda-go-alpha@npm:2.155.0-alpha.0" + peerDependencies: + aws-cdk-lib: ^2.155.0 + constructs: ^10.0.0 + checksum: 10/2dbea0a36bed606fdb31b3e1b7454d5d9dffc0ddfc0bc21f103a5b2845280657e23c22e5cea72511fb903253aba1409bc00493fa044c6de73cabe85cb4b517d6 + languageName: node + linkType: hard + "@aws-cdk/aws-lambda-python-alpha@npm:2.155.0-alpha.0": version: 2.155.0-alpha.0 resolution: "@aws-cdk/aws-lambda-python-alpha@npm:2.155.0-alpha.0" @@ -4003,6 +4013,7 @@ __metadata: version: 0.0.0-use.local resolution: "orcabus@workspace:." dependencies: + "@aws-cdk/aws-lambda-go-alpha": "npm:^2.155.0-alpha.0" "@aws-cdk/aws-lambda-python-alpha": "npm:2.155.0-alpha.0" "@aws-cdk/aws-pipes-alpha": "npm:^2.155.0-alpha.0" "@aws-cdk/aws-pipes-sources-alpha": "npm:^2.155.0-alpha.0"