Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added ora file decompression fastq pair component #624

Merged
merged 4 commits into from
Nov 2, 2024
Merged

Added ora file decompression fastq pair component #624

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
89b6b41
Added ora compression and ora decompression services
alexiswl Nov 1, 2024
8e9573a
Add Nag Suppressions on role level
alexiswl Nov 1, 2024
c467161
Nag Suppression should be applied to children
alexiswl Nov 1, 2024
79d54a4
Nag suppression add resource suppressions after creation of Ora Conta…
alexiswl Nov 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitattributes
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
/.yarn/releases/** binary
/.yarn/plugins/** binary
*.json.gz filter=lfs diff=lfs merge=lfs -text
*.tar.gz filter=lfs diff=lfs merge=lfs -text
240 changes: 240 additions & 0 deletions cdk.context.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,86 @@
}
]
},
"vpc-provider:account=843407916570:filter.tag:Name=main-vpc:region=ap-southeast-2:returnAsymmetricSubnets=true": {
"vpcId": "vpc-00eafc63c0dfca266",
"vpcCidrBlock": "10.2.0.0/16",
"ownerAccountId": "843407916570",
"availabilityZones": [],
"subnetGroups": [
{
"name": "database",
"type": "Isolated",
"subnets": [
{
"subnetId": "subnet-08edbca7f0eda37f6",
"cidr": "10.2.40.0/23",
"availabilityZone": "ap-southeast-2a",
"routeTableId": "rtb-04f3e62b4f5c4f071"
},
{
"subnetId": "subnet-00e0de28430f6e3cc",
"cidr": "10.2.42.0/23",
"availabilityZone": "ap-southeast-2b",
"routeTableId": "rtb-04f3e62b4f5c4f071"
},
{
"subnetId": "subnet-01009a3a383f4bff1",
"cidr": "10.2.44.0/23",
"availabilityZone": "ap-southeast-2c",
"routeTableId": "rtb-04f3e62b4f5c4f071"
}
]
},
{
"name": "public",
"type": "Public",
"subnets": [
{
"subnetId": "subnet-0fab038b0341872f1",
"cidr": "10.2.0.0/23",
"availabilityZone": "ap-southeast-2a",
"routeTableId": "rtb-0645235a1d1a53601"
},
{
"subnetId": "subnet-0e84dd3a07fb770f5",
"cidr": "10.2.2.0/23",
"availabilityZone": "ap-southeast-2b",
"routeTableId": "rtb-0645235a1d1a53601"
},
{
"subnetId": "subnet-093aee876a555f218",
"cidr": "10.2.4.0/23",
"availabilityZone": "ap-southeast-2c",
"routeTableId": "rtb-0645235a1d1a53601"
}
]
},
{
"name": "private",
"type": "Private",
"subnets": [
{
"subnetId": "subnet-050e6fb0f6028178b",
"cidr": "10.2.20.0/23",
"availabilityZone": "ap-southeast-2a",
"routeTableId": "rtb-06aa800ce565746f4"
},
{
"subnetId": "subnet-07b944de33d98047e",
"cidr": "10.2.22.0/23",
"availabilityZone": "ap-southeast-2b",
"routeTableId": "rtb-06aa800ce565746f4"
},
{
"subnetId": "subnet-0207da0a8acfcb732",
"cidr": "10.2.24.0/23",
"availabilityZone": "ap-southeast-2c",
"routeTableId": "rtb-06aa800ce565746f4"
}
]
}
]
},
"vpc-provider:account=455634345446:filter.tag:Name=main-vpc:filter.tag:Stack=networking:region=ap-southeast-2:returnAsymmetricSubnets=true": {
"vpcId": "vpc-0e226f3aa0f9c5781",
"vpcCidrBlock": "10.2.0.0/16",
Expand Down Expand Up @@ -159,6 +239,86 @@
}
]
},
"vpc-provider:account=455634345446:filter.tag:Name=main-vpc:region=ap-southeast-2:returnAsymmetricSubnets=true": {
"vpcId": "vpc-0e226f3aa0f9c5781",
"vpcCidrBlock": "10.2.0.0/16",
"ownerAccountId": "455634345446",
"availabilityZones": [],
"subnetGroups": [
{
"name": "database",
"type": "Isolated",
"subnets": [
{
"subnetId": "subnet-016b98cc640954651",
"cidr": "10.2.40.0/23",
"availabilityZone": "ap-southeast-2a",
"routeTableId": "rtb-014f5bf2c2d4af5e7"
},
{
"subnetId": "subnet-0c2c217219ed56bfc",
"cidr": "10.2.42.0/23",
"availabilityZone": "ap-southeast-2b",
"routeTableId": "rtb-014f5bf2c2d4af5e7"
},
{
"subnetId": "subnet-03e6fba027a1e0427",
"cidr": "10.2.44.0/23",
"availabilityZone": "ap-southeast-2c",
"routeTableId": "rtb-014f5bf2c2d4af5e7"
}
]
},
{
"name": "private",
"type": "Private",
"subnets": [
{
"subnetId": "subnet-01308be8bb704e5ef",
"cidr": "10.2.20.0/23",
"availabilityZone": "ap-southeast-2a",
"routeTableId": "rtb-0ad5a1fddb41ed1cf"
},
{
"subnetId": "subnet-0ab125fef23f8feed",
"cidr": "10.2.22.0/23",
"availabilityZone": "ap-southeast-2b",
"routeTableId": "rtb-0ad5a1fddb41ed1cf"
},
{
"subnetId": "subnet-02290481f7651e39b",
"cidr": "10.2.24.0/23",
"availabilityZone": "ap-southeast-2c",
"routeTableId": "rtb-0ad5a1fddb41ed1cf"
}
]
},
{
"name": "public",
"type": "Public",
"subnets": [
{
"subnetId": "subnet-079ba5a17c7102446",
"cidr": "10.2.0.0/23",
"availabilityZone": "ap-southeast-2a",
"routeTableId": "rtb-08cf65d720bf1e5f3"
},
{
"subnetId": "subnet-00df4e6910885d2f6",
"cidr": "10.2.2.0/23",
"availabilityZone": "ap-southeast-2b",
"routeTableId": "rtb-08cf65d720bf1e5f3"
},
{
"subnetId": "subnet-027a37df34939ab32",
"cidr": "10.2.4.0/23",
"availabilityZone": "ap-southeast-2c",
"routeTableId": "rtb-08cf65d720bf1e5f3"
}
]
}
]
},
"vpc-provider:account=472057503814:filter.tag:Name=main-vpc:filter.tag:Stack=networking:region=ap-southeast-2:returnAsymmetricSubnets=true": {
"vpcId": "vpc-0dc99f521ceaa3f2d",
"vpcCidrBlock": "10.2.0.0/16",
Expand Down Expand Up @@ -239,6 +399,86 @@
}
]
},
"vpc-provider:account=472057503814:filter.tag:Name=main-vpc:region=ap-southeast-2:returnAsymmetricSubnets=true": {
"vpcId": "vpc-0dc99f521ceaa3f2d",
"vpcCidrBlock": "10.2.0.0/16",
"ownerAccountId": "472057503814",
"availabilityZones": [],
"subnetGroups": [
{
"name": "public",
"type": "Public",
"subnets": [
{
"subnetId": "subnet-0a7fb9d501192f5ee",
"cidr": "10.2.0.0/23",
"availabilityZone": "ap-southeast-2a",
"routeTableId": "rtb-0c10b927ab0264377"
},
{
"subnetId": "subnet-0d61be977ee60a5b5",
"cidr": "10.2.2.0/23",
"availabilityZone": "ap-southeast-2b",
"routeTableId": "rtb-0c10b927ab0264377"
},
{
"subnetId": "subnet-0d8226a9b4af34507",
"cidr": "10.2.4.0/23",
"availabilityZone": "ap-southeast-2c",
"routeTableId": "rtb-0c10b927ab0264377"
}
]
},
{
"name": "database",
"type": "Isolated",
"subnets": [
{
"subnetId": "subnet-03ac51c3ab4223a1a",
"cidr": "10.2.40.0/23",
"availabilityZone": "ap-southeast-2a",
"routeTableId": "rtb-01266550d60b56ddb"
},
{
"subnetId": "subnet-06d14dcb934f22c5e",
"cidr": "10.2.42.0/23",
"availabilityZone": "ap-southeast-2b",
"routeTableId": "rtb-01266550d60b56ddb"
},
{
"subnetId": "subnet-0f5a5386ddf295579",
"cidr": "10.2.44.0/23",
"availabilityZone": "ap-southeast-2c",
"routeTableId": "rtb-01266550d60b56ddb"
}
]
},
{
"name": "private",
"type": "Private",
"subnets": [
{
"subnetId": "subnet-01be4c1109eca3446",
"cidr": "10.2.20.0/23",
"availabilityZone": "ap-southeast-2a",
"routeTableId": "rtb-067d123217c80f6bd"
},
{
"subnetId": "subnet-070a9acba78168239",
"cidr": "10.2.22.0/23",
"availabilityZone": "ap-southeast-2b",
"routeTableId": "rtb-067d123217c80f6bd"
},
{
"subnetId": "subnet-01ae2b4ad1eb584d7",
"cidr": "10.2.24.0/23",
"availabilityZone": "ap-southeast-2c",
"routeTableId": "rtb-067d123217c80f6bd"
}
]
}
]
},
"security-group:account=843407916570:region=ap-southeast-2:securityGroupName=OrcaBusSharedComputeSecurityGroup:vpcId=vpc-00eafc63c0dfca266": {
"securityGroupId": "sg-03abb47eba799e044",
"allowAllOutbound": false
Expand Down
9 changes: 9 additions & 0 deletions config/config.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,11 @@ import {
getOncoanalyserPipelineManagerStackProps,
getOncoanalyserPipelineTableStackProps,
} from './stacks/oncoanalyser';
import {
getOraCompressionIcav2PipelineManagerStackProps,
getOraCompressionIcav2PipelineTableStackProps,
} from './stacks/oraCompressionPipelineManager';
import { getOraDecompressionManagerStackProps } from './stacks/oraDecompressionPipelineManager';

interface EnvironmentConfig {
name: string;
Expand Down Expand Up @@ -92,6 +97,7 @@ export const getEnvironmentConfig = (stage: AppStage): EnvironmentConfig | null
wtsIcav2PipelineTableStackProps: getWtsIcav2PipelineTableStackProps(),
umccriseIcav2PipelineTableStackProps: getUmccriseIcav2PipelineTableStackProps(),
rnasumIcav2PipelineTableStackProps: getRnasumIcav2PipelineTableStackProps(),
oraCompressionIcav2PipelineTableStackProps: getOraCompressionIcav2PipelineTableStackProps(),
BclConvertTableStackProps: getBclConvertManagerTableStackProps(stage),
stackyStatefulTablesStackProps: getStatefulGlueStackProps(),
pierianDxPipelineTableStackProps: getPierianDxPipelineTableStackProps(),
Expand All @@ -115,6 +121,9 @@ export const getEnvironmentConfig = (stage: AppStage): EnvironmentConfig | null
pieriandxPipelineManagerStackProps: getPierianDxPipelineManagerStackProps(stage),
oncoanalyserPipelineManagerStackProps: getOncoanalyserPipelineManagerStackProps(stage),
sashPipelineManagerStackProps: getSashPipelineManagerStackProps(stage),
oraCompressionIcav2PipelineManagerStackProps:
getOraCompressionIcav2PipelineManagerStackProps(stage),
oraDecompressionManagerStackProps: getOraDecompressionManagerStackProps(stage),
eventSchemaStackProps: getEventSchemaStackProps(),
dataSchemaStackProps: getDataSchemaStackProps(),
bclConvertManagerStackProps: getBclConvertManagerStackProps(stage),
Expand Down
50 changes: 50 additions & 0 deletions config/constants.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -813,3 +813,53 @@ export const stackyAnalysisLogsUriSsmParameterName = '/orcabus/stacky/analysis_l
// stg: s3://pipeline-stg-cache-503977275616-ap-southeast-2/byob-icav2/staging/cache/__workflow_name__/__portal_run_id__/
// prod: s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/cache/__workflow_name__/__portal_run_id__/
export const stackyAnalysisCacheUriSsmParameterName = '/orcabus/stacky/analysis_cache_uri';

/*
Resources generated by the ORA Compression pipeline
*/

export const oraCompressionSSMRoot = '/orcabus/ora_compression/';

export const oraCompressionIcav2PipelineManagerDynamodbTableName =
'oraCompressionICAv2AnalysesDynamoDBTable';

// Stateful
export const oraCompressionDynamoDbTableSSMName = path.join(
oraCompressionSSMRoot,
'dynamodb_table_name'
);
export const oraCompressionDynamoDbTableSSMArn = path.join(
oraCompressionSSMRoot,
'dynamodb_table_arn'
);

// Stateless
export const oraCompressionIcav2PipelineWorkflowType = 'ora-compression';
export const oraCompressionIcav2PipelineWorkflowTypeVersion = '4.2.4--v2';
export const oraCompressionIcav2ServiceVersion = '2024.07.01';

export const oraCompressionIcav2ReadyEventSource = 'orcabus.workflowmanager';
export const oraCompressionIcav2EventSource = 'orcabus.oracompression';
export const oraCompressionIcav2EventDetailType = 'WorkflowRunStateChange';
export const oraCompressionStateMachinePrefix = 'oraCompressionSfn';

/*
Resources used by the ora compression pipeline
*/

// Release can be found here: https://github.com/umccr/cwl-ica/releases/tag/dragen-instrument-run-fastq-to-ora-pipeline%2F4.2.4__20241030041958
// Pipeline ID is: ba8f618a-842f-4a2f-9b2f-a074c0472218 // FIXME not in stg/prod
export const oraCompressionIcav2PipelineIdSSMParameterPath =
'/icav2/umccr-prod/ora_compression_pipeline_id';

// Default Reference Uri for compressing ORA files // FIXME not in stg/prod
export const oraCompressionDefaultReferenceUriSSmParameterPath =
'/icav2/umccr-prod/ora_compression_default_reference_version_uri';

/*
Resources generated by the ora decompression manager
*/
export const oraDecompressionIcav2ReadyEventSource = 'orcabus.workflowmanager';
export const oraDecompressionIcav2EventSource = 'orcabus.oradecompression';
export const oraDecompressionIcav2EventDetailType = 'FastqListRowDecompressed';
export const oraDecompressionStateMachinePrefix = 'oraDecompressionSfn';
Loading