Skip to content

Commit

Permalink
Merge pull request #506 from umccr/feature/umccrise-pipeline-manager
Browse files Browse the repository at this point in the history
initialised umccrise pipeline manager
  • Loading branch information
alexiswl authored Aug 20, 2024
2 parents 1cb82bf + 12e29e2 commit 381961e
Show file tree
Hide file tree
Showing 12 changed files with 880 additions and 1 deletion.
6 changes: 6 additions & 0 deletions config/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,10 @@ import {
getWtsIcav2PipelineManagerStackProps,
getWtsIcav2PipelineTableStackProps,
} from './stacks/wtsPipelineManager';
import {
getUmccriseIcav2PipelineManagerStackProps,
getUmccriseIcav2PipelineTableStackProps,
} from './stacks/umccrisePipelineManager';

interface EnvironmentConfig {
name: string;
Expand Down Expand Up @@ -70,6 +74,7 @@ export const getEnvironmentConfig = (stage: AppStage): EnvironmentConfig | null
wgtsQcIcav2PipelineTableStackProps: getWgtsQcIcav2PipelineTableStackProps(),
tnIcav2PipelineTableStackProps: getTnIcav2PipelineTableStackProps(),
wtsIcav2PipelineTableStackProps: getWtsIcav2PipelineTableStackProps(),
umccriseIcav2PipelineTableStackProps: getUmccriseIcav2PipelineTableStackProps(),
BclConvertTableStackProps: getBclConvertManagerTableStackProps(stage),
stackyStatefulTablesStackProps: getStatefulGlueStackProps(),
},
Expand All @@ -85,6 +90,7 @@ export const getEnvironmentConfig = (stage: AppStage): EnvironmentConfig | null
wgtsQcIcav2PipelineManagerStackProps: getWgtsQcIcav2PipelineManagerStackProps(stage),
tnIcav2PipelineManagerStackProps: getTnIcav2PipelineManagerStackProps(stage),
wtsIcav2PipelineManagerStackProps: getWtsIcav2PipelineManagerStackProps(stage),
umccriseIcav2PipelineManagerStackProps: getUmccriseIcav2PipelineManagerStackProps(stage),
eventSchemaStackProps: getEventSchemaStackProps(),
dataSchemaStackProps: getDataSchemaStackProps(),
bclConvertManagerStackProps: getBclConvertManagerStackProps(stage),
Expand Down
51 changes: 50 additions & 1 deletion config/constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -396,6 +396,56 @@ export const wtsDefaultArribaVersion = '2.4.0';

export const wtsDefaultQcReferenceSamplesVersion = '2023-07-21--4.2.4';

/*
Resources required by the umccrise stacks
*/

// Deployed under dev/stg/prod
// '
// [
// {
// "name": "202303",
// "uri": "icav2://reference-data/umccrise/202303/genomes.tar.gz"
// }
// ]
// '
export const icav2UmccriseGenomesReferenceUriMappingSSMParameterPath =
'/icav2/umccr-prod/umccrise_genomes_reference_mapping'; //

/*
Resources generated by the UMCCRise Stateful Stack
*/
export const umccriseIcav2PipelineManagerSSMRoot = '/orcabus/umccrise';

/*
UMCCRise Stateful stack
*/
export const umccriseIcav2PipelineManagerDynamodbTableName = 'umccriseICAv2AnalysesDynamoDBTable';
export const umccriseDynamoDbTableSSMName = path.join(
umccriseIcav2PipelineManagerSSMRoot,
'dynamodb_table_name'
);
export const umccriseDynamoDbTableSSMArn = path.join(
umccriseIcav2PipelineManagerSSMRoot,
'dynamodb_table_arn'
);

/*
UMCCRise Stateless stack
*/

// Deployed in dev/stg/prod
export const umccriseIcav2PipelineIdSSMParameterPath =
'/icav2/umccr-prod/umccrise_2.3.1_pipeline_id'; // 61254f38-b56e-4576-a8a1-341e5c412d11
export const umccriseIcav2PipelineWorkflowType = 'umccrise';
export const umccriseIcav2PipelineWorkflowTypeVersion = '2.3.1';
export const umccriseIcav2ServiceVersion = '2024.07.01';
export const umccriseIcav2ReadyEventSource = 'orcabus.workflowmanager';
export const umccriseIcav2EventSource = 'orcabus.umccrise';
export const umccriseIcav2EventDetailType = 'WorkflowRunStateChange';
export const umccriseStateMachinePrefix = 'umccriseSfn';
export const umccriseDefaultGenomeVersion = '202303';

// Mock Stack
export const mockEventBusName = eventBusName;
export const mockInstrumentRunTableName = 'stacky-instrument-run-table';
Expand All @@ -405,7 +455,6 @@ export const mockWgtsQcGlueTableName = 'stacky-wgts-qc-glue-table';
export const mockTnGlueTableName = 'stacky-tn-glue-table';
export const mockWtsGlueTableName = 'stacky-wts-glue-table';
export const mockUmccriseGlueTableName = 'stacky-umccrise-glue-table';

export const mockWorkflowManagerTableName = 'stacky-workflow-manager-table';

// {
Expand Down
58 changes: 58 additions & 0 deletions config/stacks/umccrisePipelineManager.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
import {
AppStage,
eventBusName,
icaEventPipeStackName,
icav2AccessTokenSecretName,
dragenIcav2ReferenceUriMappingSSMParameterPath,
umccriseIcav2PipelineIdSSMParameterPath,
umccriseIcav2PipelineManagerDynamodbTableName,
umccriseIcav2PipelineWorkflowType,
umccriseIcav2PipelineWorkflowTypeVersion,
umccriseIcav2ServiceVersion,
umccriseIcav2ReadyEventSource,
umccriseIcav2EventSource,
umccriseIcav2EventDetailType,
umccriseStateMachinePrefix,
umccriseDefaultGenomeVersion,
umccriseDynamoDbTableSSMArn,
umccriseDynamoDbTableSSMName,
} from '../constants';
import { UmccriseIcav2PipelineManagerConfig } from '../../lib/workload/stateless/stacks/umccrise-pipeline-manager/deploy';
import { UmccriseIcav2PipelineTableConfig } from '../../lib/workload/stateful/stacks/umccrise-pipeline-dynamo-db/deploy/stack';

// Stateful
export const getUmccriseIcav2PipelineTableStackProps = (): UmccriseIcav2PipelineTableConfig => {
return {
umccriseIcav2DynamodbTableArnSsmParameterPath: umccriseDynamoDbTableSSMArn,
umccriseIcav2DynamodbTableNameSsmParameterPath: umccriseDynamoDbTableSSMName,
dynamodbTableName: umccriseIcav2PipelineManagerDynamodbTableName,
};
};

// Stateless
export const getUmccriseIcav2PipelineManagerStackProps = (
stage: AppStage
): UmccriseIcav2PipelineManagerConfig => {
return {
/* ICAv2 Pipeline analysis essentials */
icav2TokenSecretId: icav2AccessTokenSecretName[stage], // "/icav2/umccr-prod/service-production-jwt-token-secret-arn"
pipelineIdSsmPath: umccriseIcav2PipelineIdSSMParameterPath, // List of parameters the workflow session state machine will need access to
/* Table to store analyis metadata */
dynamodbTableName: umccriseIcav2PipelineManagerDynamodbTableName,
/* Internal and external buses */
eventBusName: eventBusName,
icaEventPipeName: `${icaEventPipeStackName}Pipe`,
/* Event handling */
workflowType: umccriseIcav2PipelineWorkflowType,
workflowVersion: umccriseIcav2PipelineWorkflowTypeVersion,
serviceVersion: umccriseIcav2ServiceVersion,
triggerLaunchSource: umccriseIcav2ReadyEventSource,
internalEventSource: umccriseIcav2EventSource,
detailType: umccriseIcav2EventDetailType,
/* Names for statemachines */
stateMachinePrefix: umccriseStateMachinePrefix,
/* SSM Workflow Parameters */
defaultReferenceVersion: umccriseDefaultGenomeVersion,
referenceUriSsmPath: dragenIcav2ReferenceUriMappingSSMParameterPath,
};
};
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
import * as cdk from 'aws-cdk-lib';
import { Construct } from 'constructs';
import * as ssm from 'aws-cdk-lib/aws-ssm';
import { DynamodbPartitionedPipelineConstruct } from '../../../../components/dynamodb-partitioned-table';

export interface UmccriseIcav2PipelineTableConfig {
dynamodbTableName: string;
umccriseIcav2DynamodbTableArnSsmParameterPath: string;
umccriseIcav2DynamodbTableNameSsmParameterPath: string;
}

export type UmccriseIcav2PipelineTableStackProps = UmccriseIcav2PipelineTableConfig &
cdk.StackProps;

export class UmccriseIcav2PipelineTable extends cdk.Stack {
public readonly umccriseIcav2DynamodbTableArnSsmParameterPath: string;
public readonly umccriseIcav2DynamodbTableNameSsmParameterPath: string;

constructor(scope: Construct, id: string, props: UmccriseIcav2PipelineTableStackProps) {
super(scope, id, props);

/*
Initialise dynamodb table, where portal_run_id is the primary sort key
*/
const dynamodb_table = new DynamodbPartitionedPipelineConstruct(
this,
'umccrise_icav2_pipeline_table',
{
tableName: props.dynamodbTableName,
}
);

/*
Generate a ssm parameter to store the table arn so it can be referred to be other stacks
*/
this.umccriseIcav2DynamodbTableArnSsmParameterPath = new ssm.StringParameter(
this,
'umccrise_icav2_pipeline_table_arn_ssm_path',
{
parameterName: props.umccriseIcav2DynamodbTableArnSsmParameterPath,
stringValue: dynamodb_table.tableNameArn,
}
).parameterName;

this.umccriseIcav2DynamodbTableNameSsmParameterPath = new ssm.StringParameter(
this,
'umccrise_icav2_pipeline_table_name_ssm_path',
{
parameterName: props.umccriseIcav2DynamodbTableNameSsmParameterPath,
stringValue: props.dynamodbTableName,
}
).parameterName;
}
}
15 changes: 15 additions & 0 deletions lib/workload/stateful/statefulStackCollectionClass.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,10 @@ import {
WtsIcav2PipelineTable,
WtsIcav2PipelineTableStackProps,
} from './stacks/wts-dynamo-db/deploy/stack';
import {
UmccriseIcav2PipelineTable,
UmccriseIcav2PipelineTableStackProps,
} from './stacks/umccrise-pipeline-dynamo-db/deploy/stack';

export interface StatefulStackCollectionProps {
dataBucketStackProps: DataBucketStackProps;
Expand All @@ -49,6 +53,7 @@ export interface StatefulStackCollectionProps {
wgtsQcIcav2PipelineTableStackProps: WgtsQcIcav2PipelineTableStackProps;
tnIcav2PipelineTableStackProps: TnIcav2PipelineTableStackProps;
wtsIcav2PipelineTableStackProps: WtsIcav2PipelineTableStackProps;
umccriseIcav2PipelineTableStackProps: UmccriseIcav2PipelineTableStackProps;
BclConvertTableStackProps: BclConvertTableStackProps;
stackyStatefulTablesStackProps: StackyStatefulTablesStackProps;
}
Expand All @@ -66,6 +71,7 @@ export class StatefulStackCollection {
readonly wgtsQcIcav2PipelineTableStack: Stack;
readonly tnIcav2PipelineTableStack: Stack;
readonly wtsIcav2PipelineTableStack: Stack;
readonly umccriseIcav2PipelineTableStack: Stack;
readonly BclConvertTableStack: Stack;
readonly stackyStatefulTablesStack: Stack;

Expand Down Expand Up @@ -144,6 +150,15 @@ export class StatefulStackCollection {
}
);

this.umccriseIcav2PipelineTableStack = new UmccriseIcav2PipelineTable(
scope,
'UmccriseIcav2PipelineTableStack',
{
...this.createTemplateProps(env, 'UmccriseIcav2PipelineTableStack'),
...statefulConfiguration.umccriseIcav2PipelineTableStackProps,
}
);

this.BclConvertTableStack = new BclConvertTable(scope, 'BclConvertTableStack', {
...this.createTemplateProps(env, 'BclConvertTableStack'),
...statefulConfiguration.BclConvertTableStackProps,
Expand Down
Loading

0 comments on commit 381961e

Please sign in to comment.