Friday changes:

## PierianDx * Added pieriandx pipeline manager (monitors runs and sends events once the reports are generated) * Added nails for stacky ## Workflow Type Renames * wgtsQc to wgts-qc * tumor_normal to tumor-normal ## ICAv2 Resilience * Allow a ‘rerun’ of a workflow execution if the analysis id doesn’t exist (but the portal id is in the database). * Reduce concurrency in cttsov2 copy jobs to 5 * Reduce concurrency in bsshfastqcopy jobs to 5 ## Metadata * Refactored metadata tools to match new metadata manager structure in dev * Made metadata mapper lambda construct that allows user to easily collect an orcabus id from an internal id (library id / subject id / project id etc) and vice versa. ## Showers Don’t produce subject based showers. ## Stacky Refactored stacky services to not register subjects / instruments, libraries only, use a scan to find other libraries that have the same subject / on same instrument instead. Reduces number of components within the stack.
umccr · Oct 4, 2024 · 2f8477b · 2f8477b
1 parent d2dae68
commit 2f8477b
Show file tree

Hide file tree

Showing 219 changed files with 15,709 additions and 8,186 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -1,2 +1,3 @@
 /.yarn/releases/** binary
 /.yarn/plugins/** binary
+*.json.gz filter=lfs diff=lfs merge=lfs -text
diff --git a/config/config.ts b/config/config.ts
@@ -48,6 +48,11 @@ import {
   getRnasumIcav2PipelineTableStackProps,
 } from './stacks/rnasumPipelineManager';
 import { getFmAnnotatorProps } from './stacks/fmAnnotator';
+import {
+  getPierianDxPipelineManagerStackProps,
+  getPierianDxPipelineTableStackProps,
+} from './stacks/pierianDxPipelineManager';
+import { PieriandxPipelineManagerStack } from '../lib/workload/stateless/stacks/pieriandx-pipeline-manager/deploy';
 
 interface EnvironmentConfig {
   name: string;
@@ -83,6 +88,7 @@ export const getEnvironmentConfig = (stage: AppStage): EnvironmentConfig | null
       rnasumIcav2PipelineTableStackProps: getRnasumIcav2PipelineTableStackProps(),
       BclConvertTableStackProps: getBclConvertManagerTableStackProps(stage),
       stackyStatefulTablesStackProps: getStatefulGlueStackProps(),
+      pierianDxPipelineTableStackProps: getPierianDxPipelineTableStackProps(),
     },
     statelessConfig: {
       metadataManagerStackProps: getMetadataManagerStackProps(stage),
@@ -98,6 +104,7 @@ export const getEnvironmentConfig = (stage: AppStage): EnvironmentConfig | null
       wtsIcav2PipelineManagerStackProps: getWtsIcav2PipelineManagerStackProps(stage),
       umccriseIcav2PipelineManagerStackProps: getUmccriseIcav2PipelineManagerStackProps(stage),
       rnasumIcav2PipelineManagerStackProps: getRnasumIcav2PipelineManagerStackProps(stage),
+      pieriandxPipelineManagerStackProps: getPierianDxPipelineManagerStackProps(stage),
       eventSchemaStackProps: getEventSchemaStackProps(),
       dataSchemaStackProps: getDataSchemaStackProps(),
       bclConvertManagerStackProps: getBclConvertManagerStackProps(stage),

diff --git a/config/constants.ts b/config/constants.ts
@@ -291,7 +291,7 @@ External resources required by the wgtsqc Stack
 // Deployed under dev/stg/prod
 export const wgtsQcIcav2PipelineIdSSMParameterPath = '/icav2/umccr-prod/wgts_qc_4.2.4_pipeline_id'; // 03689516-b7f8-4dca-bba9-8405b85fae45
 
-export const wgtsQcIcav2PipelineWorkflowType = 'wgtsQc';
+export const wgtsQcIcav2PipelineWorkflowType = 'wgts-qc';
 export const wgtsQcIcav2PipelineWorkflowTypeVersion = '4.2.4';
 export const wgtsQcIcav2ServiceVersion = '2024.07.01';
 
@@ -324,7 +324,7 @@ TN Stateless stack
 
 // Deployed under dev/stg/prod
 export const tnIcav2PipelineIdSSMParameterPath = '/icav2/umccr-prod/tumor_normal_4.2.4_pipeline_id'; // 0f5575bc-6cf8-4a90-a80e-05088aae8ed7
-export const tnIcav2PipelineWorkflowType = 'tumor_normal';
+export const tnIcav2PipelineWorkflowType = 'tumor-normal';
 export const tnIcav2PipelineWorkflowTypeVersion = '4.2.4';
 export const tnIcav2ServiceVersion = '2024.07.01';
 export const tnIcav2ReadyEventSource = 'orcabus.workflowmanager';
@@ -500,6 +500,183 @@ export const rnasumIcav2EventDetailType = 'WorkflowRunStateChange';
 export const rnasumStateMachinePrefix = 'rnasumSfn';
 export const rnasumDefaultDatasetVersion = 'PANCAN';
 
+/*
+Ora Compression Stateless Stack
+*/
+
+// Deployed in dev
+// export const oraCompressionTarSSMParameterPath = '/icav2/umccr-prod/ora_compression_tar_uri'; // icav2://reference-data/dragen-ora/v2/ora_reference_v2.tar.gz
+
+/*
+PierianDx Stateful and Stateless stacks
+*/
+export const pieriandxPrefix = 'pieriandx';
+export const pieriandxTriggerLaunchSource = 'orcabus.workflowmanager';
+export const pieriandxWorkflowName = 'pieriandx';
+export const pieriandxWorkflowVersion = '2.1';
+export const pieriandxDetailType = 'WorkflowRunStateChange';
+export const pieriandxEventSource = 'orcabus.pieriandx';
+export const pieriandxPayloadVersion = '2024.10.01';
+export const pieriandxDynamodbTable = 'PierianDxPipelineDynamoDbTable';
+
+/*
+[
+    {
+        "panelName": "main",
+        "panelId": "tso500_DRAGEN_ctDNA_v2_1_Universityofmelbourne"  // pragma: allowlist secret
+    },
+    {
+        "panelName": "subpanel",
+        "panelId": "tso500_DRAGEN_ctDNA_v2_1_subpanel_Universityofmelbourne"  // pragma: allowlist secret
+    }
+]
+
+*/
+export const pieriandxDefaultPanelName = 'main';
+export const pieriandxPanelMapSsmParameterPath = '/umccr/orcabus/stateful/pieriandx/panel_map';
+
+/*
+[
+    {
+        "dagName": "cromwell_tso500_ctdna_workflow_1.0.4",
+        "dagDescription": "tso500_ctdna_workflow"
+    }
+]
+*/
+export const pieriandxDefaultDagName = 'cromwell_tso500_ctdna_workflow_1.0.4';
+export const pieriandxDagSsmParameterPath = '/umccr/orcabus/stateful/pieriandx/dag_map';
+
+/*
+"s3://pdx-cgwxfer-test/melbournetest" // development
+"s3://pdx-cgwxfer-test/melbournetest" // staging
+"s3://pdx-cgwxfer/melbourne" // production
+*/
+export const pieriandxS3SequencerRunRootSsmParameterPath =
+  '/umccr/orcabus/pieriandx/s3_sequencer_run_root';
+
+/*
+"[email protected]"  // development
+"[email protected]"  // staging
+"[email protected]"  // production
+*/
+export const pieriandxUserEmailSsmParameterPath = '/umccr/orcabus/pieriandx/user_email';
+
+/*
+"melbournetest"  // development
+"melbournetest"  // staging
+"melbourne" //  production
+*/
+export const pieriandxInstitutionSsmParameterPath = '/umccr/orcabus/pieriandx/institution';
+
+/*
+"https://app.uat.pieriandx.com/cgw-api/v2.0.0"  // development
+"https://app.uat.pieriandx.com/cgw-api/v2.0.0"  // staging
+"https://app.pieriandx.com/cgw-api/v2.0.0" //  production
+*/
+export const pieriandxBaseUrlSsmParameterPath = '/umccr/orcabus/pieriandx/base_url';
+
+// Constant for all environments
+export const pieriandxAuthTokeSsmParameterPath = 'collectPierianDxAccessToken';
+
+// Secret name for PierianDx S3 credentials (test bucket for dev and staging, prod bucket for prod)
+export const pieriandxS3CredentialsSecretsManagerId = 'PierianDx/S3Credentials'; // pragma: allowlist secret
+
+/*
+[
+  {
+    "project_id": "PO",
+    "panel": "subpanel",
+    "sample_type": "patientcare",
+    "is_identified": "identified",
+    "default_snomed_disease_code": null
+  },
+  {
+    "project_id": "COUMN",
+    "panel": "subpanel",
+    "sample_type": "patientcare",
+    "is_identified": "identified",
+    "default_snomed_disease_code": null
+  },
+  {
+    "project_id": "CUP",
+    "panel": "main",
+    "sample_type": "patientcare",
+    "is_identified": "identified",
+    "default_snomed_disease_code": 285645000
+  },
+  {
+    "project_id": "PPGL",
+    "panel": "main",
+    "sample_type": "patientcare",
+    "is_identified": "identified",
+    "default_snomed_disease_code": null
+  },
+  {
+    "project_id": "MESO",
+    "panel": "subpanel",
+    "sample_type": "patientcare",
+    "is_identified": "identified",
+    "default_snomed_disease_code": null
+  },
+  {
+    "project_id": "OCEANiC",
+    "panel": "subpanel",
+    "sample_type": "patientcare",
+    "is_identified": "deidentified",
+    "default_snomed_disease_code": null
+  },
+  {
+    "project_id": "SOLACE2",
+    "panel": "main",
+    "sample_type": "patientcare",
+    "is_identified": "deidentified",
+    "default_snomed_disease_code": 55342001
+  },
+  {
+    "project_id": "IMPARP",
+    "panel": "main",
+    "sample_type": "patientcare",
+    "is_identified": "deidentified",
+    "default_snomed_disease_code": 55342001
+  },
+  {
+    "project_id": "Control",
+    "panel": "main",
+    "sample_type": "validation",
+    "is_identified": "deidentified",
+    "default_snomed_disease_code": 55342001
+  },
+  {
+    "project_id": "QAP",
+    "panel": "subpanel",
+    "sample_type": "patientcare",
+    "is_identified": "identified",
+    "default_snomed_disease_code": null
+  },
+  {
+    "project_id": "iPredict2",
+    "panel": "subpanel",
+    "sample_type": "patientcare",
+    "is_identified": "identified",
+    "default_snomed_disease_code":null
+  },
+  {
+    "project_id": "*",
+    "panel": "main",
+    "sample_type": "patientcare",
+    "is_identified": "deidentified",
+    "default_snomed_disease_code": 55342001
+  }
+]
+*/
+export const pieriandxProjectInfoSsmParameterPath = '/umccr/orcabus/pieriandx/project_info';
+
+export const redcapLambdaFunctionName: Record<AppStage, string> = {
+  [AppStage.BETA]: 'redcap-apis-dev-lambda-function',
+  [AppStage.GAMMA]: 'redcap-apis-stg-lambda-function',
+  [AppStage.PROD]: 'redcap-apis-prod-lambda-function',
+};
+
 // Mock Stack
 export const mockEventBusName = eventBusName;
 export const mockInstrumentRunTableName = 'stacky-instrument-run-table';
@@ -510,6 +687,7 @@ export const mockTnGlueTableName = 'stacky-tn-glue-table';
 export const mockWtsGlueTableName = 'stacky-wts-glue-table';
 export const mockUmccriseGlueTableName = 'stacky-umccrise-glue-table';
 export const mockRnasumGlueTableName = 'stacky-rnasum-glue-table';
+export const mockPierianDxGlueTableName = 'stacky-pieriandx-glue-table';
 export const mockWorkflowManagerTableName = 'stacky-workflow-manager-table';
 
 // {

diff --git a/config/stacks/pierianDxPipelineManager.ts b/config/stacks/pierianDxPipelineManager.ts
@@ -0,0 +1,71 @@
+import {
+  AppStage,
+  eventBusName,
+  icav2AccessTokenSecretName,
+  pieriandxAuthTokeSsmParameterPath,
+  pieriandxBaseUrlSsmParameterPath,
+  pieriandxDagSsmParameterPath,
+  pieriandxDefaultDagName,
+  pieriandxDefaultPanelName,
+  pieriandxDetailType,
+  pieriandxDynamodbTable,
+  pieriandxEventSource,
+  pieriandxInstitutionSsmParameterPath,
+  pieriandxPanelMapSsmParameterPath,
+  pieriandxPayloadVersion,
+  pieriandxPrefix,
+  pieriandxS3CredentialsSecretsManagerId,
+  pieriandxS3SequencerRunRootSsmParameterPath,
+  pieriandxTriggerLaunchSource,
+  pieriandxUserEmailSsmParameterPath,
+  pieriandxWorkflowName,
+  pieriandxWorkflowVersion,
+} from '../constants';
+import { PierianDxPipelineTableConfig } from '../../lib/workload/stateful/stacks/pieriandx-pipeline-dynamo-db/deploy';
+import { PierianDxPipelineManagerConfig } from '../../lib/workload/stateless/stacks/pieriandx-pipeline-manager/deploy';
+
+// Stateful
+export const getPierianDxPipelineTableStackProps = (): PierianDxPipelineTableConfig => {
+  return {
+    dynamodbTableName: pieriandxDynamodbTable,
+  };
+};
+
+// Stateless
+export const getPierianDxPipelineManagerStackProps = (
+  stage: AppStage
+): PierianDxPipelineManagerConfig => {
+  return {
+    /* DynamoDB Table */
+    dynamodbTableName: pieriandxDynamodbTable,
+    /* Workflow knowledge */
+    workflowName: pieriandxWorkflowName,
+    workflowVersion: pieriandxWorkflowVersion,
+    /* Default values */
+    defaultDagVersion: pieriandxDefaultDagName,
+    defaultPanelName: pieriandxDefaultPanelName,
+    /* Secrets */
+    /* ICAv2 Pipeline analysis essentials */
+    icav2AccessTokenSecretId: icav2AccessTokenSecretName[stage], // "/icav2/umccr-prod/service-production-jwt-token-secret-arn"
+    pieriandxS3AccessTokenSecretId: pieriandxS3CredentialsSecretsManagerId, // "/pieriandx/s3AccessCredentials"
+    /* SSM Parameters */
+    dagSsmParameterPath: pieriandxDagSsmParameterPath,
+    panelNameSsmParameterPath: pieriandxPanelMapSsmParameterPath,
+    s3SequencerRunRootSsmParameterPath: pieriandxS3SequencerRunRootSsmParameterPath,
+    /*
+        Pieriandx specific parameters
+        */
+    pieriandxUserEmailSsmParameterPath: pieriandxUserEmailSsmParameterPath,
+    pieriandxInstitutionSsmParameterPath: pieriandxInstitutionSsmParameterPath,
+    pieriandxBaseUrlSsmParameterPath: pieriandxBaseUrlSsmParameterPath,
+    pieriandxAuthTokenCollectionLambdaFunctionName: pieriandxAuthTokeSsmParameterPath,
+    /* Event info */
+    eventDetailType: pieriandxDetailType,
+    eventBusName: eventBusName,
+    eventSource: pieriandxEventSource,
+    payloadVersion: pieriandxPayloadVersion,
+    triggerLaunchSource: pieriandxTriggerLaunchSource,
+    /* Custom */
+    prefix: pieriandxPrefix,
+  };
+};
diff --git a/config/stacks/stackyMcStackFace.ts b/config/stacks/stackyMcStackFace.ts
@@ -16,20 +16,18 @@ import {
   mockWtsGlueTableName,
   mockUmccriseGlueTableName,
   mockRnasumGlueTableName,
+  mockPierianDxGlueTableName,
+  pieriandxProjectInfoSsmParameterPath,
+  redcapLambdaFunctionName,
 } from '../constants';
 import { GlueStackConfig } from '../../lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs';
 import { StackyStatefulTablesConfig } from '../../lib/workload/stateful/stacks/stacky-mcstackface-dynamodb';
 
 export const getGlueStackProps = (stage: AppStage): GlueStackConfig => {
   return {
-    /* SSM Parameters */
-    bsshOutputFastqCopyUriSsmParameterName: mockPrimaryOutputUriSsmParameterName,
-    analysisCacheUriSsmParameterName: mockAnalysisCacheUriSsmParameterName,
-    analysisOutputUriSsmParameterName: mockAnalysisOutputUriSsmParameterName,
-    icav2ProjectIdSsmParameterName: mockIcav2ProjectIdSsmParameterName,
-    analysisLogsUriSsmParameterName: mockAnalysisLogsUriSsmParameterName,
     /* Events */
     eventBusName: mockEventBusName,
+
     /* Tables */
     inputMakerTableName: mockInputMakerTableName,
     instrumentRunTableName: mockInstrumentRunTableName,
@@ -40,8 +38,23 @@ export const getGlueStackProps = (stage: AppStage): GlueStackConfig => {
     wtsGlueTableName: mockWtsGlueTableName,
     umccriseGlueTableName: mockUmccriseGlueTableName,
     rnasumGlueTableName: mockRnasumGlueTableName,
+    pieriandxGlueTableName: mockPierianDxGlueTableName,
+
+    /* SSM Parameters */
+    analysisCacheUriSsmParameterName: mockAnalysisCacheUriSsmParameterName,
+    analysisOutputUriSsmParameterName: mockAnalysisOutputUriSsmParameterName,
+    icav2ProjectIdSsmParameterName: mockIcav2ProjectIdSsmParameterName,
+    analysisLogsUriSsmParameterName: mockAnalysisLogsUriSsmParameterName,
+
     /* Secrets */
     icav2AccessTokenSecretName: icav2AccessTokenSecretName[stage],
+
+    /* BSSH SSM Parameters */
+    bsshOutputFastqCopyUriSsmParameterName: mockPrimaryOutputUriSsmParameterName,
+
+    /* PierianDx SSM Parameters */
+    pieriandxProjectInfoSsmParameterPath: pieriandxProjectInfoSsmParameterPath,
+    redcapLambdaFunctionName: redcapLambdaFunctionName[stage],
   };
 };
 
@@ -56,5 +69,6 @@ export const getStatefulGlueStackProps = (): StackyStatefulTablesConfig => {
     dynamodbWtsGlueTableName: mockWtsGlueTableName,
     dynamodbUmccriseGlueTableName: mockUmccriseGlueTableName,
     dynamodbRnasumGlueTableName: mockRnasumGlueTableName,
+    dynamodbPieriandxGlueTableName: mockPierianDxGlueTableName,
   };
 };
diff --git a/lib/workload/components/dynamodb-icav2-handle-event-change-sfn/index.ts b/lib/workload/components/dynamodb-icav2-handle-event-change-sfn/index.ts
@@ -127,7 +127,7 @@ export class Icav2AnalysisEventHandlerConstruct extends Construct {
       Convert a workflow name to lowercase and remove any spacing
 
       This has to be in align with Python impl:
-      lib/workload/components/event-workflowdraftrunstatechange-to-workflowrunstatechange-ready/lambdas/generate_workflow_run_name_py/generate_workflow_run_name.py
+      lib/workload/components/sfn-generate-workflowrunstatechange-ready-event/lambdas/generate_workflow_run_name_py/generate_workflow_run_name.py
     */
     let _name = name.toLowerCase().replace(new RegExp(' ', 'g'), '');
     _name = _name.replace(new RegExp('\\.', 'g'), '-');