diff --git a/.gitattributes b/.gitattributes index 937c0eb37..baa1a1850 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,3 @@ /.yarn/releases/** binary /.yarn/plugins/** binary +*.json.gz filter=lfs diff=lfs merge=lfs -text diff --git a/config/config.ts b/config/config.ts index f7dd0a45a..11740875e 100644 --- a/config/config.ts +++ b/config/config.ts @@ -48,6 +48,10 @@ import { getRnasumIcav2PipelineTableStackProps, } from './stacks/rnasumPipelineManager'; import { getFmAnnotatorProps } from './stacks/fmAnnotator'; +import { + getPierianDxPipelineManagerStackProps, + getPierianDxPipelineTableStackProps, +} from './stacks/pierianDxPipelineManager'; interface EnvironmentConfig { name: string; @@ -83,6 +87,7 @@ export const getEnvironmentConfig = (stage: AppStage): EnvironmentConfig | null rnasumIcav2PipelineTableStackProps: getRnasumIcav2PipelineTableStackProps(), BclConvertTableStackProps: getBclConvertManagerTableStackProps(stage), stackyStatefulTablesStackProps: getStatefulGlueStackProps(), + pierianDxPipelineTableStackProps: getPierianDxPipelineTableStackProps(), }, statelessConfig: { metadataManagerStackProps: getMetadataManagerStackProps(stage), @@ -98,6 +103,7 @@ export const getEnvironmentConfig = (stage: AppStage): EnvironmentConfig | null wtsIcav2PipelineManagerStackProps: getWtsIcav2PipelineManagerStackProps(stage), umccriseIcav2PipelineManagerStackProps: getUmccriseIcav2PipelineManagerStackProps(stage), rnasumIcav2PipelineManagerStackProps: getRnasumIcav2PipelineManagerStackProps(stage), + pieriandxPipelineManagerStackProps: getPierianDxPipelineManagerStackProps(stage), eventSchemaStackProps: getEventSchemaStackProps(), dataSchemaStackProps: getDataSchemaStackProps(), bclConvertManagerStackProps: getBclConvertManagerStackProps(stage), diff --git a/config/constants.ts b/config/constants.ts index cc98f88e7..eb8d47cb5 100644 --- a/config/constants.ts +++ b/config/constants.ts @@ -291,7 +291,7 @@ External resources required by the wgtsqc Stack // Deployed under dev/stg/prod export const wgtsQcIcav2PipelineIdSSMParameterPath = '/icav2/umccr-prod/wgts_qc_4.2.4_pipeline_id'; // 03689516-b7f8-4dca-bba9-8405b85fae45 -export const wgtsQcIcav2PipelineWorkflowType = 'wgtsQc'; +export const wgtsQcIcav2PipelineWorkflowType = 'wgts-qc'; export const wgtsQcIcav2PipelineWorkflowTypeVersion = '4.2.4'; export const wgtsQcIcav2ServiceVersion = '2024.07.01'; @@ -324,7 +324,7 @@ TN Stateless stack // Deployed under dev/stg/prod export const tnIcav2PipelineIdSSMParameterPath = '/icav2/umccr-prod/tumor_normal_4.2.4_pipeline_id'; // 0f5575bc-6cf8-4a90-a80e-05088aae8ed7 -export const tnIcav2PipelineWorkflowType = 'tumor_normal'; +export const tnIcav2PipelineWorkflowType = 'tumor-normal'; export const tnIcav2PipelineWorkflowTypeVersion = '4.2.4'; export const tnIcav2ServiceVersion = '2024.07.01'; export const tnIcav2ReadyEventSource = 'orcabus.workflowmanager'; @@ -500,6 +500,183 @@ export const rnasumIcav2EventDetailType = 'WorkflowRunStateChange'; export const rnasumStateMachinePrefix = 'rnasumSfn'; export const rnasumDefaultDatasetVersion = 'PANCAN'; +/* +Ora Compression Stateless Stack +*/ + +// Deployed in dev +// export const oraCompressionTarSSMParameterPath = '/icav2/umccr-prod/ora_compression_tar_uri'; // icav2://reference-data/dragen-ora/v2/ora_reference_v2.tar.gz + +/* +PierianDx Stateful and Stateless stacks +*/ +export const pieriandxPrefix = 'pieriandx'; +export const pieriandxTriggerLaunchSource = 'orcabus.workflowmanager'; +export const pieriandxWorkflowName = 'pieriandx'; +export const pieriandxWorkflowVersion = '2.1'; +export const pieriandxDetailType = 'WorkflowRunStateChange'; +export const pieriandxEventSource = 'orcabus.pieriandx'; +export const pieriandxPayloadVersion = '2024.10.01'; +export const pieriandxDynamodbTable = 'PierianDxPipelineDynamoDbTable'; + +/* +[ + { + "panelName": "main", + "panelId": "tso500_DRAGEN_ctDNA_v2_1_Universityofmelbourne" // pragma: allowlist secret + }, + { + "panelName": "subpanel", + "panelId": "tso500_DRAGEN_ctDNA_v2_1_subpanel_Universityofmelbourne" // pragma: allowlist secret + } +] + +*/ +export const pieriandxDefaultPanelName = 'main'; +export const pieriandxPanelMapSsmParameterPath = '/umccr/orcabus/stateful/pieriandx/panel_map'; + +/* +[ + { + "dagName": "cromwell_tso500_ctdna_workflow_1.0.4", + "dagDescription": "tso500_ctdna_workflow" + } +] +*/ +export const pieriandxDefaultDagName = 'cromwell_tso500_ctdna_workflow_1.0.4'; +export const pieriandxDagSsmParameterPath = '/umccr/orcabus/stateful/pieriandx/dag_map'; + +/* +"s3://pdx-cgwxfer-test/melbournetest" // development +"s3://pdx-cgwxfer-test/melbournetest" // staging +"s3://pdx-cgwxfer/melbourne" // production +*/ +export const pieriandxS3SequencerRunRootSsmParameterPath = + '/umccr/orcabus/pieriandx/s3_sequencer_run_root'; + +/* +"services@umccr.org" // development +"services@umccr.org" // staging +"services@umccr.org" // production +*/ +export const pieriandxUserEmailSsmParameterPath = '/umccr/orcabus/pieriandx/user_email'; + +/* +"melbournetest" // development +"melbournetest" // staging +"melbourne" // production +*/ +export const pieriandxInstitutionSsmParameterPath = '/umccr/orcabus/pieriandx/institution'; + +/* +"https://app.uat.pieriandx.com/cgw-api/v2.0.0" // development +"https://app.uat.pieriandx.com/cgw-api/v2.0.0" // staging +"https://app.pieriandx.com/cgw-api/v2.0.0" // production +*/ +export const pieriandxBaseUrlSsmParameterPath = '/umccr/orcabus/pieriandx/base_url'; + +// Constant for all environments +export const pieriandxAuthTokeSsmParameterPath = 'collectPierianDxAccessToken'; + +// Secret name for PierianDx S3 credentials (test bucket for dev and staging, prod bucket for prod) +export const pieriandxS3CredentialsSecretsManagerId = 'PierianDx/S3Credentials'; // pragma: allowlist secret + +/* +[ + { + "project_id": "PO", + "panel": "subpanel", + "sample_type": "patientcare", + "is_identified": "identified", + "default_snomed_disease_code": null + }, + { + "project_id": "COUMN", + "panel": "subpanel", + "sample_type": "patientcare", + "is_identified": "identified", + "default_snomed_disease_code": null + }, + { + "project_id": "CUP", + "panel": "main", + "sample_type": "patientcare", + "is_identified": "identified", + "default_snomed_disease_code": 285645000 + }, + { + "project_id": "PPGL", + "panel": "main", + "sample_type": "patientcare", + "is_identified": "identified", + "default_snomed_disease_code": null + }, + { + "project_id": "MESO", + "panel": "subpanel", + "sample_type": "patientcare", + "is_identified": "identified", + "default_snomed_disease_code": null + }, + { + "project_id": "OCEANiC", + "panel": "subpanel", + "sample_type": "patientcare", + "is_identified": "deidentified", + "default_snomed_disease_code": null + }, + { + "project_id": "SOLACE2", + "panel": "main", + "sample_type": "patientcare", + "is_identified": "deidentified", + "default_snomed_disease_code": 55342001 + }, + { + "project_id": "IMPARP", + "panel": "main", + "sample_type": "patientcare", + "is_identified": "deidentified", + "default_snomed_disease_code": 55342001 + }, + { + "project_id": "Control", + "panel": "main", + "sample_type": "validation", + "is_identified": "deidentified", + "default_snomed_disease_code": 55342001 + }, + { + "project_id": "QAP", + "panel": "subpanel", + "sample_type": "patientcare", + "is_identified": "identified", + "default_snomed_disease_code": null + }, + { + "project_id": "iPredict2", + "panel": "subpanel", + "sample_type": "patientcare", + "is_identified": "identified", + "default_snomed_disease_code":null + }, + { + "project_id": "*", + "panel": "main", + "sample_type": "patientcare", + "is_identified": "deidentified", + "default_snomed_disease_code": 55342001 + } +] +*/ +export const pieriandxProjectInfoSsmParameterPath = '/umccr/orcabus/pieriandx/project_info'; + +export const redcapLambdaFunctionName: Record = { + [AppStage.BETA]: 'redcap-apis-dev-lambda-function', + [AppStage.GAMMA]: 'redcap-apis-stg-lambda-function', + [AppStage.PROD]: 'redcap-apis-prod-lambda-function', +}; + // Mock Stack export const mockEventBusName = eventBusName; export const mockInstrumentRunTableName = 'stacky-instrument-run-table'; @@ -510,6 +687,7 @@ export const mockTnGlueTableName = 'stacky-tn-glue-table'; export const mockWtsGlueTableName = 'stacky-wts-glue-table'; export const mockUmccriseGlueTableName = 'stacky-umccrise-glue-table'; export const mockRnasumGlueTableName = 'stacky-rnasum-glue-table'; +export const mockPierianDxGlueTableName = 'stacky-pieriandx-glue-table'; export const mockWorkflowManagerTableName = 'stacky-workflow-manager-table'; // { diff --git a/config/stacks/pierianDxPipelineManager.ts b/config/stacks/pierianDxPipelineManager.ts new file mode 100644 index 000000000..b1f2b31a7 --- /dev/null +++ b/config/stacks/pierianDxPipelineManager.ts @@ -0,0 +1,71 @@ +import { + AppStage, + eventBusName, + icav2AccessTokenSecretName, + pieriandxAuthTokeSsmParameterPath, + pieriandxBaseUrlSsmParameterPath, + pieriandxDagSsmParameterPath, + pieriandxDefaultDagName, + pieriandxDefaultPanelName, + pieriandxDetailType, + pieriandxDynamodbTable, + pieriandxEventSource, + pieriandxInstitutionSsmParameterPath, + pieriandxPanelMapSsmParameterPath, + pieriandxPayloadVersion, + pieriandxPrefix, + pieriandxS3CredentialsSecretsManagerId, + pieriandxS3SequencerRunRootSsmParameterPath, + pieriandxTriggerLaunchSource, + pieriandxUserEmailSsmParameterPath, + pieriandxWorkflowName, + pieriandxWorkflowVersion, +} from '../constants'; +import { PierianDxPipelineTableConfig } from '../../lib/workload/stateful/stacks/pieriandx-pipeline-dynamo-db/deploy'; +import { PierianDxPipelineManagerConfig } from '../../lib/workload/stateless/stacks/pieriandx-pipeline-manager/deploy'; + +// Stateful +export const getPierianDxPipelineTableStackProps = (): PierianDxPipelineTableConfig => { + return { + dynamodbTableName: pieriandxDynamodbTable, + }; +}; + +// Stateless +export const getPierianDxPipelineManagerStackProps = ( + stage: AppStage +): PierianDxPipelineManagerConfig => { + return { + /* DynamoDB Table */ + dynamodbTableName: pieriandxDynamodbTable, + /* Workflow knowledge */ + workflowName: pieriandxWorkflowName, + workflowVersion: pieriandxWorkflowVersion, + /* Default values */ + defaultDagVersion: pieriandxDefaultDagName, + defaultPanelName: pieriandxDefaultPanelName, + /* Secrets */ + /* ICAv2 Pipeline analysis essentials */ + icav2AccessTokenSecretId: icav2AccessTokenSecretName[stage], // "/icav2/umccr-prod/service-production-jwt-token-secret-arn" + pieriandxS3AccessTokenSecretId: pieriandxS3CredentialsSecretsManagerId, // "/pieriandx/s3AccessCredentials" + /* SSM Parameters */ + dagSsmParameterPath: pieriandxDagSsmParameterPath, + panelNameSsmParameterPath: pieriandxPanelMapSsmParameterPath, + s3SequencerRunRootSsmParameterPath: pieriandxS3SequencerRunRootSsmParameterPath, + /* + Pieriandx specific parameters + */ + pieriandxUserEmailSsmParameterPath: pieriandxUserEmailSsmParameterPath, + pieriandxInstitutionSsmParameterPath: pieriandxInstitutionSsmParameterPath, + pieriandxBaseUrlSsmParameterPath: pieriandxBaseUrlSsmParameterPath, + pieriandxAuthTokenCollectionLambdaFunctionName: pieriandxAuthTokeSsmParameterPath, + /* Event info */ + eventDetailType: pieriandxDetailType, + eventBusName: eventBusName, + eventSource: pieriandxEventSource, + payloadVersion: pieriandxPayloadVersion, + triggerLaunchSource: pieriandxTriggerLaunchSource, + /* Custom */ + prefix: pieriandxPrefix, + }; +}; diff --git a/config/stacks/stackyMcStackFace.ts b/config/stacks/stackyMcStackFace.ts index 93589e33f..2ea67a19d 100644 --- a/config/stacks/stackyMcStackFace.ts +++ b/config/stacks/stackyMcStackFace.ts @@ -16,20 +16,18 @@ import { mockWtsGlueTableName, mockUmccriseGlueTableName, mockRnasumGlueTableName, + mockPierianDxGlueTableName, + pieriandxProjectInfoSsmParameterPath, + redcapLambdaFunctionName, } from '../constants'; import { GlueStackConfig } from '../../lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs'; import { StackyStatefulTablesConfig } from '../../lib/workload/stateful/stacks/stacky-mcstackface-dynamodb'; export const getGlueStackProps = (stage: AppStage): GlueStackConfig => { return { - /* SSM Parameters */ - bsshOutputFastqCopyUriSsmParameterName: mockPrimaryOutputUriSsmParameterName, - analysisCacheUriSsmParameterName: mockAnalysisCacheUriSsmParameterName, - analysisOutputUriSsmParameterName: mockAnalysisOutputUriSsmParameterName, - icav2ProjectIdSsmParameterName: mockIcav2ProjectIdSsmParameterName, - analysisLogsUriSsmParameterName: mockAnalysisLogsUriSsmParameterName, /* Events */ eventBusName: mockEventBusName, + /* Tables */ inputMakerTableName: mockInputMakerTableName, instrumentRunTableName: mockInstrumentRunTableName, @@ -40,8 +38,23 @@ export const getGlueStackProps = (stage: AppStage): GlueStackConfig => { wtsGlueTableName: mockWtsGlueTableName, umccriseGlueTableName: mockUmccriseGlueTableName, rnasumGlueTableName: mockRnasumGlueTableName, + pieriandxGlueTableName: mockPierianDxGlueTableName, + + /* SSM Parameters */ + analysisCacheUriSsmParameterName: mockAnalysisCacheUriSsmParameterName, + analysisOutputUriSsmParameterName: mockAnalysisOutputUriSsmParameterName, + icav2ProjectIdSsmParameterName: mockIcav2ProjectIdSsmParameterName, + analysisLogsUriSsmParameterName: mockAnalysisLogsUriSsmParameterName, + /* Secrets */ icav2AccessTokenSecretName: icav2AccessTokenSecretName[stage], + + /* BSSH SSM Parameters */ + bsshOutputFastqCopyUriSsmParameterName: mockPrimaryOutputUriSsmParameterName, + + /* PierianDx SSM Parameters */ + pieriandxProjectInfoSsmParameterPath: pieriandxProjectInfoSsmParameterPath, + redcapLambdaFunctionName: redcapLambdaFunctionName[stage], }; }; @@ -56,5 +69,6 @@ export const getStatefulGlueStackProps = (): StackyStatefulTablesConfig => { dynamodbWtsGlueTableName: mockWtsGlueTableName, dynamodbUmccriseGlueTableName: mockUmccriseGlueTableName, dynamodbRnasumGlueTableName: mockRnasumGlueTableName, + dynamodbPieriandxGlueTableName: mockPierianDxGlueTableName, }; }; diff --git a/lib/workload/components/dynamodb-icav2-handle-event-change-sfn/index.ts b/lib/workload/components/dynamodb-icav2-handle-event-change-sfn/index.ts index e5f3b1a06..5efb19d8b 100644 --- a/lib/workload/components/dynamodb-icav2-handle-event-change-sfn/index.ts +++ b/lib/workload/components/dynamodb-icav2-handle-event-change-sfn/index.ts @@ -127,7 +127,7 @@ export class Icav2AnalysisEventHandlerConstruct extends Construct { Convert a workflow name to lowercase and remove any spacing This has to be in align with Python impl: - lib/workload/components/event-workflowdraftrunstatechange-to-workflowrunstatechange-ready/lambdas/generate_workflow_run_name_py/generate_workflow_run_name.py + lib/workload/components/sfn-generate-workflowrunstatechange-ready-event/lambdas/generate_workflow_run_name_py/generate_workflow_run_name.py */ let _name = name.toLowerCase().replace(new RegExp(' ', 'g'), ''); _name = _name.replace(new RegExp('\\.', 'g'), '-'); diff --git a/lib/workload/components/dynamodb-icav2-ready-event-handler-sfn/step_functions_templates/icav2_launch_workflow_and_raise_internal_event.asl.json b/lib/workload/components/dynamodb-icav2-ready-event-handler-sfn/step_functions_templates/icav2_launch_workflow_and_raise_internal_event.asl.json index b523599dc..e5045c7da 100644 --- a/lib/workload/components/dynamodb-icav2-ready-event-handler-sfn/step_functions_templates/icav2_launch_workflow_and_raise_internal_event.asl.json +++ b/lib/workload/components/dynamodb-icav2-ready-event-handler-sfn/step_functions_templates/icav2_launch_workflow_and_raise_internal_event.asl.json @@ -24,20 +24,34 @@ } } }, - "Next": "Is Portal Run ID in Database", + "Next": "Is Portal Run ID + Analysis ID in Database", "ResultPath": "$.portal_run_id_in_db_step", "ResultSelector": { "db_response.$": "$" } }, - "Is Portal Run ID in Database": { + "Is Portal Run ID + Analysis ID in Database": { "Type": "Choice", "Choices": [ { - "Not": { - "Variable": "$.portal_run_id_in_db_step.db_response.Item", - "IsPresent": true - }, + "Or": [ + { + "Not": { + "Variable": "$.portal_run_id_in_db_step.db_response.Item", + "IsPresent": true + } + }, + { + "Not": { + "Variable": "$.portal_run_id_in_db_step.db_response.Item.analysis_id", + "IsPresent": true + } + }, + { + "Variable": "$.portal_run_id_in_db_step.db_response.Item.analysis_id.S", + "StringEquals": "" + } + ], "Next": "Add Technical Tags" } ], diff --git a/lib/workload/components/event-workflowdraftrunstatechange-to-workflowrunstatechange-ready/step_functions_templates/workflowrunstatechange_draft_to_ready_step_function_template.asl.json b/lib/workload/components/event-workflowdraftrunstatechange-to-workflowrunstatechange-ready/step_functions_templates/workflowrunstatechange_draft_to_ready_step_function_template.asl.json deleted file mode 100644 index f658c798d..000000000 --- a/lib/workload/components/event-workflowdraftrunstatechange-to-workflowrunstatechange-ready/step_functions_templates/workflowrunstatechange_draft_to_ready_step_function_template.asl.json +++ /dev/null @@ -1,202 +0,0 @@ -{ - "Comment": "A description of my state machine", - "StartAt": "move inputs", - "States": { - "move inputs": { - "Type": "Pass", - "Next": "Get portal run id from payload", - "Parameters": { - "input_event_detail.$": "$" - } - }, - "Get portal run id from payload": { - "Type": "Pass", - "Next": "Get portal run Id in database (local)", - "Parameters": { - "portal_run_id.$": "$.input_event_detail.portalRunId" - }, - "ResultPath": "$.get_portal_run_id_step" - }, - "Get portal run Id in database (local)": { - "Type": "Task", - "Resource": "arn:aws:states:::dynamodb:getItem", - "Parameters": { - "TableName": "${__table_name__}", - "Key": { - "id.$": "$.get_portal_run_id_step.portal_run_id", - "id_type": "${__workflow_type_partition_name__}" - } - }, - "ResultPath": "$.get_portal_run_id_in_db_step", - "Next": "is portal run id in database" - }, - "is portal run id in database": { - "Type": "Choice", - "Choices": [ - { - "Variable": "$.get_portal_run_id_in_db_step.Item", - "IsPresent": true, - "Next": "Get Item from DataBase (local)" - } - ], - "Default": "Get Workflow Run Engine Parameters" - }, - "Get Workflow Run Engine Parameters": { - "Type": "Task", - "Resource": "arn:aws:states:::states:startExecution.sync:2", - "Parameters": { - "StateMachineArn": "${__engine_parameters_maker_state_machine_arn__}", - "Input": { - "portal_run_id.$": "$.input_event_detail.portalRunId", - "workflow_name.$": "$.input_event_detail.workflowName", - "workflow_version.$": "$.input_event_detail.workflowVersion", - "event_data_inputs.$": "$.input_event_detail.payload.data.inputs", - "ssm_parameters_list": [ - { - "engine_parameter_key": "outputUri", - "ssm_name": "${__output_uri_ssm_parameter_name__}" - }, - { - "engine_parameter_key": "logsUri", - "ssm_name": "${__logs_uri_ssm_parameter_name__}" - }, - { - "engine_parameter_key": "cacheUri", - "ssm_name": "${__cache_uri_ssm_parameter_name__}" - }, - { - "engine_parameter_key": "projectId", - "ssm_name": "${__project_id_ssm_parameter_name__}" - } - ] - } - }, - "Next": "Update workflow table database", - "ResultPath": "$.set_workflow_run_engine_parameters", - "ResultSelector": { - "engine_parameters.$": "$.Output.engine_parameters" - } - }, - "Update workflow table database": { - "Type": "Parallel", - "Branches": [ - { - "StartAt": "Initialise Event Data Item with Inputs Engine Parameters", - "States": { - "Initialise Event Data Item with Inputs Engine Parameters": { - "Type": "Task", - "Resource": "arn:aws:states:::dynamodb:updateItem", - "Parameters": { - "TableName": "${__table_name__}", - "Key": { - "id.$": "$.get_portal_run_id_step.portal_run_id", - "id_type": "${__workflow_type_partition_name__}" - }, - "UpdateExpression": "SET event_data_inputs = :event_data_inputs, workflow_run_name = :workflow_run_name, portal_run_id = :portal_run_id, event_data_engine_parameters = :event_data_engine_parameters, event_data_tags = :event_data_tags", - "ExpressionAttributeValues": { - ":portal_run_id": { - "S.$": "$.input_event_detail.portalRunId" - }, - ":workflow_run_name": { - "S.$": "$.input_event_detail.workflowRunName" - }, - ":event_data_inputs": { - "S.$": "States.JsonToString($.input_event_detail.payload.data.inputs)" - }, - ":event_data_engine_parameters": { - "S.$": "States.JsonToString($.set_workflow_run_engine_parameters.engine_parameters)" - }, - ":event_data_tags": { - "S.$": "States.JsonToString($.input_event_detail.payload.data.tags)" - } - } - }, - "ResultPath": null, - "End": true - } - } - }, - { - "StartAt": "Update portal_run table", - "States": { - "Update portal_run table": { - "Type": "Task", - "Resource": "arn:aws:states:::dynamodb:updateItem", - "Parameters": { - "TableName": "${__table_name__}", - "Key": { - "id.$": "$.input_event_detail.portalRunId", - "id_type": "${__portal_run_partition_name__}" - }, - "UpdateExpression": "SET analysis_status = :analysis_status", - "ExpressionAttributeValues": { - ":analysis_status": { - "S": "READY" - } - } - }, - "ResultPath": null, - "End": true - } - } - } - ], - "ResultPath": null, - "Next": "Wait 1 Second (post event detail output update)" - }, - "Wait 1 Second (post event detail output update)": { - "Type": "Wait", - "Seconds": 1, - "Next": "Get Item from DataBase (local)" - }, - "Get Item from DataBase (local)": { - "Type": "Task", - "Resource": "arn:aws:states:::dynamodb:getItem", - "Parameters": { - "TableName": "${__table_name__}", - "Key": { - "id.$": "$.get_portal_run_id_step.portal_run_id", - "id_type": "${__workflow_type_partition_name__}" - } - }, - "Next": "EventBridge PutEvents", - "ResultSelector": { - "inputs.$": "States.StringToJson($.Item.event_data_inputs.S)", - "engine_parameters.$": "States.StringToJson($.Item.event_data_engine_parameters.S)", - "tags.$": "States.StringToJson($.Item.event_data_tags.S)" - }, - "ResultPath": "$.get_event_data_output_step" - }, - "EventBridge PutEvents": { - "Type": "Task", - "Resource": "arn:aws:states:::events:putEvents", - "Parameters": { - "Entries": [ - { - "Source": "${__event_output_source__}", - "EventBusName": "${__event_bus_name__}", - "DetailType": "${__detail_type__}", - "Detail": { - "portalRunId.$": "$.input_event_detail.portalRunId", - "timestamp.$": "$$.State.EnteredTime", - "status": "READY", - "workflowName.$": "$.input_event_detail.workflowName", - "workflowVersion.$": "$.input_event_detail.workflowVersion", - "workflowRunName.$": "$.input_event_detail.workflowRunName", - "linkedLibraries.$": "$.input_event_detail.linkedLibraries", - "payload": { - "version": "${__payload_version__}", - "data": { - "inputs.$": "$.get_event_data_output_step.inputs", - "engineParameters.$": "$.get_event_data_output_step.engine_parameters", - "tags.$": "$.input_event_detail.payload.data.tags" - } - } - } - } - ] - }, - "End": true - } - } -} diff --git a/lib/workload/components/icav2-copy-files-batch/index.ts b/lib/workload/components/icav2-copy-files-batch/index.ts index 72eb64431..03fb9b966 100644 --- a/lib/workload/components/icav2-copy-files-batch/index.ts +++ b/lib/workload/components/icav2-copy-files-batch/index.ts @@ -24,7 +24,7 @@ export class ICAv2CopyBatchUtilityConstruct extends Construct { super(scope, id); // Manifest inverter lambda - const manifest_inverter_lambda = new PythonFunction(this, 'manifest_inverter_lambda', { + const manifestInverterLambda = new PythonFunction(this, 'manifest_inverter_lambda', { entry: path.join(__dirname, 'manifest_handler_lambda_py'), runtime: lambda.Runtime.PYTHON_3_12, architecture: lambda.Architecture.ARM_64, @@ -48,13 +48,13 @@ export class ICAv2CopyBatchUtilityConstruct extends Construct { ), // Definition Substitutions definitionSubstitutions: { - __manifest_inverter_lambda_arn__: manifest_inverter_lambda.currentVersion.functionArn, + __manifest_inverter_lambda_arn__: manifestInverterLambda.currentVersion.functionArn, __copy_single_job_state_machine_arn__: this.icav2CopyFilesSfnObj.stateMachineArn, }, }); // Add execution permissions to stateMachine role - manifest_inverter_lambda.currentVersion.grantInvoke(this.icav2CopyFilesBatchSfnObj.role); + manifestInverterLambda.currentVersion.grantInvoke(this.icav2CopyFilesBatchSfnObj.role); // Because we run a nested state machine, we need to add the permissions to the state machine role // See https://stackoverflow.com/questions/60612853/nested-step-function-in-a-step-function-unknown-error-not-authorized-to-cr diff --git a/lib/workload/components/icav2-copy-files-batch/step_functions_templates/copy_files_batch_sfn.asl.json b/lib/workload/components/icav2-copy-files-batch/step_functions_templates/copy_files_batch_sfn.asl.json index b10ecdc50..06a597be4 100644 --- a/lib/workload/components/icav2-copy-files-batch/step_functions_templates/copy_files_batch_sfn.asl.json +++ b/lib/workload/components/icav2-copy-files-batch/step_functions_templates/copy_files_batch_sfn.asl.json @@ -39,6 +39,8 @@ }, "Process Manifest": { "Type": "Map", + "ItemsPath": "$.manifest_inverted_step.manifest_inverted", + "MaxConcurrency": 5, "Iterator": { "StartAt": "Copy Single Job State Machine", "States": { @@ -63,7 +65,6 @@ } } }, - "ItemsPath": "$.manifest_inverted_step.manifest_inverted", "ResultPath": "$.job_list_with_attempt_counter", "Next": "Succeed" }, diff --git a/lib/workload/components/icav2-copy-files/check_or_launch_job_lambda_py/check_or_launch_job_lambda.py b/lib/workload/components/icav2-copy-files/check_or_launch_job_lambda_py/check_or_launch_job_lambda.py index 11fcc71a5..bedaad302 100644 --- a/lib/workload/components/icav2-copy-files/check_or_launch_job_lambda_py/check_or_launch_job_lambda.py +++ b/lib/workload/components/icav2-copy-files/check_or_launch_job_lambda_py/check_or_launch_job_lambda.py @@ -193,9 +193,11 @@ def handler(event, context): "dest_uri": dest_uri, "source_uris": source_uris, "job_id": job_id, - "failed_job_list": failed_job_list, # Empty list or list of failed jobs + # Empty list or list of failed jobs + "failed_job_list": failed_job_list, "job_status": "RUNNING", - "wait_time_seconds": wait_time_seconds + DEFAULT_WAIT_TIME_SECONDS_EXT # Wait a bit longer (an extra 10 seconds) + # Wait a bit longer (an extra 10 seconds) + "wait_time_seconds": wait_time_seconds + DEFAULT_WAIT_TIME_SECONDS_EXT } # Handle a failed job diff --git a/lib/workload/components/python-lambda-get-metadata-objects-from-samplesheet/get_metadata_objects_from_samplesheet_py/get_metadata_objects_from_samplesheet.py b/lib/workload/components/python-lambda-get-metadata-objects-from-samplesheet/get_metadata_objects_from_samplesheet_py/get_metadata_objects_from_samplesheet.py index df94480ed..11d20f6a7 100644 --- a/lib/workload/components/python-lambda-get-metadata-objects-from-samplesheet/get_metadata_objects_from_samplesheet_py/get_metadata_objects_from_samplesheet.py +++ b/lib/workload/components/python-lambda-get-metadata-objects-from-samplesheet/get_metadata_objects_from_samplesheet_py/get_metadata_objects_from_samplesheet.py @@ -8,10 +8,8 @@ import logging from typing import List, Dict -import pandas as pd - # Layer imports -from metadata_tools import get_all_libraries, get_all_specimens, get_all_subjects +from metadata_tools import get_all_libraries # Logger logger = logging.getLogger() @@ -34,93 +32,6 @@ def get_library_objs(library_id_list: List[str]) -> List[Dict]: ) -def get_specimen_objs(specimen_id_list: List[int]) -> List[Dict]: - """ - Get all specimen objects by doing a bulk download + filter rather than query 1-1 - :param specimen_id_list: - :return: - """ - specimen_df = pd.DataFrame( - filter( - lambda specimen_obj_iter: specimen_obj_iter['orcabusId'] in specimen_id_list, - get_all_specimens() - ) - ) - - return ( - specimen_df.drop_duplicates(). - sort_values(by='orcabusId'). - to_dict(orient='records') - ) - - -def get_specimen_objs_by_library_obj_list(library_obj_list: List[Dict]) -> List[Dict]: - """ - Get specimen objects by library object list - :param library_obj_list: - :return: - """ - specimen_id_list = list( - map( - lambda library_obj_iter: library_obj_iter['specimen'], - library_obj_list - ) - ) - - # Get the specimens as a dataframe - specimens_df = pd.DataFrame(get_specimen_objs(specimen_id_list)) - - return specimens_df.to_dict(orient='records') - - -def get_subject_objs_by_specimen_obj_list(specimen_obj_list: List[Dict]) -> List[Dict]: - """ - Get all subjects by a specimen object list - :param specimen_obj_list: - :return: - """ - # Convert to dataframe to we can coerce columns subjects -> subject - specimens_df = pd.DataFrame(specimen_obj_list) - - # Since we're merging onto the subject df, we want to rename id to - # specimen (since the specimen id column is also called specimen in the library dataframe - # We will also only keep specimen, and subject columns since these are the key linker between the - # subject and library databases - specimens_df.rename( - columns={ - "orcabusId": "specimen" - }, - inplace=True - ) - specimens_df = specimens_df[["specimen", "subject"]] - - # Get all subjects - all_subjects_list_dict = get_all_subjects() - all_subjects_df = pd.DataFrame(all_subjects_list_dict) - - # Merge specimens and subjects df - filtered_subjects_df = pd.merge( - all_subjects_df, - specimens_df, - left_on="orcabusId", - right_on="subject", - how='inner' - ) - - subject_id_list = filtered_subjects_df["subjectId"].tolist() - - return ( - pd.DataFrame( - filter( - lambda subject_iter: subject_iter['subjectId'] in subject_id_list, - all_subjects_list_dict - ) - ).drop_duplicates(). - sort_values(by='orcabusId'). - to_dict(orient='records') - ) - - def handler(event, context): """ Given a samplesheet dictionary, collect the sample_id attributes as library ids. @@ -131,11 +42,13 @@ def handler(event, context): :return: """ + # Get the samplesheet dictionary if "samplesheet" not in event.keys(): logger.error("Could not get samplesheet") raise KeyError samplesheet_dict = event["samplesheet"] + # Get the bclconvert_data from the samplesheet if "bclconvert_data" not in samplesheet_dict.keys(): logger.error("Could not get bclconvert_data from samplesheet") raise KeyError @@ -146,7 +59,7 @@ def handler(event, context): set( list( map( - lambda bclconvert_data_row_iter: bclconvert_data_row_iter.get("sample_id"), + lambda bclconvert_data_row_iter_: bclconvert_data_row_iter_.get("sample_id"), bclconvert_data ) ) @@ -156,22 +69,19 @@ def handler(event, context): # Get library objects library_obj_list = get_library_objs(library_id_list) - # Get specimen objects - specimen_obj_list = get_specimen_objs_by_library_obj_list(library_obj_list) - - # Get subject objects - subject_obj_list = get_subject_objs_by_specimen_obj_list(specimen_obj_list) - # Get all libraries from the database return { - "library_obj_list": library_obj_list, - "specimen_obj_list": specimen_obj_list, - "subject_obj_list": subject_obj_list + "library_obj_list": library_obj_list } # if __name__ == "__main__": # import json +# from os import environ +# environ['AWS_PROFILE'] = 'umccr-development' +# environ['AWS_REGION'] = 'ap-southeast-2' +# environ['ORCABUS_TOKEN_SECRET_ID'] = 'orcabus/token-service-jwt' +# environ['HOSTNAME_SSM_PARAMETER'] = '/hosted_zone/umccr/name' # print( # json.dumps( # handler( @@ -718,611 +628,816 @@ def handler(event, context): # # { # # "library_obj_list": [ # # { -# # "orcabusId": "lib.01J5S9C4VMJ6PZ8GJ2G189AMXX", +# # "orcabusId": "lib.01J8ES4MPZ5B201R50K42XXM4M", +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4EBXK08WDWB97BSCX1C9", +# # "projectId": "PO", +# # "name": null, +# # "description": null +# # } +# # ], +# # "sample": { +# # "orcabusId": "smp.01J8ES4MPHSX7MRCTTFWJBYTT7", +# # "sampleId": "MDX210402", +# # "externalSampleId": "ZUHR111121", +# # "source": "plasma-serum" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES4MNXJSDRR406DAXFZP2N", +# # "subjectId": "PM3045106" +# # }, # # "libraryId": "L2400102", # # "phenotype": "tumor", # # "workflow": "research", # # "quality": "borderline", # # "type": "WGS", # # "assay": "ctTSO", -# # "coverage": 50.0, -# # "projectOwner": "VCCC", -# # "projectName": "PO", -# # "specimen": "spc.01J5S9C4V269YTNA17TTP6NF76" -# # }, -# # { -# # "orcabusId": "lib.01J5S9CBG0NF8QBNVKM6ESCD60", +# # "coverage": 50.0 +# # }, +# # { +# # "orcabusId": "lib.01J8ES4XNYFP38JMDV7GMV0V3V", +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4XMWD0DH7MDRNER5TZS1", +# # "projectId": "Testing", +# # "name": null, +# # "description": null +# # } +# # ], +# # "sample": { +# # "orcabusId": "smp.01J8ES4XMDW0FV1YMWHSZZQ4TX", +# # "sampleId": "PTC_SCMM1pc2", +# # "externalSampleId": "SSq-CompMM-1pc-10646259ilm", +# # "source": "cfDNA" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES4XKHKNQ1NF8EKKACZ032", +# # "subjectId": "CMM1pc-10646259ilm" +# # }, # # "libraryId": "L2400159", # # "phenotype": "tumor", # # "workflow": "manual", # # "quality": "good", # # "type": "ctDNA", # # "assay": "ctTSOv2", -# # "coverage": 38.6, -# # "projectOwner": "UMCCR", -# # "projectName": "Testing", -# # "specimen": "spc.01J5S9CBFDVZX7ZT3Y6TH28SY4" -# # }, -# # { -# # "orcabusId": "lib.01J5S9CBHP6NSB42RVFAP9PGJP", +# # "coverage": 38.6 +# # }, +# # { +# # "orcabusId": "lib.01J8ES4XQG3MPBW94TTVT4STVG", +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4XMWD0DH7MDRNER5TZS1", +# # "projectId": "Testing", +# # "name": null, +# # "description": null +# # } +# # ], +# # "sample": { +# # "orcabusId": "smp.01J8ES4XQ071BF3WZN111SNJ2B", +# # "sampleId": "PTC_SCMM1pc3", +# # "externalSampleId": "SSq-CompMM-1pc-10646259ilm", +# # "source": "cfDNA" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES4XKHKNQ1NF8EKKACZ032", +# # "subjectId": "CMM1pc-10646259ilm" +# # }, # # "libraryId": "L2400160", # # "phenotype": "tumor", # # "workflow": "manual", # # "quality": "good", # # "type": "ctDNA", # # "assay": "ctTSOv2", -# # "coverage": 38.6, -# # "projectOwner": "UMCCR", -# # "projectName": "Testing", -# # "specimen": "spc.01J5S9CBH4V5B56CEJ5Q1XQKQ9" -# # }, -# # { -# # "orcabusId": "lib.01J5S9CBKCATYSFY40BRX6WJWX", +# # "coverage": 38.6 +# # }, +# # { +# # "orcabusId": "lib.01J8ES4XSS97XNRS8DH0B1RJRG", +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4XMWD0DH7MDRNER5TZS1", +# # "projectId": "Testing", +# # "name": null, +# # "description": null +# # } +# # ], +# # "sample": { +# # "orcabusId": "smp.01J8ES4XRG9NB38N03688M2CCB", +# # "sampleId": "PTC_SCMM1pc4", +# # "externalSampleId": "SSq-CompMM-1pc-10646259ilm", +# # "source": "cfDNA" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES4XKHKNQ1NF8EKKACZ032", +# # "subjectId": "CMM1pc-10646259ilm" +# # }, # # "libraryId": "L2400161", # # "phenotype": "tumor", # # "workflow": "manual", # # "quality": "good", # # "type": "ctDNA", # # "assay": "ctTSOv2", -# # "coverage": 38.6, -# # "projectOwner": "UMCCR", -# # "projectName": "Testing", -# # "specimen": "spc.01J5S9CBJTBJB72KJ74VSCHKJF" -# # }, -# # { -# # "orcabusId": "lib.01J5S9CBN6EAXW4AXG7TQ1H6NC", +# # "coverage": 38.6 +# # }, +# # { +# # "orcabusId": "lib.01J8ES4XXF6NMEJMM5M4GWS6KH", +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4XMWD0DH7MDRNER5TZS1", +# # "projectId": "Testing", +# # "name": null, +# # "description": null +# # } +# # ], +# # "sample": { +# # "orcabusId": "smp.01J8ES4XWXFANT7P0T3AFXA85G", +# # "sampleId": "PTC_SCMM01pc20", +# # "externalSampleId": "SSq-CompMM-0.1pc-10624819 - 20ng", +# # "source": "cfDNA" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES4XW2TXGEJBQWCVMRZRTS", +# # "subjectId": "CMM0.1pc-10624819" +# # }, # # "libraryId": "L2400162", # # "phenotype": "tumor", # # "workflow": "manual", # # "quality": "good", # # "type": "ctDNA", # # "assay": "ctTSOv2", -# # "coverage": 38.6, -# # "projectOwner": "UMCCR", -# # "projectName": "Testing", -# # "specimen": "spc.01J5S9CBMKTX5KN1XMPN479R2M" -# # }, -# # { -# # "orcabusId": "lib.01J5S9CBQFX8V1QRW7KAV3MD1W", +# # "coverage": 38.6 +# # }, +# # { +# # "orcabusId": "lib.01J8ES4XZD7T2VRPVQ1GSVZ11X", +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4XMWD0DH7MDRNER5TZS1", +# # "projectId": "Testing", +# # "name": null, +# # "description": null +# # } +# # ], +# # "sample": { +# # "orcabusId": "smp.01J8ES4XYTSVQVRSBA9M26NSZY", +# # "sampleId": "PTC_SCMM01pc15", +# # "externalSampleId": "SSq-CompMM-0.1pc-10624819 - 15ng", +# # "source": "cfDNA" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES4XW2TXGEJBQWCVMRZRTS", +# # "subjectId": "CMM0.1pc-10624819" +# # }, # # "libraryId": "L2400163", # # "phenotype": "tumor", # # "workflow": "manual", # # "quality": "good", # # "type": "ctDNA", # # "assay": "ctTSOv2", -# # "coverage": 38.6, -# # "projectOwner": "UMCCR", -# # "projectName": "Testing", -# # "specimen": "spc.01J5S9CBPSPN6S3TQCVJZF0XFE" -# # }, -# # { -# # "orcabusId": "lib.01J5S9CBS64DNTHK6CE850CCNZ", +# # "coverage": 38.6 +# # }, +# # { +# # "orcabusId": "lib.01J8ES4Y1AKAHYD9EW0TW4FBCP", +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4XMWD0DH7MDRNER5TZS1", +# # "projectId": "Testing", +# # "name": null, +# # "description": null +# # } +# # ], +# # "sample": { +# # "orcabusId": "smp.01J8ES4Y0V0ZBKAE91TDSY0BBB", +# # "sampleId": "PTC_SCMM01pc10", +# # "externalSampleId": "SSq-CompMM-0.1pc-10624819 - 10ng", +# # "source": "cfDNA" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES4XW2TXGEJBQWCVMRZRTS", +# # "subjectId": "CMM0.1pc-10624819" +# # }, # # "libraryId": "L2400164", # # "phenotype": "tumor", # # "workflow": "manual", # # "quality": "good", # # "type": "ctDNA", # # "assay": "ctTSOv2", -# # "coverage": 38.6, -# # "projectOwner": "UMCCR", -# # "projectName": "Testing", -# # "specimen": "spc.01J5S9CBRM3Y6PPF6E5NWZA7HG" -# # }, -# # { -# # "orcabusId": "lib.01J5S9CBTZRYQNTGAHPC2T601D", +# # "coverage": 38.6 +# # }, +# # { +# # "orcabusId": "lib.01J8ES4Y3ZKRX3C5JAHA5NBXV1", +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4XMWD0DH7MDRNER5TZS1", +# # "projectId": "Testing", +# # "name": null, +# # "description": null +# # } +# # ], +# # "sample": { +# # "orcabusId": "smp.01J8ES4Y37JTPEEJSED9BXH8N2", +# # "sampleId": "PTC_SCMM01pc5", +# # "externalSampleId": "SSq-CompMM-0.1pc-10624819 - 5ng", +# # "source": "cfDNA" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES4XW2TXGEJBQWCVMRZRTS", +# # "subjectId": "CMM0.1pc-10624819" +# # }, # # "libraryId": "L2400165", # # "phenotype": "tumor", # # "workflow": "manual", # # "quality": "good", # # "type": "ctDNA", # # "assay": "ctTSOv2", -# # "coverage": 38.6, -# # "projectOwner": "UMCCR", -# # "projectName": "Testing", -# # "specimen": "spc.01J5S9CBTACFBNJKE8C523B0A7" -# # }, -# # { -# # "orcabusId": "lib.01J5S9CBX10204CK7EKGTH9TMB", +# # "coverage": 38.6 +# # }, +# # { +# # "orcabusId": "lib.01J8ES4Y5D52202JVBXHJ9Q9WF", +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4XMWD0DH7MDRNER5TZS1", +# # "projectId": "Testing", +# # "name": null, +# # "description": null +# # } +# # ], +# # "sample": { +# # "orcabusId": "smp.01J8ES4Y4XK1WX4WCPD6XY8KNM", +# # "sampleId": "NTC_v2ctTSO240207", +# # "externalSampleId": "negative control", +# # "source": "water" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES4DFMNF0SX6P8P8Y9J6K1", +# # "subjectId": "negative control" +# # }, # # "libraryId": "L2400166", # # "phenotype": "negative-control", # # "workflow": "manual", # # "quality": "good", # # "type": "ctDNA", # # "assay": "ctTSOv2", -# # "coverage": 0.1, -# # "projectOwner": "UMCCR", -# # "projectName": "Testing", -# # "specimen": "spc.01J5S9CBWCGKQMG5S3ZSWA2ATE" -# # }, -# # { -# # "orcabusId": "lib.01J5S9CDF8HHG5PJE3ECJMKMY7", +# # "coverage": 0.1 +# # }, +# # { +# # "orcabusId": "lib.01J8ES4ZDRQAP2BN3SDYYV5PKW", +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4ZAWHH3FKYA2CFHSMZ4B", +# # "projectId": "CAVATAK", +# # "name": null, +# # "description": null +# # } +# # ], +# # "sample": { +# # "orcabusId": "smp.01J8ES4ZDAFRK3K3PY33F8XS0W", +# # "sampleId": "PRJ240169", +# # "externalSampleId": "AUS-006-DRW_C1D1PRE", +# # "source": "blood" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES4ZCKNW6QKP006SYNZ5RA", +# # "subjectId": "AUS-006-DRW" +# # }, # # "libraryId": "L2400191", # # "phenotype": "normal", # # "workflow": "research", # # "quality": "good", # # "type": "WGS", # # "assay": "TsqNano", -# # "coverage": 40.0, -# # "projectOwner": "TJohn", -# # "projectName": "CAVATAK", -# # "specimen": "spc.01J5S9CDEH0ATXYAK52KW807R4" -# # }, -# # { -# # "orcabusId": "lib.01J5S9CDQSSAG1WYCRWMD82Z1S", +# # "coverage": 40.0 +# # }, +# # { +# # "orcabusId": "lib.01J8ES4ZMY0G1H9MDN7K2TH9Y6", +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4ZAWHH3FKYA2CFHSMZ4B", +# # "projectId": "CAVATAK", +# # "name": null, +# # "description": null +# # } +# # ], +# # "sample": { +# # "orcabusId": "smp.01J8ES4ZMETZP255WMFC8TSCYT", +# # "sampleId": "PRJ240180", +# # "externalSampleId": "AUS-006-DRW_Day0", +# # "source": "tissue" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES4ZCKNW6QKP006SYNZ5RA", +# # "subjectId": "AUS-006-DRW" +# # }, # # "libraryId": "L2400195", # # "phenotype": "tumor", # # "workflow": "research", # # "quality": "good", # # "type": "WGS", # # "assay": "TsqNano", -# # "coverage": 80.0, -# # "projectOwner": "TJohn", -# # "projectName": "CAVATAK", -# # "specimen": "spc.01J5S9CDQ0V9T98EGRPQJAP11S" -# # }, -# # { -# # "orcabusId": "lib.01J5S9CDSJ2BGEYM8FTXGKVGV8", +# # "coverage": 80.0 +# # }, +# # { +# # "orcabusId": "lib.01J8ES4ZP88X2E17X5X1FRMTPK", +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4ZAWHH3FKYA2CFHSMZ4B", +# # "projectId": "CAVATAK", +# # "name": null, +# # "description": null +# # } +# # ], +# # "sample": { +# # "orcabusId": "smp.01J8ES4ZNT47EM37QKMT12JPPJ", +# # "sampleId": "PRJ240181", +# # "externalSampleId": "AUS-006-DRW_Day33", +# # "source": "tissue" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES4ZCKNW6QKP006SYNZ5RA", +# # "subjectId": "AUS-006-DRW" +# # }, # # "libraryId": "L2400196", # # "phenotype": "tumor", # # "workflow": "research", # # "quality": "good", # # "type": "WGS", # # "assay": "TsqNano", -# # "coverage": 80.0, -# # "projectOwner": "TJohn", -# # "projectName": "CAVATAK", -# # "specimen": "spc.01J5S9CDRZMMR9S784BYSMVWCT" -# # }, -# # { -# # "orcabusId": "lib.01J5S9CDVEHDZHZR3BZTQ7WNJQ", +# # "coverage": 80.0 +# # }, +# # { +# # "orcabusId": "lib.01J8ES4ZST489C712CG3R9NQSQ", +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4ZAWHH3FKYA2CFHSMZ4B", +# # "projectId": "CAVATAK", +# # "name": null, +# # "description": null +# # } +# # ], +# # "sample": { +# # "orcabusId": "smp.01J8ES4ZQ76H8P0Q7S618F3BMA", +# # "sampleId": "PRJ240182", +# # "externalSampleId": "AUS-007-JMA_Day0", +# # "source": "tissue" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES4ZEQ3FVD6DDVEG8MW60Q", +# # "subjectId": "AUS-007-JMA" +# # }, # # "libraryId": "L2400197", # # "phenotype": "tumor", # # "workflow": "research", # # "quality": "good", # # "type": "WGS", # # "assay": "TsqNano", -# # "coverage": 80.0, -# # "projectOwner": "TJohn", -# # "projectName": "CAVATAK", -# # "specimen": "spc.01J5S9CDTSHGYMMJHE3SXEB2JG" -# # }, -# # { -# # "orcabusId": "lib.01J5S9CDXCR7Q5K6A8VJRSMM4Q", +# # "coverage": 80.0 +# # }, +# # { +# # "orcabusId": "lib.01J8ES4ZVWA2CGBHJVKAS3Y0G9", +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4ZAWHH3FKYA2CFHSMZ4B", +# # "projectId": "CAVATAK", +# # "name": null, +# # "description": null +# # } +# # ], +# # "sample": { +# # "orcabusId": "smp.01J8ES4ZVAR9NQM55Z2TXCDY9V", +# # "sampleId": "PRJ240183", +# # "externalSampleId": "AUS-007-JMA_Day15", +# # "source": "tissue" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES4ZEQ3FVD6DDVEG8MW60Q", +# # "subjectId": "AUS-007-JMA" +# # }, # # "libraryId": "L2400198", # # "phenotype": "tumor", # # "workflow": "research", # # "quality": "good", # # "type": "WGS", # # "assay": "TsqNano", -# # "coverage": 80.0, -# # "projectOwner": "TJohn", -# # "projectName": "CAVATAK", -# # "specimen": "spc.01J5S9CDWHAYG4RRG75GYZEK25" -# # }, -# # { -# # "orcabusId": "lib.01J5S9CFX5P69S4KZRQGDFKV1N", +# # "coverage": 80.0 +# # }, +# # { +# # "orcabusId": "lib.01J8ES51V0RSVT6C7WQR72QQED", +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4EZAA5YMHX82664GJQB3", +# # "projectId": "CUP", +# # "name": null, +# # "description": null +# # } +# # ], +# # "sample": { +# # "orcabusId": "smp.01J8ES51T84KVVVSEPYQFGW0EV", +# # "sampleId": "PRJ240199", +# # "externalSampleId": "DNA188239", +# # "source": "FFPE" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES51S87R4EJ61QJ0DMDYWZ", +# # "subjectId": "SN_PMC-141" +# # }, # # "libraryId": "L2400231", # # "phenotype": "tumor", # # "workflow": "clinical", # # "quality": "poor", # # "type": "WGS", # # "assay": "TsqNano", -# # "coverage": 100.0, -# # "projectOwner": "Tothill", -# # "projectName": "CUP", -# # "specimen": "spc.01J5S9CFWAQTGK4MZB3HM5NVBC" -# # }, -# # { -# # "orcabusId": "lib.01J5S9CGCAKQWHD9RBM9VXENY9", +# # "coverage": 100.0 +# # }, +# # { +# # "orcabusId": "lib.01J8ES52889Q8826P5SH9HDPP0", +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4EZAA5YMHX82664GJQB3", +# # "projectId": "CUP", +# # "name": null, +# # "description": null +# # } +# # ], +# # "sample": { +# # "orcabusId": "smp.01J8ES527QKB5Y5RVZWZ8HQX0H", +# # "sampleId": "PRJ240643", +# # "externalSampleId": "DNA188378", +# # "source": "blood" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES51S87R4EJ61QJ0DMDYWZ", +# # "subjectId": "SN_PMC-141" +# # }, # # "libraryId": "L2400238", # # "phenotype": "normal", # # "workflow": "clinical", # # "quality": "good", # # "type": "WGS", # # "assay": "TsqNano", -# # "coverage": 40.0, -# # "projectOwner": "Tothill", -# # "projectName": "CUP", -# # "specimen": "spc.01J5S9CGBQCSQCS7XR3T89A82F" -# # }, -# # { -# # "orcabusId": "lib.01J5S9CGEM1DHRQP72EP09B2TA", +# # "coverage": 40.0 +# # }, +# # { +# # "orcabusId": "lib.01J8ES52ANMRT3B7Y96T1Y3RY8", +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4EZAA5YMHX82664GJQB3", +# # "projectId": "CUP", +# # "name": null, +# # "description": null +# # } +# # ], +# # "sample": { +# # "orcabusId": "smp.01J8ES52A5QX0GQ6RB78Z8DGYQ", +# # "sampleId": "PRJ240646", +# # "externalSampleId": "DNA189922", +# # "source": "blood" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES529GSPBV64SESK9SWD76", +# # "subjectId": "SN_PMC-145" +# # }, # # "libraryId": "L2400239", # # "phenotype": "normal", # # "workflow": "clinical", # # "quality": "good", # # "type": "WGS", # # "assay": "TsqNano", -# # "coverage": 40.0, -# # "projectOwner": "Tothill", -# # "projectName": "CUP", -# # "specimen": "spc.01J5S9CGE05BJCJ20M2KP4QWWB" -# # }, -# # { -# # "orcabusId": "lib.01J5S9CGG9N9GH5879SY6A6BJB", +# # "coverage": 40.0 +# # }, +# # { +# # "orcabusId": "lib.01J8ES52C3N585BGGY4VNXHC83", +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4EZAA5YMHX82664GJQB3", +# # "projectId": "CUP", +# # "name": null, +# # "description": null +# # } +# # ], +# # "sample": { +# # "orcabusId": "smp.01J8ES52BM8BVS3PX47E6FM7D5", +# # "sampleId": "PRJ240647", +# # "externalSampleId": "DNA189848", +# # "source": "FFPE" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES529GSPBV64SESK9SWD76", +# # "subjectId": "SN_PMC-145" +# # }, # # "libraryId": "L2400240", # # "phenotype": "tumor", # # "workflow": "clinical", # # "quality": "poor", # # "type": "WGS", # # "assay": "TsqNano", -# # "coverage": 100.0, -# # "projectOwner": "Tothill", -# # "projectName": "CUP", -# # "specimen": "spc.01J5S9CGFQM3BQKADX8TWQ4ZH5" -# # }, -# # { -# # "orcabusId": "lib.01J5S9CGJ6G09YQ9KFHPSXMMVD", +# # "coverage": 100.0 +# # }, +# # { +# # "orcabusId": "lib.01J8ES52DHAPZM6FZ0VZK89PRT", +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4FC6DVW20AR33FBX2SA8", +# # "projectId": "Control", +# # "name": null, +# # "description": null +# # } +# # ], +# # "sample": { +# # "orcabusId": "smp.01J8ES52D076FQM5K8128AQ593", +# # "sampleId": "NTC_TSqN240226", +# # "externalSampleId": "NTC_TSqN240226", +# # "source": "water" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES4DFMNF0SX6P8P8Y9J6K1", +# # "subjectId": "negative control" +# # }, # # "libraryId": "L2400241", # # "phenotype": "negative-control", # # "workflow": "control", # # "quality": "good", # # "type": "WGS", # # "assay": "TsqNano", -# # "coverage": 0.1, -# # "projectOwner": "UMCCR", -# # "projectName": "Control", -# # "specimen": "spc.01J5S9CGHK1G7YXD7C4FCXXS52" -# # }, -# # { -# # "orcabusId": "lib.01J5S9CGKWDN7STKZKQM3KH9XR", +# # "coverage": 0.1 +# # }, +# # { +# # "orcabusId": "lib.01J8ES52F2ZHRXQY1AT1N1F81F", +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4FC6DVW20AR33FBX2SA8", +# # "projectId": "Control", +# # "name": null, +# # "description": null +# # } +# # ], +# # "sample": { +# # "orcabusId": "smp.01J8ES52EEX67YRYAJS3F5GMJ5", +# # "sampleId": "PTC_TSqN240226", +# # "externalSampleId": "NA24385-3", +# # "source": "cell-line" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES4DRJ31Z2H1GJQZGVDXZR", +# # "subjectId": "NA24385" +# # }, # # "libraryId": "L2400242", # # "phenotype": "normal", # # "workflow": "control", # # "quality": "good", # # "type": "WGS", # # "assay": "TsqNano", -# # "coverage": 15.0, -# # "projectOwner": "UMCCR", -# # "projectName": "Control", -# # "specimen": "spc.01J5S9CGKAT4GHZ1VJFHVV15AD" -# # }, -# # { -# # "orcabusId": "lib.01J5S9CH2SQ0P1SF7WAT5H4DSE", +# # "coverage": 15.0 +# # }, +# # { +# # "orcabusId": "lib.01J8ES52XYMVGRB1Q458THNG4T", +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4FC6DVW20AR33FBX2SA8", +# # "projectId": "Control", +# # "name": null, +# # "description": null +# # } +# # ], +# # "sample": { +# # "orcabusId": "smp.01J8ES52XE661E8V8XTWD02QCK", +# # "sampleId": "PTC_NebRNA240226", +# # "externalSampleId": "Colo829", +# # "source": "cell-line" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES4GNVGZSJVTHGVKS9VW7F", +# # "subjectId": "Colo829" +# # }, # # "libraryId": "L2400249", # # "phenotype": "tumor", # # "workflow": "control", # # "quality": "good", # # "type": "WTS", # # "assay": "NebRNA", -# # "coverage": 1.0, -# # "projectOwner": "UMCCR", -# # "projectName": "Control", -# # "specimen": "spc.01J5S9CH267XEJP5GMZK31MJWS" -# # }, -# # { -# # "orcabusId": "lib.01J5S9CH4CYPA4SP05H8KRX4W9", +# # "coverage": 1.0 +# # }, +# # { +# # "orcabusId": "lib.01J8ES52Z2KTVVKZ2ZGVQ6YC10", +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4FH3XMPZQNDJ9J000BXX", +# # "projectId": "BPOP-retro", +# # "name": null, +# # "description": null +# # } +# # ], +# # "sample": { +# # "orcabusId": "smp.01J8ES4FP9WTFBDNGKVG3D9BD4", +# # "sampleId": "PRJ240003", +# # "externalSampleId": "3-23BCRL057T", +# # "source": "tissue" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES4FNJ2FCAK0RJST0428X0", +# # "subjectId": "23BCRL057T" +# # }, # # "libraryId": "L2400250", # # "phenotype": "tumor", # # "workflow": "research", # # "quality": "good", # # "type": "WTS", # # "assay": "NebRNA", -# # "coverage": 6.0, -# # "projectOwner": "Whittle", -# # "projectName": "BPOP-retro", -# # "specimen": "spc.01J5S9C0QC2TBZD7XA26D7WGTW" -# # }, -# # { -# # "orcabusId": "lib.01J5S9CH65E4EE5QJEJ1C60GGG", +# # "coverage": 6.0 +# # }, +# # { +# # "orcabusId": "lib.01J8ES530H895X4WA3NQ6CY2QV", +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4FH3XMPZQNDJ9J000BXX", +# # "projectId": "BPOP-retro", +# # "name": null, +# # "description": null +# # } +# # ], +# # "sample": { +# # "orcabusId": "smp.01J8ES530355YNZ3VHQQQ204PF", +# # "sampleId": "PRJ240561", +# # "externalSampleId": "4-218-004_Bx", +# # "source": "tissue" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES51WC4GV5YDJNTMAK2YY1", +# # "subjectId": "218-004" +# # }, # # "libraryId": "L2400251", # # "phenotype": "tumor", # # "workflow": "research", # # "quality": "good", # # "type": "WTS", # # "assay": "NebRNA", -# # "coverage": 6.0, -# # "projectOwner": "Whittle", -# # "projectName": "BPOP-retro", -# # "specimen": "spc.01J5S9CH5KXY3VMB9M9J2RCR7B" -# # }, -# # { -# # "orcabusId": "lib.01J5S9CH7TGZMV39Z59WJ8H5GP", +# # "coverage": 6.0 +# # }, +# # { +# # "orcabusId": "lib.01J8ES5320EWBNNYDGXF2SYJBD", +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4FH3XMPZQNDJ9J000BXX", +# # "projectId": "BPOP-retro", +# # "name": null, +# # "description": null +# # } +# # ], +# # "sample": { +# # "orcabusId": "smp.01J8ES531H420JM9MG5R4AE1AZ", +# # "sampleId": "PRJ240562", +# # "externalSampleId": "5-218-004_04", +# # "source": "tissue" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES51WC4GV5YDJNTMAK2YY1", +# # "subjectId": "218-004" +# # }, # # "libraryId": "L2400252", # # "phenotype": "tumor", # # "workflow": "research", # # "quality": "good", # # "type": "WTS", # # "assay": "NebRNA", -# # "coverage": 6.0, -# # "projectOwner": "Whittle", -# # "projectName": "BPOP-retro", -# # "specimen": "spc.01J5S9CH774HEZFEVWWP2XADK1" -# # }, -# # { -# # "orcabusId": "lib.01J5S9CH9TGMT2TJGBZX5VXHJY", +# # "coverage": 6.0 +# # }, +# # { +# # "orcabusId": "lib.01J8ES533DJZZNPP9MXYR5TRC0", +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4FH3XMPZQNDJ9J000BXX", +# # "projectId": "BPOP-retro", +# # "name": null, +# # "description": null +# # } +# # ], +# # "sample": { +# # "orcabusId": "smp.01J8ES532ZBHWY3DWY0DWQ223R", +# # "sampleId": "PRJ240566", +# # "externalSampleId": "9-218-007_Bx", +# # "source": "tissue" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES522WN7YPZS1Z9NGSPNDA", +# # "subjectId": "218-007" +# # }, # # "libraryId": "L2400253", # # "phenotype": "tumor", # # "workflow": "research", # # "quality": "good", # # "type": "WTS", # # "assay": "NebRNA", -# # "coverage": 6.0, -# # "projectOwner": "Whittle", -# # "projectName": "BPOP-retro", -# # "specimen": "spc.01J5S9CH98MQ2B1G2BQFEY0XZH" -# # }, -# # { -# # "orcabusId": "lib.01J5S9CHBGAP2XSN4TG8SAMRYY", +# # "coverage": 6.0 +# # }, +# # { +# # "orcabusId": "lib.01J8ES534XGBFYDVYV8ZG6SYS0", +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4FH3XMPZQNDJ9J000BXX", +# # "projectId": "BPOP-retro", +# # "name": null, +# # "description": null +# # } +# # ], +# # "sample": { +# # "orcabusId": "smp.01J8ES534BX7B89X5EKSCFRDDZ", +# # "sampleId": "PRJ240567", +# # "externalSampleId": "10-218-007_04", +# # "source": "tissue" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES522WN7YPZS1Z9NGSPNDA", +# # "subjectId": "218-007" +# # }, # # "libraryId": "L2400254", # # "phenotype": "tumor", # # "workflow": "research", # # "quality": "borderline", # # "type": "WTS", # # "assay": "NebRNA", -# # "coverage": 6.0, -# # "projectOwner": "Whittle", -# # "projectName": "BPOP-retro", -# # "specimen": "spc.01J5S9CHAX3XKJE5XE4VQWYN5H" -# # }, -# # { -# # "orcabusId": "lib.01J5S9CHE4ERQ4H209DH397W8A", +# # "coverage": 6.0 +# # }, +# # { +# # "orcabusId": "lib.01J8ES536AB5A5PBJ8S45SZP7Q", +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4EZAA5YMHX82664GJQB3", +# # "projectId": "CUP", +# # "name": null, +# # "description": null +# # } +# # ], +# # "sample": { +# # "orcabusId": "smp.01J8ES535VGG93023KWAFMWGH4", +# # "sampleId": "PRJ240200", +# # "externalSampleId": "RNA036747", +# # "source": "FFPE" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES51S87R4EJ61QJ0DMDYWZ", +# # "subjectId": "SN_PMC-141" +# # }, # # "libraryId": "L2400255", # # "phenotype": "tumor", # # "workflow": "clinical", # # "quality": "very-poor", # # "type": "WTS", # # "assay": "NebRNA", -# # "coverage": 6.0, -# # "projectOwner": "Tothill", -# # "projectName": "CUP", -# # "specimen": "spc.01J5S9CHDGRNK70B043K887RP2" -# # }, -# # { -# # "orcabusId": "lib.01J5S9CHFXPDGYQ8TXHRWQR3PY", +# # "coverage": 6.0 +# # }, +# # { +# # "orcabusId": "lib.01J8ES537S0W1AX9PQPST13GM9", +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4EZAA5YMHX82664GJQB3", +# # "projectId": "CUP", +# # "name": null, +# # "description": null +# # } +# # ], +# # "sample": { +# # "orcabusId": "smp.01J8ES5379C40K08YG3JDMZJN7", +# # "sampleId": "PRJ240648", +# # "externalSampleId": "RNA037080", +# # "source": "FFPE" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES529GSPBV64SESK9SWD76", +# # "subjectId": "SN_PMC-145" +# # }, # # "libraryId": "L2400256", # # "phenotype": "tumor", # # "workflow": "clinical", # # "quality": "very-poor", # # "type": "WTS", # # "assay": "NebRNA", -# # "coverage": 6.0, -# # "projectOwner": "Tothill", -# # "projectName": "CUP", -# # "specimen": "spc.01J5S9CHFAPXYKK49FAGVF5CQF" -# # }, -# # { -# # "orcabusId": "lib.01J5S9CHHNGFJN73NPRQMSYGN9", +# # "coverage": 6.0 +# # }, +# # { +# # "orcabusId": "lib.01J8ES5395KETT9T2NJSVNDKNP", +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4FC6DVW20AR33FBX2SA8", +# # "projectId": "Control", +# # "name": null, +# # "description": null +# # } +# # ], +# # "sample": { +# # "orcabusId": "smp.01J8ES538PFF6MQQ35PTC00JAY", +# # "sampleId": "NTC_NebRNA240226", +# # "externalSampleId": "NTC_NebRNA240226", +# # "source": "water" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES4DFMNF0SX6P8P8Y9J6K1", +# # "subjectId": "negative control" +# # }, # # "libraryId": "L2400257", # # "phenotype": "negative-control", # # "workflow": "control", # # "quality": "good", # # "type": "WTS", # # "assay": "NebRNA", -# # "coverage": 0.1, -# # "projectOwner": "UMCCR", -# # "projectName": "Control", -# # "specimen": "spc.01J5S9CHH24VFM443RD8Q8X4B3" -# # } -# # ], -# # "specimen_obj_list": [ -# # { -# # "orcabusId": "spc.01J5S9C0QC2TBZD7XA26D7WGTW", -# # "specimenId": "PRJ240003", -# # "source": "tissue", -# # "subject": "sbj.01J5S9C0PVB4QNVGK4Q1WSYEGV" -# # }, -# # { -# # "orcabusId": "spc.01J5S9C4V269YTNA17TTP6NF76", -# # "specimenId": "MDX210402", -# # "source": "plasma-serum", -# # "subject": "sbj.01J5S9C4TE1GCWA1QGNCWHB1Y9" -# # }, -# # { -# # "orcabusId": "spc.01J5S9CBFDVZX7ZT3Y6TH28SY4", -# # "specimenId": "PTC_SCMM1pc2", -# # "source": "cfDNA", -# # "subject": "sbj.01J5S9CBEQ3DM8XDV2G2ZQJDXB" -# # }, -# # { -# # "orcabusId": "spc.01J5S9CBH4V5B56CEJ5Q1XQKQ9", -# # "specimenId": "PTC_SCMM1pc3", -# # "source": "cfDNA", -# # "subject": "sbj.01J5S9CBEQ3DM8XDV2G2ZQJDXB" -# # }, -# # { -# # "orcabusId": "spc.01J5S9CBJTBJB72KJ74VSCHKJF", -# # "specimenId": "PTC_SCMM1pc4", -# # "source": "cfDNA", -# # "subject": "sbj.01J5S9CBEQ3DM8XDV2G2ZQJDXB" -# # }, -# # { -# # "orcabusId": "spc.01J5S9CBMKTX5KN1XMPN479R2M", -# # "specimenId": "PTC_SCMM01pc20", -# # "source": "cfDNA", -# # "subject": "sbj.01J5S9CBM3AT89QTXD7PT0BKA0" -# # }, -# # { -# # "orcabusId": "spc.01J5S9CBPSPN6S3TQCVJZF0XFE", -# # "specimenId": "PTC_SCMM01pc15", -# # "source": "cfDNA", -# # "subject": "sbj.01J5S9CBM3AT89QTXD7PT0BKA0" -# # }, -# # { -# # "orcabusId": "spc.01J5S9CBRM3Y6PPF6E5NWZA7HG", -# # "specimenId": "PTC_SCMM01pc10", -# # "source": "cfDNA", -# # "subject": "sbj.01J5S9CBM3AT89QTXD7PT0BKA0" -# # }, -# # { -# # "orcabusId": "spc.01J5S9CBTACFBNJKE8C523B0A7", -# # "specimenId": "PTC_SCMM01pc5", -# # "source": "cfDNA", -# # "subject": "sbj.01J5S9CBM3AT89QTXD7PT0BKA0" -# # }, -# # { -# # "orcabusId": "spc.01J5S9CBWCGKQMG5S3ZSWA2ATE", -# # "specimenId": "NTC_v2ctTSO240207", -# # "source": "water", -# # "subject": "sbj.01J5S9BYKC1RH7DY68GF1JNSR6" -# # }, -# # { -# # "orcabusId": "spc.01J5S9CDEH0ATXYAK52KW807R4", -# # "specimenId": "PRJ240169", -# # "source": "blood", -# # "subject": "sbj.01J5S9CDDP20JX8V63ZKMPBJQS" -# # }, -# # { -# # "orcabusId": "spc.01J5S9CDQ0V9T98EGRPQJAP11S", -# # "specimenId": "PRJ240180", -# # "source": "tissue", -# # "subject": "sbj.01J5S9CDDP20JX8V63ZKMPBJQS" -# # }, -# # { -# # "orcabusId": "spc.01J5S9CDRZMMR9S784BYSMVWCT", -# # "specimenId": "PRJ240181", -# # "source": "tissue", -# # "subject": "sbj.01J5S9CDDP20JX8V63ZKMPBJQS" -# # }, -# # { -# # "orcabusId": "spc.01J5S9CDTSHGYMMJHE3SXEB2JG", -# # "specimenId": "PRJ240182", -# # "source": "tissue", -# # "subject": "sbj.01J5S9CDG7B0KA8YEDK876VVDP" -# # }, -# # { -# # "orcabusId": "spc.01J5S9CDWHAYG4RRG75GYZEK25", -# # "specimenId": "PRJ240183", -# # "source": "tissue", -# # "subject": "sbj.01J5S9CDG7B0KA8YEDK876VVDP" -# # }, -# # { -# # "orcabusId": "spc.01J5S9CFWAQTGK4MZB3HM5NVBC", -# # "specimenId": "PRJ240199", -# # "source": "FFPE", -# # "subject": "sbj.01J5S9CFVJ9GVEHZK6CD9WAAV5" -# # }, -# # { -# # "orcabusId": "spc.01J5S9CGBQCSQCS7XR3T89A82F", -# # "specimenId": "PRJ240643", -# # "source": "blood", -# # "subject": "sbj.01J5S9CFVJ9GVEHZK6CD9WAAV5" -# # }, -# # { -# # "orcabusId": "spc.01J5S9CGE05BJCJ20M2KP4QWWB", -# # "specimenId": "PRJ240646", -# # "source": "blood", -# # "subject": "sbj.01J5S9CGDGTF5VZJSSE4ADBNJ3" -# # }, -# # { -# # "orcabusId": "spc.01J5S9CGFQM3BQKADX8TWQ4ZH5", -# # "specimenId": "PRJ240647", -# # "source": "FFPE", -# # "subject": "sbj.01J5S9CGDGTF5VZJSSE4ADBNJ3" -# # }, -# # { -# # "orcabusId": "spc.01J5S9CGHK1G7YXD7C4FCXXS52", -# # "specimenId": "NTC_TSqN240226", -# # "source": "water", -# # "subject": "sbj.01J5S9BYKC1RH7DY68GF1JNSR6" -# # }, -# # { -# # "orcabusId": "spc.01J5S9CGKAT4GHZ1VJFHVV15AD", -# # "specimenId": "PTC_TSqN240226", -# # "source": "cell-line", -# # "subject": "sbj.01J5S9BYVWZDS8AW7A94CDQBXK" -# # }, -# # { -# # "orcabusId": "spc.01J5S9CH267XEJP5GMZK31MJWS", -# # "specimenId": "PTC_NebRNA240226", -# # "source": "cell-line", -# # "subject": "sbj.01J5S9C1S3XV8PNB78XYJ1EQM1" -# # }, -# # { -# # "orcabusId": "spc.01J5S9CH5KXY3VMB9M9J2RCR7B", -# # "specimenId": "PRJ240561", -# # "source": "tissue", -# # "subject": "sbj.01J5S9CFY1BV2Z0SGKYNF1VHQN" -# # }, -# # { -# # "orcabusId": "spc.01J5S9CH774HEZFEVWWP2XADK1", -# # "specimenId": "PRJ240562", -# # "source": "tissue", -# # "subject": "sbj.01J5S9CFY1BV2Z0SGKYNF1VHQN" -# # }, -# # { -# # "orcabusId": "spc.01J5S9CH98MQ2B1G2BQFEY0XZH", -# # "specimenId": "PRJ240566", -# # "source": "tissue", -# # "subject": "sbj.01J5S9CG5GEWYBK0065C49HT23" -# # }, -# # { -# # "orcabusId": "spc.01J5S9CHAX3XKJE5XE4VQWYN5H", -# # "specimenId": "PRJ240567", -# # "source": "tissue", -# # "subject": "sbj.01J5S9CG5GEWYBK0065C49HT23" -# # }, -# # { -# # "orcabusId": "spc.01J5S9CHDGRNK70B043K887RP2", -# # "specimenId": "PRJ240200", -# # "source": "FFPE", -# # "subject": "sbj.01J5S9CFVJ9GVEHZK6CD9WAAV5" -# # }, -# # { -# # "orcabusId": "spc.01J5S9CHFAPXYKK49FAGVF5CQF", -# # "specimenId": "PRJ240648", -# # "source": "FFPE", -# # "subject": "sbj.01J5S9CGDGTF5VZJSSE4ADBNJ3" -# # }, -# # { -# # "orcabusId": "spc.01J5S9CHH24VFM443RD8Q8X4B3", -# # "specimenId": "NTC_NebRNA240226", -# # "source": "water", -# # "subject": "sbj.01J5S9BYKC1RH7DY68GF1JNSR6" -# # } -# # ], -# # "subject_obj_list": [ -# # { -# # "orcabusId": "sbj.01J5S9BYKC1RH7DY68GF1JNSR6", -# # "subjectId": "SBJ00006" -# # }, -# # { -# # "orcabusId": "sbj.01J5S9BYVWZDS8AW7A94CDQBXK", -# # "subjectId": "SBJ00005" -# # }, -# # { -# # "orcabusId": "sbj.01J5S9C0PVB4QNVGK4Q1WSYEGV", -# # "subjectId": "SBJ04488" -# # }, -# # { -# # "orcabusId": "sbj.01J5S9C1S3XV8PNB78XYJ1EQM1", -# # "subjectId": "SBJ00029" -# # }, -# # { -# # "orcabusId": "sbj.01J5S9C4TE1GCWA1QGNCWHB1Y9", -# # "subjectId": "SBJ01143" -# # }, -# # { -# # "orcabusId": "sbj.01J5S9CBEQ3DM8XDV2G2ZQJDXB", -# # "subjectId": "SBJ04407" -# # }, -# # { -# # "orcabusId": "sbj.01J5S9CBM3AT89QTXD7PT0BKA0", -# # "subjectId": "SBJ04648" -# # }, -# # { -# # "orcabusId": "sbj.01J5S9CDDP20JX8V63ZKMPBJQS", -# # "subjectId": "SBJ04653" -# # }, -# # { -# # "orcabusId": "sbj.01J5S9CDG7B0KA8YEDK876VVDP", -# # "subjectId": "SBJ04654" -# # }, -# # { -# # "orcabusId": "sbj.01J5S9CFVJ9GVEHZK6CD9WAAV5", -# # "subjectId": "SBJ04659" -# # }, -# # { -# # "orcabusId": "sbj.01J5S9CFY1BV2Z0SGKYNF1VHQN", -# # "subjectId": "SBJ04660" -# # }, -# # { -# # "orcabusId": "sbj.01J5S9CG5GEWYBK0065C49HT23", -# # "subjectId": "SBJ04661" -# # }, -# # { -# # "orcabusId": "sbj.01J5S9CGDGTF5VZJSSE4ADBNJ3", -# # "subjectId": "SBJ04662" +# # "coverage": 0.1 # # } # # ] # # } diff --git a/lib/workload/components/python-lambda-layer/index.ts b/lib/workload/components/python-lambda-layer/index.ts index 55c4f271d..b8d952b2f 100644 --- a/lib/workload/components/python-lambda-layer/index.ts +++ b/lib/workload/components/python-lambda-layer/index.ts @@ -29,7 +29,10 @@ export class PythonLambdaLayerConstruct extends Construct { return []; }, afterBundling(inputDir: string, outputDir: string): string[] { - return [`python -m pip install ${inputDir} -t ${outputDir}`]; + return [ + `python -m pip install ${inputDir} -t ${outputDir}`, + `find ${outputDir} -name 'pandas' -exec rm -rf {}/tests/ \\;`, + ]; }, }, }, diff --git a/lib/workload/components/python-lambda-metadata-mapper/index.ts b/lib/workload/components/python-lambda-metadata-mapper/index.ts new file mode 100644 index 000000000..369224988 --- /dev/null +++ b/lib/workload/components/python-lambda-metadata-mapper/index.ts @@ -0,0 +1,94 @@ +/* +Quick and dirty way to map orcabus ids to complementary ids for each +of the databases from the metadata manager + +Comes with the bells and whistles of metadata tools layer and +permissions to use the orcabus token. + +User will need to use the 'addEnvironment' method on the returned lambda object in order +to specify what command will be run + +User will need + +ENV: +CONTEXT: One of the following: + - library + - subject + - individual + - sample + - project + - contact + +FROM_ORCABUS or FROM_ID +RETURN_STR or RETURN_OBJ + +Look at ./map_metadata_py/map_metadata.py for examples of outputs + +*/ + +import { Construct } from 'constructs'; +import * as lambda from 'aws-cdk-lib/aws-lambda'; +import { PythonFunction } from '@aws-cdk/aws-lambda-python-alpha'; +import path from 'path'; +import { MetadataToolsPythonLambdaLayer } from '../python-metadata-tools-layer'; +import * as ssm from 'aws-cdk-lib/aws-ssm'; +import * as secretsmanager from 'aws-cdk-lib/aws-secretsmanager'; +import { Duration } from 'aws-cdk-lib'; + +interface MapMetadataLambdaObj { + functionNamePrefix: string; +} + +export class GetMetadataLambdaConstruct extends Construct { + public readonly lambdaObj: PythonFunction; + + // Globals + private readonly hostnameSsmParameterPath = '/hosted_zone/umccr/name'; + private readonly orcabusTokenSecretId = 'orcabus/token-service-jwt'; // pragma: allowlist secret + + constructor(scope: Construct, id: string, props: MapMetadataLambdaObj) { + super(scope, id); + + // Get the metadata layer object + const metadataLayerObj = new MetadataToolsPythonLambdaLayer(this, 'metadata-tools-layer', { + layerPrefix: 'get-library-objects', + }); + + /* + Collect the required secret and ssm parameters for getting metadata + */ + const hostnameSsmParameterObj = ssm.StringParameter.fromStringParameterName( + this, + 'hostname_ssm_parameter', + this.hostnameSsmParameterPath + ); + const orcabusTokenSecretObj = secretsmanager.Secret.fromSecretNameV2( + this, + 'orcabus_token_secret', + this.orcabusTokenSecretId + ); + + // Get library objects + this.lambdaObj = new PythonFunction(this, 'map_metadata_py', { + functionName: `${props.functionNamePrefix}-map-metadata-py`, + entry: path.join(__dirname, 'map_metadata_py'), + runtime: lambda.Runtime.PYTHON_3_12, + architecture: lambda.Architecture.ARM_64, + index: 'map_metadata.py', + handler: 'handler', + memorySize: 1024, + layers: [metadataLayerObj.lambdaLayerVersionObj], + environment: { + HOSTNAME_SSM_PARAMETER: hostnameSsmParameterObj.parameterName, + ORCABUS_TOKEN_SECRET_ID: orcabusTokenSecretObj.secretName, + }, + timeout: Duration.seconds(60), + }); + + // Allow the lambda to read the secret + orcabusTokenSecretObj.grantRead(this.lambdaObj.currentVersion); + + // Allow the lambda to read the ssm parameter + hostnameSsmParameterObj.grantRead(this.lambdaObj.currentVersion); + } +} diff --git a/lib/workload/components/python-lambda-metadata-mapper/map_metadata_py/map_metadata.py b/lib/workload/components/python-lambda-metadata-mapper/map_metadata_py/map_metadata.py new file mode 100644 index 000000000..ca6a493ab --- /dev/null +++ b/lib/workload/components/python-lambda-metadata-mapper/map_metadata_py/map_metadata.py @@ -0,0 +1,446 @@ +#!/usr/bin/env python3 + + +# !/usr/bin/env python3 + +""" +This script is used to get the subject orcabus id from the subject id. +""" + +# Standard imports +from os import environ + +# Metadata imports +from metadata_tools import ( + # Orcabus helpers + get_orcabus_token, + # Subject helpers + get_subject_from_subject_orcabus_id, get_subject_from_subject_id, + # Sample helpers + get_sample_from_sample_orcabus_id, get_sample_from_sample_id, + # Library helpers + get_library_from_library_orcabus_id, get_library_from_library_id, + # Project helpers + get_project_from_project_orcabus_id, get_project_from_project_id, + # Individual helpers + get_individual_from_individual_orcabus_id, get_individual_from_individual_id, + # Contact helpers + get_contact_from_contact_orcabus_id, get_contact_from_contact_id +) + +# Globals +ALLOWED_CONTEXTS = ['individual', 'subject', 'sample', 'library', 'project', 'contact'] + + +def handler(event, context): + """ + Lambda handler. + + # Use the environment variables to customize the behavior of the function + # Based on the use case + ENV VAR VALUE is required + ENV VAR FROM_ORCABUS or FROM_ID is required + ENV VAR CONTEXT is required, one of 'subject', 'sample', 'library', 'project' + ENV VAR RETURN_STR or RETURN_OBJ is required + + :param event: + :param context: + :return: + """ + + # Get the orcabus token + environ['ORCABUS_TOKEN'] = get_orcabus_token() + + # Get value from the event object + value = event['value'] + + # Check if FROM_ORCABUS or FROM_ID is set + # But make sure not both are set + # And that at least one is set + if ('FROM_ORCABUS' in environ and 'FROM_ID' in environ) or ( + 'FROM_ORCABUS' not in environ and 'FROM_ID' not in environ): + raise ValueError("Either FROM_ORCABUS or FROM_ID must be set, but not both.") + is_from_orcabus = 'FROM_ORCABUS' in environ + + # Get the context + # And ensure that context is one of + # 'subject', 'sample', 'library', 'project' + if 'CONTEXT' not in environ: + raise ValueError("CONTEXT must be set.") + context = environ['CONTEXT'] + if context not in ALLOWED_CONTEXTS: + raise ValueError(f"CONTEXT must be one of {' '.join(ALLOWED_CONTEXTS)}") + + # Get the return type + # And ensure that it is one of 'id', 'object' and that not both are set + if 'RETURN_STR' not in environ and 'RETURN_OBJ' not in environ: + raise ValueError("RETURN_STR or RETURN_OBJ must be set.") + if 'RETURN_STR' in environ and 'RETURN_OBJ' in environ: + raise ValueError("RETURN_STR and RETURN_OBJ cannot be set at the same time.") + is_return_obj = 'RETURN_OBJ' in environ + + # Get the object based on the context + # For each context we then check if we need to return the object or the id + # And by 'id' we mean the orcabus id or the business unique identifier + + # If the context is 'individual' + if context == 'individual': + # Get the individual object + if is_from_orcabus: + individual_obj = get_individual_from_individual_orcabus_id(value) + else: + individual_obj = get_individual_from_individual_id(value) + + # Return the individual object if RETURN_OBJ is set + # Otherwise, return the orcabus id + if is_return_obj: + return individual_obj + + if is_from_orcabus: + return { + "individual_id": individual_obj['individualId'] + } + else: + return { + "orcabus_id": individual_obj['orcabusId'] + } + + # If the context is 'subject' + if context == 'subject': + # Get the subject object + if is_from_orcabus: + subject_obj = get_subject_from_subject_orcabus_id(value) + else: + subject_obj = get_subject_from_subject_id(value) + + # Return the subject object if RETURN_OBJ is set + # Otherwise, return the orcabus id + if is_return_obj: + return subject_obj + + if is_from_orcabus: + return { + "subject_id": subject_obj['subjectId'] + } + else: + return { + "orcabus_id": subject_obj['orcabusId'] + } + + # If the context is 'sample' + if context == 'sample': + + # Get the sample object + if is_from_orcabus: + sample_obj = get_sample_from_sample_orcabus_id(value) + else: + sample_obj = get_sample_from_sample_id(value) + + # Return the sample object if RETURN_OBJ is set + # Otherwise, return the orcabus id + if is_return_obj: + return sample_obj + + if is_from_orcabus: + return { + "sample_id": sample_obj['sampleId'] + } + else: + return { + "orcabus_id": sample_obj['orcabusId'] + } + + # If the context is 'library' + if context == 'library': + if is_from_orcabus: + library_obj = get_library_from_library_orcabus_id(value) + else: + library_obj = get_library_from_library_id(value) + + if is_return_obj: + return library_obj + + if is_from_orcabus: + return { + "library_id": library_obj['libraryId'] + } + else: + return { + "orcabus_id": library_obj['orcabusId'] + } + + # If the context is 'project' + if context == 'project': + if is_from_orcabus: + project_obj = get_project_from_project_orcabus_id(value) + else: + project_obj = get_project_from_project_id(value) + + if is_return_obj: + return project_obj + + if is_from_orcabus: + return { + "project_id": project_obj['projectId'] + } + else: + return { + "orcabus_id": project_obj['orcabusId'] + } + + # If the context is 'project' + if context == 'contact': + if is_from_orcabus: + contact_obj = get_contact_from_contact_orcabus_id(value) + else: + contact_obj = get_contact_from_contact_id(value) + + if is_return_obj: + return contact_obj + + if is_from_orcabus: + return { + "contact_id": contact_obj['contactId'] + } + else: + return { + "orcabus_id": contact_obj['orcabusId'] + } + + +# if __name__ == "__main__": +# import json +# +# # Set the aws variables +# environ['AWS_PROFILE'] = 'umccr-development' +# environ['AWS_REGION'] = 'ap-southeast-2' +# environ['HOSTNAME_SSM_PARAMETER'] = '/hosted_zone/umccr/name' +# environ['ORCABUS_TOKEN_SECRET_ID'] = 'orcabus/token-service-jwt' +# +# # Set the context variables +# environ['CONTEXT'] = 'individual' +# environ['FROM_ORCABUS'] = '' +# environ['RETURN_STR'] = '' +# +# # print( +# # json.dumps( +# # handler( +# # { +# # "value": "idv.01J8EV7AVRB43911YD4WKZNCHK" +# # }, +# # None +# # ), +# # indent=4 +# # ) +# # ) +# # +# # # { +# # # "individual_id": "SBJ05695" +# # # } + +# if __name__ == "__main__": +# # Import the json module +# import json +# +# # Set the environment variables +# environ['AWS_PROFILE'] = 'umccr-development' +# environ['AWS_REGION'] = 'ap-southeast-2' +# environ['HOSTNAME_SSM_PARAMETER'] = '/hosted_zone/umccr/name' +# environ['ORCABUS_TOKEN_SECRET_ID'] = 'orcabus/token-service-jwt' +# +# # Set the context variables +# environ['CONTEXT'] = 'subject' +# environ['FROM_ID'] = '' +# environ['RETURN_OBJ'] = '' +# +# print( +# json.dumps( +# handler( +# { +# "value": "VENTURE-24004301" +# }, +# None +# ), +# indent=4 +# ) +# ) +# +# # { +# # "orcabusId": "sbj.01J8ES921MV8VCY71SJPAS7D24", +# # "individualSet": [ +# # { +# # "orcabusId": "idv.01J8ES90SEHD03JY5R0DR8K44H", +# # "individualId": null, +# # "source": "lab" +# # }, +# # { +# # "orcabusId": "idv.01J8EV7ARC2ZF9X0FA243CSXA5", +# # "individualId": "SBJ05693", +# # "source": "lab" +# # } +# # ], +# # "librarySet": [ +# # { +# # "orcabusId": "lib.01J8ES922VZT5S55T9HXPRZXC7", +# # "libraryId": "L2401462", +# # "phenotype": "normal", +# # "workflow": "qc", +# # "quality": "good", +# # "type": "WGS", +# # "assay": "TsqNano", +# # "coverage": 30.0, +# # "sample": "smp.01J8ES922C8KDSG3TWT8AEFCPZ", +# # "subject": "sbj.01J8ES921MV8VCY71SJPAS7D24" +# # } +# # ], +# # "subjectId": "VENTURE-24004301" +# # } + + +# if __name__ == "__main__": +# # Import the json module +# import json +# +# # Set the environment variables +# environ['AWS_PROFILE'] = 'umccr-development' +# environ['AWS_REGION'] = 'ap-southeast-2' +# environ['HOSTNAME_SSM_PARAMETER'] = '/hosted_zone/umccr/name' +# environ['ORCABUS_TOKEN_SECRET_ID'] = 'orcabus/token-service-jwt' +# +# # Set the context variables +# environ['CONTEXT'] = 'sample' +# environ['FROM_ID'] = '' +# environ['RETURN_OBJ'] = '' +# +# +# print( +# json.dumps( +# handler( +# { +# "value": "PTC_TSqN240923" +# }, +# None +# ), +# indent=4 +# ) +# ) +# +# # { +# # "orcabusId": "smp.01J8ES92CBMCVE09FJSPADGYG1", +# # "sampleId": "PTC_TSqN240923", +# # "externalSampleId": "NA24385", +# # "source": "cell-line" +# # } + + +# if __name__ == "__main__": +# # Set the environment variables +# environ['AWS_PROFILE'] = 'umccr-development' +# environ['AWS_REGION'] = 'ap-southeast-2' +# environ['HOSTNAME_SSM_PARAMETER'] = '/hosted_zone/umccr/name' +# environ['ORCABUS_TOKEN_SECRET_ID'] = 'orcabus/token-service-jwt' +# +# print( +# handler( +# { +# "subject_id": "VENTURE-24004301" +# }, +# None +# ) +# ) + + +# if __name__ == "__main__": +# # Import the json module +# import json +# +# # Set the environment variables +# environ['AWS_PROFILE'] = 'umccr-development' +# environ['AWS_REGION'] = 'ap-southeast-2' +# environ['HOSTNAME_SSM_PARAMETER'] = '/hosted_zone/umccr/name' +# environ['ORCABUS_TOKEN_SECRET_ID'] = 'orcabus/token-service-jwt' +# +# # Set the context variables +# environ['CONTEXT'] = 'library' +# environ['FROM_ORCABUS'] = '' +# environ['RETURN_STR'] = '' +# +# print( +# json.dumps( +# handler( +# { +# "value": "lib.01J8ES92FTH314XSPZDWBA91E2" +# }, +# None +# ), +# indent=4 +# ) +# ) +# +# # { +# # "library_id": "L2401469" +# # } + + +# if __name__ == "__main__": +# # Import the json module +# import json +# +# # Set the environment variables +# environ['AWS_PROFILE'] = 'umccr-development' +# environ['AWS_REGION'] = 'ap-southeast-2' +# environ['HOSTNAME_SSM_PARAMETER'] = '/hosted_zone/umccr/name' +# environ['ORCABUS_TOKEN_SECRET_ID'] = 'orcabus/token-service-jwt' +# +# # Set the context variables +# environ['CONTEXT'] = 'project' +# environ['FROM_ORCABUS'] = '' +# environ['RETURN_STR'] = '' +# +# print( +# json.dumps( +# handler( +# { +# "value": "prj.01J8ES70B12PTV40XAYF0GPW3C" +# }, +# None +# ), +# indent=4 +# ) +# ) +# +# # { +# # "project_id": "SEQC" +# # } + + +# if __name__ == "__main__": +# # Import the json module +# import json +# +# # Set the environment variables +# environ['AWS_PROFILE'] = 'umccr-development' +# environ['AWS_REGION'] = 'ap-southeast-2' +# environ['HOSTNAME_SSM_PARAMETER'] = '/hosted_zone/umccr/name' +# environ['ORCABUS_TOKEN_SECRET_ID'] = 'orcabus/token-service-jwt' +# +# # Set the context variables +# environ['CONTEXT'] = 'contact' +# environ['FROM_ORCABUS'] = '' +# environ['RETURN_STR'] = '' +# +# print( +# json.dumps( +# handler( +# { +# "value": "ctc.01J8ES70ANRQQ4K71HQ9PZAEM6" +# }, +# None +# ), +# indent=4 +# ) +# ) +# +# # { +# # "contact_id": "Hofmann" +# # } diff --git a/lib/workload/components/python-metadata-tools-layer/metadata_tools_layer/src/metadata_tools/__init__.py b/lib/workload/components/python-metadata-tools-layer/metadata_tools_layer/src/metadata_tools/__init__.py index a8fd46d9b..e5d62ab47 100644 --- a/lib/workload/components/python-metadata-tools-layer/metadata_tools_layer/src/metadata_tools/__init__.py +++ b/lib/workload/components/python-metadata-tools-layer/metadata_tools_layer/src/metadata_tools/__init__.py @@ -1,8 +1,12 @@ #!/usr/bin/env python +# Utils +from .utils.aws_helpers import get_orcabus_token + # Library Helpers from .utils.library_helpers import ( get_library_from_library_id, + get_library_from_library_orcabus_id, get_subject_from_library_id, get_library_type, get_library_assay_type, @@ -11,39 +15,78 @@ get_all_libraries ) -# Specimen Helpers -from .utils.specimen_helpers import ( - get_specimen_from_specimen_id, - list_libraries_in_specimen, - get_all_specimens +# Sample Helpers +from .utils.sample_helpers import ( + get_sample_from_sample_id, + get_sample_from_sample_orcabus_id, + list_libraries_in_sample, + get_all_samples ) # Subject Helpers from .utils.subject_helpers import ( get_subject_from_subject_id, - list_specimens_in_subject, + get_subject_from_subject_orcabus_id, + list_samples_in_subject, list_libraries_in_subject, get_all_subjects ) +# Project Helpers +from .utils.project_helpers import ( + get_all_projects, + get_project_from_project_id, + get_project_from_project_orcabus_id, +) + +# Individual Helpers +from .utils.individual_helpers import ( + get_individual_from_individual_id, + get_individual_from_individual_orcabus_id, + get_all_individuals +) + +# Contact helpers +from .utils.contact_helpers import ( + get_contact_from_contact_id, + get_contact_from_contact_orcabus_id, + get_all_contacts +) # Set _all__ __all__ = [ + # Utils + 'get_orcabus_token', # Library Funcs 'get_library_from_library_id', + 'get_library_from_library_orcabus_id', 'get_subject_from_library_id', 'get_library_type', 'get_library_assay_type', 'get_library_phenotype', 'get_library_workflow', 'get_all_libraries', - # Specimen Funcs - 'get_specimen_from_specimen_id', - 'list_libraries_in_specimen', - 'get_all_specimens', + # Sample Funcs + 'get_sample_from_sample_id', + 'get_sample_from_sample_orcabus_id', + 'list_libraries_in_sample', + 'list_samples_in_subject', + 'get_all_samples', # Subject Funcs 'get_subject_from_subject_id', - 'list_specimens_in_subject', + 'get_subject_from_subject_orcabus_id', 'list_libraries_in_subject', - 'get_all_subjects' + 'get_all_subjects', + # Project Funcs + 'get_all_projects', + 'get_project_from_project_id', + 'get_project_from_project_orcabus_id', + # Individual Funcs + 'get_individual_from_individual_id', + 'get_individual_from_individual_orcabus_id', + 'get_all_individuals', + # Contact Funcs + 'get_contact_from_contact_id', + 'get_contact_from_contact_orcabus_id', + 'get_all_contacts', ] diff --git a/lib/workload/components/python-metadata-tools-layer/metadata_tools_layer/src/metadata_tools/utils/aws_helpers.py b/lib/workload/components/python-metadata-tools-layer/metadata_tools_layer/src/metadata_tools/utils/aws_helpers.py index 061510bcf..0a5bf2f44 100644 --- a/lib/workload/components/python-metadata-tools-layer/metadata_tools_layer/src/metadata_tools/utils/aws_helpers.py +++ b/lib/workload/components/python-metadata-tools-layer/metadata_tools_layer/src/metadata_tools/utils/aws_helpers.py @@ -6,8 +6,6 @@ import json from os import environ -# Locals - # Type hinting if typing.TYPE_CHECKING: from mypy_boto3_secretsmanager import SecretsManagerClient diff --git a/lib/workload/components/python-metadata-tools-layer/metadata_tools_layer/src/metadata_tools/utils/contact_helpers.py b/lib/workload/components/python-metadata-tools-layer/metadata_tools_layer/src/metadata_tools/utils/contact_helpers.py new file mode 100644 index 000000000..d93ccc18a --- /dev/null +++ b/lib/workload/components/python-metadata-tools-layer/metadata_tools_layer/src/metadata_tools/utils/contact_helpers.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 + +""" +Helpers for using the contact API endpoint +""" + +# Standard imports +from typing import List, Dict +from .globals import CONTACT_ENDPOINT + +# Local imports +from .requests_helpers import get_request_response_results + + +def get_contact_from_contact_id(contact_id: str) -> Dict: + """ + Get subject from the subject id + :param contact_id: + :return: + """ + # We have an internal id, convert to int + params = { + "contact_id": contact_id + } + + # Get subject + return get_request_response_results(CONTACT_ENDPOINT, params)[0] + + +def get_contact_from_contact_orcabus_id(contact_orcabus_id: str) -> Dict: + """ + Get contact from the contact id + :param contact_orcabus_id: + :return: + """ + params = { + "orcabus_id": contact_orcabus_id.split(".")[1] + } + + # Get contact + return get_request_response_results(CONTACT_ENDPOINT, params)[0] + + +def get_all_contacts() -> List[Dict]: + """ + Get all subjects + :return: + """ + + return get_request_response_results(CONTACT_ENDPOINT) diff --git a/lib/workload/components/python-metadata-tools-layer/metadata_tools_layer/src/metadata_tools/utils/globals.py b/lib/workload/components/python-metadata-tools-layer/metadata_tools_layer/src/metadata_tools/utils/globals.py index 6ce6e8a40..a0ad2902f 100644 --- a/lib/workload/components/python-metadata-tools-layer/metadata_tools_layer/src/metadata_tools/utils/globals.py +++ b/lib/workload/components/python-metadata-tools-layer/metadata_tools_layer/src/metadata_tools/utils/globals.py @@ -4,3 +4,11 @@ # AWS PARAMETERS METADATA_SUBDOMAIN_NAME = "metadata" + +# API ENDPOINTS +LIBRARY_ENDPOINT = "api/v1/library" +SAMPLE_ENDPOINT = "api/v1/sample" +SUBJECT_ENDPOINT = "api/v1/subject" +PROJECT_ENDPOINT = "api/v1/project" +INDIVIDUAL_ENDPOINT = "api/v1/individual" +CONTACT_ENDPOINT = "api/v1/contact" \ No newline at end of file diff --git a/lib/workload/components/python-metadata-tools-layer/metadata_tools_layer/src/metadata_tools/utils/individual_helpers.py b/lib/workload/components/python-metadata-tools-layer/metadata_tools_layer/src/metadata_tools/utils/individual_helpers.py new file mode 100644 index 000000000..551e1d519 --- /dev/null +++ b/lib/workload/components/python-metadata-tools-layer/metadata_tools_layer/src/metadata_tools/utils/individual_helpers.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 + +""" +This module contains helper functions for the individual class. +""" + +#!/usr/bin/env python3 + + +# !/usr/bin/env python3 + + +""" +Helper functions for a subject +""" + +# Standard imports +from typing import Dict + +# Local imports +from .globals import INDIVIDUAL_ENDPOINT +from .requests_helpers import get_request_response_results + + +def get_individual_from_individual_id(individual_id: str) -> Dict: + """ + Get individual from the individual id + :param individual_id: + :return: + """ + # We have an internal id + params = { + "individual_id": individual_id + } + + # Get individual + return get_request_response_results(INDIVIDUAL_ENDPOINT, params)[0] + + +def get_individual_from_individual_orcabus_id(individual_orcabus_id: str) -> Dict: + """ + Get individual from the individual id + :param individual_orcabus_id: + :return: + """ + # We have an internal id + params = { + "orcabus_id": individual_orcabus_id + } + + # Get individual + return get_request_response_results(INDIVIDUAL_ENDPOINT, params)[0] + + +def get_all_individuals(): + """ + Get all samples from the sample database + :return: + """ + return get_request_response_results(INDIVIDUAL_ENDPOINT) diff --git a/lib/workload/components/python-metadata-tools-layer/metadata_tools_layer/src/metadata_tools/utils/library_helpers.py b/lib/workload/components/python-metadata-tools-layer/metadata_tools_layer/src/metadata_tools/utils/library_helpers.py index bf5d0ff49..675ae6ca1 100644 --- a/lib/workload/components/python-metadata-tools-layer/metadata_tools_layer/src/metadata_tools/utils/library_helpers.py +++ b/lib/workload/components/python-metadata-tools-layer/metadata_tools_layer/src/metadata_tools/utils/library_helpers.py @@ -1,6 +1,7 @@ #!/usr/bin/env python from typing import Union, Dict, List +from .globals import LIBRARY_ENDPOINT from .requests_helpers import get_request_response_results @@ -10,20 +11,31 @@ def get_library_from_library_id(library_id: Union[int | str]) -> Dict: :param library_id: :return: """ - endpoint = "api/v1/library" + # Get library id + # We have an internal id, convert to int + params = { + "library_id": library_id + } + + # Get library + return get_request_response_results(LIBRARY_ENDPOINT, params)[0] + +def get_library_from_library_orcabus_id(library_orcabus_id: Union[int | str]) -> Dict: + """ + Get library from the library id + :param library_orcabus_id: + :return: + """ # Get library id - if isinstance(library_id, str): - # We have an internal id, convert to int - params = { - "library_id": library_id - } - else: - endpoint = f"{endpoint}/{library_id}" - params = {} + # We have an internal id, convert to int + params = { + "orcabus_id": library_orcabus_id + } # Get library - return get_request_response_results(endpoint, params)[0] + return get_request_response_results(LIBRARY_ENDPOINT, params)[0] + def get_subject_from_library_id(library_id: Union[int | str]) -> Dict: @@ -32,20 +44,10 @@ def get_subject_from_library_id(library_id: Union[int | str]) -> Dict: :param library_id: :return: """ - from .specimen_helpers import get_specimen_from_specimen_id from .subject_helpers import get_subject_from_subject_id - # Get the specimen linked to this library id - specimen_id = get_library_from_library_id(library_id)["specimen"] - - specimen_obj = get_specimen_from_specimen_id(specimen_id) - - if "subjects" in specimen_obj.keys() and len(specimen_obj["subjects"]) > 0: - subject_id = specimen_obj["subjects"][0] - elif "subject" in specimen_obj.keys(): - subject_id = specimen_obj["subject"] - else: - raise KeyError(f"Subject not found for library id: {library_id}") + # Get the subject linked to this library id + subject_id = get_library_from_library_id(library_id)["subject"]['subjectId'] return get_subject_from_subject_id(subject_id) @@ -121,6 +123,4 @@ def get_all_libraries() -> List[Dict]: Collect all libraries from the database :return: """ - endpoint = "api/v1/library" - - return get_request_response_results(endpoint) + return get_request_response_results(LIBRARY_ENDPOINT) diff --git a/lib/workload/components/python-metadata-tools-layer/metadata_tools_layer/src/metadata_tools/utils/project_helpers.py b/lib/workload/components/python-metadata-tools-layer/metadata_tools_layer/src/metadata_tools/utils/project_helpers.py new file mode 100644 index 000000000..723f29235 --- /dev/null +++ b/lib/workload/components/python-metadata-tools-layer/metadata_tools_layer/src/metadata_tools/utils/project_helpers.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 + +""" +Helpers for using the project API endpoint +""" + +# Standard imports +from typing import List, Dict +from .globals import PROJECT_ENDPOINT + +# Local imports +from .requests_helpers import get_request_response_results + + +def get_project_from_project_id(project_id: str) -> Dict: + """ + Get subject from the subject id + :param project_id: + :return: + """ + # We have an internal id, convert to int + params = { + "project_id": project_id + } + + # Get subject + return get_request_response_results(PROJECT_ENDPOINT, params)[0] + + +def get_project_from_project_orcabus_id(project_orcabus_id: str) -> Dict: + """ + Get project from the project id + :param project_orcabus_id: + :return: + """ + params = { + "orcabus_id": project_orcabus_id.split(".")[1] + } + + # Get project + return get_request_response_results(PROJECT_ENDPOINT, params)[0] + + +def get_all_projects() -> List[Dict]: + """ + Get all subjects + :return: + """ + + return get_request_response_results(PROJECT_ENDPOINT) diff --git a/lib/workload/components/python-metadata-tools-layer/metadata_tools_layer/src/metadata_tools/utils/sample_helpers.py b/lib/workload/components/python-metadata-tools-layer/metadata_tools_layer/src/metadata_tools/utils/sample_helpers.py new file mode 100644 index 000000000..1fa84bcb0 --- /dev/null +++ b/lib/workload/components/python-metadata-tools-layer/metadata_tools_layer/src/metadata_tools/utils/sample_helpers.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 + + +# !/usr/bin/env python3 + + +""" +Helper functions for a subject +""" + +# Standard imports +from typing import List, Union, Dict + +# Local imports +from .globals import SAMPLE_ENDPOINT, LIBRARY_ENDPOINT +from .requests_helpers import get_request_response_results + + +def get_sample_from_sample_id(sample_id: str) -> Dict: + """ + Get sample from the sample id + :param sample_id: + :return: + """ + # We have an internal id + params = { + "sample_id": sample_id + } + + # Get sample + return get_request_response_results(SAMPLE_ENDPOINT, params)[0] + + +def get_sample_from_sample_orcabus_id(sample_orcabus_id: str) -> Dict: + """ + Get sample from the sample id + :param sample_orcabus_id: + :return: + """ + # We have an internal id + params = { + "orcabus_id": sample_orcabus_id + } + + # Get sample + return get_request_response_results(SAMPLE_ENDPOINT, params)[0] + + + +def list_libraries_in_sample(sample_id: Union[str, int]) -> List[Dict]: + """ + Given a sample_id, list the samples in the subject + :param sample_id: + :return: + """ + # Get the subject + return list( + filter( + lambda library_iter: library_iter.get("sample").get("sampleId") == sample_id, + get_request_response_results(LIBRARY_ENDPOINT) + ) + ) + + +def get_all_samples(): + """ + Get all samples from the sample database + :return: + """ + return get_request_response_results(SAMPLE_ENDPOINT) diff --git a/lib/workload/components/python-metadata-tools-layer/metadata_tools_layer/src/metadata_tools/utils/specimen_helpers.py b/lib/workload/components/python-metadata-tools-layer/metadata_tools_layer/src/metadata_tools/utils/specimen_helpers.py deleted file mode 100644 index cc165556a..000000000 --- a/lib/workload/components/python-metadata-tools-layer/metadata_tools_layer/src/metadata_tools/utils/specimen_helpers.py +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env python3 - - -# !/usr/bin/env python3 - - -""" -Helper functions for a subject -""" - -# Standard imports -from typing import List, Union, Dict - -# Local imports -from .requests_helpers import get_request_response_results - - -def get_specimen_from_specimen_id(specimen_id: Union[str, int]) -> Dict: - """ - Get specimen from the specimen id - :param specimen_id: - :return: - """ - endpoint = "specimen" - - # Get specimen id - if isinstance(specimen_id, str): - # We have an internal id, convert to int - params = { - "specimen_id": specimen_id - } - else: - endpoint = f"{endpoint}/{specimen_id}" - params = {} - - # Get specimen - return get_request_response_results(endpoint, params)[0] - - -def list_libraries_in_specimen(specimen_id: Union[str, int]) -> List[Dict]: - """ - Given a specimen_id id, list the specimens in the subject - :param specimen_id: - :return: - """ - - # If subject id is a string, we have the internal id (SBJ...) - specimen = get_specimen_from_specimen_id(specimen_id) - - endpoint = f"api/v1/library" - - # Get the subject - return list( - filter( - lambda library_iter: library_iter.get("specimen") == specimen.get("id"), - get_request_response_results(endpoint) - ) - ) - - -def get_all_specimens(): - """ - Get all specimens from the specimen database - :return: - """ - endpoint = "api/v1/specimen" - - return get_request_response_results(endpoint) - diff --git a/lib/workload/components/python-metadata-tools-layer/metadata_tools_layer/src/metadata_tools/utils/subject_helpers.py b/lib/workload/components/python-metadata-tools-layer/metadata_tools_layer/src/metadata_tools/utils/subject_helpers.py index e2bfa6514..f7e8a99ce 100644 --- a/lib/workload/components/python-metadata-tools-layer/metadata_tools_layer/src/metadata_tools/utils/subject_helpers.py +++ b/lib/workload/components/python-metadata-tools-layer/metadata_tools_layer/src/metadata_tools/utils/subject_helpers.py @@ -8,39 +8,50 @@ # Standard imports from typing import List, Union, Dict +from .globals import SUBJECT_ENDPOINT # Local imports from .requests_helpers import get_request_response_results -def get_subject_from_subject_id(subject_id: Union[str, int]) -> Dict: +def get_subject_from_subject_id(subject_id: str) -> Dict: """ Get subject from the subject id :param subject_id: :return: """ - endpoint = "api/v1/subject" + # We have an internal id, convert to int + params = { + "subject_id": subject_id + } + # Get subject + return get_request_response_results(SUBJECT_ENDPOINT, params)[0] + + + +def get_subject_from_subject_orcabus_id(subject_orcabus_id: str) -> Dict: + """ + Get subject from the subject id + :param subject_orcabus_id: + :return: + """ # Get subject id - if isinstance(subject_id, str): - # We have an internal id, convert to int - params = { - "subject_id": subject_id - } - else: - endpoint = f"{endpoint}/{subject_id}" - params = {} + # We have an internal id, convert to int + params = { + "orcabus_id": subject_orcabus_id + } # Get subject - return get_request_response_results(endpoint, params)[0] + return get_request_response_results(SUBJECT_ENDPOINT, params)[0] -def list_specimens_in_subject(subject_id: Union[str, int]) -> List[Dict]: +def list_samples_in_subject(subject_id: Union[str, int]) -> List[Dict]: """ - Given a subject id, list the specimens in the subject + Given a subject id, list the samples in the subject :param subject_id: :return: """ - from metadata_tools import get_all_specimens + from metadata_tools import get_all_samples # Get ID For Subject subject = get_subject_from_subject_id(subject_id) @@ -48,10 +59,12 @@ def list_specimens_in_subject(subject_id: Union[str, int]) -> List[Dict]: # Get the subject return list( filter( - lambda specimen_iter: - subject.get("id") == specimen_iter.get("subjects")[0] if "subjects" in specimen_iter.keys() - else specimen_iter.get("subject") == subject.get("id"), - get_all_specimens() + lambda sample_iter: + subject.get("id") == sample_iter.get("subjects")[0] + if "subjects" in sample_iter.keys() + else + sample_iter.get("subject") == subject.get("id"), + get_all_samples() ) ) @@ -62,14 +75,14 @@ def list_libraries_in_subject(subject_id: str) -> List[Dict]: :param subject_id: :return: """ - from .specimen_helpers import list_libraries_in_specimen + from .sample_helpers import list_libraries_in_sample library_list = [] - specimens_list = list_specimens_in_subject(subject_id) + samples_list = list_samples_in_subject(subject_id) - for specimen_iter in specimens_list: - library_list.extend(list_libraries_in_specimen(specimen_iter.get("id"))) + for sample_iter in samples_list: + library_list.extend(list_libraries_in_sample(sample_iter.get("id"))) return library_list @@ -79,7 +92,4 @@ def get_all_subjects() -> List[Dict]: Get all subjects :return: """ - - endpoint = "api/v1/subject" - - return get_request_response_results(endpoint) + return get_request_response_results(SUBJECT_ENDPOINT) diff --git a/lib/workload/components/event-workflowdraftrunstatechange-to-workflowrunstatechange-ready/Readme.md b/lib/workload/components/sfn-generate-workflowrunstatechange-ready-event/Readme.md similarity index 100% rename from lib/workload/components/event-workflowdraftrunstatechange-to-workflowrunstatechange-ready/Readme.md rename to lib/workload/components/sfn-generate-workflowrunstatechange-ready-event/Readme.md diff --git a/lib/workload/components/event-workflowdraftrunstatechange-to-workflowrunstatechange-ready/images/workflowrunstatechange_input_maker_step_function_sfn.png b/lib/workload/components/sfn-generate-workflowrunstatechange-ready-event/images/workflowrunstatechange_input_maker_step_function_sfn.png similarity index 100% rename from lib/workload/components/event-workflowdraftrunstatechange-to-workflowrunstatechange-ready/images/workflowrunstatechange_input_maker_step_function_sfn.png rename to lib/workload/components/sfn-generate-workflowrunstatechange-ready-event/images/workflowrunstatechange_input_maker_step_function_sfn.png diff --git a/lib/workload/components/event-workflowdraftrunstatechange-to-workflowrunstatechange-ready/index.ts b/lib/workload/components/sfn-generate-workflowrunstatechange-ready-event/index.ts similarity index 69% rename from lib/workload/components/event-workflowdraftrunstatechange-to-workflowrunstatechange-ready/index.ts rename to lib/workload/components/sfn-generate-workflowrunstatechange-ready-event/index.ts index d14033b58..99d09be7a 100644 --- a/lib/workload/components/event-workflowdraftrunstatechange-to-workflowrunstatechange-ready/index.ts +++ b/lib/workload/components/sfn-generate-workflowrunstatechange-ready-event/index.ts @@ -5,15 +5,12 @@ Collect the engine parameters required for the workflow run state change to be s import { Construct } from 'constructs'; import * as lambda from 'aws-cdk-lib/aws-lambda'; import * as sfn from 'aws-cdk-lib/aws-stepfunctions'; -import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; import { PythonFunction } from '@aws-cdk/aws-lambda-python-alpha'; import * as iam from 'aws-cdk-lib/aws-iam'; import * as events from 'aws-cdk-lib/aws-events'; -import * as eventsTargets from 'aws-cdk-lib/aws-events-targets'; import * as ssm from 'aws-cdk-lib/aws-ssm'; import path from 'path'; import * as cdk from 'aws-cdk-lib'; -import { PythonLambdaUuidConstruct } from '../python-lambda-uuid-generator-function'; import { PythonLambdaFlattenListOfObjectsConstruct } from '../python-lambda-flatten-list-of-objects'; import * as secretsManager from 'aws-cdk-lib/aws-secretsmanager'; import { Duration } from 'aws-cdk-lib'; @@ -22,15 +19,8 @@ export interface WorkflowRunStateChangeInternalInputMakerProps { /* Object name prefixes */ stateMachinePrefix: string; lambdaPrefix: string; - rulePrefix: string; - /* Table configs */ - tableObj: dynamodb.ITableV2; - tablePartitionName: string; /* Event trigger configs */ eventBusObj: events.IEventBus; - triggerSource: string; - triggerStatus: string; - triggerDetailType?: string; outputSource: string; /* Workflow metadata constants */ workflowName: string; @@ -45,11 +35,10 @@ export interface WorkflowRunStateChangeInternalInputMakerProps { icav2AccessTokenSecretObj: secretsManager.ISecret; } -export class WorkflowDraftRunStateChangeToWorkflowRunStateChangeReadyConstruct extends Construct { +export class GenerateWorkflowRunStateChangeReadyConstruct extends Construct { public readonly stepFunctionObj: sfn.StateMachine; - public readonly defaultTriggerDetailType = 'WorkflowDraftRunStateChange'; public readonly outputDetailType = 'WorkflowRunStateChange'; - private readonly portalRunPartitionName = 'portal_run'; + private readonly readyStatus = 'READY'; constructor(scope: Construct, id: string, props: WorkflowRunStateChangeInternalInputMakerProps) { super(scope, id); @@ -63,7 +52,7 @@ export class WorkflowDraftRunStateChangeToWorkflowRunStateChangeReadyConstruct e this, 'fill_placeholders_in_event_payload_data_lambda', { - functionName: `${props.lambdaPrefix}-fill-placeholders-in-event-payload-data`, + functionName: `${props.lambdaPrefix}-fill-engine-parameters`, entry: path.join(__dirname, 'lambdas', 'fill_placeholders_in_event_payload_data_py'), index: 'fill_placeholders_in_event_payload_data.py', runtime: lambda.Runtime.PYTHON_3_12, @@ -133,33 +122,24 @@ export class WorkflowDraftRunStateChangeToWorkflowRunStateChangeReadyConstruct e /* Part 2 - Build the AWS State Machine */ - /* Build the uuid generator lambda */ - const uuidGeneratorLambda = new PythonLambdaUuidConstruct(this, 'uuid_generator_lambda'); - - // FIXME - sfn should check that the data object in the input - // FIXME matches the dataobject in the database first before raising the event this.stepFunctionObj = new sfn.StateMachine(this, 'StateMachine', { - stateMachineName: `${props.stateMachinePrefix}-draft-to-ready-sfn`, + stateMachineName: `${props.stateMachinePrefix}-ready-sfn`, definitionBody: sfn.DefinitionBody.fromFile( path.join( __dirname, 'step_functions_templates', - 'workflowrunstatechange_draft_to_ready_step_function_template.asl.json' + 'workflowrunstatechange_generate_ready_step_function_template.asl.json' ) ), definitionSubstitutions: { - /* Lambdas */ - __generate_uuid_lambda_function_arn__: - uuidGeneratorLambda.lambdaObj.currentVersion.functionArn, - /* Table configurations */ - __table_name__: props.tableObj.tableName, - __workflow_type_partition_name__: props.tablePartitionName, - __portal_run_partition_name__: this.portalRunPartitionName, /* Event configurations */ __event_output_source__: props.outputSource, __detail_type__: this.outputDetailType, __event_bus_name__: props.eventBusObj.eventBusName, + __ready_status__: this.readyStatus, /* Workflow name */ + __workflow_name__: props.workflowName, + __workflow_version__: props.workflowVersion, __payload_version__: props.payloadVersion, /* Nested statemachine */ __engine_parameters_maker_state_machine_arn__: @@ -173,15 +153,8 @@ export class WorkflowDraftRunStateChangeToWorkflowRunStateChangeReadyConstruct e }); /* - Part 3 - Connect permissions + Part 3 - Connect permissions between state-machines */ - /* Allow step functions to invoke the lambda */ - [uuidGeneratorLambda.lambdaObj].forEach((lambdaObj) => { - lambdaObj.currentVersion.grantInvoke(this.stepFunctionObj.role); - }); - - /* Allow step function to write to table */ - props.tableObj.grantReadWriteData(this.stepFunctionObj.role); /* Allow step function to call nested state machine */ // Because we run a nested state machine, we need to add the permissions to the state machine role @@ -194,34 +167,9 @@ export class WorkflowDraftRunStateChangeToWorkflowRunStateChangeReadyConstruct e actions: ['events:PutTargets', 'events:PutRule', 'events:DescribeRule'], }) ); - engineParameterGeneratorStateMachineSfn.grantStartExecution( - this.stepFunctionObj.role - ); + engineParameterGeneratorStateMachineSfn.grantStartExecution(this.stepFunctionObj); /* Allow step function to send events */ - props.eventBusObj.grantPutEventsTo(this.stepFunctionObj.role); - - /* - Part 4 - Set up a rule to trigger the state machine - */ - const rule = new events.Rule(this, 'workflowrunstatechangeparser_event_rule', { - ruleName: `${props.rulePrefix}-rule`, - eventBus: props.eventBusObj, - eventPattern: { - source: [props.triggerSource], - detailType: [props.triggerDetailType || this.defaultTriggerDetailType], - detail: { - status: [{ 'equals-ignore-case': props.triggerStatus }], - workflowName: [{ 'equals-ignore-case': props.workflowName }], - }, - }, - }); - - // Add target of event to be the state machine - rule.addTarget( - new eventsTargets.SfnStateMachine(this.stepFunctionObj, { - input: events.RuleTargetInput.fromEventPath('$.detail'), - }) - ); + props.eventBusObj.grantPutEventsTo(this.stepFunctionObj); } } diff --git a/lib/workload/components/event-workflowdraftrunstatechange-to-workflowrunstatechange-ready/lambdas/fill_placeholders_in_event_payload_data_py/fill_placeholders_in_event_payload_data.py b/lib/workload/components/sfn-generate-workflowrunstatechange-ready-event/lambdas/fill_placeholders_in_event_payload_data_py/fill_placeholders_in_event_payload_data.py similarity index 100% rename from lib/workload/components/event-workflowdraftrunstatechange-to-workflowrunstatechange-ready/lambdas/fill_placeholders_in_event_payload_data_py/fill_placeholders_in_event_payload_data.py rename to lib/workload/components/sfn-generate-workflowrunstatechange-ready-event/lambdas/fill_placeholders_in_event_payload_data_py/fill_placeholders_in_event_payload_data.py diff --git a/lib/workload/components/event-workflowdraftrunstatechange-to-workflowrunstatechange-ready/lambdas/fill_placeholders_in_event_payload_data_py/requirements.txt b/lib/workload/components/sfn-generate-workflowrunstatechange-ready-event/lambdas/fill_placeholders_in_event_payload_data_py/requirements.txt similarity index 100% rename from lib/workload/components/event-workflowdraftrunstatechange-to-workflowrunstatechange-ready/lambdas/fill_placeholders_in_event_payload_data_py/requirements.txt rename to lib/workload/components/sfn-generate-workflowrunstatechange-ready-event/lambdas/fill_placeholders_in_event_payload_data_py/requirements.txt diff --git a/lib/workload/components/event-workflowdraftrunstatechange-to-workflowrunstatechange-ready/step_functions_templates/generate_analysis_engine_parameters_from_ssm_sfn_template.asl.json b/lib/workload/components/sfn-generate-workflowrunstatechange-ready-event/step_functions_templates/generate_analysis_engine_parameters_from_ssm_sfn_template.asl.json similarity index 98% rename from lib/workload/components/event-workflowdraftrunstatechange-to-workflowrunstatechange-ready/step_functions_templates/generate_analysis_engine_parameters_from_ssm_sfn_template.asl.json rename to lib/workload/components/sfn-generate-workflowrunstatechange-ready-event/step_functions_templates/generate_analysis_engine_parameters_from_ssm_sfn_template.asl.json index 4afdfa3ad..5dd9da428 100644 --- a/lib/workload/components/event-workflowdraftrunstatechange-to-workflowrunstatechange-ready/step_functions_templates/generate_analysis_engine_parameters_from_ssm_sfn_template.asl.json +++ b/lib/workload/components/sfn-generate-workflowrunstatechange-ready-event/step_functions_templates/generate_analysis_engine_parameters_from_ssm_sfn_template.asl.json @@ -36,10 +36,10 @@ }, "Set Map Output Blank": { "Type": "Pass", - "End": true, - "Result": { + "Parameters": { "output.$": "States.StringToJson(States.Format('\\{\"{}\":\"{}\"\\}', $.engine_parameter_key, ''))" - } + }, + "End": true }, "Get EngineParameter URI": { "Type": "Task", diff --git a/lib/workload/components/sfn-generate-workflowrunstatechange-ready-event/step_functions_templates/workflowrunstatechange_generate_ready_step_function_template.asl.json b/lib/workload/components/sfn-generate-workflowrunstatechange-ready-event/step_functions_templates/workflowrunstatechange_generate_ready_step_function_template.asl.json new file mode 100644 index 000000000..42e1e742d --- /dev/null +++ b/lib/workload/components/sfn-generate-workflowrunstatechange-ready-event/step_functions_templates/workflowrunstatechange_generate_ready_step_function_template.asl.json @@ -0,0 +1,80 @@ +{ + "Comment": "A description of my state machine", + "StartAt": "move inputs", + "States": { + "move inputs": { + "Type": "Pass", + "Next": "Get Workflow Run Engine Parameters", + "Parameters": { + "input_event_detail_draft.$": "$.StatePayload" + } + }, + "Get Workflow Run Engine Parameters": { + "Type": "Task", + "Resource": "arn:aws:states:::states:startExecution.sync:2", + "Parameters": { + "StateMachineArn": "${__engine_parameters_maker_state_machine_arn__}", + "Input": { + "portal_run_id.$": "$.input_event_detail_draft.portal_run_id", + "workflow_name": "${__workflow_name__}", + "workflow_version": "${__workflow_version__}", + "event_data_inputs.$": "$.input_event_detail_draft.data_inputs", + "ssm_parameters_list": [ + { + "engine_parameter_key": "outputUri", + "ssm_name": "${__output_uri_ssm_parameter_name__}" + }, + { + "engine_parameter_key": "logsUri", + "ssm_name": "${__logs_uri_ssm_parameter_name__}" + }, + { + "engine_parameter_key": "cacheUri", + "ssm_name": "${__cache_uri_ssm_parameter_name__}" + }, + { + "engine_parameter_key": "projectId", + "ssm_name": "${__project_id_ssm_parameter_name__}" + } + ] + } + }, + "Next": "EventBridge PutEvents", + "ResultPath": "$.set_workflow_run_engine_parameters", + "ResultSelector": { + "engine_parameters.$": "$.Output.engine_parameters" + } + }, + "EventBridge PutEvents": { + "Type": "Task", + "Resource": "arn:aws:states:::events:putEvents", + "Parameters": { + "Entries": [ + { + "Source": "${__event_output_source__}", + "EventBusName": "${__event_bus_name__}", + "DetailType": "${__detail_type__}", + "Detail": { + "portalRunId.$": "$.input_event_detail_draft.portal_run_id", + "timestamp.$": "$$.State.EnteredTime", + "status": "${__ready_status__}", + "workflowName": "${__workflow_name__}", + "workflowVersion": "${__workflow_version__}", + "workflowRunName.$": "$.input_event_detail_draft.workflow_run_name", + "linkedLibraries.$": "$.input_event_detail_draft.linked_libraries", + "payload": { + "version": "${__payload_version__}", + "data": { + "inputs.$": "$.input_event_detail_draft.data_inputs", + "engineParameters.$": "$.set_workflow_run_engine_parameters.engine_parameters", + "tags.$": "$.input_event_detail_draft.data_tags" + } + } + } + } + ] + }, + "End": true + } + } +} diff --git a/lib/workload/components/sfn-workflowdraftrunstatechange-common-preamble/index.ts b/lib/workload/components/sfn-workflowdraftrunstatechange-common-preamble/index.ts index 65cfdd601..4cfd259b8 100644 --- a/lib/workload/components/sfn-workflowdraftrunstatechange-common-preamble/index.ts +++ b/lib/workload/components/sfn-workflowdraftrunstatechange-common-preamble/index.ts @@ -8,7 +8,6 @@ This sfn works as a subfunction with the primary aim of import { Construct } from 'constructs'; import * as lambda from 'aws-cdk-lib/aws-lambda'; import * as sfn from 'aws-cdk-lib/aws-stepfunctions'; -import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; import { PythonFunction } from '@aws-cdk/aws-lambda-python-alpha'; import * as iam from 'aws-cdk-lib/aws-iam'; import path from 'path'; @@ -17,9 +16,6 @@ import { Duration } from 'aws-cdk-lib'; export interface WorkflowRunStateChangeInternalInputMakerProps { /* Object name prefixes */ stateMachinePrefix: string; - /* Table configs */ - tableObj: dynamodb.ITableV2; - portalRunTablePartitionName: string; /* Workflow metadata constants */ workflowName: string; workflowVersion: string; @@ -79,9 +75,6 @@ export class WorkflowDraftRunStateChangeCommonPreambleConstruct extends Construc portalRunIdLambda.currentVersion.functionArn, __generate_workflow_run_name_lambda_function_arn__: workflowRunNameLambda.currentVersion.functionArn, - /* Table configurations */ - __table_name__: props.tableObj.tableName, - __portal_run_partition_name__: props.portalRunTablePartitionName, /* Workflow name */ __workflow_name__: props.workflowName, __workflow_version__: props.workflowVersion, @@ -95,8 +88,5 @@ export class WorkflowDraftRunStateChangeCommonPreambleConstruct extends Construc [workflowRunNameLambda, portalRunIdLambda].forEach((lambdaObj) => { lambdaObj.currentVersion.grantInvoke(this.stepFunctionObj.role); }); - - /* Allow step function to write to table */ - props.tableObj.grantReadWriteData(this.stepFunctionObj.role); } } diff --git a/lib/workload/components/sfn-workflowdraftrunstatechange-common-preamble/step_functions_templates/workflowdraftrunstatechange_preamble_template.asl.json b/lib/workload/components/sfn-workflowdraftrunstatechange-common-preamble/step_functions_templates/workflowdraftrunstatechange_preamble_template.asl.json index a0c6aad64..4fce64ac3 100644 --- a/lib/workload/components/sfn-workflowdraftrunstatechange-common-preamble/step_functions_templates/workflowdraftrunstatechange_preamble_template.asl.json +++ b/lib/workload/components/sfn-workflowdraftrunstatechange-common-preamble/step_functions_templates/workflowdraftrunstatechange_preamble_template.asl.json @@ -59,29 +59,12 @@ "BackoffRate": 2 } ], - "Next": "Save Portal Run ID To Input Maker DB", + "Next": "Wait 1 Second", "ResultPath": "$.generate_workflow_run_name_step", "ResultSelector": { "workflow_run_name.$": "$.Payload.workflow_run_name" } }, - "Save Portal Run ID To Input Maker DB": { - "Type": "Task", - "Resource": "arn:aws:states:::dynamodb:putItem", - "Parameters": { - "TableName": "${__table_name__}", - "Item": { - "id.$": "$.generate_portal_run_id_step.portal_run_id", - "id_type": "${__portal_run_partition_name__}", - "workflow_name": "${__workflow_name__}", - "workflow_version": "${__workflow_version__}", - "workflow_run_name.$": "$.generate_workflow_run_name_step.workflow_run_name", - "status": "DRAFT" - } - }, - "Next": "Wait 1 Second", - "ResultPath": null - }, "Wait 1 Second": { "Type": "Wait", "Seconds": 1, diff --git a/lib/workload/stateful/stacks/pieriandx-pipeline-dynamo-db/deploy/index.ts b/lib/workload/stateful/stacks/pieriandx-pipeline-dynamo-db/deploy/index.ts new file mode 100644 index 000000000..079a42e2d --- /dev/null +++ b/lib/workload/stateful/stacks/pieriandx-pipeline-dynamo-db/deploy/index.ts @@ -0,0 +1,22 @@ +import * as cdk from 'aws-cdk-lib'; +import { Construct } from 'constructs'; +import { DynamodbPartitionedPipelineConstruct } from '../../../../components/dynamodb-partitioned-table'; + +export interface PierianDxPipelineTableConfig { + dynamodbTableName: string; +} + +export type PierianDxPipelineTableStackProps = PierianDxPipelineTableConfig & cdk.StackProps; + +export class PierianDxPipelineTable extends cdk.Stack { + constructor(scope: Construct, id: string, props: PierianDxPipelineTableStackProps) { + super(scope, id, props); + + /* + Initialise dynamodb table, where portal_run_id is the primary sort key + */ + new DynamodbPartitionedPipelineConstruct(this, 'pieriandx_pipeline_table', { + tableName: props.dynamodbTableName, + }); + } +} diff --git a/lib/workload/stateful/stacks/stacky-mcstackface-dynamodb/index.ts b/lib/workload/stateful/stacks/stacky-mcstackface-dynamodb/index.ts index 23f2630ff..4e66ffb97 100644 --- a/lib/workload/stateful/stacks/stacky-mcstackface-dynamodb/index.ts +++ b/lib/workload/stateful/stacks/stacky-mcstackface-dynamodb/index.ts @@ -14,6 +14,7 @@ export interface StackyStatefulTablesConfig { dynamodbWtsGlueTableName: string; dynamodbUmccriseGlueTableName: string; dynamodbRnasumGlueTableName: string; + dynamodbPieriandxGlueTableName: string; removalPolicy?: RemovalPolicy; } @@ -29,6 +30,7 @@ export class StackyStatefulTablesStack extends Stack { public readonly wtsGlueTable: dynamodb.ITableV2; public readonly umccriseGlueTable: dynamodb.ITableV2; public readonly rnasumGlueTable: dynamodb.ITableV2; + public readonly pieriandxGlueTable: dynamodb.ITableV2; constructor(scope: Construct, id: string, props: StackProps & StackyStatefulTablesStackProps) { super(scope, id, props); @@ -115,5 +117,13 @@ export class StackyStatefulTablesStack extends Stack { tableName: props.dynamodbRnasumGlueTableName, removalPolicy: props.removalPolicy, }).tableObj; + + /* + Initialise dynamodb table for the pieriandx glue service + */ + this.pieriandxGlueTable = new DynamodbPartitionedPipelineConstruct(this, 'pieriandxGlueTable', { + tableName: props.dynamodbPieriandxGlueTableName, + removalPolicy: props.removalPolicy, + }).tableObj; } } diff --git a/lib/workload/stateful/statefulStackCollectionClass.ts b/lib/workload/stateful/statefulStackCollectionClass.ts index 86d6648f9..960ae809c 100644 --- a/lib/workload/stateful/statefulStackCollectionClass.ts +++ b/lib/workload/stateful/statefulStackCollectionClass.ts @@ -45,6 +45,10 @@ import { RnasumIcav2PipelineTable, RnasumIcav2PipelineTableStackProps, } from './stacks/rnasum-pipeline-dynamo-db/deploy/stack'; +import { + PierianDxPipelineTable, + PierianDxPipelineTableStackProps, +} from './stacks/pieriandx-pipeline-dynamo-db/deploy'; export interface StatefulStackCollectionProps { dataBucketStackProps: DataBucketStackProps; @@ -61,6 +65,7 @@ export interface StatefulStackCollectionProps { rnasumIcav2PipelineTableStackProps: RnasumIcav2PipelineTableStackProps; BclConvertTableStackProps: BclConvertTableStackProps; stackyStatefulTablesStackProps: StackyStatefulTablesStackProps; + pierianDxPipelineTableStackProps: PierianDxPipelineTableStackProps; } export class StatefulStackCollection { @@ -80,6 +85,7 @@ export class StatefulStackCollection { readonly rnasumIcav2PipelineTableStack: Stack; readonly BclConvertTableStack: Stack; readonly stackyStatefulTablesStack: Stack; + readonly pierianDxPipelineTableStack: Stack; constructor( scope: Construct, @@ -178,6 +184,7 @@ export class StatefulStackCollection { ...this.createTemplateProps(env, 'BclConvertTableStack'), ...statefulConfiguration.BclConvertTableStackProps, }); + this.stackyStatefulTablesStack = new StackyStatefulTablesStack( scope, 'StackyStatefulTablesStack', @@ -186,6 +193,15 @@ export class StatefulStackCollection { ...statefulConfiguration.stackyStatefulTablesStackProps, } ); + + this.pierianDxPipelineTableStack = new PierianDxPipelineTable( + scope, + 'PierianDxPipelineTableStack', + { + ...this.createTemplateProps(env, 'PierianDxPipelineTableStack'), + ...statefulConfiguration.pierianDxPipelineTableStackProps, + } + ); } /** diff --git a/lib/workload/stateless/stacks/bssh-icav2-fastq-copy-manager/Readme.md b/lib/workload/stateless/stacks/bssh-icav2-fastq-copy-manager/Readme.md index 84f07b165..45f438ec3 100644 --- a/lib/workload/stateless/stacks/bssh-icav2-fastq-copy-manager/Readme.md +++ b/lib/workload/stateless/stacks/bssh-icav2-fastq-copy-manager/Readme.md @@ -588,7 +588,7 @@ To decode these see the headers below ## Lambdas in this directory -All lambdas run on python 3.11 or higher. +All lambdas run on python 3.12 or higher. ### Process BCLConvert Output diff --git a/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/deploy/constructs/cttsov2-icav2-manager/index.ts b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/deploy/constructs/cttsov2-icav2-manager/index.ts index 7a73e9b8f..66b676619 100644 --- a/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/deploy/constructs/cttsov2-icav2-manager/index.ts +++ b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/deploy/constructs/cttsov2-icav2-manager/index.ts @@ -36,8 +36,10 @@ interface Cttsov2Icav2PipelineManagerConstructProps { /* Extras */ // Lambdas // SFN Input Lambdas - uploadSamplesheetToCacheDirLambdaObj: PythonFunction; // __dirname + '/../../../lambdas/upload_samplesheet_to_cache_dir_py' - generateCopyManifestDictLambdaObj: PythonFunction; // __dirname + '/../../../lambdas/generate_copy_manifest_dict_py' + uploadSamplesheetToCacheDirLambdaObj: PythonFunction; + generateCopyManifestDictLambdaObj: PythonFunction; + checkNumRunningSfnsLambdaObj: PythonFunction; + getRandomNumberLambdaObj: PythonFunction; // SFN Output lambdas deleteCacheUriLambdaObj: PythonFunction; setOutputJsonLambdaObj: PythonFunction; @@ -60,7 +62,7 @@ export class Cttsov2Icav2PipelineManagerConstruct extends Construct { ); // Specify the statemachine and replace the arn placeholders with the lambda arns defined above - const configure_inputs_sfn = new sfn.StateMachine( + const configureInputsSfn = new sfn.StateMachine( this, 'cttso_v2_launch_step_functions_state_machine', { @@ -74,6 +76,10 @@ export class Cttsov2Icav2PipelineManagerConstruct extends Construct { props.generateCopyManifestDictLambdaObj.currentVersion.functionArn, __upload_samplesheet_to_cache_dir__: props.uploadSamplesheetToCacheDirLambdaObj.currentVersion.functionArn, + __get_variable_number_of_seconds_lambda_function_arn__: + props.getRandomNumberLambdaObj.currentVersion.functionArn, + __check_number_of_copy_jobs_running_lambda_function_arn__: + props.checkNumRunningSfnsLambdaObj.currentVersion.functionArn, /* Subfunction state machines */ __copy_icav2_files_state_machine_arn__: props.icav2CopyFilesStateMachineObj.stateMachineArn, @@ -84,18 +90,21 @@ export class Cttsov2Icav2PipelineManagerConstruct extends Construct { ); // Grant lambda invoke permissions to the state machine - [props.generateCopyManifestDictLambdaObj, props.uploadSamplesheetToCacheDirLambdaObj].forEach( - (lambda_obj) => { - lambda_obj.currentVersion.grantInvoke(configure_inputs_sfn.role); - } - ); + [ + props.generateCopyManifestDictLambdaObj, + props.uploadSamplesheetToCacheDirLambdaObj, + props.getRandomNumberLambdaObj, + props.checkNumRunningSfnsLambdaObj, + ].forEach((lambda_obj) => { + lambda_obj.currentVersion.grantInvoke(configureInputsSfn); + }); // Allow state machine to read/write to dynamodb table - props.dynamodbTableObj.grantReadWriteData(configure_inputs_sfn.role); + props.dynamodbTableObj.grantReadWriteData(configureInputsSfn); // Because we run a nested state machine, we need to add the permissions to the state machine role // See https://stackoverflow.com/questions/60612853/nested-step-function-in-a-step-function-unknown-error-not-authorized-to-cr - configure_inputs_sfn.addToRolePolicy( + configureInputsSfn.addToRolePolicy( new iam.PolicyStatement({ resources: [ `arn:aws:events:${cdk.Aws.REGION}:${cdk.Aws.ACCOUNT_ID}:rule/StepFunctionsGetEventsForStepFunctionsExecutionRule`, @@ -105,14 +114,40 @@ export class Cttsov2Icav2PipelineManagerConstruct extends Construct { ); // Add state machine execution permissions to stateMachine role - props.icav2CopyFilesStateMachineObj.grantStartExecution(configure_inputs_sfn.role); + props.icav2CopyFilesStateMachineObj.grantStartExecution(configureInputsSfn); + + // Update checkNumRunningSfnsLambdaObj env var to include the state machine arn of + // the icav2 copy files sfn + props.checkNumRunningSfnsLambdaObj.addEnvironment( + 'SFN_ARN', + props.icav2CopyFilesStateMachineObj.stateMachineArn + ); + + // Allow the check num running sfns lambda to list the number of running icav2 copy file sfns running + /* + // FIXME: this is the ideal setup but not approved by cdk nag, since we + // FIXME: are granting the lambda permissions to all versions of the step function + props.icav2CopyFilesStateMachineObj.grantRead( + props.checkNumRunningSfnsLambdaObj.currentVersion + ); + */ + props.checkNumRunningSfnsLambdaObj.currentVersion.addToRolePolicy( + new iam.PolicyStatement({ + actions: [ + 'states:ListActivities', + 'states:DescribeStateMachine', + 'states:DescribeActivity', + ], + resources: [props.icav2CopyFilesStateMachineObj.stateMachineArn], + }) + ); /* Part 2: Configure the lambdas and outputs step function Quite a bit more complicated than regular ICAv2 workflow setup since we need to 1. Generate the outputs json from a nextflow pipeline (which doesn't have a json outputs endpoint) 2. Delete the cache fastqs we generated in the configure inputs json step function - */ + */ // Add icav2 secrets permissions to lambdas [ @@ -122,10 +157,10 @@ export class Cttsov2Icav2PipelineManagerConstruct extends Construct { props.compressVcfLambdaObj, props.checkSuccessSampleLambdaObj, ].forEach((lambda_obj) => { - props.icav2AccessTokenSecretObj.grantRead(lambda_obj.currentVersion.role); + props.icav2AccessTokenSecretObj.grantRead(lambda_obj.currentVersion); }); - const configure_outputs_sfn = new sfn.StateMachine(this, 'sfn_configure_outputs_json', { + const configureOutputsSfn = new sfn.StateMachine(this, 'sfn_configure_outputs_json', { stateMachineName: `${props.stateMachinePrefix}-configure-outputs-json`, // defintiontemplate definitionBody: DefinitionBody.fromFile(props.generateOutputJsonSfnTemplatePath), @@ -147,7 +182,7 @@ export class Cttsov2Icav2PipelineManagerConstruct extends Construct { }); // Allow the state machine to read/write to dynamodb table - props.dynamodbTableObj.grantReadWriteData(configure_outputs_sfn.role); + props.dynamodbTableObj.grantReadWriteData(configureOutputsSfn); // Allow the state machine to invoke the lambdas [ @@ -157,7 +192,7 @@ export class Cttsov2Icav2PipelineManagerConstruct extends Construct { props.compressVcfLambdaObj, props.checkSuccessSampleLambdaObj, ].forEach((lambda_obj) => { - lambda_obj.currentVersion.grantInvoke(configure_outputs_sfn.role); + lambda_obj.currentVersion.grantInvoke(configureOutputsSfn); }); /* Add ICAv2 WfmworkflowRunStateChange wrapper around launch state machine */ @@ -179,7 +214,7 @@ export class Cttsov2Icav2PipelineManagerConstruct extends Construct { internalEventSource: props.internalEventSource, // What we push back to the orcabus /* State machines to run (underneath) */ /* The inputs generation statemachine */ - generateInputsJsonSfn: configure_inputs_sfn, + generateInputsJsonSfn: configureInputsSfn, /* Internal workflowRunStateChange event details */ workflowName: props.workflowType, workflowVersion: props.workflowVersion, @@ -188,7 +223,7 @@ export class Cttsov2Icav2PipelineManagerConstruct extends Construct { // Create statemachine for handling any state changes of the pipeline // Generate state machine for handling the external ICAv2 event - const handle_external_icav2_event_sfn = new Icav2AnalysisEventHandlerConstruct( + const handleExternalIcav2EventSfn = new Icav2AnalysisEventHandlerConstruct( this, 'handle_interop_qc_ready_event', { @@ -198,7 +233,7 @@ export class Cttsov2Icav2PipelineManagerConstruct extends Construct { eventBusName: props.eventBusObj.eventBusName, icaEventPipeName: props.icaEventPipeName, internalEventSource: props.internalEventSource, - generateOutputsJsonSfn: configure_outputs_sfn, + generateOutputsJsonSfn: configureOutputsSfn, workflowName: props.workflowType, workflowVersion: props.workflowVersion, serviceVersion: props.serviceVersion, diff --git a/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/deploy/stack.ts b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/deploy/stack.ts index a12c9defb..da35eabdb 100644 --- a/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/deploy/stack.ts +++ b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/deploy/stack.ts @@ -22,8 +22,8 @@ export interface Cttsov2Icav2PipelineManagerConfig { eventBusName: string; icaEventPipeName: string; /* - Event handling - */ + Event handling + */ workflowType: string; workflowVersion: string; serviceVersion: string; @@ -31,8 +31,8 @@ export interface Cttsov2Icav2PipelineManagerConfig { internalEventSource: string; detailType: string; /* - Names for statemachines - */ + Names for statemachines + */ stateMachinePrefix: string; } @@ -44,31 +44,31 @@ export class Cttsov2Icav2PipelineManagerStack extends cdk.Stack { super(scope, id, props); // Get dynamodb table for construct - const dynamodb_table_obj = dynamodb.TableV2.fromTableName( + const dynamodbTableObj = dynamodb.TableV2.fromTableName( this, 'dynamodb_table', props.dynamodbTableName ); // Get ICAv2 Access token secret object for construct - const icav2_access_token_secret_obj = secretsManager.Secret.fromSecretNameV2( + const icav2AccessTokenSecretObj = secretsManager.Secret.fromSecretNameV2( this, 'icav2_secrets_object', props.icav2TokenSecretId ); // Get the copy batch state machine name - const icav2_copy_files_state_machine_obj = new ICAv2CopyFilesConstruct( + const icav2CopyFilesStateMachineObj = new ICAv2CopyFilesConstruct( this, 'icav2_copy_files_state_machine_obj', { - icav2JwtSecretParameterObj: icav2_access_token_secret_obj, + icav2JwtSecretParameterObj: icav2AccessTokenSecretObj, stateMachineName: `${props.stateMachinePrefix}-icav2-copy-files-sfn`, } ); // Set ssm parameter object list - const pipeline_id_ssm_obj_list = ssm.StringParameter.fromStringParameterName( + const pipelineIdSsmObjList = ssm.StringParameter.fromStringParameterName( this, props.pipelineIdSsmPath, props.pipelineIdSsmPath @@ -79,16 +79,18 @@ export class Cttsov2Icav2PipelineManagerStack extends cdk.Stack { /* Build lambdas - */ + */ + /* Part 1: Set up the lambdas needed for the input json generation state machine Quite a bit more complicated than regular ICAv2 workflow setup since we need to 1. Convert the samplesheet from json into csv format 2. Upload the samplesheet to icav2 3. Copy fastqs into a particular directory setup type - */ + 4. Regulate how many fastqs are copied at a time given ICAv2 cannot limit this itself and falls over + */ - const generate_copy_manifest_dict_lambda_obj = new PythonFunction( + const generateCopyManifestDictLambdaObj = new PythonFunction( this, 'generate_copy_manifest_dict_lambda_python_function', { @@ -100,13 +102,13 @@ export class Cttsov2Icav2PipelineManagerStack extends cdk.Stack { memorySize: 1024, timeout: Duration.seconds(60), environment: { - ICAV2_ACCESS_TOKEN_SECRET_ID: icav2_access_token_secret_obj.secretName, + ICAV2_ACCESS_TOKEN_SECRET_ID: icav2AccessTokenSecretObj.secretName, }, } ); // upload_samplesheet_to_cache_dir_py lambda - const upload_samplesheet_to_cache_dir_lambda_obj = new PythonFunction( + const uploadSamplesheetToCacheDirLambdaObj = new PythonFunction( this, 'upload_samplesheet_to_cache_dir_lambda_python_function', { @@ -118,17 +120,43 @@ export class Cttsov2Icav2PipelineManagerStack extends cdk.Stack { memorySize: 1024, timeout: Duration.seconds(60), environment: { - ICAV2_ACCESS_TOKEN_SECRET_ID: icav2_access_token_secret_obj.secretName, + ICAV2_ACCESS_TOKEN_SECRET_ID: icav2AccessTokenSecretObj.secretName, }, } ); + const getRandomNumberLambdaObj = new PythonFunction( + this, + 'get_random_number_lambda_python_function', + { + entry: path.join(__dirname, '../lambdas/get_random_number_py'), + runtime: lambda.Runtime.PYTHON_3_12, + architecture: lambda.Architecture.ARM_64, + index: 'get_random_number.py', + handler: 'handler', + } + ); + + // We need to add SFN_ARN to the env var list once we have it + // We also need to update the permissions to allow this function to list executions + const checkNumRunningSfns = new PythonFunction( + this, + 'check_num_running_sfns_lambda_python_function', + { + entry: path.join(__dirname, '../lambdas/check_num_running_sfns_py'), + runtime: lambda.Runtime.PYTHON_3_12, + architecture: lambda.Architecture.ARM_64, + index: 'check_num_running_sfns.py', + handler: 'handler', + } + ); + /* - Part 2: Build lambdas for output json generation - */ + Part 2: Build lambdas for output json generation + */ // Delete the cache uri directory lambda - const delete_cache_uri_lambda_function = new PythonFunction( + const deleteCacheUriLambdaFunction = new PythonFunction( this, 'delete_cache_uri_lambda_python_function', { @@ -140,13 +168,13 @@ export class Cttsov2Icav2PipelineManagerStack extends cdk.Stack { memorySize: 1024, timeout: Duration.seconds(60), environment: { - ICAV2_ACCESS_TOKEN_SECRET_ID: icav2_access_token_secret_obj.secretName, + ICAV2_ACCESS_TOKEN_SECRET_ID: icav2AccessTokenSecretObj.secretName, }, } ); // Set the output json lambda - const set_output_json_lambda_function = new PythonFunction( + const setOutputJsonLambdaFunction = new PythonFunction( this, 'set_output_json_lambda_python_function', { @@ -158,13 +186,13 @@ export class Cttsov2Icav2PipelineManagerStack extends cdk.Stack { memorySize: 1024, timeout: Duration.seconds(60), environment: { - ICAV2_ACCESS_TOKEN_SECRET_ID: icav2_access_token_secret_obj.secretName, + ICAV2_ACCESS_TOKEN_SECRET_ID: icav2AccessTokenSecretObj.secretName, }, } ); // Get vcfs - const get_vcfs_lambda_function = new PythonFunction(this, 'get_vcfs_lambda_python_function', { + const getVcfsLambdaFunction = new PythonFunction(this, 'get_vcfs_lambda_python_function', { entry: path.join(__dirname, '../lambdas/find_all_vcf_files_py'), runtime: lambda.Runtime.PYTHON_3_12, architecture: lambda.Architecture.ARM_64, @@ -173,13 +201,13 @@ export class Cttsov2Icav2PipelineManagerStack extends cdk.Stack { memorySize: 1024, timeout: Duration.seconds(60), environment: { - ICAV2_ACCESS_TOKEN_SECRET_ID: icav2_access_token_secret_obj.secretName, + ICAV2_ACCESS_TOKEN_SECRET_ID: icav2AccessTokenSecretObj.secretName, }, }); // Compress vcf const architecture = lambda.Architecture.ARM_64; - const compress_vcf_lambda_function = new DockerImageFunction(this, 'compress_vcf_lambda', { + const compressVcfLambdaFunction = new DockerImageFunction(this, 'compress_vcf_lambda', { description: 'Compress Vcfs', code: DockerImageCode.fromImageAsset(path.join(__dirname, '../lambdas/compress_icav2_vcf'), { file: 'Dockerfile', @@ -193,43 +221,41 @@ export class Cttsov2Icav2PipelineManagerStack extends cdk.Stack { memorySize: 2048, // Don't want pandas to kill the lambda architecture: architecture, environment: { - ICAV2_ACCESS_TOKEN_SECRET_ID: icav2_access_token_secret_obj.secretName, + ICAV2_ACCESS_TOKEN_SECRET_ID: icav2AccessTokenSecretObj.secretName, }, }); // Check success lambda - const check_success_lambda_function = new PythonFunction( - this, - 'check_success_lambda_function', - { - entry: path.join(__dirname, '../lambdas/check_success_py'), - runtime: lambda.Runtime.PYTHON_3_12, - architecture: lambda.Architecture.ARM_64, - index: 'check_success.py', - handler: 'handler', - memorySize: 1024, - timeout: Duration.seconds(60), - environment: { - ICAV2_ACCESS_TOKEN_SECRET_ID: icav2_access_token_secret_obj.secretName, - }, - } - ); + const checkSuccessLambdaFunction = new PythonFunction(this, 'check_success_lambda_function', { + entry: path.join(__dirname, '../lambdas/check_success_py'), + runtime: lambda.Runtime.PYTHON_3_12, + architecture: lambda.Architecture.ARM_64, + index: 'check_success.py', + handler: 'handler', + memorySize: 1024, + timeout: Duration.seconds(60), + environment: { + ICAV2_ACCESS_TOKEN_SECRET_ID: icav2AccessTokenSecretObj.secretName, + }, + }); // Create the state machine to launch the nextflow workflow on ICAv2 - const cttso_v2_launch_state_machine = new Cttsov2Icav2PipelineManagerConstruct(this, id, { + const cttsov2LaunchStateMachine = new Cttsov2Icav2PipelineManagerConstruct(this, id, { /* Stack Objects */ - dynamodbTableObj: dynamodb_table_obj, - icav2AccessTokenSecretObj: icav2_access_token_secret_obj, - icav2CopyFilesStateMachineObj: icav2_copy_files_state_machine_obj.icav2CopyFilesSfnObj, - pipelineIdSsmObj: pipeline_id_ssm_obj_list, + dynamodbTableObj: dynamodbTableObj, + icav2AccessTokenSecretObj: icav2AccessTokenSecretObj, + icav2CopyFilesStateMachineObj: icav2CopyFilesStateMachineObj.icav2CopyFilesSfnObj, + pipelineIdSsmObj: pipelineIdSsmObjList, /* Lambdas paths */ - uploadSamplesheetToCacheDirLambdaObj: upload_samplesheet_to_cache_dir_lambda_obj, // __dirname + '/../../../lambdas/upload_samplesheet_to_cache_dir_py' - generateCopyManifestDictLambdaObj: generate_copy_manifest_dict_lambda_obj, // __dirname + '/../../../lambdas/generate_copy_manifest_dict_py' - deleteCacheUriLambdaObj: delete_cache_uri_lambda_function, - setOutputJsonLambdaObj: set_output_json_lambda_function, - getVcfsLambdaObj: get_vcfs_lambda_function, - compressVcfLambdaObj: compress_vcf_lambda_function, - checkSuccessSampleLambdaObj: check_success_lambda_function, + uploadSamplesheetToCacheDirLambdaObj: uploadSamplesheetToCacheDirLambdaObj, // __dirname + '/../../../lambdas/upload_samplesheet_to_cache_dir_py' + generateCopyManifestDictLambdaObj: generateCopyManifestDictLambdaObj, // __dirname + '/../../../lambdas/generate_copy_manifest_dict_py' + getRandomNumberLambdaObj: getRandomNumberLambdaObj, + checkNumRunningSfnsLambdaObj: checkNumRunningSfns, + deleteCacheUriLambdaObj: deleteCacheUriLambdaFunction, + setOutputJsonLambdaObj: setOutputJsonLambdaFunction, + getVcfsLambdaObj: getVcfsLambdaFunction, + compressVcfLambdaObj: compressVcfLambdaFunction, + checkSuccessSampleLambdaObj: checkSuccessLambdaFunction, /* Step function templates */ generateInputJsonSfnTemplatePath: path.join( __dirname, diff --git a/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/lambdas/check_num_running_sfns_py/check_num_running_sfns.py b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/lambdas/check_num_running_sfns_py/check_num_running_sfns.py new file mode 100644 index 000000000..6e3591418 --- /dev/null +++ b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/lambdas/check_num_running_sfns_py/check_num_running_sfns.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python + +""" +Given the environment variable 'SFN_ARN' which represents the ARN of a Step Function, +this script will check the number of running executions of the Step Function +and return the number of running executions +""" + +# Imports +import boto3 +import typing +from os import environ + +# Mypy type hints +if typing.TYPE_CHECKING: + from mypy_boto3_stepfunctions import SFNClient + +# Constants +MAX_CONCURRENCY_ALLOWED = 5 + + +def get_sfn_client() -> 'SFNClient': + """ + Get the Step Functions client + :return: SFNClient + """ + return boto3.client('stepfunctions') + + +def list_running_executions(sfn_arn: str) -> int: + """ + List the number of running executions + """ + sfn_client = get_sfn_client() + response = sfn_client.list_executions( + stateMachineArn=sfn_arn, + statusFilter='RUNNING' + ) + return len(response['executions']) + + + +def handler(event, context=None): + """ + Given the environment variable 'SFN_ARN' which represents the ARN of a Step Function, + this script will check the number of running executions of the Step Function + and return the number of running executions + :param event: + :param context: + :return: + """ + + sfn_arn = environ.get('SFN_ARN', None) + if not sfn_arn: + raise ValueError('SFN_ARN environment variable is required') + + # Get the number of running executions + running_executions = list_running_executions(sfn_arn) + + # Check if the number of running executions is + # less than the maximum concurrency allowed + if running_executions < MAX_CONCURRENCY_ALLOWED: + return { + "run_copy_job_step_bool": True, + } + else: + return { + "run_copy_job_step_bool": False, + } + + +# if __name__ == '__main__': +# from os import environ +# import json +# environ["AWS_PROFILE"] = "umccr-production" +# environ["AWS_DEFAULT_REGION"] = "ap-southeast-2" +# environ['SFN_ARN'] = 'arn:aws:states:ap-southeast-2:472057503814:stateMachine:cttsov2Sfn-icav2-copy-files-sfn' +# +# print( +# json.dumps( +# handler({}, None), +# indent=4 +# ) +# ) +# +# # { +# # "run_copy_job_step_bool": true +# # } diff --git a/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/lambdas/get_random_number_py/get_random_number.py b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/lambdas/get_random_number_py/get_random_number.py new file mode 100644 index 000000000..e5d91caee --- /dev/null +++ b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/lambdas/get_random_number_py/get_random_number.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 + +""" +Get a random number +""" + +# Imports +import random +from typing import Dict + +# Globals +MAX_NUMBER = 60 + + + +def handler(event, context) -> Dict[str, int]: + """ + Get a random number + """ + + return { + "random_number": random.randint(0, MAX_NUMBER) + } + + +# if __name__ == "__main__": +# import json +# print( +# json.dumps( +# handler({}, None), +# indent=4 +# ) +# ) \ No newline at end of file diff --git a/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/step_functions_templates/set_cttso_v2_nf_inputs.asl.json b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/step_functions_templates/set_cttso_v2_nf_inputs.asl.json index 4f1b13bd1..e473bac6a 100644 --- a/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/step_functions_templates/set_cttso_v2_nf_inputs.asl.json +++ b/lib/workload/stateless/stacks/cttso-v2-pipeline-manager/step_functions_templates/set_cttso_v2_nf_inputs.asl.json @@ -104,7 +104,7 @@ "BackoffRate": 2 } ], - "Next": "Copy Fastq Files to Cache Directory", + "Next": "Get Variable number of seconds", "Comment": "Generate a copy manifest object, ready to parse into the icav2 copy batch utility step function\n\nWe expect the following inputs:\n\n* cache_path\n* project_id\n* sample_id\n* fastq_list_rows\n\nAnd we expect the following outputs:\n\n* manifest", "ResultSelector": { "dest_uri.$": "$.Payload.dest_uri", @@ -112,6 +112,78 @@ }, "ResultPath": "$.generate_copy_manifest_dict_step" }, + "Get Variable number of seconds": { + "Type": "Task", + "Resource": "arn:aws:states:::lambda:invoke", + "Parameters": { + "FunctionName": "${__get_variable_number_of_seconds_lambda_function_arn__}" + }, + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 1, + "MaxAttempts": 3, + "BackoffRate": 2 + } + ], + "ResultSelector": { + "num_seconds.$": "$.Payload.random_number" + }, + "ResultPath": "$.get_variable_seconds_step", + "Next": "Wait a variable amount of time" + }, + "Wait a variable amount of time": { + "Type": "Wait", + "SecondsPath": "$.get_variable_seconds_step.num_seconds", + "Next": "Check Running Copy Fastq Job Number" + }, + "Check Running Copy Fastq Job Number": { + "Type": "Task", + "Resource": "arn:aws:states:::lambda:invoke", + "Parameters": { + "FunctionName": "${__check_number_of_copy_jobs_running_lambda_function_arn__}" + }, + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 1, + "MaxAttempts": 3, + "BackoffRate": 2 + } + ], + "ResultSelector": { + "run_copy_job_step_bool.$": "$.Payload.run_copy_job_step_bool" + }, + "ResultPath": "$.run_copy_job_step", + "Next": "Allowed to run" + }, + "Allowed to run": { + "Type": "Choice", + "Choices": [ + { + "Variable": "$.run_copy_job_step.run_copy_job_step_bool", + "BooleanEquals": false, + "Next": "Wait A Minute", + "Comment": "Too Many jobs running already" + } + ], + "Default": "Copy Fastq Files to Cache Directory" + }, + "Wait A Minute": { + "Type": "Wait", + "Seconds": 60, + "Next": "Check Running Copy Fastq Job Number" + }, "Copy Fastq Files to Cache Directory": { "Type": "Task", "End": true, diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/Readme.md b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/Readme.md new file mode 100644 index 000000000..988c1f6c7 --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/Readme.md @@ -0,0 +1,247 @@ +# PierianDx Pipeline Manager + +Giant, serverless pipeline attached to a serverless database + +## Using the AWS Launch Step function + +The AWS Launch step function takes in the following inputs + +* dag: Object + * name: string # The name of this case dag + * description: string # The description of this case dag +* case_metadata: Object + * panel_name: string # The name of this case’s panel _tso500_DRAGEN_ctDNA_v2_1_Universityofmelbourne_ # pragma: allowlist secret + * specimen_label: string # The label of the specimen _primarySpecimen_ + * sample_type: Enum # patientcare, clinical_trial, validation, proficiency_testing + * indication: String # Optional input + * disease: Object + * code: string # The disease id + * label: string # The name of the disease (optional) + * is_identified: bool # Boolean is this a boolean identified case or de-identified case + * case_accession_number: string - must be unique - uses syntax SBJID__LIBID__NNN + * specimen_type: + * code: string # The SNOMED-CT term for a specimen type + * label: Optional label for the specimen type + * external_specimen_id: string # The external specimen id + * date_accessioned: Datetime # The date the case was accessioned _2021-01-01T00:00:00Z_ + * date_collected: Datetime # The date the specimen was collected _2021-01-01T00:00:00Z_ + * date_received: Datetime # The date the specimen was received _2021-01-01T00:00:00Z_ + * gender: Enum # unknown, male, femail, unspecified, other, ambiguous, not_applicable # Optional + * ethnicity: Enum # unknown, hispanic_or_latino, not_hispanic_or_latino, not_reported # Optional + * race: Enum # american_indian_or_alaska_native, asian, black_or_african_american, native_hawaiian_or_other_pacific_islander, not_reported, unknown, white # Optional + + > Note: If the case is de-identified, the following fields are required + * study_id: String # Only required if is_identified is false + * participant_id: String # Only required if is_identified is false + + > Note: If the case is identified, the following fields are required + * date_of_birth: Datetime # Only required if is_identified is true + * first_name: String # Only required if is_identified is true + * last_name: String # Only required if is_identified is true + * medical_record_numbers: Object # Only required if is_identified is true + * mrn: string # The medical record number + * medical_facility: Object + * facility: string # The name of the facility + * hospital_number: string # The hospital number + * requesting_physicians: Object # The requesting physician - required for identified cases + * first_name: string + * last_name: string + +* data_files: Object + * microsat_output: uri + * tmb_metrics: uri + * cnv: uri + * hard_filtered: uri + * fusions: uri + * metrics_output: uri + +* samplesheet_b64gz: str +* portal_run_id: str +* sequencerrun_s3_path_root: str + +## DataBase Structure + +> IDs +* db_uuid +* portal_run_id +* case_id +* sequencerun_id +* informaticsjob_id +* report_id +* case_accession_number + +> Objects +* samplesheet_b64gz: str +* case_creation_obj: CaseCreationObject +* sequencerrun_creation_obj: SequencerRunCreationObject +* informaticsjob_creation_obj: InformaticsJobCreationObject +* data_files: List[DataFileObject] + +> Status +* job_status: waiting, ready, running, complete, failed, canceled +* report_status: waiting, ready, running, complete, failed, canceled, report_generation_complete + +## Step Function execution structure + +### Launch Overview + +This will trigger three sub-step-functions, case creation, sequencerrun creation and informatics job creation + +![images/launch_overview.png](images/launch_overview.png) + +The whole process should not take more than 30 seconds + +### Case Creation Overview + +Generate the case object, this defines the case metadata and will generate a case id (required for the informatics job section) + +![images/case_creation_overview.png](images/case_overview.png) + +### Sequencer Run Overview + +Upload the data files from ICAv2 to the PierianDx S3 bucket. + +Generate a samplesheet and upload it to the PierianDx S3 bucket. + +Create a sequencer run object, this defines the sequencer run metadata and will generate a sequencer run id (required for the informatics job section) + +![images/sequencerrun_overview.png](images/sequencerrun_overview.png) + +### Informatics Job Overview + +Generate an informatics job for the case. + +This will return an informatics job id. + +![images/informaticsjob_launch_overview.png](images/informaticsjob_launch_overview.png) + + +## Launch Execution Example + + +``` +aws events put-events --no-cli-pager --cli-input-json "$( \ + jq --raw-output --compact-output \ + ' + { + "Entries": [ + { + "EventBusName": "OrcaBusMain", + "DetailType": "workflowRunStateChange", + "Source": "orcabus.manual", + "Time": $utc_time, + "Resources": [], + "Detail": { + "portalRunId": "abcd1234", + "timestamp": "20240920T113200Z", + "status": "READY", + "workflowName": "pieriandx", + "workflowVersion": "2.1.1", + "workflowRunName": "umccr-automated--pieriandx--2-1-1--abcd1234", + "linkedLibraries": [ + { + "libraryId": "L2400161", + "orcabusId": "NA" + } + ], + "payload": { + "version": "2024.10.01", + "data": { + "inputs": { + "instrumentRunId": "231116_A01052_0172_BHVLM5DSX7", + "dagVersion": "2.1.1", + "panelVersion": "main", + "caseMetadata": { + "isIdentified": true, + "caseAccessionNumber": "SBJ04407__L2400161__V2__abcd1238", + "externalSpecimenId": "externalspecimenid", + "sampleType": "PatientCare", + "specimenLabel": "primarySpecimen", + "indication": "Test", + "diseaseCode": 64572001, + "specimenCode": 122561005, + "sampleReception": { + "dateAccessioned": "2021-01-01T00:00:00Z", + "dateCollected": "2024-02-20T20:17:00Z", + "dateReceived": "2021-01-01T00:00:00Z" + }, + "patientInformation": { + "dateOfBirth": "1970-01-01", + "firstName": "John", + "lastName": "Doe" + }, + "medicalRecordNumbers": { + "mrn": "3069999", + "medicalFacility": { + "facility": "Not Available", + "hospitalNumber": "99" + } + }, + "requestingPhysician": { + "firstName": "Meredith", + "lastName": "Gray" + } + }, + "dataFiles": { + "microsatOutputUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Logs_Intermediates/DragenCaller/L2400161/L2400161.microsat_output.json", + "tmbMetricsUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Logs_Intermediates/Tmb/L2400161/L2400161.tmb.metrics.csv", + "cnvVcfUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Results/L2400161/L2400161.cnv.vcf.gz", + "hardFilteredVcfUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Results/L2400161/L2400161.hard-filtered.vcf.gz", + "fusionsUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Results/L2400161/L2400161_Fusions.csv", + "metricsOutputUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Results/L2400161/L2400161_MetricsOutput.tsv", + "samplesheetUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Logs_Intermediates/SampleSheetValidation/SampleSheet_Intermediate.csv" + } + }, + "engineParameters": {}, + "tags": { + "libraryId": "L2400161", + "subjectId": "SBJ04407", + "instrumentRunId": "231116_A01052_0172_BHVLM5DSX7", + "isIdentified": true, + "metadataFromRedCap": true + } + } + } + } + } + ] + } + ' +)" +``` + +DAGs (PROD) + +```json +{ + "name": "cromwell_tso500_ctdna_workflow_1.0.4", + "description": "tso500_ctdna_workflow" +} +``` + +Panels (PROD) + +```json5 +[ + { + "name": "tso500_DRAGEN_ctDNA_v2_1_Universityofmelbourne", // pragma: allowlist secret + "label": "TSO500 DRAGEN ctDNA v2.1" + }, + { + "name": "tso500_ctDNA_vcf_subpanel_workflow_university_of_melbourne", + "label": "TruSight Oncology 500 ctDNA (VCF) Sub-panel" + }, + { + "name": "Melbourne_WGS_Somatic_Textual", + "label": "Melbourne WGS Somatic" + }, + { + "name": "tso500_ctDNA_vcf_workflow_university_of_melbourne", + "label": "TruSight Oncology 500 ctDNA (VCF) University of Melbourne" + }, + { + "name": "Melbourne_WGS_Somatic_Textual_SAS", + "label": "Melbourne WGS Somatic New" + } +] +``` \ No newline at end of file diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/deploy/constructs/lambda_layer.ts b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/deploy/constructs/lambda_layer.ts new file mode 100644 index 000000000..df0e89ee1 --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/deploy/constructs/lambda_layer.ts @@ -0,0 +1,40 @@ +import { Construct } from 'constructs'; +import { PythonLayerVersion } from '@aws-cdk/aws-lambda-python-alpha'; +import * as lambda from 'aws-cdk-lib/aws-lambda'; + +export interface LambdaLayerConstructProps { + layer_directory: string; +} + +export class LambdaLayerConstruct extends Construct { + public readonly lambda_layer_arn: string; + public readonly lambda_layer_version_obj: PythonLayerVersion; + + constructor(scope: Construct, id: string, props: LambdaLayerConstructProps) { + super(scope, id); + + this.lambda_layer_version_obj = new PythonLayerVersion(this, 'cttso_v2_tool_layer', { + entry: props.layer_directory, + compatibleRuntimes: [lambda.Runtime.PYTHON_3_12], + compatibleArchitectures: [lambda.Architecture.ARM_64], + license: 'GPL3', + description: 'A layer to enable the cttso_v2 manager tools layer', + bundling: { + commandHooks: { + beforeBundling(inputDir: string, outputDir: string): string[] { + return []; + }, + afterBundling(inputDir: string, outputDir: string): string[] { + return [ + `python -m pip install ${inputDir} -t ${outputDir}`, + `find ${outputDir} -type d -name "pandas" -exec echo rm -rf {}/tests \;`, + ]; + }, + }, + }, + }); + + // Set outputs + this.lambda_layer_arn = this.lambda_layer_version_obj.layerVersionArn; + } +} diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/deploy/constructs/pieriandx_launch_case_creation_step_function.ts b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/deploy/constructs/pieriandx_launch_case_creation_step_function.ts new file mode 100644 index 000000000..24fdf06ee --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/deploy/constructs/pieriandx_launch_case_creation_step_function.ts @@ -0,0 +1,67 @@ +import * as cdk from 'aws-cdk-lib'; +import { Duration } from 'aws-cdk-lib'; +import { Construct } from 'constructs'; +import * as lambda from 'aws-cdk-lib/aws-lambda'; +import * as iam from 'aws-cdk-lib/aws-iam'; +import * as sfn from 'aws-cdk-lib/aws-stepfunctions'; +import * as ssm from 'aws-cdk-lib/aws-ssm'; +import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; +import * as secretsManager from 'aws-cdk-lib/aws-secretsmanager'; +import { DefinitionBody } from 'aws-cdk-lib/aws-stepfunctions'; + +import { PythonFunction } from '@aws-cdk/aws-lambda-python-alpha'; +import { LambdaLayerConstruct } from './lambda_layer'; + +interface PieriandxLaunchCaseCreationStepFunctionConstructProps { + /* Stack Objects */ + dynamodbTableObj: dynamodb.ITableV2; + /* Lambdas paths */ + generateCaseLambdaObj: PythonFunction; + /* Step function templates */ + launchPieriandxCaseCreationStepfunctionTemplatePath: string; + /* Custom */ + prefix: string; +} + +export class PieriandxLaunchCaseCreationStepFunctionStateMachineConstruct extends Construct { + public readonly stateMachineObj: sfn.IStateMachine; + + constructor( + scope: Construct, + id: string, + props: PieriandxLaunchCaseCreationStepFunctionConstructProps + ) { + super(scope, id); + + // Specify the statemachine and replace the arn placeholders with the lambda arns defined above + const stateMachine = new sfn.StateMachine( + this, + 'pieriandx_launch_step_functions_state_machine', + { + // stateMachineName + stateMachineName: `${props.prefix}-sub-case-sfn`, + // defintiontemplate + definitionBody: DefinitionBody.fromFile( + props.launchPieriandxCaseCreationStepfunctionTemplatePath + ), + // definitionSubstitutions + definitionSubstitutions: { + /* Tables */ + __table_name__: props.dynamodbTableObj.tableName, + /* Lambdas */ + __generate_case_lambda_function_arn__: + props.generateCaseLambdaObj.currentVersion.functionArn, + }, + } + ); + + // Grant lambda invoke permissions to the state machine + props.generateCaseLambdaObj.currentVersion.grantInvoke(stateMachine); + + // Allow state machine to read/write to dynamodb table + props.dynamodbTableObj.grantReadWriteData(stateMachine); + + // Set outputs + this.stateMachineObj = stateMachine; + } +} diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/deploy/constructs/pieriandx_launch_informaticsjob_creation_step_function.ts b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/deploy/constructs/pieriandx_launch_informaticsjob_creation_step_function.ts new file mode 100644 index 000000000..6b86e5e28 --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/deploy/constructs/pieriandx_launch_informaticsjob_creation_step_function.ts @@ -0,0 +1,58 @@ +import { Construct } from 'constructs'; +import * as iam from 'aws-cdk-lib/aws-iam'; +import * as sfn from 'aws-cdk-lib/aws-stepfunctions'; +import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; +import { DefinitionBody } from 'aws-cdk-lib/aws-stepfunctions'; +import { PythonFunction } from '@aws-cdk/aws-lambda-python-alpha'; + +interface PieriandxLaunchInformaticsjobCreationStepFunctionsStateMachineConstructProps { + /* Stack Objects */ + dynamodbTableObj: dynamodb.ITableV2; + /* Lambdas paths */ + generateInformaticsjobLambdaObj: PythonFunction; // __dirname + '/../../../lambdas/upload_samplesheet_to_cache_dir' + /* Step function templates */ + launchPieriandxInformaticsjobCreationStepfunctionTemplate: string; // __dirname + '/../../../step_functions_templates/launch_pieriandx_informaticsjob_creation.asl.json' + /* Prefix */ + prefix: string; +} + +export class PieriandxLaunchInformaticsjobCreationStepFunctionsStateMachineConstruct extends Construct { + public readonly stateMachineObj: sfn.IStateMachine; + + constructor( + scope: Construct, + id: string, + props: PieriandxLaunchInformaticsjobCreationStepFunctionsStateMachineConstructProps + ) { + super(scope, id); + + // Specify the statemachine and replace the arn placeholders with the lambda arns defined above + const stateMachine = new sfn.StateMachine( + this, + 'pieriandx_launch_step_functions_state_machine', + { + // stateMachineName + stateMachineName: `${props.prefix}-sub-job-sfn`, + // defintiontemplate + definitionBody: DefinitionBody.fromFile( + props.launchPieriandxInformaticsjobCreationStepfunctionTemplate + ), + // definitionSubstitutions + definitionSubstitutions: { + __generate_informaticsjob_lambda_function_arn__: + props.generateInformaticsjobLambdaObj.currentVersion.functionArn, + __table_name__: props.dynamodbTableObj.tableName, + }, + } + ); + + // Grant lambda invoke permissions to the state machine + props.generateInformaticsjobLambdaObj.currentVersion.grantInvoke(stateMachine); + + // Allow state machine to read/write to dynamodb table + props.dynamodbTableObj.grantReadWriteData(stateMachine.role); + + // Set outputs + this.stateMachineObj = stateMachine; + } +} diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/deploy/constructs/pieriandx_launch_sequencerrun_creation_step_function.ts b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/deploy/constructs/pieriandx_launch_sequencerrun_creation_step_function.ts new file mode 100644 index 000000000..af8bf0909 --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/deploy/constructs/pieriandx_launch_sequencerrun_creation_step_function.ts @@ -0,0 +1,70 @@ +import { Construct } from 'constructs'; +import * as iam from 'aws-cdk-lib/aws-iam'; +import * as sfn from 'aws-cdk-lib/aws-stepfunctions'; +import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; +import { DefinitionBody } from 'aws-cdk-lib/aws-stepfunctions'; +import { PythonFunction } from '@aws-cdk/aws-lambda-python-alpha'; + +interface PieriandxLaunchSequencerrunCreationStepFunctionsStateMachineConstructProps { + /* Stack Objects */ + dynamodbTableObj: dynamodb.ITableV2; + /* Lambdas paths */ + uploadDataToS3LambdaObj: PythonFunction; + generateSamplesheetLambdaObj: PythonFunction; + generateSequencerrunLambdaObj: PythonFunction; + /* Step function templates */ + launchPieriandxSequencerrunCreationStepfunctionTemplate: string; // __dirname + '/../../../step_functions_templates/launch_pieriandx_sequencerrun_creation.asl.json' + /* Custom props */ + prefix: string; +} + +export class PieriandxLaunchSequencerrunCreationStepFunctionsStateMachineConstruct extends Construct { + public readonly stateMachineObj: sfn.IStateMachine; + + constructor( + scope: Construct, + id: string, + props: PieriandxLaunchSequencerrunCreationStepFunctionsStateMachineConstructProps + ) { + super(scope, id); + + // Specify the statemachine and replace the arn placeholders with the lambda arns defined above + const stateMachine = new sfn.StateMachine( + this, + 'pieriandx_launch_step_functions_state_machine', + { + // stateMachineName + stateMachineName: `${props.prefix}-sub-sqrrun-sfn`, + // defintiontemplate + definitionBody: DefinitionBody.fromFile( + props.launchPieriandxSequencerrunCreationStepfunctionTemplate + ), + // definitionSubstitutions + definitionSubstitutions: { + __upload_data_to_s3_lambda_function_arn__: + props.uploadDataToS3LambdaObj.currentVersion.functionArn, + __generate_samplesheet_lambda_function_arn__: + props.generateSamplesheetLambdaObj.currentVersion.functionArn, + __generate_sequencerrun_case_lambda_function_arn__: + props.generateSequencerrunLambdaObj.currentVersion.functionArn, + __table_name__: props.dynamodbTableObj.tableName, + }, + } + ); + + // Grant lambda invoke permissions to the state machine + [ + props.uploadDataToS3LambdaObj, + props.generateSamplesheetLambdaObj, + props.generateSequencerrunLambdaObj, + ].forEach((lambda_obj) => { + lambda_obj.currentVersion.grantInvoke(stateMachine); + }); + + // Allow state machine to read/write to dynamodb table + props.dynamodbTableObj.grantReadWriteData(stateMachine); + + // Set outputs + this.stateMachineObj = stateMachine; + } +} diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/deploy/constructs/pieriandx_launch_step_function.ts b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/deploy/constructs/pieriandx_launch_step_function.ts new file mode 100644 index 000000000..44e40692c --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/deploy/constructs/pieriandx_launch_step_function.ts @@ -0,0 +1,161 @@ +import * as cdk from 'aws-cdk-lib'; +import { Construct } from 'constructs'; +import * as iam from 'aws-cdk-lib/aws-iam'; +import * as sfn from 'aws-cdk-lib/aws-stepfunctions'; +import * as ssm from 'aws-cdk-lib/aws-ssm'; +import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; +import * as events from 'aws-cdk-lib/aws-events'; +import { DefinitionBody } from 'aws-cdk-lib/aws-stepfunctions'; + +import { PythonFunction, PythonLayerVersion } from '@aws-cdk/aws-lambda-python-alpha'; +import { LambdaLayerConstruct } from './lambda_layer'; +import * as events_targets from 'aws-cdk-lib/aws-events-targets'; + +interface PieriandxLaunchStepFunctionConstructProps { + /* Stack Objects */ + dynamodbTableObj: dynamodb.ITableV2; + /* workflow */ + workflowName: string; + workflowVersion: string; + /* lambda paths */ + generatePieriandxObjectsLambdaObj: PythonFunction; // __dirname + '/../../../lambdas/generate_trimmed_samplesheet_lambda_path' + /* Defaults */ + defaultDagVersion: string; + defaultPanelName: string; + /* SSM Parameters */ + dagSsmParameterObj: ssm.IStringParameter; + panelNameSsmParameterObj: ssm.IStringParameter; + s3SequencerRunRootSsmParameterObj: ssm.IStringParameter; + /* Step function templates */ + launchPieriandxStepfunctionTemplate: string; // __dirname + '/../../../step_functions_templates/cttso_v2_launch_workflow_state_machine.json' + launchPieriandxCaseCreationStepfunctionObj: sfn.IStateMachine; + launchPieriandxInformaticsjobCreationStepfunctionObj: sfn.IStateMachine; + launchPieriandxSequencerrunCreationStepfunctionObj: sfn.IStateMachine; + /* Events */ + payloadVersion: string; + eventBusName: string; + detailType: string; + eventSource: string; + triggerLaunchSource: string; + /* Custom */ + prefix: string; +} + +export class PieriandxLaunchStepFunctionStateMachineConstruct extends Construct { + public readonly stateMachineObj: sfn.StateMachine; + + constructor(scope: Construct, id: string, props: PieriandxLaunchStepFunctionConstructProps) { + super(scope, id); + + // Specify the statemachine and replace the arn placeholders with the lambda arns defined above + const stateMachine = new sfn.StateMachine( + this, + 'pieriandx_launch_step_functions_state_machine', + { + // state machine name + stateMachineName: `${props.prefix}-submit-sfn`, + // definition template + definitionBody: DefinitionBody.fromFile(props.launchPieriandxStepfunctionTemplate), + // definitionSubstitutions + definitionSubstitutions: { + /* Table */ + __table_name__: props.dynamodbTableObj.tableName, + __workflow_name__: props.workflowName, + __workflow_version__: props.workflowVersion, + + /* Lambda Functions */ + __generate_pieriandx_objects_lambda_function_arn__: + props.generatePieriandxObjectsLambdaObj.currentVersion.functionArn, + + /* Child step functions */ + __create_case_sfn__: props.launchPieriandxCaseCreationStepfunctionObj.stateMachineArn, + __create_informaticsjob_sfn__: + props.launchPieriandxInformaticsjobCreationStepfunctionObj.stateMachineArn, + __create_sequencerrun_sfn__: + props.launchPieriandxSequencerrunCreationStepfunctionObj.stateMachineArn, + + /* SSM Parameters */ + __dag_versions_ssm_parameter__: props.dagSsmParameterObj.parameterName, + __panel_names_ssm_parameter__: props.panelNameSsmParameterObj.parameterName, + + /* Defaults / Hardcoded values */ + __default_dag_version__: props.defaultDagVersion, + __default_panel_name__: props.defaultPanelName, + __sequencerrun_s3_path_root__: props.s3SequencerRunRootSsmParameterObj.stringValue, + + /* Events */ + __payload_version__: props.payloadVersion, + __event_bus_name__: props.eventBusName, + __event_detail_type__: props.detailType, + __event_source__: props.eventSource, + }, + } + ); + + // Grant lambda invoke permissions to the state machine + props.generatePieriandxObjectsLambdaObj.currentVersion.grantInvoke(stateMachine); + + // Grant read permissions to the ssm parameters + [ + props.dagSsmParameterObj, + props.panelNameSsmParameterObj, + props.s3SequencerRunRootSsmParameterObj, + ].forEach((ssmParameterObj: ssm.IStringParameter) => { + // Grant read permissions to the state machine + ssmParameterObj.grantRead(stateMachine); + }); + + // Allow state machine to read/write to dynamodb table + props.dynamodbTableObj.grantReadWriteData(stateMachine.role); + + // Because we run a nested state machine, we need to add the permissions to the state machine role + // See https://stackoverflow.com/questions/60612853/nested-step-function-in-a-step-function-unknown-error-not-authorized-to-cr + stateMachine.addToRolePolicy( + new iam.PolicyStatement({ + resources: [ + `arn:aws:events:${cdk.Aws.REGION}:${cdk.Aws.ACCOUNT_ID}:rule/StepFunctionsGetEventsForStepFunctionsExecutionRule`, + ], + actions: ['events:PutTargets', 'events:PutRule', 'events:DescribeRule'], + }) + ); + + // Allow sub-state launch machines to be invoked by this statemachine + [ + props.launchPieriandxCaseCreationStepfunctionObj, + props.launchPieriandxInformaticsjobCreationStepfunctionObj, + props.launchPieriandxSequencerrunCreationStepfunctionObj, + ].forEach((state_machine_obj) => { + state_machine_obj.grantStartExecution(stateMachine); + }); + + // Get event bus from event bus name + const eventBusObj = events.EventBus.fromEventBusName(this, 'eventBus', props.eventBusName); + + // Add permissions to the state machine to send events to the event bus + eventBusObj.grantPutEventsTo(stateMachine); + + // Create a rule for this state machine + const rule = new events.Rule(this, 'rule', { + eventBus: eventBusObj, + ruleName: `${props.prefix}-launch-wrsc-rule`, + eventPattern: { + source: [props.triggerLaunchSource], + detailType: [props.detailType], + detail: { + status: ['READY'], + workflowName: [{ 'equals-ignore-case': props.workflowName }], + }, + }, + }); + + /* Add rule as a target to the state machine */ + rule.addTarget( + new events_targets.SfnStateMachine(stateMachine, { + input: events.RuleTargetInput.fromEventPath('$.detail'), + }) + ); + + // Set outputs + this.stateMachineObj = stateMachine; + } +} diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/deploy/constructs/pieriandx_monitor_runs_step_function.ts b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/deploy/constructs/pieriandx_monitor_runs_step_function.ts new file mode 100644 index 000000000..15836e794 --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/deploy/constructs/pieriandx_monitor_runs_step_function.ts @@ -0,0 +1,116 @@ +import { Construct } from 'constructs'; +import * as sfn from 'aws-cdk-lib/aws-stepfunctions'; +import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; +import * as ssm from 'aws-cdk-lib/aws-ssm'; +import * as events from 'aws-cdk-lib/aws-events'; +import { DefinitionBody } from 'aws-cdk-lib/aws-stepfunctions'; +import { PythonFunction } from '@aws-cdk/aws-lambda-python-alpha'; +import { Duration } from 'aws-cdk-lib'; +import * as events_targets from 'aws-cdk-lib/aws-events-targets'; + +interface PieriandxMonitorRunsStepFunctionConstructProps { + /* Stack Objects */ + dynamodbTableObj: dynamodb.ITableV2; + + /* Workflow */ + workflowName: string; + workflowVersion: string; + + /* Lambdas paths */ + getInformaticsjobAndReportStatusLambdaObj: PythonFunction; + generateOutputPayloadDataLambdaObj: PythonFunction; + + /* Step function templates */ + launchPieriandxMonitorRunsStepfunctionTemplatePath: string; + + /* SSM Parameters */ + pierianDxBaseUrlSsmParameterObj: ssm.IStringParameter; + + /* Event Bus */ + eventBusName: string; + eventDetailType: string; + eventSource: string; + payloadVersion: string; + + /* Custom */ + prefix: string; +} + +export class PieriandxMonitorRunsStepFunctionStateMachineConstruct extends Construct { + public readonly stateMachineObj: sfn.IStateMachine; + + constructor(scope: Construct, id: string, props: PieriandxMonitorRunsStepFunctionConstructProps) { + super(scope, id); + + // Specify the statemachine and replace the arn placeholders with the lambda arns defined above + const stateMachine = new sfn.StateMachine( + this, + 'pieriandx_launch_step_functions_state_machine', + { + // State Machine Name + stateMachineName: `${props.prefix}-monitor-runs-sfn`, + // defintiontemplate + definitionBody: DefinitionBody.fromFile( + props.launchPieriandxMonitorRunsStepfunctionTemplatePath + ), + // definitionSubstitutions + definitionSubstitutions: { + /* Tables */ + __table_name__: props.dynamodbTableObj.tableName, + + /* Workflow Name */ + __workflow_name__: props.workflowName, + __workflow_version__: props.workflowVersion, + + /* Lambdas */ + __get_current_job_status_lambda_function_arn__: + props.getInformaticsjobAndReportStatusLambdaObj.currentVersion.functionArn, + __generate_data_payload_lambda_function_arn__: + props.generateOutputPayloadDataLambdaObj.currentVersion.functionArn, + + /* Hardcoded ssm parameters */ + __pieriandx_base_url__: props.pierianDxBaseUrlSsmParameterObj.stringValue, + + /* Event Bus Name */ + __event_bus_name__: props.eventBusName, + __event_detail_type__: props.eventDetailType, + __event_source__: props.eventSource, + __payload_version__: props.payloadVersion, + }, + } + ); + + // Grant lambda invoke permissions to the state machine + [ + props.getInformaticsjobAndReportStatusLambdaObj, + props.generateOutputPayloadDataLambdaObj, + ].forEach((lambdaFunction) => { + lambdaFunction.currentVersion.grantInvoke(stateMachine); + }); + + // Get the event bus from the event bus name + const eventBusObj = events.EventBus.fromEventBusName(this, 'eventBus', props.eventBusName); + + // Allow state machine to read/write to dynamodb table + props.dynamodbTableObj.grantReadWriteData(stateMachine); + + // Add rule for this sfn to run every 5 minutes + const rule = new events.Rule(this, 'rule', { + ruleName: `${props.prefix}-pieriandx-monitor-runs-rule`, + schedule: events.Schedule.rate(Duration.minutes(5)), + }); + + // Add permissions to the state machine to send events to the event bus + eventBusObj.grantPutEventsTo(stateMachine); + + /* Add rule as a target to the state machine */ + rule.addTarget( + new events_targets.SfnStateMachine(stateMachine, { + input: events.RuleTargetInput.fromEventPath('$.detail'), + }) + ); + + // Set outputs + this.stateMachineObj = stateMachine; + } +} diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/deploy/index.ts b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/deploy/index.ts new file mode 100644 index 000000000..44d53bc61 --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/deploy/index.ts @@ -0,0 +1,441 @@ +import * as cdk from 'aws-cdk-lib'; +import { Construct } from 'constructs'; +import * as ssm from 'aws-cdk-lib/aws-ssm'; +import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; +import * as secretsManager from 'aws-cdk-lib/aws-secretsmanager'; +import * as lambda from 'aws-cdk-lib/aws-lambda'; +import { PieriandxLaunchStepFunctionStateMachineConstruct } from './constructs/pieriandx_launch_step_function'; +import { PieriandxLaunchCaseCreationStepFunctionStateMachineConstruct } from './constructs/pieriandx_launch_case_creation_step_function'; +import { PieriandxLaunchInformaticsjobCreationStepFunctionsStateMachineConstruct } from './constructs/pieriandx_launch_informaticsjob_creation_step_function'; +import { PieriandxLaunchSequencerrunCreationStepFunctionsStateMachineConstruct } from './constructs/pieriandx_launch_sequencerrun_creation_step_function'; +import * as path from 'path'; +import { PythonFunction } from '@aws-cdk/aws-lambda-python-alpha'; +import { Duration } from 'aws-cdk-lib'; +import { PieriandxMonitorRunsStepFunctionStateMachineConstruct } from './constructs/pieriandx_monitor_runs_step_function'; +import { PythonLambdaLayerConstruct } from '../../../../components/python-lambda-layer'; + +export interface PierianDxPipelineManagerConfig { + /* DynamoDB Table */ + dynamodbTableName: string; + /* Workflow knowledge */ + workflowName: string; + workflowVersion: string; + /* Default values */ + defaultDagVersion: string; + defaultPanelName: string; + /* Secrets */ + icav2AccessTokenSecretId: string; // "/icav2/umccr-prod/service-production-jwt-token-secret-arn" + pieriandxS3AccessTokenSecretId: string; // "/pieriandx/s3AccessCredentials" + /* SSM Parameters */ + dagSsmParameterPath: string; + panelNameSsmParameterPath: string; + s3SequencerRunRootSsmParameterPath: string; + /* + Pieriandx specific parameters + */ + pieriandxUserEmailSsmParameterPath: string; + pieriandxInstitutionSsmParameterPath: string; + pieriandxBaseUrlSsmParameterPath: string; + pieriandxAuthTokenCollectionLambdaFunctionName: string; + /* Event info */ + eventDetailType: string; + eventBusName: string; + eventSource: string; + payloadVersion: string; + triggerLaunchSource: string; + /* Custom */ + prefix: string; +} + +export type PierianDxPipelineManagerStackProps = PierianDxPipelineManagerConfig & cdk.StackProps; + +export class PieriandxPipelineManagerStack extends cdk.Stack { + constructor(scope: Construct, id: string, props: PierianDxPipelineManagerStackProps) { + super(scope, id, props); + + // Get dynamodb table for construct + const dynamodbTableObj = dynamodb.TableV2.fromTableName( + this, + 'dynamodb_table', + props.dynamodbTableName + ); + + // Get ICAv2 Access token secret object for construct + const icav2AccessTokenSecretObj = secretsManager.Secret.fromSecretNameV2( + this, + 'Icav2SecretsObject', + props.icav2AccessTokenSecretId + ); + + const pieriandxS3AccessTokenSecretObj = secretsManager.Secret.fromSecretNameV2( + this, + 'PieriandxS3SecretsObject', + props.pieriandxS3AccessTokenSecretId + ); + + /* + Get the ssm parameters + */ + const dagSsmParameterObj = ssm.StringParameter.fromStringParameterName( + this, + 'dag', + props.dagSsmParameterPath + ); + const panelNameSsmParameterObj = ssm.StringParameter.fromStringParameterName( + this, + 'panel_name', + props.panelNameSsmParameterPath + ); + const s3SequencerRunRootSsmParameterObj = ssm.StringParameter.fromStringParameterName( + this, + 's3_sequencerrun_root', + props.s3SequencerRunRootSsmParameterPath + ); + + /* + Get the pieriandx parameters + */ + const pieriandxUserEmailSsmParameterObj = ssm.StringParameter.fromStringParameterName( + this, + 'pieriandx_useremail', + props.pieriandxUserEmailSsmParameterPath + ); + const pieriandxInstitutionSsmParameterObj = ssm.StringParameter.fromStringParameterName( + this, + 'pieriandx_institution', + props.pieriandxInstitutionSsmParameterPath + ); + const pieriandxBaseUrlSsmParameterObj = ssm.StringParameter.fromStringParameterName( + this, + 'pieriandx_baseurl', + props.pieriandxBaseUrlSsmParameterPath + ); + + // Get lambda layer object + const lambdaLayerObj = new PythonLambdaLayerConstruct(this, 'lambda_layer', { + layerName: 'pieriandx-tools-lambda-layer', + layerDirectory: path.join(__dirname, '../layers'), + layerDescription: 'PierianDx Tools Lambda Layer', + }); + + // Collect the pieriandx access token + const pieriandxTokenCollectionLambdaObj: lambda.IFunction = lambda.Function.fromFunctionName( + this, + 'pieriandx_auth_token_collection_lambda', + props.pieriandxAuthTokenCollectionLambdaFunctionName + ); + + // Set Pieriandx secret env for lambdas + const pieriandxEnvs = { + PIERIANDX_COLLECT_AUTH_TOKEN_LAMBDA_NAME: pieriandxTokenCollectionLambdaObj.functionName, + PIERIANDX_USER_EMAIL: pieriandxUserEmailSsmParameterObj.stringValue, + PIERIANDX_INSTITUTION: pieriandxInstitutionSsmParameterObj.stringValue, + PIERIANDX_BASE_URL: pieriandxBaseUrlSsmParameterObj.stringValue, + }; + + const pieriandxSecretEnvs = { + PIERIANDX_S3_ACCESS_CREDENTIALS_SECRET_ID: pieriandxS3AccessTokenSecretObj.secretName, + }; + + const icav2Envs = { + ICAV2_BASE_URL: 'https://ica.illumina.com/ica/rest', + ICAV2_ACCESS_TOKEN_SECRET_ID: icav2AccessTokenSecretObj.secretName, + }; + + /* + Build lambdas + */ + /* Part 1: Lambdas for generating the PierianDx API objects */ + const generatePieriandxObjectsLambdaObj = new PythonFunction( + this, + 'generate_pieriandx_objects_lambda_py', + { + entry: path.join(__dirname + '/../lambdas/generate_pieriandx_objects_py'), + runtime: lambda.Runtime.PYTHON_3_12, + architecture: lambda.Architecture.ARM_64, + index: 'generate_pieriandx_objects.py', + handler: 'handler', + memorySize: 1024, + layers: [lambdaLayerObj.lambdaLayerVersionObj], + timeout: Duration.seconds(20), + environment: icav2Envs, + } + ); + + /* Part 2: Lambdas used by the case creation step function */ + /* Generate case lambda object */ + const generateCaseLambdaObj = new PythonFunction(this, 'generate_case_lambda_py', { + entry: path.join(__dirname + '/../lambdas/generate_case_py'), + runtime: lambda.Runtime.PYTHON_3_12, + architecture: lambda.Architecture.ARM_64, + index: 'generate_case.py', + handler: 'handler', + memorySize: 1024, + layers: [lambdaLayerObj.lambdaLayerVersionObj], + timeout: Duration.seconds(30), + environment: pieriandxEnvs, + }); + + /* Part 3 - Lambdas used by the sequencerrun creation step function */ + /* Generate sequencerrun and samplesheet lambda objects */ + // Simple samplesheet generator object, no env or permissions needed + const generateSamplesheetLambdaObj = new PythonFunction( + this, + 'generate_samplesheet_lambda_py', + { + entry: path.join(__dirname + '/../lambdas/generate_samplesheet_py'), + runtime: lambda.Runtime.PYTHON_3_12, + architecture: lambda.Architecture.ARM_64, + index: 'generate_samplesheet.py', + handler: 'handler', + memorySize: 1024, + layers: [lambdaLayerObj.lambdaLayerVersionObj], + timeout: Duration.seconds(30), + environment: icav2Envs, + } + ); + + // Generate sequencerrun object + const generateSequencerrunLambdaObj = new PythonFunction( + this, + 'generate_sequencerrun_lambda_py', + { + entry: path.join(__dirname + '/../lambdas/generate_sequencerrun_case_py'), + runtime: lambda.Runtime.PYTHON_3_12, + architecture: lambda.Architecture.ARM_64, + index: 'generate_sequencerrun_case.py', + handler: 'handler', + memorySize: 1024, + layers: [lambdaLayerObj.lambdaLayerVersionObj], + environment: pieriandxEnvs, + } + ); + + // Upload data to S3 lambda object + const uploadDataToS3LambdaObj = new PythonFunction( + this, + 'upload_pieriandx_sample_data_to_s3_py', + { + entry: path.join(__dirname + '/../lambdas/upload_pieriandx_sample_data_to_s3_py'), + runtime: lambda.Runtime.PYTHON_3_12, + architecture: lambda.Architecture.ARM_64, + index: 'upload_pieriandx_sample_data_to_s3.py', + handler: 'handler', + memorySize: 1024, + layers: [lambdaLayerObj.lambdaLayerVersionObj], + timeout: Duration.seconds(300), + environment: { ...pieriandxEnvs, ...pieriandxSecretEnvs, ...icav2Envs }, + } + ); + + /* Part 4 - Lambdas used by the informatics job creation step function */ + /* Generate informatics job lambda object */ + const generateInformaticsjobLambdaObj = new PythonFunction( + this, + 'generate_informatics_job_lambda_py', + { + entry: path.join(__dirname + '/../lambdas/generate_informaticsjob_py'), + runtime: lambda.Runtime.PYTHON_3_12, + architecture: lambda.Architecture.ARM_64, + index: 'generate_informaticsjob.py', + handler: 'handler', + memorySize: 1024, + layers: [lambdaLayerObj.lambdaLayerVersionObj], + timeout: Duration.seconds(30), + environment: pieriandxEnvs, + } + ); + + /* Part 5 - Lambdas used by the monitor runs step function */ + const getInformaticsjobAndReportStatusLambdaObj = new PythonFunction( + this, + 'get_informaticsjob_and_report_status_lambda_obj', + { + entry: path.join(__dirname + '/../lambdas/get_informaticsjob_and_report_status_py'), + runtime: lambda.Runtime.PYTHON_3_12, + architecture: lambda.Architecture.ARM_64, + index: 'get_informaticsjob_and_report_status.py', + handler: 'handler', + memorySize: 1024, + layers: [lambdaLayerObj.lambdaLayerVersionObj], + timeout: Duration.seconds(30), + environment: pieriandxEnvs, + } + ); + + const generateOutputPayloadDataLambdaObj = new PythonFunction( + this, + 'generate_output_payload_data_lambda_obj', + { + entry: path.join(__dirname + '/../lambdas/generate_output_data_payload_py'), + runtime: lambda.Runtime.PYTHON_3_12, + architecture: lambda.Architecture.ARM_64, + index: 'generate_output_data_payload.py', + handler: 'handler', + } + ); + + /* + Give the lambda permission to access the pieriandx apis + */ + [ + generateCaseLambdaObj, + generateInformaticsjobLambdaObj, + generateSequencerrunLambdaObj, + uploadDataToS3LambdaObj, + getInformaticsjobAndReportStatusLambdaObj, + ].forEach((lambdaFunction) => { + // Give the lambda permission to access the pieriandx apis + pieriandxTokenCollectionLambdaObj.latestVersion.grantInvoke(lambdaFunction); + }); + + /* + Give the upload lambda access to the pieriandx s3 bucket + */ + // @ts-ignore + pieriandxS3AccessTokenSecretObj.grantRead(uploadDataToS3LambdaObj); + + /* + Give the lambdas permission to access the icav2 apis + */ + [ + generatePieriandxObjectsLambdaObj, + generateSamplesheetLambdaObj, + uploadDataToS3LambdaObj, + ].forEach((lambdaFunction) => { + // @ts-ignore + icav2AccessTokenSecretObj.grantRead(lambdaFunction); + }); + + /* + Generate State Machines + */ + + /* Generate case creation statemachine object */ + const pieriandxLaunchCaseCreationStateMachine = + new PieriandxLaunchCaseCreationStepFunctionStateMachineConstruct(this, 'case_creation', { + /* Stack Objects */ + dynamodbTableObj: dynamodbTableObj, + /* Lambda objs */ + generateCaseLambdaObj: generateCaseLambdaObj, + /* Step function template */ + launchPieriandxCaseCreationStepfunctionTemplatePath: path.join( + __dirname, + '/../step_function_templates/launch_pieriandx_case_creation.asl.json' + ), + /* Prefix */ + prefix: props.prefix, + }); + + /* Generate informatics job creation statemachine object */ + const pieriandxInformaticsjobCreationStateMachine = + new PieriandxLaunchInformaticsjobCreationStepFunctionsStateMachineConstruct( + this, + 'informaticsjob_creation_sfn', + { + /* Stack Objects */ + dynamodbTableObj: dynamodbTableObj, + /* Lambda paths */ + generateInformaticsjobLambdaObj: generateInformaticsjobLambdaObj, + /* Step function templates */ + launchPieriandxInformaticsjobCreationStepfunctionTemplate: path.join( + __dirname, + '/../step_function_templates/launch_pieriandx_informaticsjob_creation.asl.json' + ), + /* Custom */ + prefix: props.prefix, + } + ); + + /* Generate Sequence Run Creation StateMachine object */ + const pieriandxSequencerrunCreationStateMachine = + new PieriandxLaunchSequencerrunCreationStepFunctionsStateMachineConstruct( + this, + 'sequencerrun_creation_sfn', + { + /* Stack Objects */ + dynamodbTableObj: dynamodbTableObj, + /* Lambda paths */ + uploadDataToS3LambdaObj: uploadDataToS3LambdaObj, + generateSamplesheetLambdaObj: generateSamplesheetLambdaObj, + generateSequencerrunLambdaObj: generateSequencerrunLambdaObj, + /* Step function templates */ + launchPieriandxSequencerrunCreationStepfunctionTemplate: path.join( + __dirname, + '/../step_function_templates/launch_pieriandx_sequencerrun_creation.asl.json' + ), + /* Custom */ + prefix: props.prefix, + } + ); + + /* Generate parent statemachine object to launch pieriandx analysis */ + const pieriandxLaunchStateMachine = new PieriandxLaunchStepFunctionStateMachineConstruct( + this, + id, + { + /* Stack Objects */ + dynamodbTableObj: dynamodbTableObj, + /* Workflow information */ + workflowName: props.workflowName, + workflowVersion: props.workflowVersion, + /* Defaults */ + defaultDagVersion: props.defaultDagVersion, + defaultPanelName: props.defaultPanelName, + /* Lambdas paths */ + generatePieriandxObjectsLambdaObj: generatePieriandxObjectsLambdaObj, + /* SSM Parameters */ + dagSsmParameterObj: dagSsmParameterObj, + panelNameSsmParameterObj: panelNameSsmParameterObj, + s3SequencerRunRootSsmParameterObj: s3SequencerRunRootSsmParameterObj, + /* Step function templates */ + launchPieriandxStepfunctionTemplate: path.join( + __dirname, + '/../step_function_templates/launch_pieriandx.asl.json' + ), + /* Step function objects */ + launchPieriandxCaseCreationStepfunctionObj: + pieriandxLaunchCaseCreationStateMachine.stateMachineObj, + launchPieriandxInformaticsjobCreationStepfunctionObj: + pieriandxInformaticsjobCreationStateMachine.stateMachineObj, + launchPieriandxSequencerrunCreationStepfunctionObj: + pieriandxSequencerrunCreationStateMachine.stateMachineObj, + /* Events */ + detailType: props.eventDetailType, + eventBusName: props.eventBusName, + eventSource: props.eventSource, + payloadVersion: props.payloadVersion, + triggerLaunchSource: props.triggerLaunchSource, + /* Custom */ + prefix: props.prefix, + } + ); + + /* Create the PierianDx Monitor Runs SFN */ + const pieriandxMonitorRunsStateMachine = + new PieriandxMonitorRunsStepFunctionStateMachineConstruct(this, 'monitor_runs', { + /* Stack Objects */ + dynamodbTableObj: dynamodbTableObj, + /* workflow info */ + workflowName: props.workflowName, + workflowVersion: props.workflowVersion, + /* Lambda objs */ + getInformaticsjobAndReportStatusLambdaObj: getInformaticsjobAndReportStatusLambdaObj, + generateOutputPayloadDataLambdaObj: generateOutputPayloadDataLambdaObj, + /* SSM Parameters */ + pierianDxBaseUrlSsmParameterObj: pieriandxBaseUrlSsmParameterObj, + /* Step function template */ + launchPieriandxMonitorRunsStepfunctionTemplatePath: path.join( + __dirname, + '/../step_function_templates/monitor_runs.asl.json' + ), + /* Event info */ + eventBusName: props.eventBusName, + eventDetailType: props.eventDetailType, + eventSource: props.eventSource, + payloadVersion: props.payloadVersion, + /* Custom */ + prefix: props.prefix, + }); + } +} diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/images/case_overview.png b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/images/case_overview.png new file mode 100755 index 000000000..ffc777d80 Binary files /dev/null and b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/images/case_overview.png differ diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/images/informaticsjob_launch_overview.png b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/images/informaticsjob_launch_overview.png new file mode 100755 index 000000000..dddf4c732 Binary files /dev/null and b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/images/informaticsjob_launch_overview.png differ diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/images/launch_overview.png b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/images/launch_overview.png new file mode 100755 index 000000000..bcb95cd85 Binary files /dev/null and b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/images/launch_overview.png differ diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/images/sequencerrun_overview.png b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/images/sequencerrun_overview.png new file mode 100755 index 000000000..ebde4a3b1 Binary files /dev/null and b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/images/sequencerrun_overview.png differ diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/lambdas/generate_case_py/generate_case.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/lambdas/generate_case_py/generate_case.py new file mode 100644 index 000000000..3d6cf7e63 --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/lambdas/generate_case_py/generate_case.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python + +""" +Generate a case from a case object + +Environment variables +PIERIANDX_COLLECT_AUTH_TOKEN_LAMBDA_NAME: The secret name for the PierianDx Auth Token 'PierianDx/JwtKey' +PIERIANDX_USER_EMAIL: +PIERIANDX_INSTITUTION +PIERIANDX_BASE_URL + +A Case object will look like this + +{ + "identified": true, + "indication": "indication", + "panelName": "panelname", + "sampleType": "patientcare", + "specimens": [ + { + "accessionNumber": "caseaccessionnumber", + "dateAccessioned": "2021-01-01TZ:00+Z", + "dateReceived": "2021-01-01TZ:00+Z", + "datecollected": "2021-01-01TZ:00+Z", + "externalSpecimenId": "externalspecimenid", + "name": "panelspecimenscheme", + "type": { + "code": "specimentypecode", + "label": "specimentypelabel" + }, + "firstName": "John", + "lastName": "Doe", + "dateOfBirth": "1970-01-01", + "medicalRecordNumbers": [ + { + "mrn": "mrn", + "medicalFacility": { + "facility": "facility", + "hospitalNumber": "hospitalnumber" + } + } + ] + } + ], + "dagDescription": "dagdescription", + "dagName": "dagname", + "disease": { + "code": "diseasecode", + "label": "diseaselabel" + }, + "physicians": [ + { + "firstName": "Meredith", + "lastName": "Gray" + } + ] + } +""" + +import logging +from os import environ + +from pieriandx_pipeline_tools.utils.pieriandx_helpers import get_pieriandx_client +from requests import Response, HTTPError + +from pieriandx_pipeline_tools.utils.secretsmanager_helpers import set_pieriandx_env_vars + + +def handler(event, context): + case_creation_obj = event.get("case_creation_obj", {}) + set_pieriandx_env_vars() + pyriandx_client = get_pieriandx_client( + email=environ['PIERIANDX_USER_EMAIL'], + token=environ['PIERIANDX_USER_AUTH_TOKEN'], + instiution=environ['PIERIANDX_INSTITUTION'], + base_url=environ['PIERIANDX_BASE_URL'], + ) + + try: + response: Response = pyriandx_client._post_api( + endpoint="/case", + data=case_creation_obj + ) + response.raise_for_status() + except HTTPError as e: + logging.error(f"Failed to create case: {e}") + raise Exception(f"Failed to create case: {e}") + + if response.status_code != 200: + logging.error(f"Failed to create case: {response.json()}") + raise Exception(f"Failed to create case: {response.json()}") + + return response.json() + + +# if __name__ == "__main__": +# import json +# +# logging.basicConfig(level=logging.DEBUG) +# environ['AWS_PROFILE'] = 'umccr-development' +# environ['AWS_REGION'] = 'ap-southeast-2' +# environ['PIERIANDX_BASE_URL'] = "https://app.uat.pieriandx.com/cgw-api/v2.0.0" +# environ['PIERIANDX_COLLECT_AUTH_TOKEN_LAMBDA_NAME'] = "collectPierianDxAccessToken" +# environ['PIERIANDX_INSTITUTION'] = "melbournetest" +# environ['PIERIANDX_USER_EMAIL'] = "services@umccr.org" +# +# print( +# json.dumps( +# handler( +# { +# "case_creation_obj": { +# "identified": True, +# "indication": "Test", +# "panelName": "tso500_DRAGEN_ctDNA_v2_1_Universityofmelbourne", # // pragma: allowlist secret +# "sampleType": "patientcare", +# "specimens": [ +# { +# "accessionNumber": "SBJ04407__L2301368__V2__abcd12345", +# "dateAccessioned": "2021-01-01T00:00:00Z", +# "dateReceived": "2021-01-01T00:00:00Z", +# "datecollected": "2024-02-20T20:17:00Z", +# "externalSpecimenId": "externalspecimenid", +# "name": "primarySpecimen", +# "type": { +# "code": "122561005", +# "label": "Blood specimen from patient" +# }, +# "firstName": "John", +# "lastName": "Doe", +# "dateOfBirth": "1970-01-01", +# "medicalRecordNumbers": [ +# { +# "mrn": "3069999", +# "medicalFacility": { +# "facility": "Not Available", +# "hospitalNumber": "99" +# } +# } +# ] +# } +# ], +# "dagDescription": "tso500_ctdna_workflow", +# "dagName": "cromwell_tso500_ctdna_workflow_1.0.4", +# "disease": { +# "code": "64572001", +# "label": "Disease" +# }, +# "physicians": [ +# { +# "firstName": "Meredith", +# "lastName": "Gray" +# } +# ] +# }, +# }, +# None +# ), +# indent=2 +# ) +# ) +# +# # Yields +# # { +# # 'id': '100937', +# # 'accessionNumber': 'SBJ04405__L2301368__ot__002', +# # 'dateCreated': '2024-04-14' +# # } + +# +# if __name__ == "__main__": +# import json +# +# logging.basicConfig(level=logging.DEBUG) +# environ['AWS_PROFILE'] = 'umccr-development' +# environ['AWS_REGION'] = 'ap-southeast-2' +# environ['PIERIANDX_BASE_URL'] = "https://app.uat.pieriandx.com/cgw-api/v2.0.0" +# environ['PIERIANDX_COLLECT_AUTH_TOKEN_LAMBDA_NAME'] = "collectPierianDxAccessToken" +# environ['PIERIANDX_INSTITUTION'] = "melbournetest" +# environ['PIERIANDX_USER_EMAIL'] = "services@umccr.org" +# +# print( +# json.dumps( +# handler( +# { +# "case_creation_obj": { +# "identified": False, +# "indication": "NA", +# "panelName": "tso500_DRAGEN_ctDNA_v2_1_Universityofmelbourne", # // pragma: allowlist secret +# "sampleType": "patientcare", +# "specimens": [ +# { +# "accessionNumber": "L2400160__V2__20241003f3149835", +# "dateAccessioned": "2024-10-04T09:01:32+1000", +# "dateReceived": "2024-10-04T09:01:32+1000", +# "datecollected": "2024-10-04T09:01:32+1000", +# "externalSpecimenId": "SSq-CompMM-1pc-10646259ilm", +# "name": "primarySpecimen", +# "type": { +# "code": "122561005", +# "label": "Blood specimen from patient" +# }, +# "studyIdentifier": "Testing", +# "studySubjectIdentifier": "CMM1pc-10646259ilm" +# } +# ], +# "dagDescription": "tso500_ctdna_workflow", +# "dagName": "cromwell_tso500_ctdna_workflow_1.0.4", +# "disease": { +# "code": "55342001", +# "label": "Neoplastic disease" +# } +# } +# }, +# None +# ), +# indent=2 +# ) +# ) +# +# # Yields +# # { +# # 'id': '100937', +# # 'accessionNumber': 'SBJ04405__L2301368__ot__002', +# # 'dateCreated': '2024-04-14' +# # } diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/lambdas/generate_informaticsjob_py/generate_informaticsjob.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/lambdas/generate_informaticsjob_py/generate_informaticsjob.py new file mode 100644 index 000000000..98af0e052 --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/lambdas/generate_informaticsjob_py/generate_informaticsjob.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python + +""" +Generate a case from a case object + +Environment variables +PIERIANDX_AUTH_TOKEN_SECRET_ID: The secret name for the PierianDx Auth Token + +An informatics object will look like this + +{ + "input": [ + { + "accessionNumber": "caseaccessionnumber", + "sequencerRunInfos": [ + { + "accessionNumber": "caseaccessionnumber", + "barcode": "GACTGAGTAG+CACTATCAAC", + "lane": "1", + "sampleId": "L2301368", + "sampleType": "DNA" + } + ] + } + ] + } +""" + +import logging +from os import environ + +from pieriandx_pipeline_tools.utils.pieriandx_helpers import get_pieriandx_client +from requests import Response + +from pieriandx_pipeline_tools.utils.secretsmanager_helpers import set_pieriandx_env_vars + + +def handler(event, context): + # Get inputs + informaticsjob_creation_obj = event.get("informaticsjob_creation_obj", {}) + case_id = event.get("case_id", None) + set_pieriandx_env_vars() + pyriandx_client = get_pieriandx_client( + email=environ['PIERIANDX_USER_EMAIL'], + token=environ['PIERIANDX_USER_AUTH_TOKEN'], + instiution=environ['PIERIANDX_INSTITUTION'], + base_url=environ['PIERIANDX_BASE_URL'], + ) + response: Response = pyriandx_client._post_api( + endpoint=f"/case/{case_id}/informaticsJobs", + data=informaticsjob_creation_obj + ) + + if response.status_code != 200: + logging.error(f"Failed to create informaticsjob: {response.json()}") + raise Exception(f"Failed to create informaticsjob: {response.json()}") + + return response.json() + +# +# if __name__ == "__main__": +# import json +# +# print( +# json.dumps( +# handler( +# { +# "informaticsjob_creation_obj": { +# "input": [ +# { +# "accessionNumber": "SBJ04405__L2301368__ot__003", +# "sequencerRunInfos": [ +# { +# "runId": "231116_A01052_0172_BHVLM5DSX7__SBJ04405__L2301368__ot__003__20240415abcd0001", +# "barcode": "GACTGAGTAG-CACTATCAAC", +# "lane": "1", +# "sampleId": "L2301368", +# "sampleType": "DNA" +# } +# ] +# } +# ] +# }, +# "case_id": "100938" +# }, +# None +# ), +# indent=2 +# ) +# ) diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/lambdas/generate_output_data_payload_py/generate_output_data_payload.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/lambdas/generate_output_data_payload_py/generate_output_data_payload.py new file mode 100644 index 000000000..5b67785cf --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/lambdas/generate_output_data_payload_py/generate_output_data_payload.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 + +""" +Get the data payload python + +The outputs (if report status is complete) will comprise the following + +# Case url where pieriandx_base_url is one of https://app.pieriandx.com +{pieriandx_base_url}/cgw/order/viewOrderDetails/{case_id} +# VCF Output URL +{pieriandx_base_url}/cgw/informatics/downloadJobOutputAnalysisFile?caseId={case_id}&jobId={job_id}&accessionNumber={case_accession_number}&fileName=main.vcf +# Biomarker Report PDF URL +{pieriandx_base_url}/cgw/informatics/downloadJobOutputAnalysisFile?caseId={case_id}&jobId={job_id}&accessionNumber={case_accession_number}&fileName={sample_name}_BiomarkerReport.txt +# Report PDF URL +{pieriandx_base_url}/cgw/report/openPdfReport/{report_id} + +""" + + +def handler(event, context): + """ + Get the data payload with or without the outputs + :param event: + :param context: + :return: + """ + + # Get inputs + inputs = event.get("inputs") + engine_parameters = event.get("engine_parameters") + tags = event.get("tags") + report_status = event.get("report_status") + case_id = event.get("case_id") + job_id = event.get("job_id") + case_accession_number = event.get("case_accession_number") + report_id = event.get("report_id") + pieriandx_base_url = event.get("pieriandx_base_url") + sample_name = event.get("sample_name") + + # Initial dict + return_dict = { + "data_payload": { + "inputs": inputs, + "engineParameters": engine_parameters, + "tags": tags + } + } + + # Return if the report generation is not complete yet + if not report_status in ["report_generation_complete", "complete"]: + # Return as id + return return_dict + + # Set the outputs + return_dict["data_payload"]["outputs"] = { + "caseUrl": f"{pieriandx_base_url.replace("cgw-api/v2.0.0/", "")}/cgw/order/viewOrderDetails/{case_id}", + "vcfOutputUrl": f"{pieriandx_base_url.replace("cgw-api/v2.0.0/", "")}/cgw/informatics/downloadJobOutputAnalysisFile?caseId={case_id}&jobId={job_id}&accessionNumber={case_accession_number}&fileName=main.vcf", + "reportPdfUrl": f"{pieriandx_base_url.replace("cgw-api/v2.0.0/", "")}/cgw/report/openPdfReport/{report_id}", + "biomarkerReportUrl": f"{pieriandx_base_url.replace("cgw-api/v2.0.0/", "")}/cgw/informatics/downloadJobOutputAnalysisFile?caseId={case_id}&jobId={job_id}&accessionNumber={case_accession_number}&fileName={sample_name}_BiomarkerReport.txt" + } + + return return_dict + + +# if __name__ == "__main__": +# import json +# +# print( +# json.dumps( +# handler( +# { +# "inputs": { +# "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", +# "panelVersion": "main", +# "caseMetadata": { +# "isIdentified": False, +# "caseAccessionNumber": "L2400160__V2__2024100405a12a95", +# "externalSpecimenId": "SSq-CompMM-1pc-10646259ilm", +# "sampleType": "patientcare", +# "specimenLabel": "primarySpecimen", +# "indication": "NA", +# "diseaseCode": 55342001, +# "specimenCode": "122561005", +# "sampleReception": { +# "dateAccessioned": "2024-10-04T10:03:11+1000", +# "dateCollected": "2024-10-04T10:03:11+1000", +# "dateReceived": "2024-10-04T10:03:11+1000" +# }, +# "study": { +# "id": "Testing", +# "subjectIdentifier": "CMM1pc-10646259ilm" +# } +# }, +# "dataFiles": { +# "microsatOutputUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20241003ead8ad9f/Logs_Intermediates/DragenCaller/L2400160/L2400160.microsat_output.json", +# "tmbMetricsUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20241003ead8ad9f/Logs_Intermediates/Tmb/L2400160/L2400160.tmb.metrics.csv", +# "cnvVcfUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20241003ead8ad9f/Results/L2400160/L2400160.cnv.vcf.gz", +# "hardFilteredVcfUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20241003ead8ad9f/Results/L2400160/L2400160.hard-filtered.vcf.gz", +# "fusionsUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20241003ead8ad9f/Results/L2400160/L2400160_Fusions.csv", +# "metricsOutputUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20241003ead8ad9f/Results/L2400160/L2400160_MetricsOutput.tsv", +# "samplesheetUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20241003ead8ad9f/Logs_Intermediates/SampleSheetValidation/SampleSheet_Intermediate.csv" +# } +# }, +# "engine_parameters": { +# "caseId": "103779", +# "informaticsJobId": "46014" +# }, +# "tags": { +# "metadataFromRedCap": False, +# "isIdentified": False, +# "libraryId": "L2400160", +# "sampleType": "patientcare", +# "projectId": "Testing", +# "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC" +# }, +# "report_status": "complete", +# "case_id": "103779", +# "job_id": "46014", +# "case_accession_number": "L2400160__V2__2024100405a12a95", +# "report_id": "38152", +# "pieriandx_base_url": "https://app.uat.pieriandx.com/cgw-api/v2.0.0", +# "sample_name": "L2400160" +# }, +# None +# ), +# indent=4 +# ) +# ) +# +# # { +# # "data_payload": { +# # "inputs": { +# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", +# # "panelVersion": "main", +# # "caseMetadata": { +# # "isIdentified": false, +# # "caseAccessionNumber": "L2400160__V2__2024100405a12a95", +# # "externalSpecimenId": "SSq-CompMM-1pc-10646259ilm", +# # "sampleType": "patientcare", +# # "specimenLabel": "primarySpecimen", +# # "indication": "NA", +# # "diseaseCode": 55342001, +# # "specimenCode": "122561005", +# # "sampleReception": { +# # "dateAccessioned": "2024-10-04T10:03:11+1000", +# # "dateCollected": "2024-10-04T10:03:11+1000", +# # "dateReceived": "2024-10-04T10:03:11+1000" +# # }, +# # "study": { +# # "id": "Testing", +# # "subjectIdentifier": "CMM1pc-10646259ilm" +# # } +# # }, +# # "dataFiles": { +# # "microsatOutputUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20241003ead8ad9f/Logs_Intermediates/DragenCaller/L2400160/L2400160.microsat_output.json", +# # "tmbMetricsUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20241003ead8ad9f/Logs_Intermediates/Tmb/L2400160/L2400160.tmb.metrics.csv", +# # "cnvVcfUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20241003ead8ad9f/Results/L2400160/L2400160.cnv.vcf.gz", +# # "hardFilteredVcfUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20241003ead8ad9f/Results/L2400160/L2400160.hard-filtered.vcf.gz", +# # "fusionsUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20241003ead8ad9f/Results/L2400160/L2400160_Fusions.csv", +# # "metricsOutputUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20241003ead8ad9f/Results/L2400160/L2400160_MetricsOutput.tsv", +# # "samplesheetUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20241003ead8ad9f/Logs_Intermediates/SampleSheetValidation/SampleSheet_Intermediate.csv" +# # } +# # }, +# # "engineParameters": { +# # "caseId": "103779", +# # "informaticsJobId": "46014" +# # }, +# # "tags": { +# # "metadataFromRedCap": false, +# # "isIdentified": false, +# # "libraryId": "L2400160", +# # "sampleType": "patientcare", +# # "projectId": "Testing", +# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC" +# # }, +# # "outputs": { +# # "caseUrl": "https://app.uat.pieriandx.com/cgw-api/v2.0.0/cgw/order/viewOrderDetails/103779", +# # "vcfOutputUrl": "https://app.uat.pieriandx.com/cgw-api/v2.0.0/cgw/informatics/downloadJobOutputAnalysisFile?caseId=103779&jobId=46014&accessionNumber=L2400160__V2__2024100405a12a95&fileName=main.vcf", +# # "reportPdfUrl": "https://app.uat.pieriandx.com/cgw-api/v2.0.0/cgw/report/openPdfReport/38152", +# # "biomarkerReportUrl": "https://app.uat.pieriandx.com/cgw-api/v2.0.0/cgw/informatics/downloadJobOutputAnalysisFile?caseId=103779&jobId=46014&accessionNumber=L2400160__V2__2024100405a12a95&fileName=L2400160_BiomarkerReport.txt" +# # } +# # } +# # } diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/lambdas/generate_pieriandx_objects_py/generate_pieriandx_objects.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/lambdas/generate_pieriandx_objects_py/generate_pieriandx_objects.py new file mode 100644 index 000000000..064600506 --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/lambdas/generate_pieriandx_objects_py/generate_pieriandx_objects.py @@ -0,0 +1,623 @@ +#!/usr/bin/env python3 + +""" +Tackle workflow inputs. + +By far the most complicated lambda of the lot. + +Expects event input in the following syntax (how this is built is someone else's problem) + +* dag: Object + * name: string # The name of this case dag + * description: string # The description of this case dag +* case_metadata: Object + * panel_name: string # The name of this case’s panel + * specimen_label: string # The mapping to the panels specimen scheme + * sample_type: Enum # patientcare, clinical_trial, validation, proficiency_testing + * indication: String # Optional input + * disease: Object + * code: string # The disease id + * label: string # The name of the disease (optional) + * is_identified: bool # Boolean + * case_accession_number: string - must be unique - uses syntax SBJID__LIBID__NNN + * specimen_type: + * code: string # The SNOMED-CT term for a specimen type + * label: Optional label for the specimen type + * external_specimen_id: string # The external specimen id + * date_accessioned: Datetime # The date the case was accessioned + * date_collected: Datetime # The date the specimen was collected + * date_received: Datetime # The date the specimen was received + * gender: Enum # unknown, male, femail, unspecified, other, ambiguous, not_applicable + * ethnicity: Enum # unknown, hispanic_or_latino, not_hispanic_or_latino, not_reported + * race: Enum # american_indian_or_alaska_native, asian, black_or_african_american, native_hawaiian_or_other_pacific_islander, not_reported, unknown, white + + > Note: If the case is de-identified, the following fields are required + * study_id: String # Only required if is_identified is false + * study_subject_identifier: String # Only required if is_identified is false + + > Note: If the case is identified, the following fields are required + * date_of_birth: Datetime # Only required if is_identified is true + * first_name: String # Only required if is_identified is true + * last_name: String # Only required if is_identified is true + * medical_record_numbers: Object # Only required if is_identified is true + * mrn: string # The medical record number + * medical_facility: Object + * facility: string # The name of the facility + * hospital_number: string # The hospital number + * requesting_physician: Object + * first_name: string + * last_name: string + +* data_files: Object + * microsat_output: uri + * tmb_metrics: uri + * cnv: uri + * hard_filtered: uri + * fusions: uri + * metrics_output: uri + +* samplesheet_b64gz: str +* instrument_run_id: str +* portal_run_id: str +* sequencerrun_s3_prefix: str + +We then use pydantic to validate the input and generate the following outputs + +* case_creation_obj: A CaseCreation object +* sequencerrun_creation_obj: A SequencerrunCreation object +* informaticsjob_creation_obj: An InformaticsjobCreation object +* data_files: List of DataFile objects (each containing a src_uri, dest_uri and file_type) +* sequencerrun_s3_path_root: The root s3 path we will upload data to. +* This is the same as the input sequencerrun_s3_path but we will extend the run id to it +""" + +import logging +import pandas as pd + +from pieriandx_pipeline_tools.utils.secretsmanager_helpers import set_icav2_env_vars +from pieriandx_pipeline_tools.pieriandx_classes.data_file import DataFile, DataType +from pieriandx_pipeline_tools.pieriandx_enums.specimen_type import SpecimenType +from pieriandx_pipeline_tools.utils.samplesheet_helpers import read_v2_samplesheet + +from pieriandx_pipeline_tools.pieriandx_classes.physician import Physician +from pieriandx_pipeline_tools.pieriandx_classes.sequencerrun import SequencerrunCreation +from pieriandx_pipeline_tools.pieriandx_classes.informaticsjob import InformaticsjobCreation +from pieriandx_pipeline_tools.pieriandx_classes.specimen_sequencer_info import SpecimenSequencerInfo +from pieriandx_pipeline_tools.pieriandx_classes.dag import Dag +from pieriandx_pipeline_tools.pieriandx_classes.disease import Disease +from pieriandx_pipeline_tools.pieriandx_classes.medical_record_number import MedicalRecordNumber +from pieriandx_pipeline_tools.pieriandx_classes.medical_facility import MedicalFacility + +from pieriandx_pipeline_tools.pieriandx_enums.sequencing_type import SequencingType +from pieriandx_pipeline_tools.pieriandx_enums.sample_type import SampleType + +TOP_LEVEL_KEYS = [ + "dag", + "case_metadata", + "data_files", + "panel_name", + "instrument_run_id", + "portal_run_id", + "sequencerrun_s3_path_root", +] + +# Set basic logger +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + + + +def handler(event, context): + # Set env vars + set_icav2_env_vars() + + # Basic housekeeping + if event is None: + raise ValueError("Event is required") + if not isinstance(event, dict): + raise ValueError("Event must be a dictionary") + + # Check for top level keys + if not all([key in event for key in TOP_LEVEL_KEYS]): + logger.error(f"Could not find keys {' '.join([key for key in TOP_LEVEL_KEYS if key not in event])}") + raise ValueError("Event is missing required top level keys") + + # Read samplesheet - we need this for the sequencer run infos + v2_samplesheet_dict = read_v2_samplesheet(event.get("samplesheet_uri")) + + # Collect the tso500l_data section + if not len(v2_samplesheet_dict.get("tso500l_data")) == 1: + logger.error("Should only be one item in the tso500l data section") + raise ValueError + + tso500l_data_samplesheet_obj = v2_samplesheet_dict.get("tso500l_data")[0] + sample_name = tso500l_data_samplesheet_obj['sample_id'] + + if event.get("case_metadata").get("isIdentified"): + from pieriandx_pipeline_tools.pieriandx_classes.case import IdentifiedCaseCreation as CaseCreation + from pieriandx_pipeline_tools.pieriandx_classes.specimen import IdentifiedSpecimen as Specimen + else: + from pieriandx_pipeline_tools.pieriandx_classes.case import DeIdentifiedCaseCreation as CaseCreation + from pieriandx_pipeline_tools.pieriandx_classes.specimen import DeIdentifiedSpecimen as Specimen + + # Other imports + case_creation_obj = CaseCreation( + # Standard case collection + dag=Dag( + name=event.get("dag").get("dagName"), + description=event.get("dag").get("dagDescription") + ), + disease=Disease(code=int(event.get("case_metadata").get("diseaseCode"))), + indication=event.get("case_metadata").get("indication", None), + panel_name=event.get("panel_name"), + sample_type=SampleType(event.get("case_metadata").get("sampleType").lower()), + # Identified only fields + requesting_physician=( + Physician( + first_name=event.get("case_metadata").get("requestingPhysician").get("firstName"), + last_name=event.get("case_metadata").get("requestingPhysician").get("lastName") + ) if event.get("case_metadata").get("isIdentified") + else None + ), + # Specimen collection + specimen=Specimen( + # Standard specimen collection + case_accession_number=event.get("case_metadata").get("caseAccessionNumber"), + date_accessioned=pd.to_datetime( + event.get("case_metadata").get("sampleReception").get("dateAccessioned"), + utc=True + ), + date_received=pd.to_datetime( + event.get("case_metadata").get("sampleReception").get("dateReceived"), + utc=True + ), + date_collected=pd.to_datetime( + event.get("case_metadata").get("sampleReception").get("dateCollected"), + utc=True + ), + external_specimen_id=event.get("case_metadata").get("externalSpecimenId"), + specimen_label=event.get("case_metadata").get("specimenLabel"), + gender=event.get("case_metadata").get("gender", None), + ethnicity=event.get("case_metadata").get("ethnicity", None), + race=event.get("case_metadata").get("race", None), + specimen_type=SpecimenType(code=int(event.get("case_metadata").get("specimenCode"))), + # Identified only fields + date_of_birth=pd.to_datetime( + event.get("case_metadata").get("patientInformation", {}).get("dateOfBirth", None) + ), + first_name=event.get("case_metadata").get("patientInformation", {}).get("firstName", None), + last_name=event.get("case_metadata").get("patientInformation", {}).get("lastName", None), + medical_record_number=MedicalRecordNumber( + mrn=event.get("case_metadata").get("medicalRecordNumbers").get("mrn"), + medical_facility=( + MedicalFacility( + facility=( + event.get("case_metadata") + .get("medicalRecordNumbers") + .get("medicalFacility") + .get("facility") + ), + hospital_number=( + event.get("case_metadata") + .get("medicalRecordNumbers") + .get("medicalFacility") + .get("hospitalNumber") + ) + ) + ) + ) if event.get("case_metadata").get("isIdentified") else None, + # De-identified only fields + study_identifier=event.get("case_metadata").get("study", {}).get("id", None), + study_subject_identifier=event.get("case_metadata").get("study", {}).get("subjectIdentifier", None) + ) + ) + + # Set run id (used for sequencer run path) + run_id = "__".join( + [ + event.get("instrument_run_id"), + event.get("case_metadata").get("caseAccessionNumber"), + event.get('portal_run_id') + ] + ) + + # Get sequencer runinfo object (used for both sequencer run and informatics job creation) + specimen_sequencer_info = SpecimenSequencerInfo( + run_id=run_id, + case_accession_number=event.get("case_metadata").get("caseAccessionNumber"), + barcode=f"{tso500l_data_samplesheet_obj.get('index')}-{tso500l_data_samplesheet_obj.get('index2')}", + lane=tso500l_data_samplesheet_obj.get("lane", 1), + sample_id=tso500l_data_samplesheet_obj.get("sample_id"), + sample_type=tso500l_data_samplesheet_obj.get("sample_type") + ) + + # Sequencerrun creation object + sequencer_run_creation = SequencerrunCreation( + run_id=run_id, + specimen_sequence_info=specimen_sequencer_info, + sequencing_type=SequencingType.PAIRED_END + ) + + # Informatics job + informatics_job_creation = InformaticsjobCreation( + case_accession_number=event.get("case_metadata").get("caseAccessionNumber"), + specimen_sequencer_run_info=specimen_sequencer_info + ) + + # Add sequencerrun path + sequencerrun_s3_path = f"{event.get('sequencerrun_s3_path_root').rstrip('/')}/{run_id}" + + # Data files + data_files = list( + map( + lambda data_file_iter_kv: DataFile( + sequencerrun_path_root=sequencerrun_s3_path, + file_type=DataType(data_file_iter_kv[0]), + sample_id=tso500l_data_samplesheet_obj.get("sample_id"), + src_uri=data_file_iter_kv[1], + contents=None + ), + filter( + lambda data_file_iter_kv: ( + data_file_iter_kv[0] in + list(map(lambda enum_iter: enum_iter.value, DataType._member_map_.values())) + ), + event.get("data_files").items() + ) + ) + ) + + # Return list of objects for downstream sfns to consume + return { + "case_creation_obj": case_creation_obj.to_dict(), + "sequencerrun_creation_obj": sequencer_run_creation.to_dict(), + "informaticsjob_creation_obj": informatics_job_creation.to_dict(), + "data_files": list(map(lambda data_file_iter: data_file_iter.to_dict(), data_files)), + "sequencerrun_s3_path": sequencerrun_s3_path, + "sample_name": sample_name, + } + + + +# # Idenitified Patient +# if __name__ == "__main__": +# import json +# from os import environ +# +# environ['AWS_PROFILE'] = 'umccr-development' +# environ['AWS_REGION'] = 'ap-southeast-2' +# environ['ICAV2_ACCESS_TOKEN_SECRET_ID'] = "ICAv2JWTKey-umccr-prod-service-dev" +# print( +# json.dumps( +# handler( +# { +# "sequencerrun_s3_path_root": "s3://pdx-cgwxfer-test/melbournetest", +# "portal_run_id": "abcd1234", +# "samplesheet_uri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Logs_Intermediates/SampleSheetValidation/SampleSheet_Intermediate.csv", +# "panel_name": "tso500_DRAGEN_ctDNA_v2_1_Universityofmelbourne", # pragma: allowlist secret +# "dag": { +# "dagName": "cromwell_tso500_ctdna_workflow_1.0.4", +# "dagDescription": "tso500_ctdna_workflow" +# }, +# "data_files": { +# "microsatOutputUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Logs_Intermediates/DragenCaller/L2301368/L2301368.microsat_output.json", +# "tmbMetricsUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Logs_Intermediates/Tmb/L2301368/L2301368.tmb.metrics.csv", +# "cnvVcfUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Results/L2301368/L2301368.cnv.vcf.gz", +# "hardFilteredVcfUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Results/L2301368/L2301368.hard-filtered.vcf.gz", +# "fusionsUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Results/L2301368/L2301368_Fusions.csv", +# "metricsOutputUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Results/L2301368/L2301368_MetricsOutput.tsv", +# "samplesheetUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Logs_Intermediates/SampleSheetValidation/SampleSheet_Intermediate.csv" +# }, +# "case_metadata": { +# "isIdentified": True, +# "caseAccessionNumber": "SBJ04407__L2301368__V2__abcd1234", +# "externalSpecimenId": "externalspecimenid", +# "sampleType": "PatientCare", +# "specimenLabel": "primarySpecimen", +# "indication": "Test", +# "diseaseCode": 64572001, +# "specimenCode": 122561005, +# "sampleReception": { +# "dateAccessioned": "2021-01-01", +# "dateCollected": "2024-02-20", +# "dateReceived": "2021-01-01" +# }, +# "patientInformation": { +# "dateOfBirth": "1970-01-01", +# "firstName": "John", +# "lastName": "Doe" +# }, +# "medicalRecordNumbers": { +# "mrn": "3069999", +# "medicalFacility": { +# "facility": "Not Available", +# "hospitalNumber": "99" +# } +# }, +# "requestingPhysician": { +# "firstName": "Meredith", +# "lastName": "Gray" +# } +# }, +# "instrument_run_id": "231116_A01052_0172_BHVLM5DSX7" +# }, +# None +# ), +# indent=2 +# ) +# ) +# +# # Yields +# # { +# # "case_creation_obj": { +# # "identified": true, +# # "indication": "Test", +# # "panelName": "tso500_DRAGEN_ctDNA_v2_1_Universityofmelbourne", # pragma: allowlist secret +# # "sampleType": "patientcare", +# # "specimens": [ +# # { +# # "accessionNumber": "SBJ04407__L2301368__V2__abcd1234", +# # "dateAccessioned": "2021-01-01T00:00:00Z", +# # "dateReceived": "2021-01-01T00:00:00Z", +# # "datecollected": "2024-02-20T00:00:00Z", +# # "externalSpecimenId": "externalspecimenid", +# # "name": "primarySpecimen", +# # "type": { +# # "code": "122561005", +# # "label": "Blood specimen from patient" +# # }, +# # "firstName": "John", +# # "lastName": "Doe", +# # "dateOfBirth": "1970-01-01", +# # "medicalRecordNumbers": [ +# # { +# # "mrn": "3069999", +# # "medicalFacility": { +# # "facility": "Not Available", +# # "hospitalNumber": "99" +# # } +# # } +# # ] +# # } +# # ], +# # "dagDescription": "tso500_ctdna_workflow", +# # "dagName": "cromwell_tso500_ctdna_workflow_1.0.4", +# # "disease": { +# # "code": "64572001", +# # "label": "Disease" +# # }, +# # "physicians": [ +# # { +# # "firstName": "Meredith", +# # "lastName": "Gray" +# # } +# # ] +# # }, +# # "sequencerrun_creation_obj": { +# # "runId": "231116_A01052_0172_BHVLM5DSX7__SBJ04407__L2301368__V2__abcd1234__abcd1234", +# # "specimens": [ +# # { +# # "accessionNumber": "SBJ04407__L2301368__V2__abcd1234", +# # "barcode": "CCATCATTAG-AGAGGCAACC", +# # "lane": "1", +# # "sampleId": "L2400161", +# # "sampleType": "DNA" +# # } +# # ], +# # "type": "pairedEnd" +# # }, +# # "informaticsjob_creation_obj": { +# # "input": [ +# # { +# # "accessionNumber": "SBJ04407__L2301368__V2__abcd1234", +# # "sequencerRunInfos": [ +# # { +# # "runId": "231116_A01052_0172_BHVLM5DSX7__SBJ04407__L2301368__V2__abcd1234__abcd1234", +# # "barcode": "CCATCATTAG-AGAGGCAACC", +# # "lane": "1", +# # "sampleId": "L2400161", +# # "sampleType": "DNA" +# # } +# # ] +# # } +# # ] +# # }, +# # "data_files": [ +# # { +# # "src_uri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Logs_Intermediates/DragenCaller/L2301368/L2301368.microsat_output.json", +# # "dest_uri": "s3://pdx-cgwxfer-test/melbournetest/231116_A01052_0172_BHVLM5DSX7__SBJ04407__L2301368__V2__abcd1234__abcd1234/Data/Intensities/BaseCalls/L2400161.microsat_output.json", +# # "needs_decompression": false, +# # "contents": null +# # }, +# # { +# # "src_uri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Logs_Intermediates/Tmb/L2301368/L2301368.tmb.metrics.csv", +# # "dest_uri": "s3://pdx-cgwxfer-test/melbournetest/231116_A01052_0172_BHVLM5DSX7__SBJ04407__L2301368__V2__abcd1234__abcd1234/Data/Intensities/BaseCalls/L2400161.tmb.metrics.csv", +# # "needs_decompression": false, +# # "contents": null +# # }, +# # { +# # "src_uri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Results/L2301368/L2301368.cnv.vcf.gz", +# # "dest_uri": "s3://pdx-cgwxfer-test/melbournetest/231116_A01052_0172_BHVLM5DSX7__SBJ04407__L2301368__V2__abcd1234__abcd1234/Data/Intensities/BaseCalls/L2400161.cnv.vcf", +# # "needs_decompression": true, +# # "contents": null +# # }, +# # { +# # "src_uri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Results/L2301368/L2301368.hard-filtered.vcf.gz", +# # "dest_uri": "s3://pdx-cgwxfer-test/melbournetest/231116_A01052_0172_BHVLM5DSX7__SBJ04407__L2301368__V2__abcd1234__abcd1234/Data/Intensities/BaseCalls/L2400161.hard-filtered.vcf", +# # "needs_decompression": true, +# # "contents": null +# # }, +# # { +# # "src_uri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Results/L2301368/L2301368_Fusions.csv", +# # "dest_uri": "s3://pdx-cgwxfer-test/melbournetest/231116_A01052_0172_BHVLM5DSX7__SBJ04407__L2301368__V2__abcd1234__abcd1234/Data/Intensities/BaseCalls/L2400161_Fusions.csv", +# # "needs_decompression": false, +# # "contents": null +# # }, +# # { +# # "src_uri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Results/L2301368/L2301368_MetricsOutput.tsv", +# # "dest_uri": "s3://pdx-cgwxfer-test/melbournetest/231116_A01052_0172_BHVLM5DSX7__SBJ04407__L2301368__V2__abcd1234__abcd1234/Data/Intensities/BaseCalls/L2400161_MetricsOutput.tsv", +# # "needs_decompression": false, +# # "contents": null +# # } +# # ], +# # "sequencerrun_s3_path": "s3://pdx-cgwxfer-test/melbournetest/231116_A01052_0172_BHVLM5DSX7__SBJ04407__L2301368__V2__abcd1234__abcd1234", +# # "sample_name": "L2400161" +# # } + +# +# # # De-Idenitified Patient +# if __name__ == "__main__": +# import json +# from os import environ +# +# environ['AWS_PROFILE'] = 'umccr-development' +# environ['AWS_REGION'] = 'ap-southeast-2' +# environ['ICAV2_ACCESS_TOKEN_SECRET_ID'] = "ICAv2JWTKey-umccr-prod-service-dev" +# print( +# json.dumps( +# handler( +# { +# "sequencerrun_s3_path_root": "s3://pdx-cgwxfer-test/melbournetest", +# "portal_run_id": "20241003f44a5496", +# "samplesheet_uri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20241003ead8ad9f/Logs_Intermediates/SampleSheetValidation/SampleSheet_Intermediate.csv", +# "panel_name": "tso500_DRAGEN_ctDNA_v2_1_Universityofmelbourne", # pragma: allowlist secret +# "dag": { +# "dagName": "cromwell_tso500_ctdna_workflow_1.0.4", +# "dagDescription": "tso500_ctdna_workflow" +# }, +# "data_files": { +# "microsatOutputUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20241003ead8ad9f/Logs_Intermediates/DragenCaller/L2400160/L2400160.microsat_output.json", +# "tmbMetricsUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20241003ead8ad9f/Logs_Intermediates/Tmb/L2400160/L2400160.tmb.metrics.csv", +# "cnvVcfUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20241003ead8ad9f/Results/L2400160/L2400160.cnv.vcf.gz", +# "hardFilteredVcfUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20241003ead8ad9f/Results/L2400160/L2400160.hard-filtered.vcf.gz", +# "fusionsUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20241003ead8ad9f/Results/L2400160/L2400160_Fusions.csv", +# "metricsOutputUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20241003ead8ad9f/Results/L2400160/L2400160_MetricsOutput.tsv", +# "samplesheetUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20241003ead8ad9f/Logs_Intermediates/SampleSheetValidation/SampleSheet_Intermediate.csv" +# }, +# "case_metadata": { +# "isIdentified": False, +# "caseAccessionNumber": "L2400160__V2__20241003f44a5496", +# "externalSpecimenId": "SSq-CompMM-1pc-10646259ilm", +# "sampleType": "patientcare", +# "specimenLabel": "primarySpecimen", +# "indication": None, +# "diseaseCode": 55342001, +# "specimenCode": "122561005", +# "sampleReception": { +# "dateAccessioned": "2024-10-03", +# "dateCollected": "2024-10-03", +# "dateReceived": "2024-10-03" +# }, +# "study": { +# "id": "Testing", +# "subjectIdentifier": "CMM1pc-10646259ilm" +# } +# }, +# "instrument_run_id": "240229_A00130_0288_BH5HM2DSXC" +# }, +# None +# ), +# indent=2 +# ) +# ) +# +# # Yields +# # { +# # "case_creation_obj": { +# # "identified": false, +# # "panelName": "tso500_DRAGEN_ctDNA_v2_1_Universityofmelbourne", # pragma: allowlist secret +# # "sampleType": "patientcare", +# # "specimens": [ +# # { +# # "accessionNumber": "L2400160__V2__20241003f44a5496", +# # "dateAccessioned": "2024-10-03T00:00:00Z", +# # "dateReceived": "2024-10-03T00:00:00Z", +# # "datecollected": "2024-10-03T00:00:00Z", +# # "externalSpecimenId": "SSq-CompMM-1pc-10646259ilm", +# # "name": "primarySpecimen", +# # "type": { +# # "code": "122561005", +# # "label": "Blood specimen from patient" +# # }, +# # "studyIdentifier": "Testing", +# # "studySubjectIdentifier": "CMM1pc-10646259ilm" +# # } +# # ], +# # "dagDescription": "tso500_ctdna_workflow", +# # "dagName": "cromwell_tso500_ctdna_workflow_1.0.4", +# # "disease": { +# # "code": "55342001", +# # "label": "Neoplastic disease" +# # } +# # }, +# # "sequencerrun_creation_obj": { +# # "runId": "240229_A00130_0288_BH5HM2DSXC__L2400160__V2__20241003f44a5496__20241003f44a5496", +# # "specimens": [ +# # { +# # "accessionNumber": "L2400160__V2__20241003f44a5496", +# # "barcode": "AGAGGCAACC-CCATCATTAG", +# # "lane": "1", +# # "sampleId": "L2400160", +# # "sampleType": "DNA" +# # } +# # ], +# # "type": "pairedEnd" +# # }, +# # "informaticsjob_creation_obj": { +# # "input": [ +# # { +# # "accessionNumber": "L2400160__V2__20241003f44a5496", +# # "sequencerRunInfos": [ +# # { +# # "runId": "240229_A00130_0288_BH5HM2DSXC__L2400160__V2__20241003f44a5496__20241003f44a5496", +# # "barcode": "AGAGGCAACC-CCATCATTAG", +# # "lane": "1", +# # "sampleId": "L2400160", +# # "sampleType": "DNA" +# # } +# # ] +# # } +# # ] +# # }, +# # "data_files": [ +# # { +# # "src_uri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20241003ead8ad9f/Logs_Intermediates/DragenCaller/L2400160/L2400160.microsat_output.json", +# # "dest_uri": "s3://pdx-cgwxfer-test/melbournetest/240229_A00130_0288_BH5HM2DSXC__L2400160__V2__20241003f44a5496__20241003f44a5496/Data/Intensities/BaseCalls/L2400160.microsat_output.json", +# # "needs_decompression": false, +# # "contents": null +# # }, +# # { +# # "src_uri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20241003ead8ad9f/Logs_Intermediates/Tmb/L2400160/L2400160.tmb.metrics.csv", +# # "dest_uri": "s3://pdx-cgwxfer-test/melbournetest/240229_A00130_0288_BH5HM2DSXC__L2400160__V2__20241003f44a5496__20241003f44a5496/Data/Intensities/BaseCalls/L2400160.tmb.metrics.csv", +# # "needs_decompression": false, +# # "contents": null +# # }, +# # { +# # "src_uri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20241003ead8ad9f/Results/L2400160/L2400160.cnv.vcf.gz", +# # "dest_uri": "s3://pdx-cgwxfer-test/melbournetest/240229_A00130_0288_BH5HM2DSXC__L2400160__V2__20241003f44a5496__20241003f44a5496/Data/Intensities/BaseCalls/L2400160.cnv.vcf", +# # "needs_decompression": true, +# # "contents": null +# # }, +# # { +# # "src_uri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20241003ead8ad9f/Results/L2400160/L2400160.hard-filtered.vcf.gz", +# # "dest_uri": "s3://pdx-cgwxfer-test/melbournetest/240229_A00130_0288_BH5HM2DSXC__L2400160__V2__20241003f44a5496__20241003f44a5496/Data/Intensities/BaseCalls/L2400160.hard-filtered.vcf", +# # "needs_decompression": true, +# # "contents": null +# # }, +# # { +# # "src_uri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20241003ead8ad9f/Results/L2400160/L2400160_Fusions.csv", +# # "dest_uri": "s3://pdx-cgwxfer-test/melbournetest/240229_A00130_0288_BH5HM2DSXC__L2400160__V2__20241003f44a5496__20241003f44a5496/Data/Intensities/BaseCalls/L2400160_Fusions.csv", +# # "needs_decompression": false, +# # "contents": null +# # }, +# # { +# # "src_uri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20241003ead8ad9f/Results/L2400160/L2400160_MetricsOutput.tsv", +# # "dest_uri": "s3://pdx-cgwxfer-test/melbournetest/240229_A00130_0288_BH5HM2DSXC__L2400160__V2__20241003f44a5496__20241003f44a5496/Data/Intensities/BaseCalls/L2400160_MetricsOutput.tsv", +# # "needs_decompression": false, +# # "contents": null +# # } +# # ], +# # "sequencerrun_s3_path": "s3://pdx-cgwxfer-test/melbournetest/240229_A00130_0288_BH5HM2DSXC__L2400160__V2__20241003f44a5496__20241003f44a5496", +# # "sample_name": "L2400160" +# # } diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/lambdas/generate_samplesheet_py/generate_samplesheet.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/lambdas/generate_samplesheet_py/generate_samplesheet.py new file mode 100644 index 000000000..573866520 --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/lambdas/generate_samplesheet_py/generate_samplesheet.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 + +""" +Generate a samplesheet from a cttso v2 samplesheet file + +{ + "samplesheet_uri": "s3://.../Logs_Intermediates/SampleSheet_Validation/SampleSheet_Intermediate.csv" +} + +Returns + +{ + "samplesheet_str": "" +} + +""" + +# Standard imports +from typing import Dict +import logging +import boto3 +from os import environ + +# Custom libraries +from v2_samplesheet_maker.functions.v2_samplesheet_writer import v2_samplesheet_writer + +# Local imports +from pieriandx_pipeline_tools.utils.samplesheet_helpers import read_v2_samplesheet + +# Globals +ICAV2_BASE_URL = "https://ica.illumina.com/ica/rest" + +# Set loggers +logger = logging.getLogger() +logger.setLevel(logging.INFO) + + +def get_secrets_manager_client() -> 'SecretsManagerClient': + """ + Return Secrets Manager client + """ + return boto3.client("secretsmanager") + + +def get_secret(secret_id: str) -> str: + """ + Return secret value + """ + return get_secrets_manager_client().get_secret_value(SecretId=secret_id)["SecretString"] + + +# Functions +def set_icav2_env_vars(): + """ + Set the icav2 environment variables + :return: + """ + environ["ICAV2_BASE_URL"] = ICAV2_BASE_URL + environ["ICAV2_ACCESS_TOKEN"] = get_secret( + environ["ICAV2_ACCESS_TOKEN_SECRET_ID"] + ) + + +def handler(event, context) -> Dict[str, str]: + # Set ICAv2 env variables + logger.info("Setting icav2 env vars from secrets manager") + set_icav2_env_vars() + + # Get the samplesheet uri + samplesheet_uri = event.get("samplesheet_uri", None) + + # Get the samplesheet as an icav2 projectdata object + samplesheet_dict = read_v2_samplesheet(samplesheet_uri) + + # Convert to string (as a samplesheet) + samplesheet_str = str(v2_samplesheet_writer(samplesheet_dict).read()) + + # Replace TSO500L_Data header line + # Sample_ID,Sample_Type,Lane,Index,Index2,I7_Index_ID,I5_Index_ID + # With + # Sample_ID,Sample_Type,Lane,index,index2,I7_Index_ID,I5_Index_ID + # Without changing the Index1Cycles and Index2Cycles of the Reads section + # Hacky and dirty workaround required because PierianDx is not able to handle Index / Index2 in uppercase + # Assumes Index and Index2 fall within the middle of the index line + samplesheet_str = samplesheet_str.replace(',Index', ',index') + + return { + "samplesheet_str": samplesheet_str + } + + +# if __name__ == "__main__": +# import json +# from os import environ +# +# environ['ICAV2_ACCESS_TOKEN_SECRET_ID'] = "ICAv2JWTKey-umccr-prod-service-dev" +# environ['AWS_REGION'] = "ap-southeast-2" +# environ['AWS_PROFILE'] = 'umccr-development' +# print( +# json.dumps( +# handler( +# { +# "samplesheet_uri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Logs_Intermediates/SampleSheetValidation/SampleSheet_Intermediate.csv" +# }, +# None +# ), +# indent=2 +# ) +# ) +# +# # Yields +# # { +# # "samplesheet_str": "[Header]\nFileFormatVersion,2\nRunName,240229_A00130_0288_BH5HM2DSXC\nInstrumentType,NovaSeq\n\n[TSO500L_Settings]\n\n\n[TSO500L_Data]\nSample_ID,index_ID,Sample_Type,index,index2,I7_Index_ID,I5_Index_ID\nL2400161,UDP0019,DNA,CCATCATTAG,AGAGGCAACC,UDP0019,UDP0019\n" +# # } + diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/lambdas/generate_sequencerrun_case_py/generate_sequencerrun_case.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/lambdas/generate_sequencerrun_case_py/generate_sequencerrun_case.py new file mode 100644 index 000000000..459388cc2 --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/lambdas/generate_sequencerrun_case_py/generate_sequencerrun_case.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python + +""" +Generate a case from a case object + +Environment variables +PIERIANDX_COLLECT_AUTH_TOKEN_LAMBDA_NAME: The lambda used to collect the auth token + +A sequencerrun object will look like this + +{ + "runId": "20201203_A00123_0001_BHJGJFDS__caseaccessionnumber__20240411235959", + "specimens": [ + { + "accessionNumber": "caseaccessionnumber", + "barcode": "GACTGAGTAG+CACTATCAAC", + "lane": "1", + "sampleId": "L2301368", + "sampleType": "DNA" + } + ], + "type": "pairedEnd" +} +""" + +# Standard Imports +import logging +from requests import Response +from os import environ + + +# Local imports +from pieriandx_pipeline_tools.utils.pieriandx_helpers import get_pieriandx_client +from pieriandx_pipeline_tools.utils.secretsmanager_helpers import set_pieriandx_env_vars + + +def handler(event, context): + # Get inputs + sequencerrun_creation_obj = event.get("sequencerrun_creation_obj", {}) + + set_pieriandx_env_vars() + pyriandx_client = get_pieriandx_client( + email=environ['PIERIANDX_USER_EMAIL'], + token=environ['PIERIANDX_USER_AUTH_TOKEN'], + instiution=environ['PIERIANDX_INSTITUTION'], + base_url=environ['PIERIANDX_BASE_URL'], + ) + + response: Response = pyriandx_client._post_api( + endpoint=f"/sequencerRun", + data=sequencerrun_creation_obj + ) + + if response.status_code != 200: + logging.error(f"Failed to create sequencerrun: {response.json()}") + raise Exception(f"Failed to create sequencerrun: {response.json()}") + + return response.json() + +# +# if __name__ == "__main__": +# import json +# print( +# json.dumps( +# handler( +# { +# "sequencerrun_creation_obj": { +# "runId": "231116_A01052_0172_BHVLM5DSX7__SBJ04405__L2301368__ot__003__20240415abcd0001", +# "specimens": [ +# { +# "accessionNumber": "SBJ04405__L2301368__ot__003", +# "barcode": "GACTGAGTAG-CACTATCAAC", +# "lane": "1", +# "sampleId": "L2301368", +# "sampleType": "DNA" +# } +# ], +# "type": "pairedEnd" +# } +# }, +# None +# ), +# indent=2 +# ) +# ) +# +# # Yields +# # { +# # "id": "38862" +# # } \ No newline at end of file diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/lambdas/get_informaticsjob_and_report_status_py/get_informaticsjob_and_report_status.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/lambdas/get_informaticsjob_and_report_status_py/get_informaticsjob_and_report_status.py new file mode 100644 index 000000000..88141a68f --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/lambdas/get_informaticsjob_and_report_status_py/get_informaticsjob_and_report_status.py @@ -0,0 +1,301 @@ +#!/usr/bin/env python3 + +""" +Get informatics job status + +Given a case id and an informatics job id, return the status of the job + +The job status can be one of the following: +* waiting # PROCESSING +* ready # PROCESSING +* running # PROCESSING +* complete # TERMINAL +* failed # TERMINAL +* canceled # TERMINAL + +If the job is complete, check the reports for the case to see if the report generation is also complete + +Also return the DynamoDB object to the expression reference and the object dict since +the object can vary depending on what status the job / report is at + +job_status: STR VALUE OF THE JOB STATUS +job_status_bool: BOOL VALUE OF THE JOB STATUS TRUE IF COMPLETE, FALSE IF FAILED, NONE OTHERWISE +report_id: INT VALUE OF THE REPORT ID +report_status: STR VALUE OF THE REPORT STATUS +report_status_bool: BOOL VALUE OF THE REPORT STATUS TRUE IF COMPLETE, FALSE IF FAILED, NONE OTHERWISE +job_status_changed: BOOL VALUE OF WHETHER THE JOB STATUS HAS CHANGED TRUE OR FALSE +expression_attribute_values_dict: DICT OF THE EXPRESSION ATTRIBUTE VALUES FOR DYNAMODB UPDATE EXPRESSION +update_expression_str: STR OF THE UPDATE EXPRESSION FOR DYNAMODB + +""" + +# Standard imports +import logging +from os import environ + +# Layer imports +from pieriandx_pipeline_tools.utils.pieriandx_helpers import get_pieriandx_client +from pieriandx_pipeline_tools.utils.secretsmanager_helpers import set_pieriandx_env_vars + +# Set logger +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +JOB_STATUS_BOOL = { + "waiting": None, + "ready": None, + "running": None, + "complete": True, + "failed": False, + "canceled": False +} + + +REPORT_STATUS_BOOL = { + "waiting": None, + "ready": None, + "running": None, + "report_generation_complete": True, + "complete": True, + "failed": False, + "canceled": False +} + + +def handler(event, context): + """ + Get informatics job status + Args: + event: + context: + + Returns: + + """ + + # Get event values + case_id = event.get("case_id", None) + job_id = event.get("informaticsjob_id", None) + report_id = event.get("report_id", None) + current_job_status = event.get("current_job_status", None) + current_report_status = event.get("current_report_status", None) + + # Initialise job status + job_status = None + + # Cannot query the job id directly, instead query the case id and get the job id from there + set_pieriandx_env_vars() + + # Set pyriandx client + pyriandx_client = get_pieriandx_client( + email=environ['PIERIANDX_USER_EMAIL'], + token=environ['PIERIANDX_USER_AUTH_TOKEN'], + instiution=environ['PIERIANDX_INSTITUTION'], + base_url=environ['PIERIANDX_BASE_URL'], + ) + + case_data = pyriandx_client._get_api( + endpoint=f"/case/{case_id}", + ) + + if report_id is None or report_id == -1: + # Get the informatics job object + try: + informaticsjob_obj = next( + filter( + lambda informaticsjob_iter: int(informaticsjob_iter.get("id")) == int(job_id), + case_data.get("informaticsJobs") + ) + ) + except StopIteration: + logger.error(f"Failed to get informatics job {job_id}") + return { + "status": "failed", + "message": f"Failed to get informatics job {job_id} from the case id {case_id}" + } + + # Get job status + job_status = informaticsjob_obj.get("status") + + # Job has either failed or is incomplete - return as is + if ( + ( + # Job not yet complete + not JOB_STATUS_BOOL[job_status] + ) or + ( + # Reports empty + case_data.get("reports") is None + ) or + ( + # Reports length is empty + len(case_data.get("reports")) == 0 + ) + ): + # Set the expression attribute values dict + expression_attribute_values_dict = { + ":job_status": { + "S": job_status + } + } + update_expression_str = "SET job_status = :job_status" + + if JOB_STATUS_BOOL[job_status] is not None: + expression_attribute_values_dict[":job_status_bool"] = { + "BOOL": JOB_STATUS_BOOL[job_status] + } + update_expression_str = f"{update_expression_str}, job_status_bool = :job_status_bool" + + if JOB_STATUS_BOOL[job_status] is False: + expression_attribute_values_dict[":workflow_status"] = { + "S": "FAILED" + } + update_expression_str = f"{update_expression_str}, workflow_status = :workflow_status" + + # Return the job status + return { + "job_status": job_status, + "job_status_bool": JOB_STATUS_BOOL[job_status], + "report_id": None, + "report_status": None, + "report_status_bool": None, + "job_status_changed": False if job_status == current_job_status else True, + "expression_attribute_values_dict": expression_attribute_values_dict, + "update_expression_str": update_expression_str + } + + # Job is complete and reports not empty, check reports + reportjob_obj = case_data.get("reports")[0] + + else: + # Report id is not None, get the report object + try: + reportjob_obj = next( + filter( + lambda reportjob_iter: int(reportjob_iter.get("id")) == int(report_id), + case_data.get("reports") + ) + ) + except StopIteration: + logger.error(f"Failed to get report id {report_id}") + return { + "status": "failed", + "message": f"Failed to get report id {report_id} from the case id {case_id}" + } + + # Reinitialise job status + job_status = "complete" if job_status is None else job_status + + # Get report status + report_status = reportjob_obj.get("status") + + # Return the job status with the report status + # expression_attribute_values_dict + # update_expression_str + + # Set the expression attribute values dict + expression_attribute_values_dict = { + ":job_status": { + "S": job_status + }, + ":report_id": { + "S": reportjob_obj.get("id") + }, + ":report_status": { + "S": report_status + }, + } + update_expression_str = "SET job_status = :job_status, report_id = :report_id, report_status = :report_status" + + # Add the bool values if they are not None + if JOB_STATUS_BOOL[job_status] is not None: + expression_attribute_values_dict[":job_status_bool"] = { + "BOOL": JOB_STATUS_BOOL[job_status] + } + update_expression_str = f"{update_expression_str}, job_status_bool = :job_status_bool" + + if REPORT_STATUS_BOOL[report_status] is not None: + expression_attribute_values_dict[":report_status_bool"] = { + "BOOL": REPORT_STATUS_BOOL[report_status] + } + update_expression_str = f"{update_expression_str}, report_status_bool = :report_status_bool" + + # Add the workflow status + # If one of the job status or report status are false, then the workflow status is failed + if JOB_STATUS_BOOL[job_status] is False or REPORT_STATUS_BOOL[report_status] is False: + expression_attribute_values_dict[":workflow_status"] = { + "S": "FAILED" + } + update_expression_str = f"{update_expression_str}, workflow_status = :workflow_status" + # If both the job status and report status are true, then the workflow status is complete + elif JOB_STATUS_BOOL[job_status] is True and REPORT_STATUS_BOOL[report_status] is True: + expression_attribute_values_dict[":workflow_status"] = { + "S": "COMPLETE" + } + update_expression_str = f"{update_expression_str}, workflow_status = :workflow_status" + + return { + "job_status": job_status, + "job_status_bool": JOB_STATUS_BOOL[job_status], + "report_id": reportjob_obj.get("id"), + "report_status": report_status, + "report_status_bool": REPORT_STATUS_BOOL[report_status], + "job_status_changed": False if report_status == current_report_status else True, + "expression_attribute_values_dict": expression_attribute_values_dict, + "update_expression_str": update_expression_str + } + + + +# if __name__ == "__main__": +# import json +# from os import environ +# environ['AWS_PROFILE'] = 'umccr-development' +# environ['AWS_REGION'] = 'ap-southeast-2' +# environ['PIERIANDX_BASE_URL'] = "https://app.uat.pieriandx.com/cgw-api/v2.0.0" +# environ['PIERIANDX_COLLECT_AUTH_TOKEN_LAMBDA_NAME'] = "collectPierianDxAccessToken" +# environ['PIERIANDX_INSTITUTION'] = "melbournetest" +# environ['PIERIANDX_USER_EMAIL'] = "services@umccr.org" +# print( +# json.dumps( +# handler( +# { +# "current_report_status": "", +# "informaticsjob_id": 45813, +# "report_id": -1, +# "case_id": 103511, +# "current_job_status": "waiting" +# }, +# None +# ), +# indent=2 +# ) +# ) +# +# # { +# # "job_status": "complete", +# # "job_status_bool": true, +# # "report_id": "37981", +# # "report_status": "complete", +# # "report_status_bool": true, +# # "job_status_changed": false, +# # "expression_attribute_values_dict": { +# # ":job_status": { +# # "S": "complete" +# # }, +# # ":report_status": { +# # "S": "complete" +# # }, +# # ":job_status_bool": { +# # "BOOL": true +# # }, +# # ":report_status_bool": { +# # "BOOL": true +# # }, +# # ":workflow_status": { +# # "S": "COMPLETE" +# # } +# # }, +# # "update_expression_str": "SET job_status = :job_status, report_status = :report_status, job_status_bool = :job_status_bool, report_status_bool = :report_status_bool, workflow_status = :workflow_status" +# # } diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/lambdas/upload_pieriandx_sample_data_to_s3_py/upload_pieriandx_sample_data_to_s3.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/lambdas/upload_pieriandx_sample_data_to_s3_py/upload_pieriandx_sample_data_to_s3.py new file mode 100644 index 000000000..23d6f945e --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/lambdas/upload_pieriandx_sample_data_to_s3_py/upload_pieriandx_sample_data_to_s3.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 + +""" + +Upload a file to pieriandx sample data s3 bucket + +Given an icav2 uri, and a destination uri, download and upload the file into the destination uri + +If needs_decompression is set to true, the downloaded file will be decompressed before uploading + +Environment variables required are: +PIERIANDX_S3_ACCESS_CREDENTIALS_SECRET_ID -> The secret id for the s3 access credentials +ICAV2_ACCESS_TOKEN_SECRET_ID -> The secret id for the icav2 access token + +Input will look like this + +{ + "src_uri": "icav2://project-id/path/to/sample-microsat_output.txt", + "dest_uri": "s3://pieriandx/melbourne/20201203_A00123_0001_BHJGJFDS__caseaccessionnumber__20240411235959/L2301368.microsat_output.json", + "needs_decompression": false, + "contents": null +} + +""" + +from tempfile import TemporaryDirectory +from pathlib import Path +from urllib.parse import urlparse +from wrapica.project_data import read_icav2_file_contents, convert_uri_to_project_data_obj + +from pieriandx_pipeline_tools.utils.s3_helpers import set_s3_access_cred_env_vars, upload_file +from pieriandx_pipeline_tools.utils.secretsmanager_helpers import set_icav2_env_vars +from pieriandx_pipeline_tools.utils.compression_helpers import decompress_file + + +def handler(event, context): + """ + Upload pieriandx sample data to s3 bucket + Args: + event: + context: + + Returns: + + """ + + # Set env vars + set_icav2_env_vars() + set_s3_access_cred_env_vars() + + # Get uris + needs_decompression = event.get("needs_decompression", False) + dest_uri = event.get("dest_uri") + dest_bucket = urlparse(dest_uri).netloc + dest_key = urlparse(dest_uri).path + + if event.get("src_uri", None) is not None: + icav2_data_obj = convert_uri_to_project_data_obj(event.get("src_uri")) + + with TemporaryDirectory() as temp_dir: + # Set output path + output_path = Path(temp_dir) / icav2_data_obj.data.details.name + + # Read icav2 file contents + read_icav2_file_contents( + project_id=icav2_data_obj.project_id, + data_id=icav2_data_obj.data.id, + output_path=output_path + ) + + if needs_decompression: + decompress_file(output_path, output_path.parent / output_path.name.replace(".gz", "")) + output_path = output_path.parent / output_path.name.replace(".gz", "") + + if output_path.name.endswith("MetricsOutput.tsv"): + with open(output_path, "r") as f: + contents = f.read() + contents = contents.replace('[Run QC Metrics]', '[Run Metrics]') + with open(output_path, "w") as f: + f.write(contents) + + # Upload to s3 + upload_file(dest_bucket, dest_key, output_path) + else: + contents = event.get("contents") + + with TemporaryDirectory() as temp_dir: + output_path = Path(temp_dir) / Path(dest_key).name + output_path.write_text(contents) + + upload_file(dest_bucket, dest_key, output_path) + + +# if __name__ == "__main__": +# from os import environ +# environ["AWS_PROFILE"] = 'umccr-development' +# environ['AWS_REGION'] = 'ap-southeast-2' +# environ["ICAV2_ACCESS_TOKEN_SECRET_ID"] = "ICAv2JWTKey-umccr-prod-service-dev" +# environ['PIERIANDX_S3_ACCESS_CREDENTIALS_SECRET_ID'] = "PierianDx/S3Credentials" +# +# handler( +# { +# "needs_decompression": False, +# "src_uri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Results/L2400161/L2400161_MetricsOutput.tsv", +# "contents": None, +# "dest_uri": "s3://pdx-cgwxfer-test/melbournetest/231116_A01052_0172_BHVLM5DSX7__SBJ04407__L2400161__V2__abcd1235__abcd1234/Data/Intensities/BaseCalls/L2400161_MetricsOutput.tsv" +# }, +# None +# ) diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/pyproject.toml b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/pyproject.toml new file mode 100644 index 000000000..1a201ac47 --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/pyproject.toml @@ -0,0 +1,39 @@ +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +name = "pieriandx_pipeline_tools" +version = "0.0.1" +description = "PierianDx Pipeline Lambda Layers" +license = "GPL-3.0-or-later" +authors = [ + "Alexis Lucattini" +] +homepage = "https://github.com/umccr/orcabus" +repository = "https://github.com/umccr/orcabus" + +[tool.poetry.dependencies] +python = "^3.12, <3.13" +boto3 = "^1.28" +botocore = "^1.31" +aws_requests_auth = "^0.4.3" +v2_samplesheet_maker = "^4.2.4" +wrapica = ">=2.27.1.post20240830140737" +# Git dependencies +pyriandx = { git = "https://github.com/umccr/pyriandx.git", branch = "0.4.0" } +pandas = "^2.2.2" +urllib3 = "^1.26" + +[tool.poetry.group.dev] +optional = true + +[tool.poetry.group.dev.dependencies] +pyarrow = "^15.0.0" # Pandas throws a warning if this is not installed +pytest = "^7.0.0" # For testing only +# For typehinting only, not required at runtime +mypy-boto3-ssm = "^1.34" +mypy-boto3-s3 = "^1.34" +mypy-boto3-secretsmanager = "^1.34" +mypy-boto3-stepfunctions = "^1.34" +mypy-boto3-lambda = "^1.34" diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/__init__.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_classes/__init__.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_classes/__init__.py new file mode 100644 index 000000000..ba4263622 --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_classes/__init__.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 + +from copy import deepcopy +from typing import Dict + +from pydantic import BaseModel + + +class BaseClass: + _model = None + + def __init__(self, **kwargs): + self._raw_kwargs_dict = deepcopy(kwargs) + self.section_dict = {} + kwargs_dict = deepcopy(kwargs) + for key in self._model.model_fields.keys(): + if key in kwargs_dict.keys(): + setattr(self, key, kwargs.pop(key, None)) + + if not hasattr(self, key) or getattr(self, key) is None: + # Set default attribute + setattr(self, key, self._model.model_fields[key].default) + + self.validate_model() + + self.coerce_values() + + self.build_section_dict() + + def validate_model(self): + """ + Validate inputs against pydantic model of class + :return: + """ + pass #self._model.model_validate(self) + + def build_section_dict(self): + # Collect original objects + self.section_dict = self._model(**self.get_dict_object()).to_dict() + self.section_dict = self.filter_dict(self.section_dict) + + def get_dict_object(self): + def get_dict_object_recursively(dict_object): + if isinstance(dict_object, BaseClass): + return dict_object.get_dict_object() + # if isinstance(dict_object, BaseModel): + # pass + elif isinstance(dict_object, dict): + return { + key: get_dict_object_recursively(value) + for key, value in dict_object.items() + } + elif isinstance(dict_object, list): + return [ + get_dict_object_recursively(value) + for value in dict_object + ] + else: + return dict_object + + return { + kv[0]: get_dict_object_recursively(kv[1]) + for kv in filter( + lambda kv_iter: not kv_iter[0] in ["section_dict", "_raw_kwargs_dict"], + self.__dict__.items() + ) + } + + def filter_dict(self, initial_dict) -> Dict: + """ + Filter out any values that are None + :param initial_dict: + :return: + """ + return dict( + filter( + lambda kv: kv[1] is not None, + initial_dict.items() + ) + ) + + def coerce_values(self): + # Coerce with model dump + coerced_dict = self._model(**self.get_dict_object()).model_dump() + + for key, value in coerced_dict.items(): + self.__setattr__(key, value) + + def to_dict(self) -> Dict: + return self.section_dict diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_classes/case.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_classes/case.py new file mode 100644 index 000000000..8524cf62c --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_classes/case.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +from . import BaseClass + + +class CaseCreation(BaseClass): + from ..pieriandx_models.case_creation import CaseCreation + _model = CaseCreation + + +class IdentifiedCaseCreation(BaseClass): + from ..pieriandx_models.case_creation import IdentifiedCaseCreation + _model = IdentifiedCaseCreation + + +class DeIdentifiedCaseCreation(BaseClass): + from ..pieriandx_models.case_creation import DeIdentifiedCaseCreation + _model = DeIdentifiedCaseCreation diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_classes/dag.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_classes/dag.py new file mode 100644 index 000000000..84c08a3b6 --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_classes/dag.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python + +from . import BaseClass + + +class Dag(BaseClass): + from ..pieriandx_models.dag import Dag + _model = Dag + + diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_classes/data_file.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_classes/data_file.py new file mode 100644 index 000000000..3abb95c13 --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_classes/data_file.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python + +from enum import Enum +from typing import Dict, Optional + + +class DataType(Enum): + MICROSAT_OUTPUT = "microsatOutputUri" + TMB_METRICS = "tmbMetricsUri" + CNV = "cnvVcfUri" + HARD_FILTERED = "hardFilteredVcfUri" + FUSIONS = "fusionsUri" + METRICS_OUTPUT = "metricsOutputUri" + + +class DataNameSuffixByDataType(Enum): + MICROSAT_OUTPUT = ".microsat_output.json" + TMB_METRICS = ".tmb.metrics.csv" + CNV = ".cnv.vcf" + HARD_FILTERED = ".hard-filtered.vcf" + FUSIONS = "_Fusions.csv" + METRICS_OUTPUT = "_MetricsOutput.tsv" + + +PATH_EXTENSION = "Data/Intensities/BaseCalls" + + +class DataFile: + + def __init__( + self, + sequencerrun_path_root, + file_type: DataType, + sample_id: str, + src_uri: Optional[str] = None, + contents: Optional[str] = None, + ): + # Initialise the class variables + self.sequencerrun_path_root = sequencerrun_path_root + self.file_type = file_type + self.sample_id = sample_id + self.src_uri = src_uri + self.contents = contents + + # Make sure that the src_uri or contents are provided + if self.src_uri is None and self.contents is None: + raise ValueError("Either src_uri or contents must be provided") + + # Determine compression status by src uri file extension + if self.src_uri.endswith(".gz"): + self.needs_decompression = True + else: + self.needs_decompression = False + + # Determine the destination uri + self.dest_uri = ( + self.sequencerrun_path_root.rstrip("/") + "/" + PATH_EXTENSION + "/" + + self.sample_id + DataNameSuffixByDataType[self.file_type.name].value + ) + + def to_dict(self) -> Dict: + return { + "src_uri": self.src_uri, + "dest_uri": self.dest_uri, + "needs_decompression": self.needs_decompression, + "contents": self.contents + } + diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_classes/disease.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_classes/disease.py new file mode 100644 index 000000000..17496f9bf --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_classes/disease.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python + +from . import BaseClass + + +class Disease(BaseClass): + from ..pieriandx_models.disease import Disease + _model = Disease + + diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_classes/informaticsjob.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_classes/informaticsjob.py new file mode 100644 index 000000000..5ac24bb6a --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_classes/informaticsjob.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python + +from . import BaseClass + + +class InformaticsjobCreation(BaseClass): + from ..pieriandx_models.informaticsjob_creation import InformaticsJobCreation + _model = InformaticsJobCreation diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_classes/medical_facility.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_classes/medical_facility.py new file mode 100644 index 000000000..58cabfedc --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_classes/medical_facility.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python + +from . import BaseClass + + +class MedicalFacility(BaseClass): + from ..pieriandx_models.medical_facility import MedicalFacility + _model = MedicalFacility + + diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_classes/medical_record_number.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_classes/medical_record_number.py new file mode 100644 index 000000000..bf58a9dbb --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_classes/medical_record_number.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python + +from . import BaseClass + + +class MedicalRecordNumber(BaseClass): + from ..pieriandx_models.medical_record_number import MedicalRecordNumber + _model = MedicalRecordNumber + + diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_classes/physician.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_classes/physician.py new file mode 100644 index 000000000..810ca75b8 --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_classes/physician.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python + +from . import BaseClass + + +class Physician(BaseClass): + from ..pieriandx_models.physician import Physician + _model = Physician + + diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_classes/sequencerrun.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_classes/sequencerrun.py new file mode 100644 index 000000000..ab766f64b --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_classes/sequencerrun.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python + +from . import BaseClass + + +class SequencerrunCreation(BaseClass): + from ..pieriandx_models.sequencerrun_creation import SequencerrunCreation + _model = SequencerrunCreation diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_classes/specimen.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_classes/specimen.py new file mode 100644 index 000000000..849c8bf41 --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_classes/specimen.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python + +from . import BaseClass + + +class Specimen(BaseClass): + from ..pieriandx_models.specimen import Specimen + _model = Specimen + + +class IdentifiedSpecimen(Specimen): + from ..pieriandx_models.specimen import IdentifiedSpecimen + _model = IdentifiedSpecimen + + +class DeIdentifiedSpecimen(Specimen): + from ..pieriandx_models.specimen import DeIdentifiedSpecimen + _model = DeIdentifiedSpecimen + + diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_classes/specimen_sequencer_info.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_classes/specimen_sequencer_info.py new file mode 100644 index 000000000..27dcbac74 --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_classes/specimen_sequencer_info.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python + +from . import BaseClass + + +class SpecimenSequencerInfo(BaseClass): + from ..pieriandx_models.specimen_sequencer_info import SpecimenSequencerInfo + _model = SpecimenSequencerInfo diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_enums/__init__.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_enums/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_enums/ethnicity.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_enums/ethnicity.py new file mode 100644 index 000000000..b02bd3bdd --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_enums/ethnicity.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python + +from enum import Enum + + +class Ethnicity(Enum): + HISPANIC_OR_LATINO = "hispanic_or_latino" + NOT_HISPANIC_OR_LATINO = "not_hispanic_or_latino" + NOT_REPORTED = "not_reported" + UNKNOWN = "unknown" diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_enums/gender.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_enums/gender.py new file mode 100644 index 000000000..233e50505 --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_enums/gender.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 + +from enum import Enum + + +class Gender(Enum): + UNKNOWN = "unknown" + MALE = "male" + FEMALE = "female" + UNSPECIFIED = "unspecified" + OTHER = "other" + AMBIGUOUS = "ambiguous" + NOT_APPLICABLE = "not_applicable" diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_enums/race.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_enums/race.py new file mode 100644 index 000000000..d6ef3d5ff --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_enums/race.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python + + +from enum import Enum + + +class Race(Enum): + AMERICAN_INDIAN_OR_ALASKA_NATIVE = "american_indian_or_alaska_native" + ASIAN = "asian" + BLACK_OR_AFRICAN_AMERICAN = "black_or_african_american" + NATIVE_HAWAIIAN_OR_OTHER_PACIFIC_ISLANDER = "native_hawaiian_or_other_pacific_islander" + NOT_REPORTED = "not_reported" + UNKNOWN = "unknown" + WHITE = "white" diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_enums/sample_type.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_enums/sample_type.py new file mode 100644 index 000000000..519427b2f --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_enums/sample_type.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 + +from enum import Enum + + +class SampleType(Enum): + PATIENTCARE = 'patientcare' + CLINICAL_TRIAL = 'clinical_trial' + VALIDATION = 'validation' + PROFICIENCY_TESTING = 'proficiency_testing' diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_enums/sequencing_type.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_enums/sequencing_type.py new file mode 100644 index 000000000..fd98ff11a --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_enums/sequencing_type.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python3 + +from enum import Enum + + +class SequencingType(Enum): + PAIRED_END = "pairedEnd" + SINGLE_END = "singleEnd" diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_enums/specimen_type.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_enums/specimen_type.py new file mode 100644 index 000000000..8aa6dbe4c --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_enums/specimen_type.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 + +from pydantic import BaseModel, ConfigDict +from typing import Optional + +from pieriandx_pipeline_tools.pieriandx_lookup.get_specimen_label import get_specimen_label_from_specimen_code + + +class SpecimenType(BaseModel): + code: int + label: Optional[str] = None + + model_config = ConfigDict(from_attributes=True) + + def to_dict(self): + return dict( + filter( + lambda dict_object: dict_object[1] is not None, + { + "code": str(self.code), + "label": + self.label if self.label is not None + else get_specimen_label_from_specimen_code(int(self.code)) + }.items() + ) + ) diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_lookup/__init__.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_lookup/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_lookup/get_disease_label.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_lookup/get_disease_label.py new file mode 100644 index 000000000..fc6dbf73b --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_lookup/get_disease_label.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 + +""" +Given a disease code, get the disease label +""" + +# Imports +from pathlib import Path +import pandas as pd +from tempfile import NamedTemporaryFile + +from ..utils.compression_helpers import decompress_file + +# Compressed version of +# https://velserapm.atlassian.net/wiki/download/attachments/86704490/SNOMED_CT%20Disease_trees.xlsx?version=1&modificationDate=1561395438000&api=v2 +SNOMED_CT_DISEASE_TREE_FILE = Path(__file__).parent / "snomed_ct_disease_tree.json.gz" + + +def get_disease_tree() -> pd.DataFrame: + """ + Returns a dataframe with the following columns + * Code + * CodeSystem + * Label + :return: + """ + # Decompress the disease tree file into a temp file + decompressed_disease_tree_file = NamedTemporaryFile(suffix=".json") + decompress_file(SNOMED_CT_DISEASE_TREE_FILE, Path(decompressed_disease_tree_file.name)) + + return pd.read_json(decompressed_disease_tree_file.name) + + +def get_disease_label_from_disease_code(disease_code: int) -> str: + """ + Given the disease code, get the disease label + :param disease_code: + :return: + """ + + # Get the disease tree + disease_tree_df = get_disease_tree() + + # Query disease code + query_df = disease_tree_df.query( + f"Code=={disease_code}" + ) + + # Assert that the query df is of length 1 + assert query_df.shape[0] == 1, f"Failed to get disease code {disease_code}" + + # Return the label + return query_df['Label'].item() diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_lookup/get_specimen_label.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_lookup/get_specimen_label.py new file mode 100644 index 000000000..5a2f656b5 --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_lookup/get_specimen_label.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 + +""" +Given a specimen code, get the specimen label +""" + +#!/usr/bin/env python3 + +""" +Given a disease code, get the disease label +""" + +# Imports +from pathlib import Path +import pandas as pd +from tempfile import NamedTemporaryFile +from ..utils.compression_helpers import decompress_file + +# Compressed version of +# https://velserapm.atlassian.net/wiki/download/attachments/86704490/SnomedCT-Term_For_SpecimenType.xls?version=1&modificationDate=1561395451000&api=v2 +SNOMED_CT_SPECIMEN_TYPE_FILE = Path(__file__).parent / "snomed_ct_specimen_type.json.gz" + + +def get_specimen_df() -> pd.DataFrame: + """ + Returns a dataframe with the following columns + * Code + * CodeLabel + * CodeSystem + :return: + """ + # Decompress the specimen file into a temp file + decompressed_specimen_df_file = NamedTemporaryFile(suffix=".json") + decompress_file(SNOMED_CT_SPECIMEN_TYPE_FILE, Path(decompressed_specimen_df_file.name)) + + return pd.read_json(decompressed_specimen_df_file.name) + + +def get_specimen_label_from_specimen_code(specimen_code: int) -> str: + """ + Given the specimen code, get the specimen label + :param specimen_code: + :return: + """ + + # Get the disease tree + specimen_tree = get_specimen_df() + + # Query disease code + query_df = specimen_tree.query( + f"Code=={specimen_code}" + ) + + # Assert that the query df is of length 1 + assert query_df.shape[0] == 1, f"Failed to get specimen code {specimen_code}" + + # Return the label + return query_df['CodeLabel'].item() diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_lookup/snomed_ct_disease_tree.json.gz b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_lookup/snomed_ct_disease_tree.json.gz new file mode 100755 index 000000000..4e329f792 --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_lookup/snomed_ct_disease_tree.json.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:612df3709957814cd15577a2112653c21ab7e3f1dca95ad055c60a6041250966 +size 1237630 diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_lookup/snomed_ct_specimen_type.json.gz b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_lookup/snomed_ct_specimen_type.json.gz new file mode 100755 index 000000000..136132538 --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_lookup/snomed_ct_specimen_type.json.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cee28e63d0cf996a192d152f9ca1aa79bd3e4a6fcd4751f8e1f3da712903af39 +size 27435 diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_models/__init__.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_models/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_models/case_creation.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_models/case_creation.py new file mode 100644 index 000000000..682eac509 --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_models/case_creation.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 + +from typing import Optional, Dict +from pydantic import BaseModel, ConfigDict +# Local imported attributes +from .dag import Dag +from .disease import Disease + +from pieriandx_pipeline_tools.pieriandx_enums.sample_type import SampleType +from .specimen import Specimen +from .physician import Physician +from .specimen import IdentifiedSpecimen, DeIdentifiedSpecimen + + +class CaseCreation(BaseModel): + dag: Dag + disease: Disease + is_identified: bool + indication: Optional[str] = None + panel_name: str + sample_type: SampleType + specimen: Specimen + + # Model configuration + model_config = ConfigDict(from_attributes=True) + + def to_dict(self) -> Dict: + # Initialise case dict + case_dict = { + "identified": self.is_identified, + "indication": self.indication, + "panelName": self.panel_name, + "sampleType": self.sample_type.value, + "specimens": [self.specimen.to_dict()], + } + + # Update dag + case_dict.update( + self.dag.to_dict() + ) + + # Update disease + case_dict.update( + { + "disease": self.disease.to_dict() + } + ) + + return case_dict + + +class IdentifiedCaseCreation(CaseCreation): + requesting_physician: Physician + specimen: IdentifiedSpecimen + is_identified: bool = True + + # Model configuration + model_config = ConfigDict(from_attributes=True) + + def to_dict(self) -> Dict: + # Get the dictionary from the parent class + initial_dict = super().to_dict() + + # Update the dictionary with the physician and specimen + initial_dict.update( + { + "physicians": [self.requesting_physician.to_dict()], + } + ) + + return initial_dict + + +class DeIdentifiedCaseCreation(CaseCreation): + specimen: DeIdentifiedSpecimen + is_identified: bool = False + + # Model configuration + model_config = ConfigDict(from_attributes=True) diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_models/dag.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_models/dag.py new file mode 100644 index 000000000..e16f15891 --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_models/dag.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 + +from typing import Dict +from pydantic import BaseModel, ConfigDict + + +class Dag(BaseModel): + # Attributes + name: str + description: str + + # Model configuration + model_config = ConfigDict(from_attributes=True) + + def to_dict(self) -> Dict: + return { + "dagDescription": self.description, + "dagName": self.name, + } diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_models/disease.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_models/disease.py new file mode 100644 index 000000000..8394d6936 --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_models/disease.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 + +from typing import Optional +from pydantic import BaseModel, ConfigDict + + + +class Disease(BaseModel): + # Attributes + code: int + label: Optional[str] = None + + # Model configuration + model_config = ConfigDict(from_attributes=True) + + def to_dict(self): + from ..pieriandx_lookup.get_disease_label import get_disease_label_from_disease_code + return dict( + filter( + lambda dict_object: dict_object[1] is not None, + { + "code": str(self.code), + "label": + self.label if self.label is not None + else get_disease_label_from_disease_code(int(self.code)) + }.items() + ) + ) diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_models/informaticsjob_creation.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_models/informaticsjob_creation.py new file mode 100644 index 000000000..76eaab728 --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_models/informaticsjob_creation.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 + +from pydantic import BaseModel, ConfigDict +from typing import Dict +from .specimen_sequencer_info import SpecimenSequencerInfo + + +class InformaticsJobCreation(BaseModel): + # Local imported attributes + + case_accession_number: str + specimen_sequencer_run_info: SpecimenSequencerInfo + + # Model configuration + model_config = ConfigDict(from_attributes=True) + + def to_dict(self) -> Dict: + # Initialise case dict + return { + "input": [ + { + "accessionNumber": self.case_accession_number, + "sequencerRunInfos": [ + self.specimen_sequencer_run_info.to_informaticsjob_dict() + ] + } + ] + } diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_models/medical_facility.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_models/medical_facility.py new file mode 100644 index 000000000..76213d93c --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_models/medical_facility.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python + +from typing import Optional +from pydantic import BaseModel + + +class MedicalFacility(BaseModel): + facility: Optional[str] + hospital_number: str + + def to_dict(self): + return dict( + filter( + lambda dict_object: dict_object[1] is not None, + { + "facility": self.facility, + "hospitalNumber": self.hospital_number + }.items() + ) + ) diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_models/medical_record_number.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_models/medical_record_number.py new file mode 100644 index 000000000..4331e98b9 --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_models/medical_record_number.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python +from pydantic import BaseModel +from .medical_facility import MedicalFacility + + +class MedicalRecordNumber(BaseModel): + mrn: str + medical_facility: MedicalFacility + + def to_dict(self): + return { + "mrn": self.mrn, + "medicalFacility": self.medical_facility.to_dict() + } diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_models/physician.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_models/physician.py new file mode 100644 index 000000000..ca461453c --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_models/physician.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +from typing import Dict +from pydantic import BaseModel, ConfigDict + + +class Physician(BaseModel): + # Attributes + first_name: str + last_name: str + + # Model configuration + model_config = ConfigDict(from_attributes=True) + + def to_dict(self) -> Dict: + return { + "firstName": self.first_name, + "lastName": self.last_name + } diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_models/sequencerrun_creation.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_models/sequencerrun_creation.py new file mode 100644 index 000000000..e16811edb --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_models/sequencerrun_creation.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 + +from pydantic import BaseModel, ConfigDict +from .specimen_sequencer_info import SpecimenSequencerInfo +from pieriandx_pipeline_tools.pieriandx_enums.sequencing_type import SequencingType + + +class SequencerrunCreation(BaseModel): + + # Sequencer run information + run_id: str + specimen_sequence_info: SpecimenSequencerInfo + sequencing_type: SequencingType + + # Model configuration + model_config = ConfigDict(from_attributes=True) + + def to_dict(self): + return { + "runId": self.run_id, + "specimens": [self.specimen_sequence_info.to_dict()], + "type": self.sequencing_type.value + } diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_models/specimen.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_models/specimen.py new file mode 100644 index 000000000..1956aeaf5 --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_models/specimen.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +from typing import Optional, Dict + +from pydantic import BaseModel, ConfigDict +from datetime import datetime, timezone +# Local imported attributes +from pieriandx_pipeline_tools.pieriandx_enums.ethnicity import Ethnicity +from pieriandx_pipeline_tools.pieriandx_enums.race import Race +from pieriandx_pipeline_tools.pieriandx_enums.gender import Gender +from pieriandx_pipeline_tools.pieriandx_enums.specimen_type import SpecimenType +from .medical_record_number import MedicalRecordNumber + + +class Specimen(BaseModel): + case_accession_number: str + date_accessioned: datetime + date_received: datetime + date_collected: datetime + ethnicity: Optional[Ethnicity] = None + external_specimen_id: str + specimen_label: str + race: Optional[Race] = None + gender: Optional[Gender] = None + hl_7_specimen_id: Optional[str] = None + specimen_type: SpecimenType + + # Model configuration + model_config = ConfigDict(from_attributes=True) + + def to_dict(self) -> Dict: + return dict( + filter( + lambda dict_object: dict_object[1] is not None, + { + "accessionNumber": self.case_accession_number, + "dateAccessioned": self.date_accessioned.astimezone(timezone.utc).isoformat(sep="T", timespec="seconds").replace("+00:00", "Z"), + "dateReceived": self.date_received.astimezone(timezone.utc).isoformat(sep="T", timespec="seconds").replace("+00:00", "Z"), + "datecollected": self.date_collected.astimezone(timezone.utc).isoformat(sep="T", timespec="seconds").replace("+00:00", "Z"), + "ethnicity": self.ethnicity.value if self.ethnicity is not None else None, + "externalSpecimenId": self.external_specimen_id, + "gender": self.gender.value if self.gender is not None else None, + "hl7SpecimenId": self.hl_7_specimen_id, + "name": self.specimen_label, + "race": self.race.value if self.race is not None else None, + "type": self.specimen_type.to_dict() + }.items() + ) + ) + + +class IdentifiedSpecimen(Specimen): + # Required for Identified specimens + first_name: str + last_name: str + date_of_birth: datetime + medical_record_number: MedicalRecordNumber + + # Model configuration + model_config = ConfigDict(from_attributes=True) + + def to_dict(self): + return dict( + filter( + lambda dict_object: dict_object[1] is not None, + { + **super().to_dict(), + "firstName": self.first_name, + "lastName": self.last_name, + "dateOfBirth": self.date_of_birth.strftime("%Y-%m-%d"), + "medicalRecordNumbers": [self.medical_record_number.to_dict()] + }.items() + ) + ) + + +class DeIdentifiedSpecimen(Specimen): + # Required for De-Identified specimens + study_identifier: str + study_subject_identifier: str + + # Model configuration + model_config = ConfigDict(from_attributes=True) + + def to_dict(self): + return dict( + filter( + lambda dict_object: dict_object[1] is not None, + { + **super().to_dict(), + "studyIdentifier": self.study_identifier, + "studySubjectIdentifier": self.study_subject_identifier, + }.items() + ) + ) diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_models/specimen_sequencer_info.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_models/specimen_sequencer_info.py new file mode 100644 index 000000000..51255c9bd --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/pieriandx_models/specimen_sequencer_info.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 + +from pydantic import BaseModel, ConfigDict +from typing import Dict + +class SpecimenSequencerInfo(BaseModel): + # Model configuration + model_config = ConfigDict(from_attributes=True) + + run_id: str + case_accession_number: str + barcode: str + lane: int + sample_id: str + sample_type: str + + def to_dict(self) -> Dict: + return { + "accessionNumber": self.case_accession_number, + "barcode": self.barcode, + "lane": str(self.lane), + "sampleId": self.sample_id, + "sampleType": self.sample_type + } + + def to_informaticsjob_dict(self) -> Dict: + return { + "runId": self.run_id, + "barcode": self.barcode, + "lane": str(self.lane), + "sampleId": self.sample_id, + "sampleType": self.sample_type + } diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/utils/compression_helpers.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/utils/compression_helpers.py new file mode 100644 index 000000000..8deca2b34 --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/utils/compression_helpers.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 + +""" +Miscellaneous utilities for parsing through compressed strings +""" + +import json +from base64 import b64encode, b64decode +import gzip +from pathlib import Path +from typing import Dict, List, Union + + +def compress_dict(input_dict: Union[Dict, List]) -> str: + """ + Given a json input, compress to a base64 encoded string + + param: input_dict: input dictionary to compress + + Returns: gzipped compressed base64 encoded string + """ + + # Compress + return b64encode( + gzip.compress( + json.dumps(input_dict).encode('utf-8') + ) + ).decode("utf-8") + + +def decompress_dict(input_compressed_b64gz_str: str) -> Union[Dict, List]: + """ + Given a base64 encoded string, decompress and return the original dictionary + Args: + input_compressed_b64gz_str: + + Returns: decompressed dictionary or list + """ + + # Decompress + return json.loads( + gzip.decompress( + b64decode(input_compressed_b64gz_str.encode('utf-8')) + ) + ) + + +def decompress_file(input_file: Path, output_file: Path): + """ + Given a gzipped compressed file as an input, decompress and write to output file + Args: + input_file: + output_file: + + Returns: + + """ + with gzip.open(input_file, 'rb') as f_in: + with open(output_file, 'wb') as f_out: + f_out.write(f_in.read()) diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/utils/lambda_helpers.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/utils/lambda_helpers.py new file mode 100644 index 000000000..6310af5b1 --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/utils/lambda_helpers.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 + +""" +Lambda +""" + + +import boto3 +import typing + +if typing.TYPE_CHECKING: + from mypy_boto3_lambda import LambdaClient + from mypy_boto3_lambda.type_defs import InvocationResponseTypeDef + + +def get_aws_lambda_client() -> 'LambdaClient': + return boto3.client('lambda') + + +def run_lambda_function(function_name: str, payload: str) -> str: + client = get_aws_lambda_client() + response: InvocationResponseTypeDef = client.invoke( + FunctionName=function_name, + InvocationType='RequestResponse', + Payload=payload + ) + return response['Payload'].read().decode('utf-8') diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/utils/pieriandx_helpers.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/utils/pieriandx_helpers.py new file mode 100644 index 000000000..3d8c66696 --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/utils/pieriandx_helpers.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 + +""" +Use low-level pyriandx commands +""" + +from pyriandx.client import Client +from os import environ +import logging + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +def get_pieriandx_client( + email: str = environ.get("PIERIANDX_USER_EMAIL", None), + token: str = environ.get("PIERIANDX_USER_AUTH_TOKEN", None), + instiution: str = environ.get("PIERIANDX_INSTITUTION", None), + base_url: str = environ.get("PIERIANDX_BASE_URL", None) +) -> Client: + """ + Get the pieriandx client, validate environment variables + PIERIANDX_BASE_URL + PIERIANDX_INSTITUTION + PIERIANDX_USER_EMAIL + PIERIANDX_USER_AUTH_TOKEN + :return: + """ + + missing_env_vars = False + + # Check inputs + if email is None: + logger.error(f"Please set the environment variable 'PIERIANDX_USER_EMAIL'") + missing_env_vars = True + if token is None: + logger.error(f"Please set the environment variable 'PIERIANDX_USER_AUTH_TOKEN'") + missing_env_vars = True + if instiution is None: + logger.error(f"Please set the environment variable 'PIERIANDX_INSTITUTION'") + missing_env_vars = True + if base_url is None: + logger.error(f"Please set the environment variable 'PIERIANDX_BASE_URL'") + missing_env_vars = True + + if missing_env_vars: + logger.error("Missing PIERIANDX environment variable") + raise EnvironmentError + + # Return client object + return Client( + email=email, + key=token, + institution=instiution, + base_url=base_url, + key_is_auth_token=True + ) diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/utils/s3_helpers.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/utils/s3_helpers.py new file mode 100644 index 000000000..dd17f71e0 --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/utils/s3_helpers.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python + +""" +Download file from s3 +""" + +import typing +from pathlib import Path + +import boto3 +from os import environ + +if typing.TYPE_CHECKING: + from mypy_boto3_s3 import S3Client + + +def get_s3_client() -> 'S3Client': + return boto3.client( + 's3', + aws_access_key_id=environ['AWS_ACCESS_KEY_ID'], + aws_secret_access_key=environ['AWS_SECRET_ACCESS_KEY'] + ) + + +def upload_file(bucket: str, key: str, input_file_path: Path) -> None: + s3 = get_s3_client() + s3.upload_file(str(input_file_path), bucket, key.lstrip("/")) + + +def set_s3_access_cred_env_vars(): + from .secretsmanager_helpers import get_pieriandx_s3_access_credentials + access_creds = get_pieriandx_s3_access_credentials() + + for key, value in access_creds.items(): + environ[key] = value diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/utils/samplesheet_helpers.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/utils/samplesheet_helpers.py new file mode 100644 index 000000000..73592d5d1 --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/utils/samplesheet_helpers.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python + +# Standard imports +from typing import Dict +from pathlib import Path +from tempfile import NamedTemporaryFile + +# Custom imports +from wrapica.project_data import ( + read_icav2_file_contents, convert_uri_to_project_data_obj, + ProjectData +) +from v2_samplesheet_maker.functions.v2_samplesheet_reader import v2_samplesheet_reader + + +def read_v2_samplesheet(samplesheet_uri: str) -> Dict: + """ + Read in a v2 samplesheet from the given uri and return as a dictionary + + Args: + samplesheet_uri: Path to the samplesheet s3 or icav2 uri + + Returns: + + """ + + # Initialise tempfile object + temp_file = NamedTemporaryFile(suffix=".csv") + + # Get samplesheet uri as an icav2 projectdata object + icav2_project_data_obj: ProjectData = convert_uri_to_project_data_obj(samplesheet_uri) + + # Write icav2 file contents to temp file + read_icav2_file_contents( + icav2_project_data_obj.project_id, + icav2_project_data_obj.data.id, + output_path=Path(temp_file.name) + ) + + # Read the v2 samplesheet from the temp file + return v2_samplesheet_reader( + Path(temp_file.name) + ) diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/utils/secretsmanager_helpers.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/utils/secretsmanager_helpers.py new file mode 100644 index 000000000..d8200ce9d --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/utils/secretsmanager_helpers.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python + +import typing +from copy import copy +from typing import Dict +import boto3 +from time import sleep +from os import environ +import json + +if typing.TYPE_CHECKING: + from mypy_boto3_secretsmanager import SecretsManagerClient + from mypy_boto3_secretsmanager.type_defs import GetSecretValueResponseTypeDef + +ICAV2_BASE_URL = "https://ica.illumina.com/ica/rest" + + +def get_secrets_manager_client() -> 'SecretsManagerClient': + return boto3.client('secretsmanager') + + +def get_secret_value_from_aws_secrets_manager(secret_id: str) -> str: + client = get_secrets_manager_client() + + secret_value_obj: GetSecretValueResponseTypeDef = client.get_secret_value(SecretId=secret_id) + + return secret_value_obj['SecretString'] + + +def get_pieriandx_auth_token() -> str: + from .lambda_helpers import run_lambda_function + collection_token_lambda = environ.get("PIERIANDX_COLLECT_AUTH_TOKEN_LAMBDA_NAME") + + auth_token = run_lambda_function(collection_token_lambda, "") + + while auth_token is None or auth_token == 'null' or json.loads(auth_token).get("auth_token") is None: + sleep(5) + auth_token = run_lambda_function(collection_token_lambda, "") + + return json.loads(auth_token).get("auth_token") + + +def get_pieriandx_s3_access_credentials() -> Dict: + secret_id = environ.get("PIERIANDX_S3_ACCESS_CREDENTIALS_SECRET_ID") + + access_credentials = get_secret_value_from_aws_secrets_manager(secret_id) + + access_credentials_dict = json.loads(access_credentials) + + for key in copy(access_credentials_dict).keys(): + access_credentials_dict[key.replace("s3", "aws").upper()] = access_credentials_dict.pop(key) + + return access_credentials_dict + + +def set_pieriandx_env_vars(): + environ["PIERIANDX_USER_AUTH_TOKEN"] = get_pieriandx_auth_token() + + +def set_icav2_env_vars(): + """ + Set the icav2 environment variables + :return: + """ + environ["ICAV2_BASE_URL"] = ICAV2_BASE_URL + environ["ICAV2_ACCESS_TOKEN"] = get_secret_value_from_aws_secrets_manager( + environ["ICAV2_ACCESS_TOKEN_SECRET_ID"] + ) + + diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/step_function_templates/launch_pieriandx.asl.json b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/step_function_templates/launch_pieriandx.asl.json new file mode 100644 index 000000000..428266574 --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/step_function_templates/launch_pieriandx.asl.json @@ -0,0 +1,451 @@ +{ + "Comment": "A description of my state machine", + "StartAt": "move_inputs", + "States": { + "move_inputs": { + "Type": "Pass", + "Parameters": { + "workflow_inputs.$": "$" + }, + "Next": "Get attributes from SSM" + }, + "Get attributes from SSM": { + "Type": "Parallel", + "Next": "generate_pieriandx_objects", + "Branches": [ + { + "StartAt": "Get Available Dag Versions", + "States": { + "Get Available Dag Versions": { + "Type": "Task", + "Parameters": { + "Name": "${__dag_versions_ssm_parameter__}" + }, + "Resource": "arn:aws:states:::aws-sdk:ssm:getParameter", + "Next": "Dag Version set", + "ResultPath": "$.get_dag_versions_from_ssm_parameter_step", + "ResultSelector": { + "dag_versions_uri_map.$": "States.StringToJson($.Parameter.Value)" + } + }, + "Dag Version set": { + "Type": "Choice", + "Choices": [ + { + "Variable": "$.workflow_inputs.dagVersion", + "IsPresent": true, + "Comment": "Dag Version Is Set", + "Next": "Get Dag Version Value" + } + ], + "Default": "Get Default Dag Version Value" + }, + "Get Default Dag Version Value": { + "Type": "Pass", + "Next": "Get Dag Dict from SSM Parameter", + "Parameters": { + "dag_version": "${__default_dag_version__}" + }, + "ResultPath": "$.get_dag_version_step" + }, + "Get Dag Version Value": { + "Type": "Pass", + "Next": "Get Dag Dict from SSM Parameter", + "Parameters": { + "dag_version.$": "$.workflow_inputs.dagVersion" + }, + "ResultPath": "$.get_dag_version_step" + }, + "Get Dag Dict from SSM Parameter": { + "Type": "Pass", + "End": true, + "Parameters": { + "dag_version_obj.$": "States.ArrayGetItem($.get_dag_versions_from_ssm_parameter_step.dag_versions_uri_map[?(@.dagName==$.get_dag_version_step.dag_version)], 0)" + }, + "ResultPath": "$.get_dag_version_step" + } + } + }, + { + "StartAt": "Get available panel names", + "States": { + "Get available panel names": { + "Type": "Task", + "Parameters": { + "Name": "${__panel_names_ssm_parameter__}" + }, + "Resource": "arn:aws:states:::aws-sdk:ssm:getParameter", + "Next": "Panel Name Set", + "ResultSelector": { + "panel_names_map.$": "States.StringToJson($.Parameter.Value)" + }, + "ResultPath": "$.get_panel_name_versions_from_ssm_parameter_step" + }, + "Panel Name Set": { + "Type": "Choice", + "Choices": [ + { + "Variable": "$.workflowInputs.panelName", + "IsPresent": true, + "Next": "Get Panel Name from inputs", + "Comment": "Panel Name is set" + } + ], + "Default": "Get Default Panel Name" + }, + "Get Default Panel Name": { + "Type": "Pass", + "Next": "Get Panel Name Value from SSM Parameter", + "Parameters": { + "panel_name": "${__default_panel_name__}" + }, + "ResultPath": "$.get_panel_name_step" + }, + "Get Panel Name from inputs": { + "Type": "Pass", + "Next": "Get Panel Name Value from SSM Parameter", + "Parameters": { + "panel_name.$": "$.workflow_inputs.panelName" + }, + "ResultPath": "$.get_panel_name_step" + }, + "Get Panel Name Value from SSM Parameter": { + "Type": "Pass", + "End": true, + "Parameters": { + "panel_name.$": "States.ArrayGetItem($.get_panel_name_versions_from_ssm_parameter_step.panel_names_map[?(@.panelName==$.get_panel_name_step.panel_name)].panelId, 0)" + }, + "ResultPath": "$.get_panel_name_step" + } + } + } + ], + "ResultSelector": { + "dag_version_obj.$": "$[0].get_dag_version_step.dag_version_obj", + "panel_name.$": "$[1].get_panel_name_step.panel_name" + }, + "ResultPath": "$.get_ssm_parameters_step" + }, + "generate_pieriandx_objects": { + "Type": "Task", + "Resource": "arn:aws:states:::lambda:invoke", + "Parameters": { + "FunctionName": "${__generate_pieriandx_objects_lambda_function_arn__}", + "Payload": { + "dag.$": "$.get_ssm_parameters_step.dag_version_obj", + "panel_name.$": "$.get_ssm_parameters_step.panel_name", + "case_metadata.$": "$.workflow_inputs.payload.data.inputs.caseMetadata", + "data_files.$": "$.workflow_inputs.payload.data.inputs.dataFiles", + "samplesheet_uri.$": "$.workflow_inputs.payload.data.inputs.dataFiles.samplesheetUri", + "sequencerrun_s3_path_root": "${__sequencerrun_s3_path_root__}", + "instrument_run_id.$": "$.workflow_inputs.payload.data.inputs.instrumentRunId", + "portal_run_id.$": "$.workflow_inputs.portalRunId" + } + }, + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 1, + "MaxAttempts": 3, + "BackoffRate": 2 + } + ], + "ResultPath": "$.generate_pieriandx_objects_step", + "ResultSelector": { + "case_creation_obj.$": "$.Payload.case_creation_obj", + "sequencerrun_creation_obj.$": "$.Payload.sequencerrun_creation_obj", + "informaticsjob_creation_obj.$": "$.Payload.informaticsjob_creation_obj", + "data_files.$": "$.Payload.data_files", + "sequencerrun_s3_path.$": "$.Payload.sequencerrun_s3_path", + "sample_name.$": "$.Payload.sample_name" + }, + "Next": "add_pieriandx_db_entries" + }, + "add_pieriandx_db_entries": { + "Type": "Parallel", + "Branches": [ + { + "StartAt": "add_case_accession_number_partition_key", + "States": { + "add_case_accession_number_partition_key": { + "Type": "Task", + "Resource": "arn:aws:states:::dynamodb:putItem", + "Parameters": { + "TableName": "${__table_name__}", + "Item": { + "id.$": "$.workflow_inputs.payload.data.inputs.caseMetadata.caseAccessionNumber", + "id_type": "case_accession_number", + "portal_run_id": { + "S.$": "$.workflow_inputs.portalRunId" + } + } + }, + "End": true + } + } + }, + { + "StartAt": "add_portal_run_id_partition_key", + "States": { + "add_portal_run_id_partition_key": { + "Type": "Task", + "Resource": "arn:aws:states:::dynamodb:putItem", + "Parameters": { + "TableName": "${__table_name__}", + "Item": { + "id.$": "$.workflow_inputs.portalRunId", + "id_type": "portal_run_id", + "state_machine_execution_arn": { + "S.$": "$$.Execution.Id" + }, + "workflow_run_name": { + "S.$": "$.workflow_inputs.workflowRunName" + }, + "linked_libraries": { + "S.$": "States.JsonToString($.workflow_inputs.linkedLibraries)" + }, + "case_accession_number": { + "S.$": "$.workflow_inputs.payload.data.inputs.caseMetadata.caseAccessionNumber" + }, + "event_inputs": { + "S.$": "States.JsonToString($.workflow_inputs.payload.data.inputs)" + }, + "engine_parameters": { + "S.$": "States.JsonToString($.workflow_inputs.payload.data.engineParameters)" + }, + "tags": { + "S.$": "States.JsonToString($.workflow_inputs.payload.data.tags)" + }, + "data_files": { + "S.$": "States.JsonToString($.generate_pieriandx_objects_step.data_files)" + }, + "sample_name": { + "S.$": "$.generate_pieriandx_objects_step.sample_name" + }, + "samplesheet_uri": { + "S.$": "$.workflow_inputs.payload.data.inputs.dataFiles.samplesheetUri" + }, + "sequencerrun_s3_path": { + "S.$": "$.generate_pieriandx_objects_step.sequencerrun_s3_path" + }, + "case_creation_obj": { + "S.$": "States.JsonToString($.generate_pieriandx_objects_step.case_creation_obj)" + }, + "case_id": { + "N": "-1" + }, + "sequencerrun_creation_obj": { + "S.$": "States.JsonToString($.generate_pieriandx_objects_step.sequencerrun_creation_obj)" + }, + "sequencerrun_id": { + "N": "-1" + }, + "informaticsjob_creation_obj": { + "S.$": "States.JsonToString($.generate_pieriandx_objects_step.informaticsjob_creation_obj)" + }, + "informaticsjob_id": { + "N": "-1" + }, + "report_id": { + "N": "-1" + }, + "job_status": { + "S": "" + }, + "report_status": { + "S": "" + } + } + }, + "End": true + } + } + } + ], + "ResultPath": "$.add_pieriandx_db_entries_step", + "Next": "wait_one_second" + }, + "wait_one_second": { + "Type": "Wait", + "Seconds": 1, + "Comment": "Allow the database to sync\n", + "Next": "create_pieriandx_prelaunch_objects" + }, + "create_pieriandx_prelaunch_objects": { + "Type": "Parallel", + "Branches": [ + { + "StartAt": "create_case_execution", + "States": { + "create_case_execution": { + "Type": "Task", + "Resource": "arn:aws:states:::states:startExecution.sync:2", + "Parameters": { + "StateMachineArn": "${__create_case_sfn__}", + "Input": { + "portal_run_id.$": "$.workflow_inputs.portalRunId" + } + }, + "ResultPath": "$.create_case_execution_step", + "End": true + } + } + }, + { + "StartAt": "create_sequencerrun_execution", + "States": { + "create_sequencerrun_execution": { + "Type": "Task", + "Resource": "arn:aws:states:::states:startExecution.sync:2", + "Parameters": { + "StateMachineArn": "${__create_sequencerrun_sfn__}", + "Input": { + "portal_run_id.$": "$.workflow_inputs.portalRunId" + } + }, + "ResultPath": "$.create_sequencerrun_execution_step", + "End": true + } + } + } + ], + "ResultPath": "$.create_pieriandx_prelaunch_objects_step", + "Next": "create_informaticsjob_execution" + }, + "create_informaticsjob_execution": { + "Type": "Task", + "Resource": "arn:aws:states:::states:startExecution.sync:2", + "Parameters": { + "StateMachineArn": "${__create_informaticsjob_sfn__}", + "Input": { + "portal_run_id.$": "$.workflow_inputs.portalRunId" + } + }, + "ResultPath": "$.create_informaticsjob_execution_step", + "Next": "Get Portal Run Id" + }, + "Get Portal Run Id": { + "Type": "Task", + "Resource": "arn:aws:states:::dynamodb:getItem", + "Parameters": { + "TableName": "${__table_name__}", + "Key": { + "id.$": "$.workflow_inputs.portalRunId", + "id_type": "portal_run_id" + } + }, + "Next": "Set engine parameters", + "ResultPath": "$.get_portal_run_id_db_step" + }, + "Set engine parameters": { + "Type": "Pass", + "Next": "Update dbs", + "ResultPath": "$.set_engine_parameters_step", + "Parameters": { + "engine_parameters.$": "States.JsonMerge($.workflow_inputs.payload.data.engineParameters, States.StringToJson(States.Format('\\{\"{}\":\"{}\",\"{}\":\"{}\"\\}', 'caseId', $.get_portal_run_id_db_step.Item.case_id.N, 'informaticsJobId', $.get_portal_run_id_db_step.Item.informaticsjob_id.N)), false)" + } + }, + "Update dbs": { + "Type": "Parallel", + "Branches": [ + { + "StartAt": "Register running informatics job", + "States": { + "Register running informatics job": { + "Type": "Task", + "Resource": "arn:aws:states:::dynamodb:putItem", + "Parameters": { + "TableName": "${__table_name__}", + "Item": { + "id.$": "$.workflow_inputs.portalRunId", + "id_type": "running_jobs", + "case_id": { + "N.$": "States.Format('{}', $.get_portal_run_id_db_step.Item.case_id.N)" + }, + "informaticsjob_id": { + "N.$": "States.Format('{}', $.get_portal_run_id_db_step.Item.informaticsjob_id.N)" + }, + "report_id": { + "N": "-1" + }, + "job_status": { + "S": "waiting" + }, + "report_status": { + "S": "" + }, + "workflow_status": { + "S": "RUNNING" + } + } + }, + "ResultPath": null, + "End": true + } + } + }, + { + "StartAt": "Update engine parameters", + "States": { + "Update engine parameters": { + "Type": "Task", + "Resource": "arn:aws:states:::dynamodb:updateItem", + "Parameters": { + "TableName": "${__table_name__}", + "Key": { + "id.$": "$.workflow_inputs.portalRunId", + "id_type": "portal_run_id" + }, + "UpdateExpression": "SET engine_parameters = :engine_parameters", + "ExpressionAttributeValues": { + ":engine_parameters": { + "S.$": "States.JsonToString($.set_engine_parameters_step.engine_parameters)" + } + } + }, + "End": true + } + } + } + ], + "ResultPath": null, + "Next": "Initialised PierianDx Job" + }, + "Initialised PierianDx Job": { + "Type": "Task", + "Resource": "arn:aws:states:::events:putEvents", + "Parameters": { + "Entries": [ + { + "DetailType": "${__event_detail_type__}", + "EventBusName": "${__event_bus_name__}", + "Source": "${__event_source__}", + "Detail": { + "portalRunId.$": "$.workflow_inputs.portalRunId", + "timestamp.$": "$$.State.EnteredTime", + "status": "RUNNING", + "workflowName": "${__workflow_name__}", + "workflowVersion": "${__workflow_version__}", + "workflowRunName.$": "$.workflow_inputs.workflowRunName", + "linkedLibraries.$": "$.workflow_inputs.linkedLibraries", + "payload": { + "version": "${__payload_version__}", + "data": { + "inputs.$": "$.workflow_inputs.payload.data.inputs", + "engineParameters.$": "$.set_engine_parameters_step.engine_parameters", + "tags.$": "$.workflow_inputs.payload.data.tags" + } + } + } + } + ] + }, + "End": true + } + } +} diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/step_function_templates/launch_pieriandx_case_creation.asl.json b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/step_function_templates/launch_pieriandx_case_creation.asl.json new file mode 100644 index 000000000..df3dd4f9a --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/step_function_templates/launch_pieriandx_case_creation.asl.json @@ -0,0 +1,114 @@ +{ + "Comment": "A description of my state machine", + "StartAt": "move_inputs", + "States": { + "move_inputs": { + "Type": "Pass", + "Parameters": { + "workflow_inputs.$": "$" + }, + "Next": "get_case_creation_object" + }, + "get_case_creation_object": { + "Type": "Task", + "Resource": "arn:aws:states:::dynamodb:getItem", + "Parameters": { + "TableName": "${__table_name__}", + "Key": { + "id.$": "$.workflow_inputs.portal_run_id", + "id_type": "portal_run_id" + } + }, + "ResultPath": "$.get_case_creation_object_step", + "ResultSelector": { + "portal_run_id.$": "$.Item.id.S", + "case_creation_obj.$": "States.StringToJson($.Item.case_creation_obj.S)" + }, + "Next": "generate_case" + }, + "generate_case": { + "Type": "Task", + "Resource": "arn:aws:states:::lambda:invoke", + "Parameters": { + "Payload": { + "case_creation_obj.$": "$.get_case_creation_object_step.case_creation_obj" + }, + "FunctionName": "${__generate_case_lambda_function_arn__}" + }, + "TimeoutSeconds": 30, + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 1, + "MaxAttempts": 3, + "BackoffRate": 2 + } + ], + "Next": "update_database", + "ResultPath": "$.generate_case_step", + "ResultSelector": { + "case_id.$": "$.Payload.id" + } + }, + "update_database": { + "Type": "Parallel", + "Branches": [ + { + "StartAt": "add_case_id_partition_key", + "States": { + "add_case_id_partition_key": { + "Type": "Task", + "Resource": "arn:aws:states:::dynamodb:putItem", + "Parameters": { + "TableName": "${__table_name__}", + "Item": { + "id.$": "$.generate_case_step.case_id", + "id_type": "case_id", + "portal_run_id": { + "S.$": "$.workflow_inputs.portal_run_id" + } + } + }, + "End": true + } + } + }, + { + "StartAt": "update_portal_run_id_partition_key_with_case_id", + "States": { + "update_portal_run_id_partition_key_with_case_id": { + "Type": "Task", + "Resource": "arn:aws:states:::dynamodb:updateItem", + "Parameters": { + "TableName": "${__table_name__}", + "Key": { + "id.$": "$.workflow_inputs.portal_run_id", + "id_type": "portal_run_id" + }, + "UpdateExpression": "SET case_id = :case_id", + "ExpressionAttributeValues": { + ":case_id": { + "N.$": "$.generate_case_step.case_id" + } + } + }, + "End": true + } + } + } + ], + "Next": "wait_one_second" + }, + "wait_one_second": { + "Type": "Wait", + "Seconds": 1, + "End": true, + "Comment": "Wait for database to sync" + } + } +} diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/step_function_templates/launch_pieriandx_informaticsjob_creation.asl.json b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/step_function_templates/launch_pieriandx_informaticsjob_creation.asl.json new file mode 100644 index 000000000..603f47d5b --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/step_function_templates/launch_pieriandx_informaticsjob_creation.asl.json @@ -0,0 +1,119 @@ +{ + "Comment": "A description of my state machine", + "StartAt": "move_inputs", + "States": { + "move_inputs": { + "Type": "Pass", + "Parameters": { + "workflow_inputs.$": "$" + }, + "Next": "get_informaticsjob_object" + }, + "get_informaticsjob_object": { + "Type": "Task", + "Resource": "arn:aws:states:::dynamodb:getItem", + "Parameters": { + "TableName": "${__table_name__}", + "Key": { + "id.$": "$.workflow_inputs.portal_run_id", + "id_type": "portal_run_id" + } + }, + "ResultSelector": { + "portal_run_id.$": "$.Item.id.S", + "informaticsjob_creation_obj.$": "States.StringToJson($.Item.informaticsjob_creation_obj.S)", + "case_id.$": "$.Item.case_id.N" + }, + "Next": "generate_informaticsjob", + "ResultPath": "$.get_informaticsjob_object" + }, + "generate_informaticsjob": { + "Type": "Task", + "Resource": "arn:aws:states:::lambda:invoke", + "TimeoutSeconds": 30, + "Parameters": { + "FunctionName": "${__generate_informaticsjob_lambda_function_arn__}", + "Payload": { + "informaticsjob_creation_obj.$": "$.get_informaticsjob_object.informaticsjob_creation_obj", + "case_id.$": "$.get_informaticsjob_object.case_id" + } + }, + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 1, + "MaxAttempts": 3, + "BackoffRate": 2 + } + ], + "Next": "update_database", + "ResultSelector": { + "informaticsjob_id.$": "$.Payload.jobId" + }, + "ResultPath": "$.generate_informaticsjob_step" + }, + "update_database": { + "Type": "Parallel", + "Branches": [ + { + "StartAt": "add_informaticsjob_id_partition_key", + "States": { + "add_informaticsjob_id_partition_key": { + "Type": "Task", + "Resource": "arn:aws:states:::dynamodb:putItem", + "Parameters": { + "TableName": "${__table_name__}", + "Item": { + "id.$": "$.generate_informaticsjob_step.informaticsjob_id", + "id_type": "informaticsjob_id", + "portal_run_id": { + "S.$": "$.workflow_inputs.portal_run_id" + } + } + }, + "End": true + } + } + }, + { + "StartAt": "update_portal_run_id_partition_key_with_informaticsjob_id", + "States": { + "update_portal_run_id_partition_key_with_informaticsjob_id": { + "Type": "Task", + "Resource": "arn:aws:states:::dynamodb:updateItem", + "Parameters": { + "TableName": "${__table_name__}", + "Key": { + "id.$": "$.workflow_inputs.portal_run_id", + "id_type": "portal_run_id" + }, + "UpdateExpression": "SET informaticsjob_id = :informaticsjob_id, job_status = :job_status", + "ExpressionAttributeValues": { + ":informaticsjob_id": { + "N.$": "$.generate_informaticsjob_step.informaticsjob_id" + }, + ":job_status": { + "S": "waiting" + } + } + }, + "End": true + } + } + } + ], + "ResultPath": "$.update_database", + "Next": "wait_one_second" + }, + "wait_one_second": { + "Type": "Wait", + "Seconds": 1, + "End": true + } + } +} diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/step_function_templates/launch_pieriandx_sequencerrun_creation.asl.json b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/step_function_templates/launch_pieriandx_sequencerrun_creation.asl.json new file mode 100644 index 000000000..dd5ea644d --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/step_function_templates/launch_pieriandx_sequencerrun_creation.asl.json @@ -0,0 +1,280 @@ +{ + "Comment": "A description of my state machine", + "StartAt": "move_inputs", + "States": { + "move_inputs": { + "Type": "Pass", + "Parameters": { + "workflow_inputs.$": "$" + }, + "Next": "get_sequencerrun_creation_object" + }, + "get_sequencerrun_creation_object": { + "Type": "Task", + "Resource": "arn:aws:states:::dynamodb:getItem", + "Parameters": { + "TableName": "${__table_name__}", + "Key": { + "id.$": "$.workflow_inputs.portal_run_id", + "id_type": "portal_run_id" + } + }, + "ResultPath": "$.get_sequencerrun_creation_object_step", + "ResultSelector": { + "portal_run_id": "$.Item.id.S", + "sequencerrun_creation_obj.$": "States.StringToJson($.Item.sequencerrun_creation_obj.S)", + "data_files.$": "States.StringToJson($.Item.data_files.S)", + "samplesheet_uri.$": "$.Item.samplesheet_uri.S", + "sequencerrun_s3_path.$": "$.Item.sequencerrun_s3_path.S" + }, + "Next": "upload_data" + }, + "upload_data": { + "Type": "Parallel", + "Branches": [ + { + "StartAt": "upload_data_files_to_s3", + "States": { + "upload_data_files_to_s3": { + "Type": "Map", + "ItemProcessor": { + "ProcessorConfig": { + "Mode": "INLINE" + }, + "StartAt": "upload_pieriandx_sample_data_to_s3", + "States": { + "upload_pieriandx_sample_data_to_s3": { + "Type": "Task", + "Resource": "arn:aws:states:::lambda:invoke", + "OutputPath": "$.Payload", + "Parameters": { + "Payload": { + "src_uri.$": "$.src_uri", + "dest_uri.$": "$.dest_uri", + "needs_decompression.$": "$.needs_decompression", + "contents.$": "$.contents" + }, + "FunctionName": "${__upload_data_to_s3_lambda_function_arn__}" + }, + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 1, + "MaxAttempts": 3, + "BackoffRate": 2 + } + ], + "End": true + } + } + }, + "ItemsPath": "$.get_sequencerrun_creation_object_step.data_files", + "End": true + } + } + }, + { + "StartAt": "generate_samplesheet_str", + "States": { + "generate_samplesheet_str": { + "Type": "Task", + "Resource": "arn:aws:states:::lambda:invoke", + "Parameters": { + "FunctionName": "${__generate_samplesheet_lambda_function_arn__}", + "Payload": { + "samplesheet_uri.$": "$.get_sequencerrun_creation_object_step.samplesheet_uri" + } + }, + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 1, + "MaxAttempts": 3, + "BackoffRate": 2 + } + ], + "ResultSelector": { + "samplesheet_str.$": "$.Payload.samplesheet_str" + }, + "ResultPath": "$.generate_samplesheet_str_step", + "Next": "upload_samplesheet_str_to_s3" + }, + "upload_samplesheet_str_to_s3": { + "Type": "Task", + "Resource": "arn:aws:states:::lambda:invoke", + "OutputPath": "$.Payload", + "Parameters": { + "FunctionName": "${__upload_data_to_s3_lambda_function_arn__}", + "Payload": { + "dest_uri.$": "States.Format('{}/SampleSheet.csv', $.get_sequencerrun_creation_object_step.sequencerrun_s3_path)", + "contents.$": "$.generate_samplesheet_str_step.samplesheet_str" + } + }, + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 1, + "MaxAttempts": 3, + "BackoffRate": 2 + } + ], + "End": true + } + } + } + ], + "ResultPath": "$.upload_data_step", + "Next": "generate_sequencerrun_creation_obj" + }, + "generate_sequencerrun_creation_obj": { + "Type": "Task", + "Resource": "arn:aws:states:::lambda:invoke", + "Parameters": { + "FunctionName": "${__generate_sequencerrun_case_lambda_function_arn__}", + "Payload": { + "sequencerrun_creation_obj.$": "$.get_sequencerrun_creation_object_step.sequencerrun_creation_obj" + } + }, + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 1, + "MaxAttempts": 3, + "BackoffRate": 2 + } + ], + "ResultPath": "$.generate_sequencerrun_creation_object_step", + "ResultSelector": { + "sequencerrun_id.$": "$.Payload.id" + }, + "Next": "update_database" + }, + "update_database": { + "Type": "Parallel", + "Branches": [ + { + "StartAt": "add_sequencer_run_partition_key", + "States": { + "add_sequencer_run_partition_key": { + "Type": "Task", + "Resource": "arn:aws:states:::dynamodb:putItem", + "Parameters": { + "TableName": "${__table_name__}", + "Item": { + "id.$": "$.generate_sequencerrun_creation_object_step.sequencerrun_id", + "id_type": "sequencerrun_id", + "portal_run_id": { + "S.$": "$.workflow_inputs.portal_run_id" + } + } + }, + "End": true + } + } + }, + { + "StartAt": "update_portal_run_id_partition_key_with_sequencerrun_id", + "States": { + "update_portal_run_id_partition_key_with_sequencerrun_id": { + "Type": "Task", + "Resource": "arn:aws:states:::dynamodb:updateItem", + "Parameters": { + "TableName": "${__table_name__}", + "Key": { + "id.$": "$.workflow_inputs.portal_run_id", + "id_type": "portal_run_id" + }, + "UpdateExpression": "SET sequencerrun_id = :sequencerrun_id", + "ExpressionAttributeValues": { + ":sequencerrun_id": { + "N.$": "$.generate_sequencerrun_creation_object_step.sequencerrun_id" + } + } + }, + "End": true + } + } + } + ], + "Next": "add_vcf_workflow_txt", + "ResultPath": "$.update_database_step" + }, + "add_vcf_workflow_txt": { + "Type": "Task", + "Resource": "arn:aws:states:::lambda:invoke", + "Parameters": { + "FunctionName": "${__upload_data_to_s3_lambda_function_arn__}", + "Payload": { + "contents": "", + "dest_uri.$": "States.Format('{}/VcfWorkflow.txt', $.get_sequencerrun_creation_object_step.sequencerrun_s3_path)" + } + }, + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 1, + "MaxAttempts": 3, + "BackoffRate": 2 + } + ], + "ResultPath": "$.add_vcf_workflow_txt_step", + "Next": "add_done_txt" + }, + "add_done_txt": { + "Type": "Task", + "Resource": "arn:aws:states:::lambda:invoke", + "Parameters": { + "FunctionName": "${__upload_data_to_s3_lambda_function_arn__}", + "Payload": { + "contents": "", + "dest_uri.$": "States.Format('{}/done.txt', $.get_sequencerrun_creation_object_step.sequencerrun_s3_path)" + } + }, + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 1, + "MaxAttempts": 3, + "BackoffRate": 2 + } + ], + "ResultPath": "$.add_done_txt_step", + "Next": "wait_one_second" + }, + "wait_one_second": { + "Type": "Wait", + "Seconds": 1, + "End": true + } + } +} diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/step_function_templates/monitor_runs.asl.json b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/step_function_templates/monitor_runs.asl.json new file mode 100644 index 000000000..1bfb364fd --- /dev/null +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/step_function_templates/monitor_runs.asl.json @@ -0,0 +1,277 @@ +{ + "Comment": "A description of my state machine", + "StartAt": "Get all current running jobs", + "States": { + "Get all current running jobs": { + "Type": "Task", + "Parameters": { + "TableName": "${__table_name__}", + "ExpressionAttributeValues": { + ":id_type": { + "S": "running_jobs" + } + }, + "ExpressionAttributeNames": { + "#id_type": "id_type" + }, + "FilterExpression": "#id_type = :id_type" + }, + "Resource": "arn:aws:states:::aws-sdk:dynamodb:scan", + "ResultPath": "$.get_current_running_jobs_step", + "Next": "Check items not empty" + }, + "Check items not empty": { + "Type": "Choice", + "Choices": [ + { + "Variable": "$.get_current_running_jobs_step.Items", + "IsPresent": true, + "Comment": "Items list exists", + "Next": "Get num of running jobs" + } + ], + "Default": "Pass" + }, + "Get num of running jobs": { + "Type": "Pass", + "Next": "Check items list is more than 0", + "Parameters": { + "num_jobs.$": "States.ArrayLength($.get_current_running_jobs_step.Items)" + }, + "ResultPath": "$.get_num_jobs_step" + }, + "Check items list is more than 0": { + "Type": "Choice", + "Choices": [ + { + "Variable": "$.get_num_jobs_step.num_jobs", + "NumericGreaterThan": 0, + "Next": "Iterate running jobs partition", + "Comment": "At least one job running" + } + ], + "Default": "Pass" + }, + "Iterate running jobs partition": { + "Type": "Map", + "ItemsPath": "$.get_current_running_jobs_step.Items", + "ItemSelector": { + "job_db_item.$": "$$.Map.Item.Value" + }, + "ItemProcessor": { + "ProcessorConfig": { + "Mode": "INLINE" + }, + "StartAt": "Get Current Job Status", + "States": { + "Get Current Job Status": { + "Type": "Task", + "Resource": "arn:aws:states:::lambda:invoke", + "Parameters": { + "FunctionName": "${__get_current_job_status_lambda_function_arn__}", + "Payload": { + "current_job_status.$": "$.job_db_item.job_status.S", + "current_report_status.$": "$.job_db_item.report_status.S", + "case_id.$": "States.StringToJson($.job_db_item.case_id.N)", + "informaticsjob_id.$": "States.StringToJson($.job_db_item.informaticsjob_id.N)", + "report_id.$": "States.StringToJson($.job_db_item.report_id.N)" + } + }, + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 1, + "MaxAttempts": 3, + "BackoffRate": 2 + } + ], + "ResultSelector": { + "job_status.$": "$.Payload.job_status", + "job_status_bool.$": "$.Payload.job_status_bool", + "report_id.$": "$.Payload.report_id", + "report_status.$": "$.Payload.report_status", + "report_status_bool.$": "$.Payload.report_status_bool", + "expression_attribute_values_dict.$": "$.Payload.expression_attribute_values_dict", + "update_expression_str.$": "$.Payload.update_expression_str", + "job_status_changed.$": "$.Payload.job_status_changed" + }, + "ResultPath": "$.get_current_status_step", + "Next": "Job Status Changed" + }, + "Job Status Changed": { + "Type": "Choice", + "Choices": [ + { + "Variable": "$.get_current_status_step.job_status_changed", + "BooleanEquals": true, + "Next": "Update Changes", + "Comment": "The job status has changed" + } + ], + "Default": "No Change" + }, + "Update Changes": { + "Type": "Parallel", + "Branches": [ + { + "StartAt": "Update portal run id job status", + "States": { + "Update portal run id job status": { + "Type": "Task", + "Resource": "arn:aws:states:::dynamodb:updateItem", + "Parameters": { + "TableName": "${__table_name__}", + "Key": { + "id.$": "$.job_db_item.id.S", + "id_type": "portal_run_id" + }, + "UpdateExpression.$": "$.get_current_status_step.update_expression_str", + "ExpressionAttributeValues.$": "$.get_current_status_step.expression_attribute_values_dict" + }, + "End": true + } + } + }, + { + "StartAt": "Is Report Status Terminal", + "States": { + "Is Report Status Terminal": { + "Type": "Choice", + "Choices": [ + { + "Variable": "$.get_current_status_step.report_status_bool", + "IsBoolean": true, + "Next": "Delete Row from Running Jobs Partition", + "Comment": "Report Status Terminal" + } + ], + "Default": "Update running jobs partition row" + }, + "Update running jobs partition row": { + "Type": "Task", + "Resource": "arn:aws:states:::dynamodb:updateItem", + "Parameters": { + "TableName": "${__table_name__}", + "Key": { + "id.$": "$.job_db_item.id.S", + "id_type": "running_jobs" + }, + "UpdateExpression.$": "$.get_current_status_step.update_expression_str", + "ExpressionAttributeValues.$": "$.get_current_status_step.expression_attribute_values_dict" + }, + "End": true + }, + "Delete Row from Running Jobs Partition": { + "Type": "Task", + "Resource": "arn:aws:states:::dynamodb:deleteItem", + "Parameters": { + "TableName": "${__table_name__}", + "Key": { + "id.$": "$.job_db_item.id.S", + "id_type": "running_jobs" + } + }, + "End": true + } + } + } + ], + "ResultPath": null, + "Next": "Get Portal Run ID Row Partition" + }, + "Get Portal Run ID Row Partition": { + "Type": "Task", + "Resource": "arn:aws:states:::dynamodb:getItem", + "Parameters": { + "TableName": "${__table_name__}", + "Key": { + "id.$": "$.job_db_item.id.S", + "id_type": "portal_run_id" + } + }, + "ResultPath": "$.get_portal_run_db_step", + "Next": "Generate Data Payload" + }, + "Generate Data Payload": { + "Type": "Task", + "Resource": "arn:aws:states:::lambda:invoke", + "Parameters": { + "FunctionName": "${__generate_data_payload_lambda_function_arn__}", + "Payload": { + "inputs.$": "States.StringToJson($.get_portal_run_db_step.Item.event_inputs.S)", + "engine_parameters.$": "States.StringToJson($.get_portal_run_db_step.Item.engine_parameters.S)", + "tags.$": "States.StringToJson($.get_portal_run_db_step.Item.tags.S)", + "report_status.$": "$.get_portal_run_db_step.Item.report_status.S", + "case_id.$": "$.get_portal_run_db_step.Item.case_id.N", + "job_id.$": "$.get_portal_run_db_step.Item.informaticsjob_id.N", + "case_accession_number.$": "$.get_portal_run_db_step.Item.case_accession_number.S", + "report_id.$": "$.get_portal_run_db_step.Item.report_id.N", + "pieriandx_base_url": "https://app.uat.pieriandx.com/cgw-api/v2.0.0", + "sample_name.$": "$.get_portal_run_db_step.Item.sample_name.S" + } + }, + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 1, + "MaxAttempts": 3, + "BackoffRate": 2 + } + ], + "Next": "Push event of Informatics Job Change", + "ResultPath": "$.get_data_payload_step", + "ResultSelector": { + "data_payload.$": "$.Payload.data_payload" + } + }, + "Push event of Informatics Job Change": { + "Type": "Task", + "Resource": "arn:aws:states:::events:putEvents", + "Parameters": { + "Entries": [ + { + "DetailType": "${__event_detail_type__}", + "EventBusName": "${__event_bus_name__}", + "Source": "${__event_source__}", + "Detail": { + "portalRunId.$": "$.get_portal_run_db_step.Item.id.S", + "timestamp.$": "$$.State.EnteredTime", + "status.$": "$.get_portal_run_db_step.Item.workflow_status.S", + "workflowName": "${__workflow_name__}", + "workflowVersion": "${__workflow_version__}", + "workflowRunName.$": "$.get_portal_run_db_step.Item.workflow_run_name.S", + "linkedLibraries.$": "States.StringToJson($.get_portal_run_db_step.Item.linkedLibraries.S)", + "payload": { + "version": "${__payload_version__}", + "data.$": "$.get_data_payload_step.data_payload" + } + } + } + ] + }, + "End": true + }, + "No Change": { + "Type": "Pass", + "End": true + } + } + }, + "End": true + }, + "Pass": { + "Type": "Pass", + "End": true + } + } +} diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/clag/part_1/samplesheet-event-shower/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/clag/part_1/samplesheet-event-shower/index.ts index e30728d3c..dfbcd7e52 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/clag/part_1/samplesheet-event-shower/index.ts +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/clag/part_1/samplesheet-event-shower/index.ts @@ -20,7 +20,7 @@ export class NewSamplesheetEventShowerConstruct extends Construct { prefix: 'clag-new-ss-event-shower', // Tables tablePartition: { - samplesheetByInstrumentRun: 'samplesheet_by_instrument_run', + instrumentRun: 'instrument_run', subject: 'subject', library: 'library', project: 'project', @@ -149,8 +149,8 @@ export class NewSamplesheetEventShowerConstruct extends Construct { __table_name__: props.tableObj.tableName, __subject_table_partition_name__: this.newSamplesheetEventShowerMap.tablePartition.subject, __library_table_partition_name__: this.newSamplesheetEventShowerMap.tablePartition.library, - __samplesheet_table_partition_name__: - this.newSamplesheetEventShowerMap.tablePartition.samplesheetByInstrumentRun, + __instrument_run_table_partition_name__: + this.newSamplesheetEventShowerMap.tablePartition.instrumentRun, __project_table_partition_name__: this.newSamplesheetEventShowerMap.tablePartition.project, // Lambdas diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/clag/part_1/samplesheet-event-shower/lambdas/generate_event_data_objects_py/generate_event_data_objects.py b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/clag/part_1/samplesheet-event-shower/lambdas/generate_event_data_objects_py/generate_event_data_objects.py index ec8a2859d..37b204e98 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/clag/part_1/samplesheet-event-shower/lambdas/generate_event_data_objects_py/generate_event_data_objects.py +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/clag/part_1/samplesheet-event-shower/lambdas/generate_event_data_objects_py/generate_event_data_objects.py @@ -3,59 +3,21 @@ """ Generate event data objects -Given an instrument run id, samplesheet, library_obj_list, specimen_obj_list, subject_obj_list, +Given an instrument run id, samplesheet, library_obj_list, sample_obj_list, subject_obj_list, Generate the events for * Start of the SampleSheet Shower -* Subject Event Data Objects (subject id, plus event data) * Library Event Data Objects (library id, plus event data) * End of the SampleSheet Shower """ -from typing import Dict, List - - -def generate_subject_event_data_object_from_subject(subject_obj: Dict, instrument_run_id: str) -> Dict: - return { - "event_data": { - "instrumentRunId": instrument_run_id, - "subject": { - "orcabusId": subject_obj.get("orcabusId"), - "subjectId": subject_obj.get("subjectId") - } - } - } - - -def get_specimen_obj_from_library_obj(library_obj: Dict, specimen_obj_list: List[Dict]) -> Dict: - """ - Given a list of specimen objects, filter - :param library_obj: - :param specimen_obj_list: - :return: - """ - return next( - filter( - lambda specimen_object_iter: specimen_object_iter.get("orcabusId") == library_obj.get("specimen"), - specimen_obj_list - ) - ) - -def get_subject_obj_from_specimen_obj(specimen_obj: Dict, subject_obj_list: List[Dict]) -> Dict: - """ - Given a specimen object, return the subject id - :param specimen_obj: - :param subject_obj_list: - :return: - """ - return next( - filter( - lambda subject_obj_iter: subject_obj_iter.get("orcabusId") == specimen_obj.get("subject"), - subject_obj_list - ) - ) +# Imports +from typing import Dict, List +import pandas as pd +from more_itertools import flatten +# Functions def get_library_bclconvert_rows(library_obj: Dict, bclconvert_data: List[Dict]) -> List[Dict]: """ Get library bclconvert rows @@ -71,19 +33,15 @@ def get_library_bclconvert_rows(library_obj: Dict, bclconvert_data: List[Dict]) ) -def generate_library_event_data_object_from_library_specimen_and_subject( +def generate_library_event_data_object_from_library( library_obj: Dict, instrument_run_id: str, - specimen_obj: Dict, - subject_obj: Dict, bclconvert_rows: List[Dict] ) -> Dict: """ Generate library event data object from library specimen and subject :param library_obj: :param instrument_run_id: - :param specimen_obj: - :param subject_obj: :param bclconvert_rows: :return: """ @@ -97,16 +55,6 @@ def generate_library_event_data_object_from_library_specimen_and_subject( "type": library_obj.get("type", None), "assay": library_obj.get("assay", None), "coverage": library_obj.get("coverage", None), - "projectOwner": library_obj.get("projectOwner", None), - "projectName": library_obj.get("projectName", None), - "specimen": { - "orcabusId": specimen_obj.get("orcabusId"), - "specimenId": specimen_obj.get("specimenId") - }, - "subject": { - "orcabusId": subject_obj.get("orcabusId"), - "subjectId": subject_obj.get("subjectId") - } } # Trim library event object to non-null values @@ -164,8 +112,11 @@ def generate_library_event_data_object_from_library_specimen_and_subject( "event_data": { "instrumentRunId": instrument_run_id, "library": library_event_obj, + "sample": library_obj.get("sample", None), + "subject": library_obj.get("subject", None), + "projectSet": library_obj.get("projectSet", None), "bclconvertDataRows": bclconvert_data_rows_event_obj, - "fastqListRows": fastq_list_row_ids + "fastqListRows": fastq_list_row_ids, } } @@ -187,22 +138,31 @@ def handler(event, context): # Get the library object list library_obj_list = event['library_obj_list'] + library_set = list(set(list( + map( + lambda library_obj_iter_: library_obj_iter_.get("orcabusId"), + library_obj_list + ) + ))) - # Get the specimen object list - specimen_obj_list = event['specimen_obj_list'] - - # Get the subject object list - subject_obj_list = event['subject_obj_list'] + # Get sample set from the library object list + sample_set = list(set(list( + filter( + lambda sample_iter_: sample_iter_ is not None, + map( + lambda library_obj_iter_: library_obj_iter_.get("sample", {}).get("orcabusId", None), + library_obj_list + ) + ) + ))) - # For each subject, generate the subject event data object - subject_event_data_list = list( + # Get the subject set list + subject_set = list(set(list( map( - lambda subject_obj_iter: ( - generate_subject_event_data_object_from_subject(subject_obj_iter, instrument_run_id) - ), - subject_obj_list + lambda library_obj_iter_: library_obj_iter_.get("subject", {}).get("orcabusId", None), + library_obj_list ) - ) + ))) library_event_data_list = [] @@ -210,56 +170,71 @@ def handler(event, context): # Get the bclconvert data rows bclconvert_rows = get_library_bclconvert_rows(library_obj, samplesheet['bclconvert_data']) - # Get the specimen object - specimen_obj = get_specimen_obj_from_library_obj(library_obj, specimen_obj_list) - - # Get the subject object - subject_obj = get_subject_obj_from_specimen_obj(specimen_obj, subject_obj_list) - # Generate the library event data object library_event_data_list.append( - generate_library_event_data_object_from_library_specimen_and_subject( + generate_library_event_data_object_from_library( library_obj, instrument_run_id, - specimen_obj, - subject_obj, bclconvert_rows ) ) # Generate project data level events - project_list = list( - set( - map( - lambda library_iter: ( - library_iter.get("projectOwner"), library_iter.get("projectName") - ), - library_obj_list + project_obj_list = list( + flatten( + list( + map( + lambda library_iter_: library_iter_.get('projectSet'), + library_obj_list + ) ) ) ) + # Drop duplicates + project_obj_list = pd.DataFrame( + project_obj_list + ).drop_duplicates( + subset='orcabusId' + ).to_dict( + orient='records' + ) + + # Generate project set + project_set = list(set(list( + map( + lambda project_obj_iter_: project_obj_iter_.get("orcabusId"), + project_obj_list + ) + ))) + project_event_data_list = [] - for project_owner, project_name in project_list: + for project_obj in project_obj_list: + # Get the library set for this project + library_proj_obj_list = list( + filter( + lambda library_obj_iter: any( + map( + lambda library_project_iter_: library_project_iter_['orcabusId'] == project_obj['orcabusId'], + library_obj_iter.get("projectSet") + ) + ), + library_obj_list + ) + ) + project_event_data_list.append( { "event_data": { "instrumentRunId": instrument_run_id, - "projectOwner": project_owner, - "projectName": project_name, - "libraries": list( + "project": project_obj, + "librarySet": list( map( lambda library_obj_iter: { "orcabusId": library_obj_iter.get("orcabusId"), "libraryId": library_obj_iter.get("libraryId") }, - filter( - lambda library_obj_iter: ( - library_obj_iter.get("projectOwner") == project_owner and - library_obj_iter.get("projectName") == project_name - ), - library_obj_list - ) + library_proj_obj_list ) ) } @@ -280,2592 +255,3022 @@ def handler(event, context): "start_samplesheet_shower_event_data": start_shower_event_data, "complete_samplesheet_shower_event_data": complete_shower_event_data, "project_event_data_list": project_event_data_list, - "subject_event_data_list": subject_event_data_list, - "library_event_data_list": library_event_data_list + "library_event_data_list": library_event_data_list, + "library_set": library_set, + "subject_set": subject_set, + "project_set": project_set, + "sample_set": sample_set, } -if __name__ == "__main__": - import json - - print( - json.dumps( - handler( - { - "instrument_run_id": "240424_A01052_0193_BH7JMMDRX5", - "samplesheet": - { - "header": { - "file_format_version": 2, - "run_name": "Tsqn240214-26-ctTSOv2_29Feb24", - "instrument_type": "NovaSeq" - }, - "reads": { - "read_1_cycles": 151, - "read_2_cycles": 151, - "index_1_cycles": 10, - "index_2_cycles": 10 - }, - "bclconvert_settings": { - "minimum_trimmed_read_length": 35, - "minimum_adapter_overlap": 3, - "mask_short_reads": 35, - "software_version": "4.2.7" - }, - "bclconvert_data": [ - { - "lane": 1, - "sample_id": "L2400102", - "index": "GAATTCGT", - "index2": "TTATGAGT", - "override_cycles": "U7N1Y143;I8N2;I8N2;U7N1Y143" - }, - { - "lane": 1, - "sample_id": "L2400159", - "index": "GAGAATGGTT", - "index2": "TTGCTGCCGA", - "override_cycles": "U7N1Y143;I10;I10;U7N1Y143", - "adapter_read_1": "CTGTCTCTTATACACATCT", - "adapter_read_2": "CTGTCTCTTATACACATCT" - }, - { - "lane": 1, - "sample_id": "L2400160", - "index": "AGAGGCAACC", - "index2": "CCATCATTAG", - "override_cycles": "U7N1Y143;I10;I10;U7N1Y143", - "adapter_read_1": "CTGTCTCTTATACACATCT", - "adapter_read_2": "CTGTCTCTTATACACATCT" - }, - { - "lane": 1, - "sample_id": "L2400161", - "index": "CCATCATTAG", - "index2": "AGAGGCAACC", - "override_cycles": "U7N1Y143;I10;I10;U7N1Y143", - "adapter_read_1": "CTGTCTCTTATACACATCT", - "adapter_read_2": "CTGTCTCTTATACACATCT" - }, - { - "lane": 1, - "sample_id": "L2400162", - "index": "GATAGGCCGA", - "index2": "GCCATGTGCG", - "override_cycles": "U7N1Y143;I10;I10;U7N1Y143", - "adapter_read_1": "CTGTCTCTTATACACATCT", - "adapter_read_2": "CTGTCTCTTATACACATCT" - }, - { - "lane": 1, - "sample_id": "L2400163", - "index": "ATGGTTGACT", - "index2": "AGGACAGGCC", - "override_cycles": "U7N1Y143;I10;I10;U7N1Y143", - "adapter_read_1": "CTGTCTCTTATACACATCT", - "adapter_read_2": "CTGTCTCTTATACACATCT" - }, - { - "lane": 1, - "sample_id": "L2400164", - "index": "TATTGCGCTC", - "index2": "CCTAACACAG", - "override_cycles": "U7N1Y143;I10;I10;U7N1Y143", - "adapter_read_1": "CTGTCTCTTATACACATCT", - "adapter_read_2": "CTGTCTCTTATACACATCT" - }, - { - "lane": 1, - "sample_id": "L2400166", - "index": "TTCTACATAC", - "index2": "TTACAGTTAG", - "override_cycles": "U7N1Y143;I10;I10;U7N1Y143", - "adapter_read_1": "CTGTCTCTTATACACATCT", - "adapter_read_2": "CTGTCTCTTATACACATCT" - }, - { - "lane": 2, - "sample_id": "L2400195", - "index": "ATGAGGCC", - "index2": "CAATTAAC", - "override_cycles": "Y151;I8N2;I8N2;Y151" - }, - { - "lane": 2, - "sample_id": "L2400196", - "index": "ACTAAGAT", - "index2": "CCGCGGTT", - "override_cycles": "Y151;I8N2;I8N2;Y151" - }, - { - "lane": 2, - "sample_id": "L2400197", - "index": "GTCGGAGC", - "index2": "TTATAACC", - "override_cycles": "Y151;I8N2;I8N2;Y151" - }, - { - "lane": 2, - "sample_id": "L2400231", - "index": "TCGTAGTG", - "index2": "CCAAGTCT", - "override_cycles": "Y151;I8N2;I8N2;Y151" - }, - { - "lane": 2, - "sample_id": "L2400238", - "index": "GGAGCGTC", - "index2": "GCACGGAC", - "override_cycles": "Y151;I8N2;I8N2;Y151" - }, - { - "lane": 2, - "sample_id": "L2400239", - "index": "ATGGCATG", - "index2": "GGTACCTT", - "override_cycles": "Y151;I8N2;I8N2;Y151" - }, - { - "lane": 2, - "sample_id": "L2400240", - "index": "GCAATGCA", - "index2": "AACGTTCC", - "override_cycles": "Y151;I8N2;I8N2;Y151" - }, - { - "lane": 3, - "sample_id": "L2400195", - "index": "ATGAGGCC", - "index2": "CAATTAAC", - "override_cycles": "Y151;I8N2;I8N2;Y151" - }, - { - "lane": 3, - "sample_id": "L2400196", - "index": "ACTAAGAT", - "index2": "CCGCGGTT", - "override_cycles": "Y151;I8N2;I8N2;Y151" - }, - { - "lane": 3, - "sample_id": "L2400197", - "index": "GTCGGAGC", - "index2": "TTATAACC", - "override_cycles": "Y151;I8N2;I8N2;Y151" - }, - { - "lane": 3, - "sample_id": "L2400231", - "index": "TCGTAGTG", - "index2": "CCAAGTCT", - "override_cycles": "Y151;I8N2;I8N2;Y151" - }, - { - "lane": 3, - "sample_id": "L2400238", - "index": "GGAGCGTC", - "index2": "GCACGGAC", - "override_cycles": "Y151;I8N2;I8N2;Y151" - }, - { - "lane": 3, - "sample_id": "L2400239", - "index": "ATGGCATG", - "index2": "GGTACCTT", - "override_cycles": "Y151;I8N2;I8N2;Y151" - }, - { - "lane": 3, - "sample_id": "L2400240", - "index": "GCAATGCA", - "index2": "AACGTTCC", - "override_cycles": "Y151;I8N2;I8N2;Y151" - }, - { - "lane": 4, - "sample_id": "L2400165", - "index": "ACGCCTTGTT", - "index2": "ACGTTCCTTA", - "override_cycles": "U7N1Y143;I10;I10;U7N1Y143", - "adapter_read_1": "CTGTCTCTTATACACATCT", - "adapter_read_2": "CTGTCTCTTATACACATCT" - }, - { - "lane": 4, - "sample_id": "L2400191", - "index": "GCACGGAC", - "index2": "TGCGAGAC", - "override_cycles": "Y151;I8N2;I8N2;Y151" - }, - { - "lane": 4, - "sample_id": "L2400197", - "index": "GTCGGAGC", - "index2": "TTATAACC", - "override_cycles": "Y151;I8N2;I8N2;Y151" - }, - { - "lane": 4, - "sample_id": "L2400198", - "index": "CTTGGTAT", - "index2": "GGACTTGG", - "override_cycles": "Y151;I8N2;I8N2;Y151" - }, - { - "lane": 4, - "sample_id": "L2400241", - "index": "GTTCCAAT", - "index2": "GCAGAATT", - "override_cycles": "Y151;I8N2;I8N2;Y151" - }, - { - "lane": 4, - "sample_id": "L2400242", - "index": "ACCTTGGC", - "index2": "ATGAGGCC", - "override_cycles": "Y151;I8N2;I8N2;Y151" - }, - { - "lane": 4, - "sample_id": "L2400249", - "index": "AGTTTCGA", - "index2": "CCTACGAT", - "override_cycles": "Y151;I8N2;I8N2;Y151" - }, - { - "lane": 4, - "sample_id": "L2400250", - "index": "GAACCTCT", - "index2": "GTCTGCGC", - "override_cycles": "Y151;I8N2;I8N2;Y151" - }, - { - "lane": 4, - "sample_id": "L2400251", - "index": "GCCCAGTG", - "index2": "CCGCAATT", - "override_cycles": "Y151;I8N2;I8N2;Y151" - }, - { - "lane": 4, - "sample_id": "L2400252", - "index": "TGACAGCT", - "index2": "CCCGTAGG", - "override_cycles": "Y151;I8N2;I8N2;Y151" - }, - { - "lane": 4, - "sample_id": "L2400253", - "index": "CATCACCC", - "index2": "ATATAGCA", - "override_cycles": "Y151;I8N2;I8N2;Y151" - }, - { - "lane": 4, - "sample_id": "L2400254", - "index": "CTGGAGTA", - "index2": "GTTCGGTT", - "override_cycles": "Y151;I8N2;I8N2;Y151" - }, - { - "lane": 4, - "sample_id": "L2400255", - "index": "GATCCGGG", - "index2": "AAGCAGGT", - "override_cycles": "Y151;I8N2;I8N2;Y151" - }, - { - "lane": 4, - "sample_id": "L2400256", - "index": "AACACCTG", - "index2": "CGCATGGG", - "override_cycles": "Y151;I8N2;I8N2;Y151" - }, - { - "lane": 4, - "sample_id": "L2400257", - "index": "GTGACGTT", - "index2": "TCCCAGAT", - "override_cycles": "Y151;I8N2;I8N2;Y151" - } - ], - "cloud_settings": { - "generated_version": "0.0.0", - "cloud_workflow": "ica_workflow_1", - "bclconvert_pipeline": "urn:ilmn:ica:pipeline:bf93b5cf-cb27-4dfa-846e-acd6eb081aca#BclConvert_v4_2_7" - }, - "cloud_data": [ - { - "sample_id": "L2400102", - "library_name": "L2400102_GAATTCGT_TTATGAGT", - "library_prep_kit_name": "ctTSO" - }, - { - "sample_id": "L2400159", - "library_name": "L2400159_GAGAATGGTT_TTGCTGCCGA", - "library_prep_kit_name": "ctTSOv2" - }, - { - "sample_id": "L2400160", - "library_name": "L2400160_AGAGGCAACC_CCATCATTAG", - "library_prep_kit_name": "ctTSOv2" - }, - { - "sample_id": "L2400161", - "library_name": "L2400161_CCATCATTAG_AGAGGCAACC", - "library_prep_kit_name": "ctTSOv2" - }, - { - "sample_id": "L2400162", - "library_name": "L2400162_GATAGGCCGA_GCCATGTGCG", - "library_prep_kit_name": "ctTSOv2" - }, - { - "sample_id": "L2400163", - "library_name": "L2400163_ATGGTTGACT_AGGACAGGCC", - "library_prep_kit_name": "ctTSOv2" - }, - { - "sample_id": "L2400164", - "library_name": "L2400164_TATTGCGCTC_CCTAACACAG", - "library_prep_kit_name": "ctTSOv2" - }, - { - "sample_id": "L2400165", - "library_name": "L2400165_ACGCCTTGTT_ACGTTCCTTA", - "library_prep_kit_name": "ctTSOv2" - }, - { - "sample_id": "L2400166", - "library_name": "L2400166_TTCTACATAC_TTACAGTTAG", - "library_prep_kit_name": "ctTSOv2" - }, - { - "sample_id": "L2400191", - "library_name": "L2400191_GCACGGAC_TGCGAGAC", - "library_prep_kit_name": "TsqNano" - }, - { - "sample_id": "L2400195", - "library_name": "L2400195_ATGAGGCC_CAATTAAC", - "library_prep_kit_name": "TsqNano" - }, - { - "sample_id": "L2400196", - "library_name": "L2400196_ACTAAGAT_CCGCGGTT", - "library_prep_kit_name": "TsqNano" - }, - { - "sample_id": "L2400197", - "library_name": "L2400197_GTCGGAGC_TTATAACC", - "library_prep_kit_name": "TsqNano" - }, - { - "sample_id": "L2400198", - "library_name": "L2400198_CTTGGTAT_GGACTTGG", - "library_prep_kit_name": "TsqNano" - }, - { - "sample_id": "L2400231", - "library_name": "L2400231_TCGTAGTG_CCAAGTCT", - "library_prep_kit_name": "TsqNano" - }, - { - "sample_id": "L2400238", - "library_name": "L2400238_GGAGCGTC_GCACGGAC", - "library_prep_kit_name": "TsqNano" - }, - { - "sample_id": "L2400239", - "library_name": "L2400239_ATGGCATG_GGTACCTT", - "library_prep_kit_name": "TsqNano" - }, - { - "sample_id": "L2400240", - "library_name": "L2400240_GCAATGCA_AACGTTCC", - "library_prep_kit_name": "TsqNano" - }, - { - "sample_id": "L2400241", - "library_name": "L2400241_GTTCCAAT_GCAGAATT", - "library_prep_kit_name": "TsqNano" - }, - { - "sample_id": "L2400242", - "library_name": "L2400242_ACCTTGGC_ATGAGGCC", - "library_prep_kit_name": "TsqNano" - }, - { - "sample_id": "L2400249", - "library_name": "L2400249_AGTTTCGA_CCTACGAT", - "library_prep_kit_name": "NebRNA" - }, - { - "sample_id": "L2400250", - "library_name": "L2400250_GAACCTCT_GTCTGCGC", - "library_prep_kit_name": "NebRNA" - }, - { - "sample_id": "L2400251", - "library_name": "L2400251_GCCCAGTG_CCGCAATT", - "library_prep_kit_name": "NebRNA" - }, - { - "sample_id": "L2400252", - "library_name": "L2400252_TGACAGCT_CCCGTAGG", - "library_prep_kit_name": "NebRNA" - }, - { - "sample_id": "L2400253", - "library_name": "L2400253_CATCACCC_ATATAGCA", - "library_prep_kit_name": "NebRNA" - }, - { - "sample_id": "L2400254", - "library_name": "L2400254_CTGGAGTA_GTTCGGTT", - "library_prep_kit_name": "NebRNA" - }, - { - "sample_id": "L2400255", - "library_name": "L2400255_GATCCGGG_AAGCAGGT", - "library_prep_kit_name": "NebRNA" - }, - { - "sample_id": "L2400256", - "library_name": "L2400256_AACACCTG_CGCATGGG", - "library_prep_kit_name": "NebRNA" - }, - { - "sample_id": "L2400257", - "library_name": "L2400257_GTGACGTT_TCCCAGAT", - "library_prep_kit_name": "NebRNA" - } - ] - }, - "library_obj_list": [ - { - "orcabusId": "lib.01J5S9C4VMJ6PZ8GJ2G189AMXX", - "libraryId": "L2400102", - "phenotype": "tumor", - "workflow": "research", - "quality": "borderline", - "type": "WGS", - "assay": "ctTSO", - "coverage": 50.0, - "projectOwner": "VCCC", - "projectName": "PO", - "specimen": "spc.01J5S9C4V269YTNA17TTP6NF76" - }, - { - "orcabusId": "lib.01J5S9CBG0NF8QBNVKM6ESCD60", - "libraryId": "L2400159", - "phenotype": "tumor", - "workflow": "manual", - "quality": "good", - "type": "ctDNA", - "assay": "ctTSOv2", - "coverage": 38.6, - "projectOwner": "UMCCR", - "projectName": "Testing", - "specimen": "spc.01J5S9CBFDVZX7ZT3Y6TH28SY4" - }, - { - "orcabusId": "lib.01J5S9CBHP6NSB42RVFAP9PGJP", - "libraryId": "L2400160", - "phenotype": "tumor", - "workflow": "manual", - "quality": "good", - "type": "ctDNA", - "assay": "ctTSOv2", - "coverage": 38.6, - "projectOwner": "UMCCR", - "projectName": "Testing", - "specimen": "spc.01J5S9CBH4V5B56CEJ5Q1XQKQ9" - }, - { - "orcabusId": "lib.01J5S9CBKCATYSFY40BRX6WJWX", - "libraryId": "L2400161", - "phenotype": "tumor", - "workflow": "manual", - "quality": "good", - "type": "ctDNA", - "assay": "ctTSOv2", - "coverage": 38.6, - "projectOwner": "UMCCR", - "projectName": "Testing", - "specimen": "spc.01J5S9CBJTBJB72KJ74VSCHKJF" - }, - { - "orcabusId": "lib.01J5S9CBN6EAXW4AXG7TQ1H6NC", - "libraryId": "L2400162", - "phenotype": "tumor", - "workflow": "manual", - "quality": "good", - "type": "ctDNA", - "assay": "ctTSOv2", - "coverage": 38.6, - "projectOwner": "UMCCR", - "projectName": "Testing", - "specimen": "spc.01J5S9CBMKTX5KN1XMPN479R2M" - }, - { - "orcabusId": "lib.01J5S9CBQFX8V1QRW7KAV3MD1W", - "libraryId": "L2400163", - "phenotype": "tumor", - "workflow": "manual", - "quality": "good", - "type": "ctDNA", - "assay": "ctTSOv2", - "coverage": 38.6, - "projectOwner": "UMCCR", - "projectName": "Testing", - "specimen": "spc.01J5S9CBPSPN6S3TQCVJZF0XFE" - }, - { - "orcabusId": "lib.01J5S9CBS64DNTHK6CE850CCNZ", - "libraryId": "L2400164", - "phenotype": "tumor", - "workflow": "manual", - "quality": "good", - "type": "ctDNA", - "assay": "ctTSOv2", - "coverage": 38.6, - "projectOwner": "UMCCR", - "projectName": "Testing", - "specimen": "spc.01J5S9CBRM3Y6PPF6E5NWZA7HG" - }, - { - "orcabusId": "lib.01J5S9CBTZRYQNTGAHPC2T601D", - "libraryId": "L2400165", - "phenotype": "tumor", - "workflow": "manual", - "quality": "good", - "type": "ctDNA", - "assay": "ctTSOv2", - "coverage": 38.6, - "projectOwner": "UMCCR", - "projectName": "Testing", - "specimen": "spc.01J5S9CBTACFBNJKE8C523B0A7" - }, - { - "orcabusId": "lib.01J5S9CBX10204CK7EKGTH9TMB", - "libraryId": "L2400166", - "phenotype": "negative-control", - "workflow": "manual", - "quality": "good", - "type": "ctDNA", - "assay": "ctTSOv2", - "coverage": 0.1, - "projectOwner": "UMCCR", - "projectName": "Testing", - "specimen": "spc.01J5S9CBWCGKQMG5S3ZSWA2ATE" - }, - { - "orcabusId": "lib.01J5S9CDF8HHG5PJE3ECJMKMY7", - "libraryId": "L2400191", - "phenotype": "normal", - "workflow": "research", - "quality": "good", - "type": "WGS", - "assay": "TsqNano", - "coverage": 40.0, - "projectOwner": "TJohn", - "projectName": "CAVATAK", - "specimen": "spc.01J5S9CDEH0ATXYAK52KW807R4" - }, - { - "orcabusId": "lib.01J5S9CDQSSAG1WYCRWMD82Z1S", - "libraryId": "L2400195", - "phenotype": "tumor", - "workflow": "research", - "quality": "good", - "type": "WGS", - "assay": "TsqNano", - "coverage": 80.0, - "projectOwner": "TJohn", - "projectName": "CAVATAK", - "specimen": "spc.01J5S9CDQ0V9T98EGRPQJAP11S" - }, - { - "orcabusId": "lib.01J5S9CDSJ2BGEYM8FTXGKVGV8", - "libraryId": "L2400196", - "phenotype": "tumor", - "workflow": "research", - "quality": "good", - "type": "WGS", - "assay": "TsqNano", - "coverage": 80.0, - "projectOwner": "TJohn", - "projectName": "CAVATAK", - "specimen": "spc.01J5S9CDRZMMR9S784BYSMVWCT" - }, - { - "orcabusId": "lib.01J5S9CDVEHDZHZR3BZTQ7WNJQ", - "libraryId": "L2400197", - "phenotype": "tumor", - "workflow": "research", - "quality": "good", - "type": "WGS", - "assay": "TsqNano", - "coverage": 80.0, - "projectOwner": "TJohn", - "projectName": "CAVATAK", - "specimen": "spc.01J5S9CDTSHGYMMJHE3SXEB2JG" - }, - { - "orcabusId": "lib.01J5S9CDXCR7Q5K6A8VJRSMM4Q", - "libraryId": "L2400198", - "phenotype": "tumor", - "workflow": "research", - "quality": "good", - "type": "WGS", - "assay": "TsqNano", - "coverage": 80.0, - "projectOwner": "TJohn", - "projectName": "CAVATAK", - "specimen": "spc.01J5S9CDWHAYG4RRG75GYZEK25" - }, - { - "orcabusId": "lib.01J5S9CFX5P69S4KZRQGDFKV1N", - "libraryId": "L2400231", - "phenotype": "tumor", - "workflow": "clinical", - "quality": "poor", - "type": "WGS", - "assay": "TsqNano", - "coverage": 100.0, - "projectOwner": "Tothill", - "projectName": "CUP", - "specimen": "spc.01J5S9CFWAQTGK4MZB3HM5NVBC" - }, - { - "orcabusId": "lib.01J5S9CGCAKQWHD9RBM9VXENY9", - "libraryId": "L2400238", - "phenotype": "normal", - "workflow": "clinical", - "quality": "good", - "type": "WGS", - "assay": "TsqNano", - "coverage": 40.0, - "projectOwner": "Tothill", - "projectName": "CUP", - "specimen": "spc.01J5S9CGBQCSQCS7XR3T89A82F" - }, - { - "orcabusId": "lib.01J5S9CGEM1DHRQP72EP09B2TA", - "libraryId": "L2400239", - "phenotype": "normal", - "workflow": "clinical", - "quality": "good", - "type": "WGS", - "assay": "TsqNano", - "coverage": 40.0, - "projectOwner": "Tothill", - "projectName": "CUP", - "specimen": "spc.01J5S9CGE05BJCJ20M2KP4QWWB" - }, - { - "orcabusId": "lib.01J5S9CGG9N9GH5879SY6A6BJB", - "libraryId": "L2400240", - "phenotype": "tumor", - "workflow": "clinical", - "quality": "poor", - "type": "WGS", - "assay": "TsqNano", - "coverage": 100.0, - "projectOwner": "Tothill", - "projectName": "CUP", - "specimen": "spc.01J5S9CGFQM3BQKADX8TWQ4ZH5" - }, - { - "orcabusId": "lib.01J5S9CGJ6G09YQ9KFHPSXMMVD", - "libraryId": "L2400241", - "phenotype": "negative-control", - "workflow": "control", - "quality": "good", - "type": "WGS", - "assay": "TsqNano", - "coverage": 0.1, - "projectOwner": "UMCCR", - "projectName": "Control", - "specimen": "spc.01J5S9CGHK1G7YXD7C4FCXXS52" - }, - { - "orcabusId": "lib.01J5S9CGKWDN7STKZKQM3KH9XR", - "libraryId": "L2400242", - "phenotype": "normal", - "workflow": "control", - "quality": "good", - "type": "WGS", - "assay": "TsqNano", - "coverage": 15.0, - "projectOwner": "UMCCR", - "projectName": "Control", - "specimen": "spc.01J5S9CGKAT4GHZ1VJFHVV15AD" - }, - { - "orcabusId": "lib.01J5S9CH2SQ0P1SF7WAT5H4DSE", - "libraryId": "L2400249", - "phenotype": "tumor", - "workflow": "control", - "quality": "good", - "type": "WTS", - "assay": "NebRNA", - "coverage": 1.0, - "projectOwner": "UMCCR", - "projectName": "Control", - "specimen": "spc.01J5S9CH267XEJP5GMZK31MJWS" - }, - { - "orcabusId": "lib.01J5S9CH4CYPA4SP05H8KRX4W9", - "libraryId": "L2400250", - "phenotype": "tumor", - "workflow": "research", - "quality": "good", - "type": "WTS", - "assay": "NebRNA", - "coverage": 6.0, - "projectOwner": "Whittle", - "projectName": "BPOP-retro", - "specimen": "spc.01J5S9C0QC2TBZD7XA26D7WGTW" - }, - { - "orcabusId": "lib.01J5S9CH65E4EE5QJEJ1C60GGG", - "libraryId": "L2400251", - "phenotype": "tumor", - "workflow": "research", - "quality": "good", - "type": "WTS", - "assay": "NebRNA", - "coverage": 6.0, - "projectOwner": "Whittle", - "projectName": "BPOP-retro", - "specimen": "spc.01J5S9CH5KXY3VMB9M9J2RCR7B" - }, - { - "orcabusId": "lib.01J5S9CH7TGZMV39Z59WJ8H5GP", - "libraryId": "L2400252", - "phenotype": "tumor", - "workflow": "research", - "quality": "good", - "type": "WTS", - "assay": "NebRNA", - "coverage": 6.0, - "projectOwner": "Whittle", - "projectName": "BPOP-retro", - "specimen": "spc.01J5S9CH774HEZFEVWWP2XADK1" - }, - { - "orcabusId": "lib.01J5S9CH9TGMT2TJGBZX5VXHJY", - "libraryId": "L2400253", - "phenotype": "tumor", - "workflow": "research", - "quality": "good", - "type": "WTS", - "assay": "NebRNA", - "coverage": 6.0, - "projectOwner": "Whittle", - "projectName": "BPOP-retro", - "specimen": "spc.01J5S9CH98MQ2B1G2BQFEY0XZH" - }, - { - "orcabusId": "lib.01J5S9CHBGAP2XSN4TG8SAMRYY", - "libraryId": "L2400254", - "phenotype": "tumor", - "workflow": "research", - "quality": "borderline", - "type": "WTS", - "assay": "NebRNA", - "coverage": 6.0, - "projectOwner": "Whittle", - "projectName": "BPOP-retro", - "specimen": "spc.01J5S9CHAX3XKJE5XE4VQWYN5H" - }, - { - "orcabusId": "lib.01J5S9CHE4ERQ4H209DH397W8A", - "libraryId": "L2400255", - "phenotype": "tumor", - "workflow": "clinical", - "quality": "very-poor", - "type": "WTS", - "assay": "NebRNA", - "coverage": 6.0, - "projectOwner": "Tothill", - "projectName": "CUP", - "specimen": "spc.01J5S9CHDGRNK70B043K887RP2" - }, - { - "orcabusId": "lib.01J5S9CHFXPDGYQ8TXHRWQR3PY", - "libraryId": "L2400256", - "phenotype": "tumor", - "workflow": "clinical", - "quality": "very-poor", - "type": "WTS", - "assay": "NebRNA", - "coverage": 6.0, - "projectOwner": "Tothill", - "projectName": "CUP", - "specimen": "spc.01J5S9CHFAPXYKK49FAGVF5CQF" - }, - { - "orcabusId": "lib.01J5S9CHHNGFJN73NPRQMSYGN9", - "libraryId": "L2400257", - "phenotype": "negative-control", - "workflow": "control", - "quality": "good", - "type": "WTS", - "assay": "NebRNA", - "coverage": 0.1, - "projectOwner": "UMCCR", - "projectName": "Control", - "specimen": "spc.01J5S9CHH24VFM443RD8Q8X4B3" - } - ], - "specimen_obj_list": [ - { - "orcabusId": "spc.01J5S9C0QC2TBZD7XA26D7WGTW", - "specimenId": "PRJ240003", - "source": "tissue", - "subject": "sbj.01J5S9C0PVB4QNVGK4Q1WSYEGV" - }, - { - "orcabusId": "spc.01J5S9C4V269YTNA17TTP6NF76", - "specimenId": "MDX210402", - "source": "plasma-serum", - "subject": "sbj.01J5S9C4TE1GCWA1QGNCWHB1Y9" - }, - { - "orcabusId": "spc.01J5S9CBFDVZX7ZT3Y6TH28SY4", - "specimenId": "PTC_SCMM1pc2", - "source": "cfDNA", - "subject": "sbj.01J5S9CBEQ3DM8XDV2G2ZQJDXB" - }, - { - "orcabusId": "spc.01J5S9CBH4V5B56CEJ5Q1XQKQ9", - "specimenId": "PTC_SCMM1pc3", - "source": "cfDNA", - "subject": "sbj.01J5S9CBEQ3DM8XDV2G2ZQJDXB" - }, - { - "orcabusId": "spc.01J5S9CBJTBJB72KJ74VSCHKJF", - "specimenId": "PTC_SCMM1pc4", - "source": "cfDNA", - "subject": "sbj.01J5S9CBEQ3DM8XDV2G2ZQJDXB" - }, - { - "orcabusId": "spc.01J5S9CBMKTX5KN1XMPN479R2M", - "specimenId": "PTC_SCMM01pc20", - "source": "cfDNA", - "subject": "sbj.01J5S9CBM3AT89QTXD7PT0BKA0" - }, - { - "orcabusId": "spc.01J5S9CBPSPN6S3TQCVJZF0XFE", - "specimenId": "PTC_SCMM01pc15", - "source": "cfDNA", - "subject": "sbj.01J5S9CBM3AT89QTXD7PT0BKA0" - }, - { - "orcabusId": "spc.01J5S9CBRM3Y6PPF6E5NWZA7HG", - "specimenId": "PTC_SCMM01pc10", - "source": "cfDNA", - "subject": "sbj.01J5S9CBM3AT89QTXD7PT0BKA0" - }, - { - "orcabusId": "spc.01J5S9CBTACFBNJKE8C523B0A7", - "specimenId": "PTC_SCMM01pc5", - "source": "cfDNA", - "subject": "sbj.01J5S9CBM3AT89QTXD7PT0BKA0" - }, - { - "orcabusId": "spc.01J5S9CBWCGKQMG5S3ZSWA2ATE", - "specimenId": "NTC_v2ctTSO240207", - "source": "water", - "subject": "sbj.01J5S9BYKC1RH7DY68GF1JNSR6" - }, - { - "orcabusId": "spc.01J5S9CDEH0ATXYAK52KW807R4", - "specimenId": "PRJ240169", - "source": "blood", - "subject": "sbj.01J5S9CDDP20JX8V63ZKMPBJQS" - }, - { - "orcabusId": "spc.01J5S9CDQ0V9T98EGRPQJAP11S", - "specimenId": "PRJ240180", - "source": "tissue", - "subject": "sbj.01J5S9CDDP20JX8V63ZKMPBJQS" - }, - { - "orcabusId": "spc.01J5S9CDRZMMR9S784BYSMVWCT", - "specimenId": "PRJ240181", - "source": "tissue", - "subject": "sbj.01J5S9CDDP20JX8V63ZKMPBJQS" - }, - { - "orcabusId": "spc.01J5S9CDTSHGYMMJHE3SXEB2JG", - "specimenId": "PRJ240182", - "source": "tissue", - "subject": "sbj.01J5S9CDG7B0KA8YEDK876VVDP" - }, - { - "orcabusId": "spc.01J5S9CDWHAYG4RRG75GYZEK25", - "specimenId": "PRJ240183", - "source": "tissue", - "subject": "sbj.01J5S9CDG7B0KA8YEDK876VVDP" - }, - { - "orcabusId": "spc.01J5S9CFWAQTGK4MZB3HM5NVBC", - "specimenId": "PRJ240199", - "source": "FFPE", - "subject": "sbj.01J5S9CFVJ9GVEHZK6CD9WAAV5" - }, - { - "orcabusId": "spc.01J5S9CGBQCSQCS7XR3T89A82F", - "specimenId": "PRJ240643", - "source": "blood", - "subject": "sbj.01J5S9CFVJ9GVEHZK6CD9WAAV5" - }, - { - "orcabusId": "spc.01J5S9CGE05BJCJ20M2KP4QWWB", - "specimenId": "PRJ240646", - "source": "blood", - "subject": "sbj.01J5S9CGDGTF5VZJSSE4ADBNJ3" - }, - { - "orcabusId": "spc.01J5S9CGFQM3BQKADX8TWQ4ZH5", - "specimenId": "PRJ240647", - "source": "FFPE", - "subject": "sbj.01J5S9CGDGTF5VZJSSE4ADBNJ3" - }, - { - "orcabusId": "spc.01J5S9CGHK1G7YXD7C4FCXXS52", - "specimenId": "NTC_TSqN240226", - "source": "water", - "subject": "sbj.01J5S9BYKC1RH7DY68GF1JNSR6" - }, - { - "orcabusId": "spc.01J5S9CGKAT4GHZ1VJFHVV15AD", - "specimenId": "PTC_TSqN240226", - "source": "cell-line", - "subject": "sbj.01J5S9BYVWZDS8AW7A94CDQBXK" - }, - { - "orcabusId": "spc.01J5S9CH267XEJP5GMZK31MJWS", - "specimenId": "PTC_NebRNA240226", - "source": "cell-line", - "subject": "sbj.01J5S9C1S3XV8PNB78XYJ1EQM1" - }, - { - "orcabusId": "spc.01J5S9CH5KXY3VMB9M9J2RCR7B", - "specimenId": "PRJ240561", - "source": "tissue", - "subject": "sbj.01J5S9CFY1BV2Z0SGKYNF1VHQN" - }, - { - "orcabusId": "spc.01J5S9CH774HEZFEVWWP2XADK1", - "specimenId": "PRJ240562", - "source": "tissue", - "subject": "sbj.01J5S9CFY1BV2Z0SGKYNF1VHQN" - }, - { - "orcabusId": "spc.01J5S9CH98MQ2B1G2BQFEY0XZH", - "specimenId": "PRJ240566", - "source": "tissue", - "subject": "sbj.01J5S9CG5GEWYBK0065C49HT23" - }, - { - "orcabusId": "spc.01J5S9CHAX3XKJE5XE4VQWYN5H", - "specimenId": "PRJ240567", - "source": "tissue", - "subject": "sbj.01J5S9CG5GEWYBK0065C49HT23" - }, - { - "orcabusId": "spc.01J5S9CHDGRNK70B043K887RP2", - "specimenId": "PRJ240200", - "source": "FFPE", - "subject": "sbj.01J5S9CFVJ9GVEHZK6CD9WAAV5" - }, - { - "orcabusId": "spc.01J5S9CHFAPXYKK49FAGVF5CQF", - "specimenId": "PRJ240648", - "source": "FFPE", - "subject": "sbj.01J5S9CGDGTF5VZJSSE4ADBNJ3" - }, - { - "orcabusId": "spc.01J5S9CHH24VFM443RD8Q8X4B3", - "specimenId": "NTC_NebRNA240226", - "source": "water", - "subject": "sbj.01J5S9BYKC1RH7DY68GF1JNSR6" - } - ], - "subject_obj_list": [ - { - "orcabusId": "sbj.01J5S9BYKC1RH7DY68GF1JNSR6", - "subjectId": "SBJ00006" - }, - { - "orcabusId": "sbj.01J5S9BYVWZDS8AW7A94CDQBXK", - "subjectId": "SBJ00005" - }, - { - "orcabusId": "sbj.01J5S9C0PVB4QNVGK4Q1WSYEGV", - "subjectId": "SBJ04488" - }, - { - "orcabusId": "sbj.01J5S9C1S3XV8PNB78XYJ1EQM1", - "subjectId": "SBJ00029" - }, - { - "orcabusId": "sbj.01J5S9C4TE1GCWA1QGNCWHB1Y9", - "subjectId": "SBJ01143" - }, - { - "orcabusId": "sbj.01J5S9CBEQ3DM8XDV2G2ZQJDXB", - "subjectId": "SBJ04407" - }, - { - "orcabusId": "sbj.01J5S9CBM3AT89QTXD7PT0BKA0", - "subjectId": "SBJ04648" - }, - { - "orcabusId": "sbj.01J5S9CDDP20JX8V63ZKMPBJQS", - "subjectId": "SBJ04653" - }, - { - "orcabusId": "sbj.01J5S9CDG7B0KA8YEDK876VVDP", - "subjectId": "SBJ04654" - }, - { - "orcabusId": "sbj.01J5S9CFVJ9GVEHZK6CD9WAAV5", - "subjectId": "SBJ04659" - }, - { - "orcabusId": "sbj.01J5S9CFY1BV2Z0SGKYNF1VHQN", - "subjectId": "SBJ04660" - }, - { - "orcabusId": "sbj.01J5S9CG5GEWYBK0065C49HT23", - "subjectId": "SBJ04661" - }, - { - "orcabusId": "sbj.01J5S9CGDGTF5VZJSSE4ADBNJ3", - "subjectId": "SBJ04662" - } - ] - }, - None - ), - indent=2 - ) - ) - # { - # "start_samplesheet_shower_event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5" - # }, - # "complete_samplesheet_shower_event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5" - # }, - # "project_event_data_list": [ - # { - # "event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", - # "projectOwner": "Tothill", - # "projectName": "CUP", - # "libraries": [ - # { - # "orcabusId": "lib.01J5S9CFX5P69S4KZRQGDFKV1N", - # "libraryId": "L2400231" - # }, - # { - # "orcabusId": "lib.01J5S9CGCAKQWHD9RBM9VXENY9", - # "libraryId": "L2400238" - # }, - # { - # "orcabusId": "lib.01J5S9CGEM1DHRQP72EP09B2TA", - # "libraryId": "L2400239" - # }, - # { - # "orcabusId": "lib.01J5S9CGG9N9GH5879SY6A6BJB", - # "libraryId": "L2400240" - # }, - # { - # "orcabusId": "lib.01J5S9CHE4ERQ4H209DH397W8A", - # "libraryId": "L2400255" - # }, - # { - # "orcabusId": "lib.01J5S9CHFXPDGYQ8TXHRWQR3PY", - # "libraryId": "L2400256" - # } - # ] - # } - # }, - # { - # "event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", - # "projectOwner": "VCCC", - # "projectName": "PO", - # "libraries": [ - # { - # "orcabusId": "lib.01J5S9C4VMJ6PZ8GJ2G189AMXX", - # "libraryId": "L2400102" - # } - # ] - # } - # }, - # { - # "event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", - # "projectOwner": "TJohn", - # "projectName": "CAVATAK", - # "libraries": [ - # { - # "orcabusId": "lib.01J5S9CDF8HHG5PJE3ECJMKMY7", - # "libraryId": "L2400191" - # }, - # { - # "orcabusId": "lib.01J5S9CDQSSAG1WYCRWMD82Z1S", - # "libraryId": "L2400195" - # }, - # { - # "orcabusId": "lib.01J5S9CDSJ2BGEYM8FTXGKVGV8", - # "libraryId": "L2400196" - # }, - # { - # "orcabusId": "lib.01J5S9CDVEHDZHZR3BZTQ7WNJQ", - # "libraryId": "L2400197" - # }, - # { - # "orcabusId": "lib.01J5S9CDXCR7Q5K6A8VJRSMM4Q", - # "libraryId": "L2400198" - # } - # ] - # } - # }, - # { - # "event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", - # "projectOwner": "UMCCR", - # "projectName": "Control", - # "libraries": [ - # { - # "orcabusId": "lib.01J5S9CGJ6G09YQ9KFHPSXMMVD", - # "libraryId": "L2400241" - # }, - # { - # "orcabusId": "lib.01J5S9CGKWDN7STKZKQM3KH9XR", - # "libraryId": "L2400242" - # }, - # { - # "orcabusId": "lib.01J5S9CH2SQ0P1SF7WAT5H4DSE", - # "libraryId": "L2400249" - # }, - # { - # "orcabusId": "lib.01J5S9CHHNGFJN73NPRQMSYGN9", - # "libraryId": "L2400257" - # } - # ] - # } - # }, - # { - # "event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", - # "projectOwner": "Whittle", - # "projectName": "BPOP-retro", - # "libraries": [ - # { - # "orcabusId": "lib.01J5S9CH4CYPA4SP05H8KRX4W9", - # "libraryId": "L2400250" - # }, - # { - # "orcabusId": "lib.01J5S9CH65E4EE5QJEJ1C60GGG", - # "libraryId": "L2400251" - # }, - # { - # "orcabusId": "lib.01J5S9CH7TGZMV39Z59WJ8H5GP", - # "libraryId": "L2400252" - # }, - # { - # "orcabusId": "lib.01J5S9CH9TGMT2TJGBZX5VXHJY", - # "libraryId": "L2400253" - # }, - # { - # "orcabusId": "lib.01J5S9CHBGAP2XSN4TG8SAMRYY", - # "libraryId": "L2400254" - # } - # ] - # } - # }, - # { - # "event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", - # "projectOwner": "UMCCR", - # "projectName": "Testing", - # "libraries": [ - # { - # "orcabusId": "lib.01J5S9CBG0NF8QBNVKM6ESCD60", - # "libraryId": "L2400159" - # }, - # { - # "orcabusId": "lib.01J5S9CBHP6NSB42RVFAP9PGJP", - # "libraryId": "L2400160" - # }, - # { - # "orcabusId": "lib.01J5S9CBKCATYSFY40BRX6WJWX", - # "libraryId": "L2400161" - # }, - # { - # "orcabusId": "lib.01J5S9CBN6EAXW4AXG7TQ1H6NC", - # "libraryId": "L2400162" - # }, - # { - # "orcabusId": "lib.01J5S9CBQFX8V1QRW7KAV3MD1W", - # "libraryId": "L2400163" - # }, - # { - # "orcabusId": "lib.01J5S9CBS64DNTHK6CE850CCNZ", - # "libraryId": "L2400164" - # }, - # { - # "orcabusId": "lib.01J5S9CBTZRYQNTGAHPC2T601D", - # "libraryId": "L2400165" - # }, - # { - # "orcabusId": "lib.01J5S9CBX10204CK7EKGTH9TMB", - # "libraryId": "L2400166" - # } - # ] - # } - # } - # ], - # "subject_event_data_list": [ - # { - # "event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", - # "subject": { - # "orcabusId": "sbj.01J5S9BYKC1RH7DY68GF1JNSR6", - # "subjectId": "SBJ00006" - # } - # } - # }, - # { - # "event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", - # "subject": { - # "orcabusId": "sbj.01J5S9BYVWZDS8AW7A94CDQBXK", - # "subjectId": "SBJ00005" - # } - # } - # }, - # { - # "event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", - # "subject": { - # "orcabusId": "sbj.01J5S9C0PVB4QNVGK4Q1WSYEGV", - # "subjectId": "SBJ04488" - # } - # } - # }, - # { - # "event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", - # "subject": { - # "orcabusId": "sbj.01J5S9C1S3XV8PNB78XYJ1EQM1", - # "subjectId": "SBJ00029" - # } - # } - # }, - # { - # "event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", - # "subject": { - # "orcabusId": "sbj.01J5S9C4TE1GCWA1QGNCWHB1Y9", - # "subjectId": "SBJ01143" - # } - # } - # }, - # { - # "event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", - # "subject": { - # "orcabusId": "sbj.01J5S9CBEQ3DM8XDV2G2ZQJDXB", - # "subjectId": "SBJ04407" - # } - # } - # }, - # { - # "event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", - # "subject": { - # "orcabusId": "sbj.01J5S9CBM3AT89QTXD7PT0BKA0", - # "subjectId": "SBJ04648" - # } - # } - # }, - # { - # "event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", - # "subject": { - # "orcabusId": "sbj.01J5S9CDDP20JX8V63ZKMPBJQS", - # "subjectId": "SBJ04653" - # } - # } - # }, - # { - # "event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", - # "subject": { - # "orcabusId": "sbj.01J5S9CDG7B0KA8YEDK876VVDP", - # "subjectId": "SBJ04654" - # } - # } - # }, - # { - # "event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", - # "subject": { - # "orcabusId": "sbj.01J5S9CFVJ9GVEHZK6CD9WAAV5", - # "subjectId": "SBJ04659" - # } - # } - # }, - # { - # "event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", - # "subject": { - # "orcabusId": "sbj.01J5S9CFY1BV2Z0SGKYNF1VHQN", - # "subjectId": "SBJ04660" - # } - # } - # }, - # { - # "event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", - # "subject": { - # "orcabusId": "sbj.01J5S9CG5GEWYBK0065C49HT23", - # "subjectId": "SBJ04661" - # } - # } - # }, - # { - # "event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", - # "subject": { - # "orcabusId": "sbj.01J5S9CGDGTF5VZJSSE4ADBNJ3", - # "subjectId": "SBJ04662" - # } - # } - # } - # ], - # "library_event_data_list": [ - # { - # "event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", - # "library": { - # "orcabusId": "lib.01J5S9C4VMJ6PZ8GJ2G189AMXX", - # "libraryId": "L2400102", - # "phenotype": "tumor", - # "workflow": "research", - # "quality": "borderline", - # "type": "WGS", - # "assay": "ctTSO", - # "coverage": 50.0, - # "projectOwner": "VCCC", - # "projectName": "PO", - # "specimen": { - # "orcabusId": "spc.01J5S9C4V269YTNA17TTP6NF76", - # "specimenId": "MDX210402" - # }, - # "subject": { - # "orcabusId": "sbj.01J5S9C4TE1GCWA1QGNCWHB1Y9", - # "subjectId": "SBJ01143" - # } - # }, - # "bclconvertDataRows": [ - # { - # "sampleId": "L2400102", - # "index": "GAATTCGT", - # "index2": "TTATGAGT", - # "lane": 1, - # "overrideCycles": "U7N1Y143;I8N2;I8N2;U7N1Y143" - # } - # ], - # "fastqListRows": [ - # { - # "fastqListRowRgid": "GAATTCGT.TTATGAGT.1.240424_A01052_0193_BH7JMMDRX5.L2400102" - # } - # ] - # } - # }, - # { - # "event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", - # "library": { - # "orcabusId": "lib.01J5S9CBG0NF8QBNVKM6ESCD60", - # "libraryId": "L2400159", - # "phenotype": "tumor", - # "workflow": "manual", - # "quality": "good", - # "type": "ctDNA", - # "assay": "ctTSOv2", - # "coverage": 38.6, - # "projectOwner": "UMCCR", - # "projectName": "Testing", - # "specimen": { - # "orcabusId": "spc.01J5S9CBFDVZX7ZT3Y6TH28SY4", - # "specimenId": "PTC_SCMM1pc2" - # }, - # "subject": { - # "orcabusId": "sbj.01J5S9CBEQ3DM8XDV2G2ZQJDXB", - # "subjectId": "SBJ04407" - # } - # }, - # "bclconvertDataRows": [ - # { - # "sampleId": "L2400159", - # "index": "GAGAATGGTT", - # "index2": "TTGCTGCCGA", - # "lane": 1, - # "overrideCycles": "U7N1Y143;I10;I10;U7N1Y143" - # } - # ], - # "fastqListRows": [ - # { - # "fastqListRowRgid": "GAGAATGGTT.TTGCTGCCGA.1.240424_A01052_0193_BH7JMMDRX5.L2400159" - # } - # ] - # } - # }, - # { - # "event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", - # "library": { - # "orcabusId": "lib.01J5S9CBHP6NSB42RVFAP9PGJP", - # "libraryId": "L2400160", - # "phenotype": "tumor", - # "workflow": "manual", - # "quality": "good", - # "type": "ctDNA", - # "assay": "ctTSOv2", - # "coverage": 38.6, - # "projectOwner": "UMCCR", - # "projectName": "Testing", - # "specimen": { - # "orcabusId": "spc.01J5S9CBH4V5B56CEJ5Q1XQKQ9", - # "specimenId": "PTC_SCMM1pc3" - # }, - # "subject": { - # "orcabusId": "sbj.01J5S9CBEQ3DM8XDV2G2ZQJDXB", - # "subjectId": "SBJ04407" - # } - # }, - # "bclconvertDataRows": [ - # { - # "sampleId": "L2400160", - # "index": "AGAGGCAACC", - # "index2": "CCATCATTAG", - # "lane": 1, - # "overrideCycles": "U7N1Y143;I10;I10;U7N1Y143" - # } - # ], - # "fastqListRows": [ - # { - # "fastqListRowRgid": "AGAGGCAACC.CCATCATTAG.1.240424_A01052_0193_BH7JMMDRX5.L2400160" - # } - # ] - # } - # }, - # { - # "event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", - # "library": { - # "orcabusId": "lib.01J5S9CBKCATYSFY40BRX6WJWX", - # "libraryId": "L2400161", - # "phenotype": "tumor", - # "workflow": "manual", - # "quality": "good", - # "type": "ctDNA", - # "assay": "ctTSOv2", - # "coverage": 38.6, - # "projectOwner": "UMCCR", - # "projectName": "Testing", - # "specimen": { - # "orcabusId": "spc.01J5S9CBJTBJB72KJ74VSCHKJF", - # "specimenId": "PTC_SCMM1pc4" - # }, - # "subject": { - # "orcabusId": "sbj.01J5S9CBEQ3DM8XDV2G2ZQJDXB", - # "subjectId": "SBJ04407" - # } - # }, - # "bclconvertDataRows": [ - # { - # "sampleId": "L2400161", - # "index": "CCATCATTAG", - # "index2": "AGAGGCAACC", - # "lane": 1, - # "overrideCycles": "U7N1Y143;I10;I10;U7N1Y143" - # } - # ], - # "fastqListRows": [ - # { - # "fastqListRowRgid": "CCATCATTAG.AGAGGCAACC.1.240424_A01052_0193_BH7JMMDRX5.L2400161" - # } - # ] - # } - # }, - # { - # "event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", - # "library": { - # "orcabusId": "lib.01J5S9CBN6EAXW4AXG7TQ1H6NC", - # "libraryId": "L2400162", - # "phenotype": "tumor", - # "workflow": "manual", - # "quality": "good", - # "type": "ctDNA", - # "assay": "ctTSOv2", - # "coverage": 38.6, - # "projectOwner": "UMCCR", - # "projectName": "Testing", - # "specimen": { - # "orcabusId": "spc.01J5S9CBMKTX5KN1XMPN479R2M", - # "specimenId": "PTC_SCMM01pc20" - # }, - # "subject": { - # "orcabusId": "sbj.01J5S9CBM3AT89QTXD7PT0BKA0", - # "subjectId": "SBJ04648" - # } - # }, - # "bclconvertDataRows": [ - # { - # "sampleId": "L2400162", - # "index": "GATAGGCCGA", - # "index2": "GCCATGTGCG", - # "lane": 1, - # "overrideCycles": "U7N1Y143;I10;I10;U7N1Y143" - # } - # ], - # "fastqListRows": [ - # { - # "fastqListRowRgid": "GATAGGCCGA.GCCATGTGCG.1.240424_A01052_0193_BH7JMMDRX5.L2400162" - # } - # ] - # } - # }, - # { - # "event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", - # "library": { - # "orcabusId": "lib.01J5S9CBQFX8V1QRW7KAV3MD1W", - # "libraryId": "L2400163", - # "phenotype": "tumor", - # "workflow": "manual", - # "quality": "good", - # "type": "ctDNA", - # "assay": "ctTSOv2", - # "coverage": 38.6, - # "projectOwner": "UMCCR", - # "projectName": "Testing", - # "specimen": { - # "orcabusId": "spc.01J5S9CBPSPN6S3TQCVJZF0XFE", - # "specimenId": "PTC_SCMM01pc15" - # }, - # "subject": { - # "orcabusId": "sbj.01J5S9CBM3AT89QTXD7PT0BKA0", - # "subjectId": "SBJ04648" - # } - # }, - # "bclconvertDataRows": [ - # { - # "sampleId": "L2400163", - # "index": "ATGGTTGACT", - # "index2": "AGGACAGGCC", - # "lane": 1, - # "overrideCycles": "U7N1Y143;I10;I10;U7N1Y143" - # } - # ], - # "fastqListRows": [ - # { - # "fastqListRowRgid": "ATGGTTGACT.AGGACAGGCC.1.240424_A01052_0193_BH7JMMDRX5.L2400163" - # } - # ] - # } - # }, - # { - # "event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", - # "library": { - # "orcabusId": "lib.01J5S9CBS64DNTHK6CE850CCNZ", - # "libraryId": "L2400164", - # "phenotype": "tumor", - # "workflow": "manual", - # "quality": "good", - # "type": "ctDNA", - # "assay": "ctTSOv2", - # "coverage": 38.6, - # "projectOwner": "UMCCR", - # "projectName": "Testing", - # "specimen": { - # "orcabusId": "spc.01J5S9CBRM3Y6PPF6E5NWZA7HG", - # "specimenId": "PTC_SCMM01pc10" - # }, - # "subject": { - # "orcabusId": "sbj.01J5S9CBM3AT89QTXD7PT0BKA0", - # "subjectId": "SBJ04648" - # } - # }, - # "bclconvertDataRows": [ - # { - # "sampleId": "L2400164", - # "index": "TATTGCGCTC", - # "index2": "CCTAACACAG", - # "lane": 1, - # "overrideCycles": "U7N1Y143;I10;I10;U7N1Y143" - # } - # ], - # "fastqListRows": [ - # { - # "fastqListRowRgid": "TATTGCGCTC.CCTAACACAG.1.240424_A01052_0193_BH7JMMDRX5.L2400164" - # } - # ] - # } - # }, - # { - # "event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", - # "library": { - # "orcabusId": "lib.01J5S9CBTZRYQNTGAHPC2T601D", - # "libraryId": "L2400165", - # "phenotype": "tumor", - # "workflow": "manual", - # "quality": "good", - # "type": "ctDNA", - # "assay": "ctTSOv2", - # "coverage": 38.6, - # "projectOwner": "UMCCR", - # "projectName": "Testing", - # "specimen": { - # "orcabusId": "spc.01J5S9CBTACFBNJKE8C523B0A7", - # "specimenId": "PTC_SCMM01pc5" - # }, - # "subject": { - # "orcabusId": "sbj.01J5S9CBM3AT89QTXD7PT0BKA0", - # "subjectId": "SBJ04648" - # } - # }, - # "bclconvertDataRows": [ - # { - # "sampleId": "L2400165", - # "index": "ACGCCTTGTT", - # "index2": "ACGTTCCTTA", - # "lane": 4, - # "overrideCycles": "U7N1Y143;I10;I10;U7N1Y143" - # } - # ], - # "fastqListRows": [ - # { - # "fastqListRowRgid": "ACGCCTTGTT.ACGTTCCTTA.4.240424_A01052_0193_BH7JMMDRX5.L2400165" - # } - # ] - # } - # }, - # { - # "event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", - # "library": { - # "orcabusId": "lib.01J5S9CBX10204CK7EKGTH9TMB", - # "libraryId": "L2400166", - # "phenotype": "negative-control", - # "workflow": "manual", - # "quality": "good", - # "type": "ctDNA", - # "assay": "ctTSOv2", - # "coverage": 0.1, - # "projectOwner": "UMCCR", - # "projectName": "Testing", - # "specimen": { - # "orcabusId": "spc.01J5S9CBWCGKQMG5S3ZSWA2ATE", - # "specimenId": "NTC_v2ctTSO240207" - # }, - # "subject": { - # "orcabusId": "sbj.01J5S9BYKC1RH7DY68GF1JNSR6", - # "subjectId": "SBJ00006" - # } - # }, - # "bclconvertDataRows": [ - # { - # "sampleId": "L2400166", - # "index": "TTCTACATAC", - # "index2": "TTACAGTTAG", - # "lane": 1, - # "overrideCycles": "U7N1Y143;I10;I10;U7N1Y143" - # } - # ], - # "fastqListRows": [ - # { - # "fastqListRowRgid": "TTCTACATAC.TTACAGTTAG.1.240424_A01052_0193_BH7JMMDRX5.L2400166" - # } - # ] - # } - # }, - # { - # "event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", - # "library": { - # "orcabusId": "lib.01J5S9CDF8HHG5PJE3ECJMKMY7", - # "libraryId": "L2400191", - # "phenotype": "normal", - # "workflow": "research", - # "quality": "good", - # "type": "WGS", - # "assay": "TsqNano", - # "coverage": 40.0, - # "projectOwner": "TJohn", - # "projectName": "CAVATAK", - # "specimen": { - # "orcabusId": "spc.01J5S9CDEH0ATXYAK52KW807R4", - # "specimenId": "PRJ240169" - # }, - # "subject": { - # "orcabusId": "sbj.01J5S9CDDP20JX8V63ZKMPBJQS", - # "subjectId": "SBJ04653" - # } - # }, - # "bclconvertDataRows": [ - # { - # "sampleId": "L2400191", - # "index": "GCACGGAC", - # "index2": "TGCGAGAC", - # "lane": 4, - # "overrideCycles": "Y151;I8N2;I8N2;Y151" - # } - # ], - # "fastqListRows": [ - # { - # "fastqListRowRgid": "GCACGGAC.TGCGAGAC.4.240424_A01052_0193_BH7JMMDRX5.L2400191" - # } - # ] - # } - # }, - # { - # "event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", - # "library": { - # "orcabusId": "lib.01J5S9CDQSSAG1WYCRWMD82Z1S", - # "libraryId": "L2400195", - # "phenotype": "tumor", - # "workflow": "research", - # "quality": "good", - # "type": "WGS", - # "assay": "TsqNano", - # "coverage": 80.0, - # "projectOwner": "TJohn", - # "projectName": "CAVATAK", - # "specimen": { - # "orcabusId": "spc.01J5S9CDQ0V9T98EGRPQJAP11S", - # "specimenId": "PRJ240180" - # }, - # "subject": { - # "orcabusId": "sbj.01J5S9CDDP20JX8V63ZKMPBJQS", - # "subjectId": "SBJ04653" - # } - # }, - # "bclconvertDataRows": [ - # { - # "sampleId": "L2400195", - # "index": "ATGAGGCC", - # "index2": "CAATTAAC", - # "lane": 2, - # "overrideCycles": "Y151;I8N2;I8N2;Y151" - # }, - # { - # "sampleId": "L2400195", - # "index": "ATGAGGCC", - # "index2": "CAATTAAC", - # "lane": 3, - # "overrideCycles": "Y151;I8N2;I8N2;Y151" - # } - # ], - # "fastqListRows": [ - # { - # "fastqListRowRgid": "ATGAGGCC.CAATTAAC.2.240424_A01052_0193_BH7JMMDRX5.L2400195" - # }, - # { - # "fastqListRowRgid": "ATGAGGCC.CAATTAAC.3.240424_A01052_0193_BH7JMMDRX5.L2400195" - # } - # ] - # } - # }, - # { - # "event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", - # "library": { - # "orcabusId": "lib.01J5S9CDSJ2BGEYM8FTXGKVGV8", - # "libraryId": "L2400196", - # "phenotype": "tumor", - # "workflow": "research", - # "quality": "good", - # "type": "WGS", - # "assay": "TsqNano", - # "coverage": 80.0, - # "projectOwner": "TJohn", - # "projectName": "CAVATAK", - # "specimen": { - # "orcabusId": "spc.01J5S9CDRZMMR9S784BYSMVWCT", - # "specimenId": "PRJ240181" - # }, - # "subject": { - # "orcabusId": "sbj.01J5S9CDDP20JX8V63ZKMPBJQS", - # "subjectId": "SBJ04653" - # } - # }, - # "bclconvertDataRows": [ - # { - # "sampleId": "L2400196", - # "index": "ACTAAGAT", - # "index2": "CCGCGGTT", - # "lane": 2, - # "overrideCycles": "Y151;I8N2;I8N2;Y151" - # }, - # { - # "sampleId": "L2400196", - # "index": "ACTAAGAT", - # "index2": "CCGCGGTT", - # "lane": 3, - # "overrideCycles": "Y151;I8N2;I8N2;Y151" - # } - # ], - # "fastqListRows": [ - # { - # "fastqListRowRgid": "ACTAAGAT.CCGCGGTT.2.240424_A01052_0193_BH7JMMDRX5.L2400196" - # }, - # { - # "fastqListRowRgid": "ACTAAGAT.CCGCGGTT.3.240424_A01052_0193_BH7JMMDRX5.L2400196" - # } - # ] - # } - # }, - # { - # "event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", - # "library": { - # "orcabusId": "lib.01J5S9CDVEHDZHZR3BZTQ7WNJQ", - # "libraryId": "L2400197", - # "phenotype": "tumor", - # "workflow": "research", - # "quality": "good", - # "type": "WGS", - # "assay": "TsqNano", - # "coverage": 80.0, - # "projectOwner": "TJohn", - # "projectName": "CAVATAK", - # "specimen": { - # "orcabusId": "spc.01J5S9CDTSHGYMMJHE3SXEB2JG", - # "specimenId": "PRJ240182" - # }, - # "subject": { - # "orcabusId": "sbj.01J5S9CDG7B0KA8YEDK876VVDP", - # "subjectId": "SBJ04654" - # } - # }, - # "bclconvertDataRows": [ - # { - # "sampleId": "L2400197", - # "index": "GTCGGAGC", - # "index2": "TTATAACC", - # "lane": 2, - # "overrideCycles": "Y151;I8N2;I8N2;Y151" - # }, - # { - # "sampleId": "L2400197", - # "index": "GTCGGAGC", - # "index2": "TTATAACC", - # "lane": 3, - # "overrideCycles": "Y151;I8N2;I8N2;Y151" - # }, - # { - # "sampleId": "L2400197", - # "index": "GTCGGAGC", - # "index2": "TTATAACC", - # "lane": 4, - # "overrideCycles": "Y151;I8N2;I8N2;Y151" - # } - # ], - # "fastqListRows": [ - # { - # "fastqListRowRgid": "GTCGGAGC.TTATAACC.2.240424_A01052_0193_BH7JMMDRX5.L2400197" - # }, - # { - # "fastqListRowRgid": "GTCGGAGC.TTATAACC.3.240424_A01052_0193_BH7JMMDRX5.L2400197" - # }, - # { - # "fastqListRowRgid": "GTCGGAGC.TTATAACC.4.240424_A01052_0193_BH7JMMDRX5.L2400197" - # } - # ] - # } - # }, - # { - # "event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", - # "library": { - # "orcabusId": "lib.01J5S9CDXCR7Q5K6A8VJRSMM4Q", - # "libraryId": "L2400198", - # "phenotype": "tumor", - # "workflow": "research", - # "quality": "good", - # "type": "WGS", - # "assay": "TsqNano", - # "coverage": 80.0, - # "projectOwner": "TJohn", - # "projectName": "CAVATAK", - # "specimen": { - # "orcabusId": "spc.01J5S9CDWHAYG4RRG75GYZEK25", - # "specimenId": "PRJ240183" - # }, - # "subject": { - # "orcabusId": "sbj.01J5S9CDG7B0KA8YEDK876VVDP", - # "subjectId": "SBJ04654" - # } - # }, - # "bclconvertDataRows": [ - # { - # "sampleId": "L2400198", - # "index": "CTTGGTAT", - # "index2": "GGACTTGG", - # "lane": 4, - # "overrideCycles": "Y151;I8N2;I8N2;Y151" - # } - # ], - # "fastqListRows": [ - # { - # "fastqListRowRgid": "CTTGGTAT.GGACTTGG.4.240424_A01052_0193_BH7JMMDRX5.L2400198" - # } - # ] - # } - # }, - # { - # "event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", - # "library": { - # "orcabusId": "lib.01J5S9CFX5P69S4KZRQGDFKV1N", - # "libraryId": "L2400231", - # "phenotype": "tumor", - # "workflow": "clinical", - # "quality": "poor", - # "type": "WGS", - # "assay": "TsqNano", - # "coverage": 100.0, - # "projectOwner": "Tothill", - # "projectName": "CUP", - # "specimen": { - # "orcabusId": "spc.01J5S9CFWAQTGK4MZB3HM5NVBC", - # "specimenId": "PRJ240199" - # }, - # "subject": { - # "orcabusId": "sbj.01J5S9CFVJ9GVEHZK6CD9WAAV5", - # "subjectId": "SBJ04659" - # } - # }, - # "bclconvertDataRows": [ - # { - # "sampleId": "L2400231", - # "index": "TCGTAGTG", - # "index2": "CCAAGTCT", - # "lane": 2, - # "overrideCycles": "Y151;I8N2;I8N2;Y151" - # }, - # { - # "sampleId": "L2400231", - # "index": "TCGTAGTG", - # "index2": "CCAAGTCT", - # "lane": 3, - # "overrideCycles": "Y151;I8N2;I8N2;Y151" - # } - # ], - # "fastqListRows": [ - # { - # "fastqListRowRgid": "TCGTAGTG.CCAAGTCT.2.240424_A01052_0193_BH7JMMDRX5.L2400231" - # }, - # { - # "fastqListRowRgid": "TCGTAGTG.CCAAGTCT.3.240424_A01052_0193_BH7JMMDRX5.L2400231" - # } - # ] - # } - # }, - # { - # "event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", - # "library": { - # "orcabusId": "lib.01J5S9CGCAKQWHD9RBM9VXENY9", - # "libraryId": "L2400238", - # "phenotype": "normal", - # "workflow": "clinical", - # "quality": "good", - # "type": "WGS", - # "assay": "TsqNano", - # "coverage": 40.0, - # "projectOwner": "Tothill", - # "projectName": "CUP", - # "specimen": { - # "orcabusId": "spc.01J5S9CGBQCSQCS7XR3T89A82F", - # "specimenId": "PRJ240643" - # }, - # "subject": { - # "orcabusId": "sbj.01J5S9CFVJ9GVEHZK6CD9WAAV5", - # "subjectId": "SBJ04659" - # } - # }, - # "bclconvertDataRows": [ - # { - # "sampleId": "L2400238", - # "index": "GGAGCGTC", - # "index2": "GCACGGAC", - # "lane": 2, - # "overrideCycles": "Y151;I8N2;I8N2;Y151" - # }, - # { - # "sampleId": "L2400238", - # "index": "GGAGCGTC", - # "index2": "GCACGGAC", - # "lane": 3, - # "overrideCycles": "Y151;I8N2;I8N2;Y151" - # } - # ], - # "fastqListRows": [ - # { - # "fastqListRowRgid": "GGAGCGTC.GCACGGAC.2.240424_A01052_0193_BH7JMMDRX5.L2400238" - # }, - # { - # "fastqListRowRgid": "GGAGCGTC.GCACGGAC.3.240424_A01052_0193_BH7JMMDRX5.L2400238" - # } - # ] - # } - # }, - # { - # "event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", - # "library": { - # "orcabusId": "lib.01J5S9CGEM1DHRQP72EP09B2TA", - # "libraryId": "L2400239", - # "phenotype": "normal", - # "workflow": "clinical", - # "quality": "good", - # "type": "WGS", - # "assay": "TsqNano", - # "coverage": 40.0, - # "projectOwner": "Tothill", - # "projectName": "CUP", - # "specimen": { - # "orcabusId": "spc.01J5S9CGE05BJCJ20M2KP4QWWB", - # "specimenId": "PRJ240646" - # }, - # "subject": { - # "orcabusId": "sbj.01J5S9CGDGTF5VZJSSE4ADBNJ3", - # "subjectId": "SBJ04662" - # } - # }, - # "bclconvertDataRows": [ - # { - # "sampleId": "L2400239", - # "index": "ATGGCATG", - # "index2": "GGTACCTT", - # "lane": 2, - # "overrideCycles": "Y151;I8N2;I8N2;Y151" - # }, - # { - # "sampleId": "L2400239", - # "index": "ATGGCATG", - # "index2": "GGTACCTT", - # "lane": 3, - # "overrideCycles": "Y151;I8N2;I8N2;Y151" - # } - # ], - # "fastqListRows": [ - # { - # "fastqListRowRgid": "ATGGCATG.GGTACCTT.2.240424_A01052_0193_BH7JMMDRX5.L2400239" - # }, - # { - # "fastqListRowRgid": "ATGGCATG.GGTACCTT.3.240424_A01052_0193_BH7JMMDRX5.L2400239" - # } - # ] - # } - # }, - # { - # "event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", - # "library": { - # "orcabusId": "lib.01J5S9CGG9N9GH5879SY6A6BJB", - # "libraryId": "L2400240", - # "phenotype": "tumor", - # "workflow": "clinical", - # "quality": "poor", - # "type": "WGS", - # "assay": "TsqNano", - # "coverage": 100.0, - # "projectOwner": "Tothill", - # "projectName": "CUP", - # "specimen": { - # "orcabusId": "spc.01J5S9CGFQM3BQKADX8TWQ4ZH5", - # "specimenId": "PRJ240647" - # }, - # "subject": { - # "orcabusId": "sbj.01J5S9CGDGTF5VZJSSE4ADBNJ3", - # "subjectId": "SBJ04662" - # } - # }, - # "bclconvertDataRows": [ - # { - # "sampleId": "L2400240", - # "index": "GCAATGCA", - # "index2": "AACGTTCC", - # "lane": 2, - # "overrideCycles": "Y151;I8N2;I8N2;Y151" - # }, - # { - # "sampleId": "L2400240", - # "index": "GCAATGCA", - # "index2": "AACGTTCC", - # "lane": 3, - # "overrideCycles": "Y151;I8N2;I8N2;Y151" - # } - # ], - # "fastqListRows": [ - # { - # "fastqListRowRgid": "GCAATGCA.AACGTTCC.2.240424_A01052_0193_BH7JMMDRX5.L2400240" - # }, - # { - # "fastqListRowRgid": "GCAATGCA.AACGTTCC.3.240424_A01052_0193_BH7JMMDRX5.L2400240" - # } - # ] - # } - # }, - # { - # "event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", - # "library": { - # "orcabusId": "lib.01J5S9CGJ6G09YQ9KFHPSXMMVD", - # "libraryId": "L2400241", - # "phenotype": "negative-control", - # "workflow": "control", - # "quality": "good", - # "type": "WGS", - # "assay": "TsqNano", - # "coverage": 0.1, - # "projectOwner": "UMCCR", - # "projectName": "Control", - # "specimen": { - # "orcabusId": "spc.01J5S9CGHK1G7YXD7C4FCXXS52", - # "specimenId": "NTC_TSqN240226" - # }, - # "subject": { - # "orcabusId": "sbj.01J5S9BYKC1RH7DY68GF1JNSR6", - # "subjectId": "SBJ00006" - # } - # }, - # "bclconvertDataRows": [ - # { - # "sampleId": "L2400241", - # "index": "GTTCCAAT", - # "index2": "GCAGAATT", - # "lane": 4, - # "overrideCycles": "Y151;I8N2;I8N2;Y151" - # } - # ], - # "fastqListRows": [ - # { - # "fastqListRowRgid": "GTTCCAAT.GCAGAATT.4.240424_A01052_0193_BH7JMMDRX5.L2400241" - # } - # ] - # } - # }, - # { - # "event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", - # "library": { - # "orcabusId": "lib.01J5S9CGKWDN7STKZKQM3KH9XR", - # "libraryId": "L2400242", - # "phenotype": "normal", - # "workflow": "control", - # "quality": "good", - # "type": "WGS", - # "assay": "TsqNano", - # "coverage": 15.0, - # "projectOwner": "UMCCR", - # "projectName": "Control", - # "specimen": { - # "orcabusId": "spc.01J5S9CGKAT4GHZ1VJFHVV15AD", - # "specimenId": "PTC_TSqN240226" - # }, - # "subject": { - # "orcabusId": "sbj.01J5S9BYVWZDS8AW7A94CDQBXK", - # "subjectId": "SBJ00005" - # } - # }, - # "bclconvertDataRows": [ - # { - # "sampleId": "L2400242", - # "index": "ACCTTGGC", - # "index2": "ATGAGGCC", - # "lane": 4, - # "overrideCycles": "Y151;I8N2;I8N2;Y151" - # } - # ], - # "fastqListRows": [ - # { - # "fastqListRowRgid": "ACCTTGGC.ATGAGGCC.4.240424_A01052_0193_BH7JMMDRX5.L2400242" - # } - # ] - # } - # }, - # { - # "event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", - # "library": { - # "orcabusId": "lib.01J5S9CH2SQ0P1SF7WAT5H4DSE", - # "libraryId": "L2400249", - # "phenotype": "tumor", - # "workflow": "control", - # "quality": "good", - # "type": "WTS", - # "assay": "NebRNA", - # "coverage": 1.0, - # "projectOwner": "UMCCR", - # "projectName": "Control", - # "specimen": { - # "orcabusId": "spc.01J5S9CH267XEJP5GMZK31MJWS", - # "specimenId": "PTC_NebRNA240226" - # }, - # "subject": { - # "orcabusId": "sbj.01J5S9C1S3XV8PNB78XYJ1EQM1", - # "subjectId": "SBJ00029" - # } - # }, - # "bclconvertDataRows": [ - # { - # "sampleId": "L2400249", - # "index": "AGTTTCGA", - # "index2": "CCTACGAT", - # "lane": 4, - # "overrideCycles": "Y151;I8N2;I8N2;Y151" - # } - # ], - # "fastqListRows": [ - # { - # "fastqListRowRgid": "AGTTTCGA.CCTACGAT.4.240424_A01052_0193_BH7JMMDRX5.L2400249" - # } - # ] - # } - # }, - # { - # "event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", - # "library": { - # "orcabusId": "lib.01J5S9CH4CYPA4SP05H8KRX4W9", - # "libraryId": "L2400250", - # "phenotype": "tumor", - # "workflow": "research", - # "quality": "good", - # "type": "WTS", - # "assay": "NebRNA", - # "coverage": 6.0, - # "projectOwner": "Whittle", - # "projectName": "BPOP-retro", - # "specimen": { - # "orcabusId": "spc.01J5S9C0QC2TBZD7XA26D7WGTW", - # "specimenId": "PRJ240003" - # }, - # "subject": { - # "orcabusId": "sbj.01J5S9C0PVB4QNVGK4Q1WSYEGV", - # "subjectId": "SBJ04488" - # } - # }, - # "bclconvertDataRows": [ - # { - # "sampleId": "L2400250", - # "index": "GAACCTCT", - # "index2": "GTCTGCGC", - # "lane": 4, - # "overrideCycles": "Y151;I8N2;I8N2;Y151" - # } - # ], - # "fastqListRows": [ - # { - # "fastqListRowRgid": "GAACCTCT.GTCTGCGC.4.240424_A01052_0193_BH7JMMDRX5.L2400250" - # } - # ] - # } - # }, - # { - # "event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", - # "library": { - # "orcabusId": "lib.01J5S9CH65E4EE5QJEJ1C60GGG", - # "libraryId": "L2400251", - # "phenotype": "tumor", - # "workflow": "research", - # "quality": "good", - # "type": "WTS", - # "assay": "NebRNA", - # "coverage": 6.0, - # "projectOwner": "Whittle", - # "projectName": "BPOP-retro", - # "specimen": { - # "orcabusId": "spc.01J5S9CH5KXY3VMB9M9J2RCR7B", - # "specimenId": "PRJ240561" - # }, - # "subject": { - # "orcabusId": "sbj.01J5S9CFY1BV2Z0SGKYNF1VHQN", - # "subjectId": "SBJ04660" - # } - # }, - # "bclconvertDataRows": [ - # { - # "sampleId": "L2400251", - # "index": "GCCCAGTG", - # "index2": "CCGCAATT", - # "lane": 4, - # "overrideCycles": "Y151;I8N2;I8N2;Y151" - # } - # ], - # "fastqListRows": [ - # { - # "fastqListRowRgid": "GCCCAGTG.CCGCAATT.4.240424_A01052_0193_BH7JMMDRX5.L2400251" - # } - # ] - # } - # }, - # { - # "event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", - # "library": { - # "orcabusId": "lib.01J5S9CH7TGZMV39Z59WJ8H5GP", - # "libraryId": "L2400252", - # "phenotype": "tumor", - # "workflow": "research", - # "quality": "good", - # "type": "WTS", - # "assay": "NebRNA", - # "coverage": 6.0, - # "projectOwner": "Whittle", - # "projectName": "BPOP-retro", - # "specimen": { - # "orcabusId": "spc.01J5S9CH774HEZFEVWWP2XADK1", - # "specimenId": "PRJ240562" - # }, - # "subject": { - # "orcabusId": "sbj.01J5S9CFY1BV2Z0SGKYNF1VHQN", - # "subjectId": "SBJ04660" - # } - # }, - # "bclconvertDataRows": [ - # { - # "sampleId": "L2400252", - # "index": "TGACAGCT", - # "index2": "CCCGTAGG", - # "lane": 4, - # "overrideCycles": "Y151;I8N2;I8N2;Y151" - # } - # ], - # "fastqListRows": [ - # { - # "fastqListRowRgid": "TGACAGCT.CCCGTAGG.4.240424_A01052_0193_BH7JMMDRX5.L2400252" - # } - # ] - # } - # }, - # { - # "event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", - # "library": { - # "orcabusId": "lib.01J5S9CH9TGMT2TJGBZX5VXHJY", - # "libraryId": "L2400253", - # "phenotype": "tumor", - # "workflow": "research", - # "quality": "good", - # "type": "WTS", - # "assay": "NebRNA", - # "coverage": 6.0, - # "projectOwner": "Whittle", - # "projectName": "BPOP-retro", - # "specimen": { - # "orcabusId": "spc.01J5S9CH98MQ2B1G2BQFEY0XZH", - # "specimenId": "PRJ240566" - # }, - # "subject": { - # "orcabusId": "sbj.01J5S9CG5GEWYBK0065C49HT23", - # "subjectId": "SBJ04661" - # } - # }, - # "bclconvertDataRows": [ - # { - # "sampleId": "L2400253", - # "index": "CATCACCC", - # "index2": "ATATAGCA", - # "lane": 4, - # "overrideCycles": "Y151;I8N2;I8N2;Y151" - # } - # ], - # "fastqListRows": [ - # { - # "fastqListRowRgid": "CATCACCC.ATATAGCA.4.240424_A01052_0193_BH7JMMDRX5.L2400253" - # } - # ] - # } - # }, - # { - # "event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", - # "library": { - # "orcabusId": "lib.01J5S9CHBGAP2XSN4TG8SAMRYY", - # "libraryId": "L2400254", - # "phenotype": "tumor", - # "workflow": "research", - # "quality": "borderline", - # "type": "WTS", - # "assay": "NebRNA", - # "coverage": 6.0, - # "projectOwner": "Whittle", - # "projectName": "BPOP-retro", - # "specimen": { - # "orcabusId": "spc.01J5S9CHAX3XKJE5XE4VQWYN5H", - # "specimenId": "PRJ240567" - # }, - # "subject": { - # "orcabusId": "sbj.01J5S9CG5GEWYBK0065C49HT23", - # "subjectId": "SBJ04661" - # } - # }, - # "bclconvertDataRows": [ - # { - # "sampleId": "L2400254", - # "index": "CTGGAGTA", - # "index2": "GTTCGGTT", - # "lane": 4, - # "overrideCycles": "Y151;I8N2;I8N2;Y151" - # } - # ], - # "fastqListRows": [ - # { - # "fastqListRowRgid": "CTGGAGTA.GTTCGGTT.4.240424_A01052_0193_BH7JMMDRX5.L2400254" - # } - # ] - # } - # }, - # { - # "event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", - # "library": { - # "orcabusId": "lib.01J5S9CHE4ERQ4H209DH397W8A", - # "libraryId": "L2400255", - # "phenotype": "tumor", - # "workflow": "clinical", - # "quality": "very-poor", - # "type": "WTS", - # "assay": "NebRNA", - # "coverage": 6.0, - # "projectOwner": "Tothill", - # "projectName": "CUP", - # "specimen": { - # "orcabusId": "spc.01J5S9CHDGRNK70B043K887RP2", - # "specimenId": "PRJ240200" - # }, - # "subject": { - # "orcabusId": "sbj.01J5S9CFVJ9GVEHZK6CD9WAAV5", - # "subjectId": "SBJ04659" - # } - # }, - # "bclconvertDataRows": [ - # { - # "sampleId": "L2400255", - # "index": "GATCCGGG", - # "index2": "AAGCAGGT", - # "lane": 4, - # "overrideCycles": "Y151;I8N2;I8N2;Y151" - # } - # ], - # "fastqListRows": [ - # { - # "fastqListRowRgid": "GATCCGGG.AAGCAGGT.4.240424_A01052_0193_BH7JMMDRX5.L2400255" - # } - # ] - # } - # }, - # { - # "event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", - # "library": { - # "orcabusId": "lib.01J5S9CHFXPDGYQ8TXHRWQR3PY", - # "libraryId": "L2400256", - # "phenotype": "tumor", - # "workflow": "clinical", - # "quality": "very-poor", - # "type": "WTS", - # "assay": "NebRNA", - # "coverage": 6.0, - # "projectOwner": "Tothill", - # "projectName": "CUP", - # "specimen": { - # "orcabusId": "spc.01J5S9CHFAPXYKK49FAGVF5CQF", - # "specimenId": "PRJ240648" - # }, - # "subject": { - # "orcabusId": "sbj.01J5S9CGDGTF5VZJSSE4ADBNJ3", - # "subjectId": "SBJ04662" - # } - # }, - # "bclconvertDataRows": [ - # { - # "sampleId": "L2400256", - # "index": "AACACCTG", - # "index2": "CGCATGGG", - # "lane": 4, - # "overrideCycles": "Y151;I8N2;I8N2;Y151" - # } - # ], - # "fastqListRows": [ - # { - # "fastqListRowRgid": "AACACCTG.CGCATGGG.4.240424_A01052_0193_BH7JMMDRX5.L2400256" - # } - # ] - # } - # }, - # { - # "event_data": { - # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", - # "library": { - # "orcabusId": "lib.01J5S9CHHNGFJN73NPRQMSYGN9", - # "libraryId": "L2400257", - # "phenotype": "negative-control", - # "workflow": "control", - # "quality": "good", - # "type": "WTS", - # "assay": "NebRNA", - # "coverage": 0.1, - # "projectOwner": "UMCCR", - # "projectName": "Control", - # "specimen": { - # "orcabusId": "spc.01J5S9CHH24VFM443RD8Q8X4B3", - # "specimenId": "NTC_NebRNA240226" - # }, - # "subject": { - # "orcabusId": "sbj.01J5S9BYKC1RH7DY68GF1JNSR6", - # "subjectId": "SBJ00006" - # } - # }, - # "bclconvertDataRows": [ - # { - # "sampleId": "L2400257", - # "index": "GTGACGTT", - # "index2": "TCCCAGAT", - # "lane": 4, - # "overrideCycles": "Y151;I8N2;I8N2;Y151" - # } - # ], - # "fastqListRows": [ - # { - # "fastqListRowRgid": "GTGACGTT.TCCCAGAT.4.240424_A01052_0193_BH7JMMDRX5.L2400257" - # } - # ] - # } - # } - # ] - # } +# if __name__ == "__main__": +# import json +# +# print( +# json.dumps( +# handler( +# { +# "instrument_run_id": "240424_A01052_0193_BH7JMMDRX5", +# "samplesheet": +# { +# "header": { +# "file_format_version": 2, +# "run_name": "Tsqn240214-26-ctTSOv2_29Feb24", +# "instrument_type": "NovaSeq" +# }, +# "reads": { +# "read_1_cycles": 151, +# "read_2_cycles": 151, +# "index_1_cycles": 10, +# "index_2_cycles": 10 +# }, +# "bclconvert_settings": { +# "minimum_trimmed_read_length": 35, +# "minimum_adapter_overlap": 3, +# "mask_short_reads": 35, +# "software_version": "4.2.7" +# }, +# "bclconvert_data": [ +# { +# "lane": 1, +# "sample_id": "L2400102", +# "index": "GAATTCGT", +# "index2": "TTATGAGT", +# "override_cycles": "U7N1Y143;I8N2;I8N2;U7N1Y143" +# }, +# { +# "lane": 1, +# "sample_id": "L2400159", +# "index": "GAGAATGGTT", +# "index2": "TTGCTGCCGA", +# "override_cycles": "U7N1Y143;I10;I10;U7N1Y143", +# "adapter_read_1": "CTGTCTCTTATACACATCT", +# "adapter_read_2": "CTGTCTCTTATACACATCT" +# }, +# { +# "lane": 1, +# "sample_id": "L2400160", +# "index": "AGAGGCAACC", +# "index2": "CCATCATTAG", +# "override_cycles": "U7N1Y143;I10;I10;U7N1Y143", +# "adapter_read_1": "CTGTCTCTTATACACATCT", +# "adapter_read_2": "CTGTCTCTTATACACATCT" +# }, +# { +# "lane": 1, +# "sample_id": "L2400161", +# "index": "CCATCATTAG", +# "index2": "AGAGGCAACC", +# "override_cycles": "U7N1Y143;I10;I10;U7N1Y143", +# "adapter_read_1": "CTGTCTCTTATACACATCT", +# "adapter_read_2": "CTGTCTCTTATACACATCT" +# }, +# { +# "lane": 1, +# "sample_id": "L2400162", +# "index": "GATAGGCCGA", +# "index2": "GCCATGTGCG", +# "override_cycles": "U7N1Y143;I10;I10;U7N1Y143", +# "adapter_read_1": "CTGTCTCTTATACACATCT", +# "adapter_read_2": "CTGTCTCTTATACACATCT" +# }, +# { +# "lane": 1, +# "sample_id": "L2400163", +# "index": "ATGGTTGACT", +# "index2": "AGGACAGGCC", +# "override_cycles": "U7N1Y143;I10;I10;U7N1Y143", +# "adapter_read_1": "CTGTCTCTTATACACATCT", +# "adapter_read_2": "CTGTCTCTTATACACATCT" +# }, +# { +# "lane": 1, +# "sample_id": "L2400164", +# "index": "TATTGCGCTC", +# "index2": "CCTAACACAG", +# "override_cycles": "U7N1Y143;I10;I10;U7N1Y143", +# "adapter_read_1": "CTGTCTCTTATACACATCT", +# "adapter_read_2": "CTGTCTCTTATACACATCT" +# }, +# { +# "lane": 1, +# "sample_id": "L2400166", +# "index": "TTCTACATAC", +# "index2": "TTACAGTTAG", +# "override_cycles": "U7N1Y143;I10;I10;U7N1Y143", +# "adapter_read_1": "CTGTCTCTTATACACATCT", +# "adapter_read_2": "CTGTCTCTTATACACATCT" +# }, +# { +# "lane": 2, +# "sample_id": "L2400195", +# "index": "ATGAGGCC", +# "index2": "CAATTAAC", +# "override_cycles": "Y151;I8N2;I8N2;Y151" +# }, +# { +# "lane": 2, +# "sample_id": "L2400196", +# "index": "ACTAAGAT", +# "index2": "CCGCGGTT", +# "override_cycles": "Y151;I8N2;I8N2;Y151" +# }, +# { +# "lane": 2, +# "sample_id": "L2400197", +# "index": "GTCGGAGC", +# "index2": "TTATAACC", +# "override_cycles": "Y151;I8N2;I8N2;Y151" +# }, +# { +# "lane": 2, +# "sample_id": "L2400231", +# "index": "TCGTAGTG", +# "index2": "CCAAGTCT", +# "override_cycles": "Y151;I8N2;I8N2;Y151" +# }, +# { +# "lane": 2, +# "sample_id": "L2400238", +# "index": "GGAGCGTC", +# "index2": "GCACGGAC", +# "override_cycles": "Y151;I8N2;I8N2;Y151" +# }, +# { +# "lane": 2, +# "sample_id": "L2400239", +# "index": "ATGGCATG", +# "index2": "GGTACCTT", +# "override_cycles": "Y151;I8N2;I8N2;Y151" +# }, +# { +# "lane": 2, +# "sample_id": "L2400240", +# "index": "GCAATGCA", +# "index2": "AACGTTCC", +# "override_cycles": "Y151;I8N2;I8N2;Y151" +# }, +# { +# "lane": 3, +# "sample_id": "L2400195", +# "index": "ATGAGGCC", +# "index2": "CAATTAAC", +# "override_cycles": "Y151;I8N2;I8N2;Y151" +# }, +# { +# "lane": 3, +# "sample_id": "L2400196", +# "index": "ACTAAGAT", +# "index2": "CCGCGGTT", +# "override_cycles": "Y151;I8N2;I8N2;Y151" +# }, +# { +# "lane": 3, +# "sample_id": "L2400197", +# "index": "GTCGGAGC", +# "index2": "TTATAACC", +# "override_cycles": "Y151;I8N2;I8N2;Y151" +# }, +# { +# "lane": 3, +# "sample_id": "L2400231", +# "index": "TCGTAGTG", +# "index2": "CCAAGTCT", +# "override_cycles": "Y151;I8N2;I8N2;Y151" +# }, +# { +# "lane": 3, +# "sample_id": "L2400238", +# "index": "GGAGCGTC", +# "index2": "GCACGGAC", +# "override_cycles": "Y151;I8N2;I8N2;Y151" +# }, +# { +# "lane": 3, +# "sample_id": "L2400239", +# "index": "ATGGCATG", +# "index2": "GGTACCTT", +# "override_cycles": "Y151;I8N2;I8N2;Y151" +# }, +# { +# "lane": 3, +# "sample_id": "L2400240", +# "index": "GCAATGCA", +# "index2": "AACGTTCC", +# "override_cycles": "Y151;I8N2;I8N2;Y151" +# }, +# { +# "lane": 4, +# "sample_id": "L2400165", +# "index": "ACGCCTTGTT", +# "index2": "ACGTTCCTTA", +# "override_cycles": "U7N1Y143;I10;I10;U7N1Y143", +# "adapter_read_1": "CTGTCTCTTATACACATCT", +# "adapter_read_2": "CTGTCTCTTATACACATCT" +# }, +# { +# "lane": 4, +# "sample_id": "L2400191", +# "index": "GCACGGAC", +# "index2": "TGCGAGAC", +# "override_cycles": "Y151;I8N2;I8N2;Y151" +# }, +# { +# "lane": 4, +# "sample_id": "L2400197", +# "index": "GTCGGAGC", +# "index2": "TTATAACC", +# "override_cycles": "Y151;I8N2;I8N2;Y151" +# }, +# { +# "lane": 4, +# "sample_id": "L2400198", +# "index": "CTTGGTAT", +# "index2": "GGACTTGG", +# "override_cycles": "Y151;I8N2;I8N2;Y151" +# }, +# { +# "lane": 4, +# "sample_id": "L2400241", +# "index": "GTTCCAAT", +# "index2": "GCAGAATT", +# "override_cycles": "Y151;I8N2;I8N2;Y151" +# }, +# { +# "lane": 4, +# "sample_id": "L2400242", +# "index": "ACCTTGGC", +# "index2": "ATGAGGCC", +# "override_cycles": "Y151;I8N2;I8N2;Y151" +# }, +# { +# "lane": 4, +# "sample_id": "L2400249", +# "index": "AGTTTCGA", +# "index2": "CCTACGAT", +# "override_cycles": "Y151;I8N2;I8N2;Y151" +# }, +# { +# "lane": 4, +# "sample_id": "L2400250", +# "index": "GAACCTCT", +# "index2": "GTCTGCGC", +# "override_cycles": "Y151;I8N2;I8N2;Y151" +# }, +# { +# "lane": 4, +# "sample_id": "L2400251", +# "index": "GCCCAGTG", +# "index2": "CCGCAATT", +# "override_cycles": "Y151;I8N2;I8N2;Y151" +# }, +# { +# "lane": 4, +# "sample_id": "L2400252", +# "index": "TGACAGCT", +# "index2": "CCCGTAGG", +# "override_cycles": "Y151;I8N2;I8N2;Y151" +# }, +# { +# "lane": 4, +# "sample_id": "L2400253", +# "index": "CATCACCC", +# "index2": "ATATAGCA", +# "override_cycles": "Y151;I8N2;I8N2;Y151" +# }, +# { +# "lane": 4, +# "sample_id": "L2400254", +# "index": "CTGGAGTA", +# "index2": "GTTCGGTT", +# "override_cycles": "Y151;I8N2;I8N2;Y151" +# }, +# { +# "lane": 4, +# "sample_id": "L2400255", +# "index": "GATCCGGG", +# "index2": "AAGCAGGT", +# "override_cycles": "Y151;I8N2;I8N2;Y151" +# }, +# { +# "lane": 4, +# "sample_id": "L2400256", +# "index": "AACACCTG", +# "index2": "CGCATGGG", +# "override_cycles": "Y151;I8N2;I8N2;Y151" +# }, +# { +# "lane": 4, +# "sample_id": "L2400257", +# "index": "GTGACGTT", +# "index2": "TCCCAGAT", +# "override_cycles": "Y151;I8N2;I8N2;Y151" +# } +# ], +# "cloud_settings": { +# "generated_version": "0.0.0", +# "cloud_workflow": "ica_workflow_1", +# "bclconvert_pipeline": "urn:ilmn:ica:pipeline:bf93b5cf-cb27-4dfa-846e-acd6eb081aca#BclConvert_v4_2_7" +# }, +# "cloud_data": [ +# { +# "sample_id": "L2400102", +# "library_name": "L2400102_GAATTCGT_TTATGAGT", +# "library_prep_kit_name": "ctTSO" +# }, +# { +# "sample_id": "L2400159", +# "library_name": "L2400159_GAGAATGGTT_TTGCTGCCGA", +# "library_prep_kit_name": "ctTSOv2" +# }, +# { +# "sample_id": "L2400160", +# "library_name": "L2400160_AGAGGCAACC_CCATCATTAG", +# "library_prep_kit_name": "ctTSOv2" +# }, +# { +# "sample_id": "L2400161", +# "library_name": "L2400161_CCATCATTAG_AGAGGCAACC", +# "library_prep_kit_name": "ctTSOv2" +# }, +# { +# "sample_id": "L2400162", +# "library_name": "L2400162_GATAGGCCGA_GCCATGTGCG", +# "library_prep_kit_name": "ctTSOv2" +# }, +# { +# "sample_id": "L2400163", +# "library_name": "L2400163_ATGGTTGACT_AGGACAGGCC", +# "library_prep_kit_name": "ctTSOv2" +# }, +# { +# "sample_id": "L2400164", +# "library_name": "L2400164_TATTGCGCTC_CCTAACACAG", +# "library_prep_kit_name": "ctTSOv2" +# }, +# { +# "sample_id": "L2400165", +# "library_name": "L2400165_ACGCCTTGTT_ACGTTCCTTA", +# "library_prep_kit_name": "ctTSOv2" +# }, +# { +# "sample_id": "L2400166", +# "library_name": "L2400166_TTCTACATAC_TTACAGTTAG", +# "library_prep_kit_name": "ctTSOv2" +# }, +# { +# "sample_id": "L2400191", +# "library_name": "L2400191_GCACGGAC_TGCGAGAC", +# "library_prep_kit_name": "TsqNano" +# }, +# { +# "sample_id": "L2400195", +# "library_name": "L2400195_ATGAGGCC_CAATTAAC", +# "library_prep_kit_name": "TsqNano" +# }, +# { +# "sample_id": "L2400196", +# "library_name": "L2400196_ACTAAGAT_CCGCGGTT", +# "library_prep_kit_name": "TsqNano" +# }, +# { +# "sample_id": "L2400197", +# "library_name": "L2400197_GTCGGAGC_TTATAACC", +# "library_prep_kit_name": "TsqNano" +# }, +# { +# "sample_id": "L2400198", +# "library_name": "L2400198_CTTGGTAT_GGACTTGG", +# "library_prep_kit_name": "TsqNano" +# }, +# { +# "sample_id": "L2400231", +# "library_name": "L2400231_TCGTAGTG_CCAAGTCT", +# "library_prep_kit_name": "TsqNano" +# }, +# { +# "sample_id": "L2400238", +# "library_name": "L2400238_GGAGCGTC_GCACGGAC", +# "library_prep_kit_name": "TsqNano" +# }, +# { +# "sample_id": "L2400239", +# "library_name": "L2400239_ATGGCATG_GGTACCTT", +# "library_prep_kit_name": "TsqNano" +# }, +# { +# "sample_id": "L2400240", +# "library_name": "L2400240_GCAATGCA_AACGTTCC", +# "library_prep_kit_name": "TsqNano" +# }, +# { +# "sample_id": "L2400241", +# "library_name": "L2400241_GTTCCAAT_GCAGAATT", +# "library_prep_kit_name": "TsqNano" +# }, +# { +# "sample_id": "L2400242", +# "library_name": "L2400242_ACCTTGGC_ATGAGGCC", +# "library_prep_kit_name": "TsqNano" +# }, +# { +# "sample_id": "L2400249", +# "library_name": "L2400249_AGTTTCGA_CCTACGAT", +# "library_prep_kit_name": "NebRNA" +# }, +# { +# "sample_id": "L2400250", +# "library_name": "L2400250_GAACCTCT_GTCTGCGC", +# "library_prep_kit_name": "NebRNA" +# }, +# { +# "sample_id": "L2400251", +# "library_name": "L2400251_GCCCAGTG_CCGCAATT", +# "library_prep_kit_name": "NebRNA" +# }, +# { +# "sample_id": "L2400252", +# "library_name": "L2400252_TGACAGCT_CCCGTAGG", +# "library_prep_kit_name": "NebRNA" +# }, +# { +# "sample_id": "L2400253", +# "library_name": "L2400253_CATCACCC_ATATAGCA", +# "library_prep_kit_name": "NebRNA" +# }, +# { +# "sample_id": "L2400254", +# "library_name": "L2400254_CTGGAGTA_GTTCGGTT", +# "library_prep_kit_name": "NebRNA" +# }, +# { +# "sample_id": "L2400255", +# "library_name": "L2400255_GATCCGGG_AAGCAGGT", +# "library_prep_kit_name": "NebRNA" +# }, +# { +# "sample_id": "L2400256", +# "library_name": "L2400256_AACACCTG_CGCATGGG", +# "library_prep_kit_name": "NebRNA" +# }, +# { +# "sample_id": "L2400257", +# "library_name": "L2400257_GTGACGTT_TCCCAGAT", +# "library_prep_kit_name": "NebRNA" +# } +# ] +# }, +# "library_obj_list": [ +# { +# "orcabusId": "lib.01J8ES4MPZ5B201R50K42XXM4M", +# "projectSet": [ +# { +# "orcabusId": "prj.01J8ES4EBXK08WDWB97BSCX1C9", +# "projectId": "PO", +# "name": None, +# "description": None +# } +# ], +# "sample": { +# "orcabusId": "smp.01J8ES4MPHSX7MRCTTFWJBYTT7", +# "sampleId": "MDX210402", +# "externalSampleId": "ZUHR111121", +# "source": "plasma-serum" +# }, +# "subject": { +# "orcabusId": "sbj.01J8ES4MNXJSDRR406DAXFZP2N", +# "subjectId": "PM3045106" +# }, +# "libraryId": "L2400102", +# "phenotype": "tumor", +# "workflow": "research", +# "quality": "borderline", +# "type": "WGS", +# "assay": "ctTSO", +# "coverage": 50.0 +# }, +# { +# "orcabusId": "lib.01J8ES4XNYFP38JMDV7GMV0V3V", +# "projectSet": [ +# { +# "orcabusId": "prj.01J8ES4XMWD0DH7MDRNER5TZS1", +# "projectId": "Testing", +# "name": None, +# "description": None +# } +# ], +# "sample": { +# "orcabusId": "smp.01J8ES4XMDW0FV1YMWHSZZQ4TX", +# "sampleId": "PTC_SCMM1pc2", +# "externalSampleId": "SSq-CompMM-1pc-10646259ilm", +# "source": "cfDNA" +# }, +# "subject": { +# "orcabusId": "sbj.01J8ES4XKHKNQ1NF8EKKACZ032", +# "subjectId": "CMM1pc-10646259ilm" +# }, +# "libraryId": "L2400159", +# "phenotype": "tumor", +# "workflow": "manual", +# "quality": "good", +# "type": "ctDNA", +# "assay": "ctTSOv2", +# "coverage": 38.6 +# }, +# { +# "orcabusId": "lib.01J8ES4XQG3MPBW94TTVT4STVG", +# "projectSet": [ +# { +# "orcabusId": "prj.01J8ES4XMWD0DH7MDRNER5TZS1", +# "projectId": "Testing", +# "name": None, +# "description": None +# } +# ], +# "sample": { +# "orcabusId": "smp.01J8ES4XQ071BF3WZN111SNJ2B", +# "sampleId": "PTC_SCMM1pc3", +# "externalSampleId": "SSq-CompMM-1pc-10646259ilm", +# "source": "cfDNA" +# }, +# "subject": { +# "orcabusId": "sbj.01J8ES4XKHKNQ1NF8EKKACZ032", +# "subjectId": "CMM1pc-10646259ilm" +# }, +# "libraryId": "L2400160", +# "phenotype": "tumor", +# "workflow": "manual", +# "quality": "good", +# "type": "ctDNA", +# "assay": "ctTSOv2", +# "coverage": 38.6 +# }, +# { +# "orcabusId": "lib.01J8ES4XSS97XNRS8DH0B1RJRG", +# "projectSet": [ +# { +# "orcabusId": "prj.01J8ES4XMWD0DH7MDRNER5TZS1", +# "projectId": "Testing", +# "name": None, +# "description": None +# } +# ], +# "sample": { +# "orcabusId": "smp.01J8ES4XRG9NB38N03688M2CCB", +# "sampleId": "PTC_SCMM1pc4", +# "externalSampleId": "SSq-CompMM-1pc-10646259ilm", +# "source": "cfDNA" +# }, +# "subject": { +# "orcabusId": "sbj.01J8ES4XKHKNQ1NF8EKKACZ032", +# "subjectId": "CMM1pc-10646259ilm" +# }, +# "libraryId": "L2400161", +# "phenotype": "tumor", +# "workflow": "manual", +# "quality": "good", +# "type": "ctDNA", +# "assay": "ctTSOv2", +# "coverage": 38.6 +# }, +# { +# "orcabusId": "lib.01J8ES4XXF6NMEJMM5M4GWS6KH", +# "projectSet": [ +# { +# "orcabusId": "prj.01J8ES4XMWD0DH7MDRNER5TZS1", +# "projectId": "Testing", +# "name": None, +# "description": None +# } +# ], +# "sample": { +# "orcabusId": "smp.01J8ES4XWXFANT7P0T3AFXA85G", +# "sampleId": "PTC_SCMM01pc20", +# "externalSampleId": "SSq-CompMM-0.1pc-10624819 - 20ng", +# "source": "cfDNA" +# }, +# "subject": { +# "orcabusId": "sbj.01J8ES4XW2TXGEJBQWCVMRZRTS", +# "subjectId": "CMM0.1pc-10624819" +# }, +# "libraryId": "L2400162", +# "phenotype": "tumor", +# "workflow": "manual", +# "quality": "good", +# "type": "ctDNA", +# "assay": "ctTSOv2", +# "coverage": 38.6 +# }, +# { +# "orcabusId": "lib.01J8ES4XZD7T2VRPVQ1GSVZ11X", +# "projectSet": [ +# { +# "orcabusId": "prj.01J8ES4XMWD0DH7MDRNER5TZS1", +# "projectId": "Testing", +# "name": None, +# "description": None +# } +# ], +# "sample": { +# "orcabusId": "smp.01J8ES4XYTSVQVRSBA9M26NSZY", +# "sampleId": "PTC_SCMM01pc15", +# "externalSampleId": "SSq-CompMM-0.1pc-10624819 - 15ng", +# "source": "cfDNA" +# }, +# "subject": { +# "orcabusId": "sbj.01J8ES4XW2TXGEJBQWCVMRZRTS", +# "subjectId": "CMM0.1pc-10624819" +# }, +# "libraryId": "L2400163", +# "phenotype": "tumor", +# "workflow": "manual", +# "quality": "good", +# "type": "ctDNA", +# "assay": "ctTSOv2", +# "coverage": 38.6 +# }, +# { +# "orcabusId": "lib.01J8ES4Y1AKAHYD9EW0TW4FBCP", +# "projectSet": [ +# { +# "orcabusId": "prj.01J8ES4XMWD0DH7MDRNER5TZS1", +# "projectId": "Testing", +# "name": None, +# "description": None +# } +# ], +# "sample": { +# "orcabusId": "smp.01J8ES4Y0V0ZBKAE91TDSY0BBB", +# "sampleId": "PTC_SCMM01pc10", +# "externalSampleId": "SSq-CompMM-0.1pc-10624819 - 10ng", +# "source": "cfDNA" +# }, +# "subject": { +# "orcabusId": "sbj.01J8ES4XW2TXGEJBQWCVMRZRTS", +# "subjectId": "CMM0.1pc-10624819" +# }, +# "libraryId": "L2400164", +# "phenotype": "tumor", +# "workflow": "manual", +# "quality": "good", +# "type": "ctDNA", +# "assay": "ctTSOv2", +# "coverage": 38.6 +# }, +# { +# "orcabusId": "lib.01J8ES4Y3ZKRX3C5JAHA5NBXV1", +# "projectSet": [ +# { +# "orcabusId": "prj.01J8ES4XMWD0DH7MDRNER5TZS1", +# "projectId": "Testing", +# "name": None, +# "description": None +# } +# ], +# "sample": { +# "orcabusId": "smp.01J8ES4Y37JTPEEJSED9BXH8N2", +# "sampleId": "PTC_SCMM01pc5", +# "externalSampleId": "SSq-CompMM-0.1pc-10624819 - 5ng", +# "source": "cfDNA" +# }, +# "subject": { +# "orcabusId": "sbj.01J8ES4XW2TXGEJBQWCVMRZRTS", +# "subjectId": "CMM0.1pc-10624819" +# }, +# "libraryId": "L2400165", +# "phenotype": "tumor", +# "workflow": "manual", +# "quality": "good", +# "type": "ctDNA", +# "assay": "ctTSOv2", +# "coverage": 38.6 +# }, +# { +# "orcabusId": "lib.01J8ES4Y5D52202JVBXHJ9Q9WF", +# "projectSet": [ +# { +# "orcabusId": "prj.01J8ES4XMWD0DH7MDRNER5TZS1", +# "projectId": "Testing", +# "name": None, +# "description": None +# } +# ], +# "sample": { +# "orcabusId": "smp.01J8ES4Y4XK1WX4WCPD6XY8KNM", +# "sampleId": "NTC_v2ctTSO240207", +# "externalSampleId": "negative control", +# "source": "water" +# }, +# "subject": { +# "orcabusId": "sbj.01J8ES4DFMNF0SX6P8P8Y9J6K1", +# "subjectId": "negative control" +# }, +# "libraryId": "L2400166", +# "phenotype": "negative-control", +# "workflow": "manual", +# "quality": "good", +# "type": "ctDNA", +# "assay": "ctTSOv2", +# "coverage": 0.1 +# }, +# { +# "orcabusId": "lib.01J8ES4ZDRQAP2BN3SDYYV5PKW", +# "projectSet": [ +# { +# "orcabusId": "prj.01J8ES4ZAWHH3FKYA2CFHSMZ4B", +# "projectId": "CAVATAK", +# "name": None, +# "description": None +# } +# ], +# "sample": { +# "orcabusId": "smp.01J8ES4ZDAFRK3K3PY33F8XS0W", +# "sampleId": "PRJ240169", +# "externalSampleId": "AUS-006-DRW_C1D1PRE", +# "source": "blood" +# }, +# "subject": { +# "orcabusId": "sbj.01J8ES4ZCKNW6QKP006SYNZ5RA", +# "subjectId": "AUS-006-DRW" +# }, +# "libraryId": "L2400191", +# "phenotype": "normal", +# "workflow": "research", +# "quality": "good", +# "type": "WGS", +# "assay": "TsqNano", +# "coverage": 40.0 +# }, +# { +# "orcabusId": "lib.01J8ES4ZMY0G1H9MDN7K2TH9Y6", +# "projectSet": [ +# { +# "orcabusId": "prj.01J8ES4ZAWHH3FKYA2CFHSMZ4B", +# "projectId": "CAVATAK", +# "name": None, +# "description": None +# } +# ], +# "sample": { +# "orcabusId": "smp.01J8ES4ZMETZP255WMFC8TSCYT", +# "sampleId": "PRJ240180", +# "externalSampleId": "AUS-006-DRW_Day0", +# "source": "tissue" +# }, +# "subject": { +# "orcabusId": "sbj.01J8ES4ZCKNW6QKP006SYNZ5RA", +# "subjectId": "AUS-006-DRW" +# }, +# "libraryId": "L2400195", +# "phenotype": "tumor", +# "workflow": "research", +# "quality": "good", +# "type": "WGS", +# "assay": "TsqNano", +# "coverage": 80.0 +# }, +# { +# "orcabusId": "lib.01J8ES4ZP88X2E17X5X1FRMTPK", +# "projectSet": [ +# { +# "orcabusId": "prj.01J8ES4ZAWHH3FKYA2CFHSMZ4B", +# "projectId": "CAVATAK", +# "name": None, +# "description": None +# } +# ], +# "sample": { +# "orcabusId": "smp.01J8ES4ZNT47EM37QKMT12JPPJ", +# "sampleId": "PRJ240181", +# "externalSampleId": "AUS-006-DRW_Day33", +# "source": "tissue" +# }, +# "subject": { +# "orcabusId": "sbj.01J8ES4ZCKNW6QKP006SYNZ5RA", +# "subjectId": "AUS-006-DRW" +# }, +# "libraryId": "L2400196", +# "phenotype": "tumor", +# "workflow": "research", +# "quality": "good", +# "type": "WGS", +# "assay": "TsqNano", +# "coverage": 80.0 +# }, +# { +# "orcabusId": "lib.01J8ES4ZST489C712CG3R9NQSQ", +# "projectSet": [ +# { +# "orcabusId": "prj.01J8ES4ZAWHH3FKYA2CFHSMZ4B", +# "projectId": "CAVATAK", +# "name": None, +# "description": None +# } +# ], +# "sample": { +# "orcabusId": "smp.01J8ES4ZQ76H8P0Q7S618F3BMA", +# "sampleId": "PRJ240182", +# "externalSampleId": "AUS-007-JMA_Day0", +# "source": "tissue" +# }, +# "subject": { +# "orcabusId": "sbj.01J8ES4ZEQ3FVD6DDVEG8MW60Q", +# "subjectId": "AUS-007-JMA" +# }, +# "libraryId": "L2400197", +# "phenotype": "tumor", +# "workflow": "research", +# "quality": "good", +# "type": "WGS", +# "assay": "TsqNano", +# "coverage": 80.0 +# }, +# { +# "orcabusId": "lib.01J8ES4ZVWA2CGBHJVKAS3Y0G9", +# "projectSet": [ +# { +# "orcabusId": "prj.01J8ES4ZAWHH3FKYA2CFHSMZ4B", +# "projectId": "CAVATAK", +# "name": None, +# "description": None +# } +# ], +# "sample": { +# "orcabusId": "smp.01J8ES4ZVAR9NQM55Z2TXCDY9V", +# "sampleId": "PRJ240183", +# "externalSampleId": "AUS-007-JMA_Day15", +# "source": "tissue" +# }, +# "subject": { +# "orcabusId": "sbj.01J8ES4ZEQ3FVD6DDVEG8MW60Q", +# "subjectId": "AUS-007-JMA" +# }, +# "libraryId": "L2400198", +# "phenotype": "tumor", +# "workflow": "research", +# "quality": "good", +# "type": "WGS", +# "assay": "TsqNano", +# "coverage": 80.0 +# }, +# { +# "orcabusId": "lib.01J8ES51V0RSVT6C7WQR72QQED", +# "projectSet": [ +# { +# "orcabusId": "prj.01J8ES4EZAA5YMHX82664GJQB3", +# "projectId": "CUP", +# "name": None, +# "description": None +# } +# ], +# "sample": { +# "orcabusId": "smp.01J8ES51T84KVVVSEPYQFGW0EV", +# "sampleId": "PRJ240199", +# "externalSampleId": "DNA188239", +# "source": "FFPE" +# }, +# "subject": { +# "orcabusId": "sbj.01J8ES51S87R4EJ61QJ0DMDYWZ", +# "subjectId": "SN_PMC-141" +# }, +# "libraryId": "L2400231", +# "phenotype": "tumor", +# "workflow": "clinical", +# "quality": "poor", +# "type": "WGS", +# "assay": "TsqNano", +# "coverage": 100.0 +# }, +# { +# "orcabusId": "lib.01J8ES52889Q8826P5SH9HDPP0", +# "projectSet": [ +# { +# "orcabusId": "prj.01J8ES4EZAA5YMHX82664GJQB3", +# "projectId": "CUP", +# "name": None, +# "description": None +# } +# ], +# "sample": { +# "orcabusId": "smp.01J8ES527QKB5Y5RVZWZ8HQX0H", +# "sampleId": "PRJ240643", +# "externalSampleId": "DNA188378", +# "source": "blood" +# }, +# "subject": { +# "orcabusId": "sbj.01J8ES51S87R4EJ61QJ0DMDYWZ", +# "subjectId": "SN_PMC-141" +# }, +# "libraryId": "L2400238", +# "phenotype": "normal", +# "workflow": "clinical", +# "quality": "good", +# "type": "WGS", +# "assay": "TsqNano", +# "coverage": 40.0 +# }, +# { +# "orcabusId": "lib.01J8ES52ANMRT3B7Y96T1Y3RY8", +# "projectSet": [ +# { +# "orcabusId": "prj.01J8ES4EZAA5YMHX82664GJQB3", +# "projectId": "CUP", +# "name": None, +# "description": None +# } +# ], +# "sample": { +# "orcabusId": "smp.01J8ES52A5QX0GQ6RB78Z8DGYQ", +# "sampleId": "PRJ240646", +# "externalSampleId": "DNA189922", +# "source": "blood" +# }, +# "subject": { +# "orcabusId": "sbj.01J8ES529GSPBV64SESK9SWD76", +# "subjectId": "SN_PMC-145" +# }, +# "libraryId": "L2400239", +# "phenotype": "normal", +# "workflow": "clinical", +# "quality": "good", +# "type": "WGS", +# "assay": "TsqNano", +# "coverage": 40.0 +# }, +# { +# "orcabusId": "lib.01J8ES52C3N585BGGY4VNXHC83", +# "projectSet": [ +# { +# "orcabusId": "prj.01J8ES4EZAA5YMHX82664GJQB3", +# "projectId": "CUP", +# "name": None, +# "description": None +# } +# ], +# "sample": { +# "orcabusId": "smp.01J8ES52BM8BVS3PX47E6FM7D5", +# "sampleId": "PRJ240647", +# "externalSampleId": "DNA189848", +# "source": "FFPE" +# }, +# "subject": { +# "orcabusId": "sbj.01J8ES529GSPBV64SESK9SWD76", +# "subjectId": "SN_PMC-145" +# }, +# "libraryId": "L2400240", +# "phenotype": "tumor", +# "workflow": "clinical", +# "quality": "poor", +# "type": "WGS", +# "assay": "TsqNano", +# "coverage": 100.0 +# }, +# { +# "orcabusId": "lib.01J8ES52DHAPZM6FZ0VZK89PRT", +# "projectSet": [ +# { +# "orcabusId": "prj.01J8ES4FC6DVW20AR33FBX2SA8", +# "projectId": "Control", +# "name": None, +# "description": None +# } +# ], +# "sample": { +# "orcabusId": "smp.01J8ES52D076FQM5K8128AQ593", +# "sampleId": "NTC_TSqN240226", +# "externalSampleId": "NTC_TSqN240226", +# "source": "water" +# }, +# "subject": { +# "orcabusId": "sbj.01J8ES4DFMNF0SX6P8P8Y9J6K1", +# "subjectId": "negative control" +# }, +# "libraryId": "L2400241", +# "phenotype": "negative-control", +# "workflow": "control", +# "quality": "good", +# "type": "WGS", +# "assay": "TsqNano", +# "coverage": 0.1 +# }, +# { +# "orcabusId": "lib.01J8ES52F2ZHRXQY1AT1N1F81F", +# "projectSet": [ +# { +# "orcabusId": "prj.01J8ES4FC6DVW20AR33FBX2SA8", +# "projectId": "Control", +# "name": None, +# "description": None +# } +# ], +# "sample": { +# "orcabusId": "smp.01J8ES52EEX67YRYAJS3F5GMJ5", +# "sampleId": "PTC_TSqN240226", +# "externalSampleId": "NA24385-3", +# "source": "cell-line" +# }, +# "subject": { +# "orcabusId": "sbj.01J8ES4DRJ31Z2H1GJQZGVDXZR", +# "subjectId": "NA24385" +# }, +# "libraryId": "L2400242", +# "phenotype": "normal", +# "workflow": "control", +# "quality": "good", +# "type": "WGS", +# "assay": "TsqNano", +# "coverage": 15.0 +# }, +# { +# "orcabusId": "lib.01J8ES52XYMVGRB1Q458THNG4T", +# "projectSet": [ +# { +# "orcabusId": "prj.01J8ES4FC6DVW20AR33FBX2SA8", +# "projectId": "Control", +# "name": None, +# "description": None +# } +# ], +# "sample": { +# "orcabusId": "smp.01J8ES52XE661E8V8XTWD02QCK", +# "sampleId": "PTC_NebRNA240226", +# "externalSampleId": "Colo829", +# "source": "cell-line" +# }, +# "subject": { +# "orcabusId": "sbj.01J8ES4GNVGZSJVTHGVKS9VW7F", +# "subjectId": "Colo829" +# }, +# "libraryId": "L2400249", +# "phenotype": "tumor", +# "workflow": "control", +# "quality": "good", +# "type": "WTS", +# "assay": "NebRNA", +# "coverage": 1.0 +# }, +# { +# "orcabusId": "lib.01J8ES52Z2KTVVKZ2ZGVQ6YC10", +# "projectSet": [ +# { +# "orcabusId": "prj.01J8ES4FH3XMPZQNDJ9J000BXX", +# "projectId": "BPOP-retro", +# "name": None, +# "description": None +# } +# ], +# "sample": { +# "orcabusId": "smp.01J8ES4FP9WTFBDNGKVG3D9BD4", +# "sampleId": "PRJ240003", +# "externalSampleId": "3-23BCRL057T", +# "source": "tissue" +# }, +# "subject": { +# "orcabusId": "sbj.01J8ES4FNJ2FCAK0RJST0428X0", +# "subjectId": "23BCRL057T" +# }, +# "libraryId": "L2400250", +# "phenotype": "tumor", +# "workflow": "research", +# "quality": "good", +# "type": "WTS", +# "assay": "NebRNA", +# "coverage": 6.0 +# }, +# { +# "orcabusId": "lib.01J8ES530H895X4WA3NQ6CY2QV", +# "projectSet": [ +# { +# "orcabusId": "prj.01J8ES4FH3XMPZQNDJ9J000BXX", +# "projectId": "BPOP-retro", +# "name": None, +# "description": None +# } +# ], +# "sample": { +# "orcabusId": "smp.01J8ES530355YNZ3VHQQQ204PF", +# "sampleId": "PRJ240561", +# "externalSampleId": "4-218-004_Bx", +# "source": "tissue" +# }, +# "subject": { +# "orcabusId": "sbj.01J8ES51WC4GV5YDJNTMAK2YY1", +# "subjectId": "218-004" +# }, +# "libraryId": "L2400251", +# "phenotype": "tumor", +# "workflow": "research", +# "quality": "good", +# "type": "WTS", +# "assay": "NebRNA", +# "coverage": 6.0 +# }, +# { +# "orcabusId": "lib.01J8ES5320EWBNNYDGXF2SYJBD", +# "projectSet": [ +# { +# "orcabusId": "prj.01J8ES4FH3XMPZQNDJ9J000BXX", +# "projectId": "BPOP-retro", +# "name": None, +# "description": None +# } +# ], +# "sample": { +# "orcabusId": "smp.01J8ES531H420JM9MG5R4AE1AZ", +# "sampleId": "PRJ240562", +# "externalSampleId": "5-218-004_04", +# "source": "tissue" +# }, +# "subject": { +# "orcabusId": "sbj.01J8ES51WC4GV5YDJNTMAK2YY1", +# "subjectId": "218-004" +# }, +# "libraryId": "L2400252", +# "phenotype": "tumor", +# "workflow": "research", +# "quality": "good", +# "type": "WTS", +# "assay": "NebRNA", +# "coverage": 6.0 +# }, +# { +# "orcabusId": "lib.01J8ES533DJZZNPP9MXYR5TRC0", +# "projectSet": [ +# { +# "orcabusId": "prj.01J8ES4FH3XMPZQNDJ9J000BXX", +# "projectId": "BPOP-retro", +# "name": None, +# "description": None +# } +# ], +# "sample": { +# "orcabusId": "smp.01J8ES532ZBHWY3DWY0DWQ223R", +# "sampleId": "PRJ240566", +# "externalSampleId": "9-218-007_Bx", +# "source": "tissue" +# }, +# "subject": { +# "orcabusId": "sbj.01J8ES522WN7YPZS1Z9NGSPNDA", +# "subjectId": "218-007" +# }, +# "libraryId": "L2400253", +# "phenotype": "tumor", +# "workflow": "research", +# "quality": "good", +# "type": "WTS", +# "assay": "NebRNA", +# "coverage": 6.0 +# }, +# { +# "orcabusId": "lib.01J8ES534XGBFYDVYV8ZG6SYS0", +# "projectSet": [ +# { +# "orcabusId": "prj.01J8ES4FH3XMPZQNDJ9J000BXX", +# "projectId": "BPOP-retro", +# "name": None, +# "description": None +# } +# ], +# "sample": { +# "orcabusId": "smp.01J8ES534BX7B89X5EKSCFRDDZ", +# "sampleId": "PRJ240567", +# "externalSampleId": "10-218-007_04", +# "source": "tissue" +# }, +# "subject": { +# "orcabusId": "sbj.01J8ES522WN7YPZS1Z9NGSPNDA", +# "subjectId": "218-007" +# }, +# "libraryId": "L2400254", +# "phenotype": "tumor", +# "workflow": "research", +# "quality": "borderline", +# "type": "WTS", +# "assay": "NebRNA", +# "coverage": 6.0 +# }, +# { +# "orcabusId": "lib.01J8ES536AB5A5PBJ8S45SZP7Q", +# "projectSet": [ +# { +# "orcabusId": "prj.01J8ES4EZAA5YMHX82664GJQB3", +# "projectId": "CUP", +# "name": None, +# "description": None +# } +# ], +# "sample": { +# "orcabusId": "smp.01J8ES535VGG93023KWAFMWGH4", +# "sampleId": "PRJ240200", +# "externalSampleId": "RNA036747", +# "source": "FFPE" +# }, +# "subject": { +# "orcabusId": "sbj.01J8ES51S87R4EJ61QJ0DMDYWZ", +# "subjectId": "SN_PMC-141" +# }, +# "libraryId": "L2400255", +# "phenotype": "tumor", +# "workflow": "clinical", +# "quality": "very-poor", +# "type": "WTS", +# "assay": "NebRNA", +# "coverage": 6.0 +# }, +# { +# "orcabusId": "lib.01J8ES537S0W1AX9PQPST13GM9", +# "projectSet": [ +# { +# "orcabusId": "prj.01J8ES4EZAA5YMHX82664GJQB3", +# "projectId": "CUP", +# "name": None, +# "description": None +# } +# ], +# "sample": { +# "orcabusId": "smp.01J8ES5379C40K08YG3JDMZJN7", +# "sampleId": "PRJ240648", +# "externalSampleId": "RNA037080", +# "source": "FFPE" +# }, +# "subject": { +# "orcabusId": "sbj.01J8ES529GSPBV64SESK9SWD76", +# "subjectId": "SN_PMC-145" +# }, +# "libraryId": "L2400256", +# "phenotype": "tumor", +# "workflow": "clinical", +# "quality": "very-poor", +# "type": "WTS", +# "assay": "NebRNA", +# "coverage": 6.0 +# }, +# { +# "orcabusId": "lib.01J8ES5395KETT9T2NJSVNDKNP", +# "projectSet": [ +# { +# "orcabusId": "prj.01J8ES4FC6DVW20AR33FBX2SA8", +# "projectId": "Control", +# "name": None, +# "description": None +# } +# ], +# "sample": { +# "orcabusId": "smp.01J8ES538PFF6MQQ35PTC00JAY", +# "sampleId": "NTC_NebRNA240226", +# "externalSampleId": "NTC_NebRNA240226", +# "source": "water" +# }, +# "subject": { +# "orcabusId": "sbj.01J8ES4DFMNF0SX6P8P8Y9J6K1", +# "subjectId": "negative control" +# }, +# "libraryId": "L2400257", +# "phenotype": "negative-control", +# "workflow": "control", +# "quality": "good", +# "type": "WTS", +# "assay": "NebRNA", +# "coverage": 0.1 +# } +# ], +# }, +# None +# ), +# indent=2 +# ) +# ) +# # { +# # "start_samplesheet_shower_event_data": { +# # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5" +# # }, +# # "complete_samplesheet_shower_event_data": { +# # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5" +# # }, +# # "project_event_data_list": [ +# # { +# # "event_data": { +# # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", +# # "project": { +# # "orcabusId": "prj.01J8ES4EBXK08WDWB97BSCX1C9", +# # "projectId": "PO", +# # "name": null, +# # "description": null +# # }, +# # "librarySet": [ +# # { +# # "orcabusId": "lib.01J8ES4MPZ5B201R50K42XXM4M", +# # "libraryId": "L2400102" +# # } +# # ] +# # } +# # }, +# # { +# # "event_data": { +# # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", +# # "project": { +# # "orcabusId": "prj.01J8ES4XMWD0DH7MDRNER5TZS1", +# # "projectId": "Testing", +# # "name": null, +# # "description": null +# # }, +# # "librarySet": [ +# # { +# # "orcabusId": "lib.01J8ES4XNYFP38JMDV7GMV0V3V", +# # "libraryId": "L2400159" +# # }, +# # { +# # "orcabusId": "lib.01J8ES4XQG3MPBW94TTVT4STVG", +# # "libraryId": "L2400160" +# # }, +# # { +# # "orcabusId": "lib.01J8ES4XSS97XNRS8DH0B1RJRG", +# # "libraryId": "L2400161" +# # }, +# # { +# # "orcabusId": "lib.01J8ES4XXF6NMEJMM5M4GWS6KH", +# # "libraryId": "L2400162" +# # }, +# # { +# # "orcabusId": "lib.01J8ES4XZD7T2VRPVQ1GSVZ11X", +# # "libraryId": "L2400163" +# # }, +# # { +# # "orcabusId": "lib.01J8ES4Y1AKAHYD9EW0TW4FBCP", +# # "libraryId": "L2400164" +# # }, +# # { +# # "orcabusId": "lib.01J8ES4Y3ZKRX3C5JAHA5NBXV1", +# # "libraryId": "L2400165" +# # }, +# # { +# # "orcabusId": "lib.01J8ES4Y5D52202JVBXHJ9Q9WF", +# # "libraryId": "L2400166" +# # } +# # ] +# # } +# # }, +# # { +# # "event_data": { +# # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", +# # "project": { +# # "orcabusId": "prj.01J8ES4ZAWHH3FKYA2CFHSMZ4B", +# # "projectId": "CAVATAK", +# # "name": null, +# # "description": null +# # }, +# # "librarySet": [ +# # { +# # "orcabusId": "lib.01J8ES4ZDRQAP2BN3SDYYV5PKW", +# # "libraryId": "L2400191" +# # }, +# # { +# # "orcabusId": "lib.01J8ES4ZMY0G1H9MDN7K2TH9Y6", +# # "libraryId": "L2400195" +# # }, +# # { +# # "orcabusId": "lib.01J8ES4ZP88X2E17X5X1FRMTPK", +# # "libraryId": "L2400196" +# # }, +# # { +# # "orcabusId": "lib.01J8ES4ZST489C712CG3R9NQSQ", +# # "libraryId": "L2400197" +# # }, +# # { +# # "orcabusId": "lib.01J8ES4ZVWA2CGBHJVKAS3Y0G9", +# # "libraryId": "L2400198" +# # } +# # ] +# # } +# # }, +# # { +# # "event_data": { +# # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", +# # "project": { +# # "orcabusId": "prj.01J8ES4EZAA5YMHX82664GJQB3", +# # "projectId": "CUP", +# # "name": null, +# # "description": null +# # }, +# # "librarySet": [ +# # { +# # "orcabusId": "lib.01J8ES51V0RSVT6C7WQR72QQED", +# # "libraryId": "L2400231" +# # }, +# # { +# # "orcabusId": "lib.01J8ES52889Q8826P5SH9HDPP0", +# # "libraryId": "L2400238" +# # }, +# # { +# # "orcabusId": "lib.01J8ES52ANMRT3B7Y96T1Y3RY8", +# # "libraryId": "L2400239" +# # }, +# # { +# # "orcabusId": "lib.01J8ES52C3N585BGGY4VNXHC83", +# # "libraryId": "L2400240" +# # }, +# # { +# # "orcabusId": "lib.01J8ES536AB5A5PBJ8S45SZP7Q", +# # "libraryId": "L2400255" +# # }, +# # { +# # "orcabusId": "lib.01J8ES537S0W1AX9PQPST13GM9", +# # "libraryId": "L2400256" +# # } +# # ] +# # } +# # }, +# # { +# # "event_data": { +# # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", +# # "project": { +# # "orcabusId": "prj.01J8ES4FC6DVW20AR33FBX2SA8", +# # "projectId": "Control", +# # "name": null, +# # "description": null +# # }, +# # "librarySet": [ +# # { +# # "orcabusId": "lib.01J8ES52DHAPZM6FZ0VZK89PRT", +# # "libraryId": "L2400241" +# # }, +# # { +# # "orcabusId": "lib.01J8ES52F2ZHRXQY1AT1N1F81F", +# # "libraryId": "L2400242" +# # }, +# # { +# # "orcabusId": "lib.01J8ES52XYMVGRB1Q458THNG4T", +# # "libraryId": "L2400249" +# # }, +# # { +# # "orcabusId": "lib.01J8ES5395KETT9T2NJSVNDKNP", +# # "libraryId": "L2400257" +# # } +# # ] +# # } +# # }, +# # { +# # "event_data": { +# # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", +# # "project": { +# # "orcabusId": "prj.01J8ES4FH3XMPZQNDJ9J000BXX", +# # "projectId": "BPOP-retro", +# # "name": null, +# # "description": null +# # }, +# # "librarySet": [ +# # { +# # "orcabusId": "lib.01J8ES52Z2KTVVKZ2ZGVQ6YC10", +# # "libraryId": "L2400250" +# # }, +# # { +# # "orcabusId": "lib.01J8ES530H895X4WA3NQ6CY2QV", +# # "libraryId": "L2400251" +# # }, +# # { +# # "orcabusId": "lib.01J8ES5320EWBNNYDGXF2SYJBD", +# # "libraryId": "L2400252" +# # }, +# # { +# # "orcabusId": "lib.01J8ES533DJZZNPP9MXYR5TRC0", +# # "libraryId": "L2400253" +# # }, +# # { +# # "orcabusId": "lib.01J8ES534XGBFYDVYV8ZG6SYS0", +# # "libraryId": "L2400254" +# # } +# # ] +# # } +# # } +# # ], +# # "library_event_data_list": [ +# # { +# # "event_data": { +# # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", +# # "library": { +# # "orcabusId": "lib.01J8ES4MPZ5B201R50K42XXM4M", +# # "libraryId": "L2400102", +# # "phenotype": "tumor", +# # "workflow": "research", +# # "quality": "borderline", +# # "type": "WGS", +# # "assay": "ctTSO", +# # "coverage": 50.0 +# # }, +# # "sample": { +# # "orcabusId": "smp.01J8ES4MPHSX7MRCTTFWJBYTT7", +# # "sampleId": "MDX210402", +# # "externalSampleId": "ZUHR111121", +# # "source": "plasma-serum" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES4MNXJSDRR406DAXFZP2N", +# # "subjectId": "PM3045106" +# # }, +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4EBXK08WDWB97BSCX1C9", +# # "projectId": "PO", +# # "name": null, +# # "description": null +# # } +# # ], +# # "bclconvertDataRows": [ +# # { +# # "sampleId": "L2400102", +# # "index": "GAATTCGT", +# # "index2": "TTATGAGT", +# # "lane": 1, +# # "overrideCycles": "U7N1Y143;I8N2;I8N2;U7N1Y143" +# # } +# # ], +# # "fastqListRows": [ +# # { +# # "fastqListRowRgid": "GAATTCGT.TTATGAGT.1.240424_A01052_0193_BH7JMMDRX5.L2400102" +# # } +# # ] +# # } +# # }, +# # { +# # "event_data": { +# # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", +# # "library": { +# # "orcabusId": "lib.01J8ES4XNYFP38JMDV7GMV0V3V", +# # "libraryId": "L2400159", +# # "phenotype": "tumor", +# # "workflow": "manual", +# # "quality": "good", +# # "type": "ctDNA", +# # "assay": "ctTSOv2", +# # "coverage": 38.6 +# # }, +# # "sample": { +# # "orcabusId": "smp.01J8ES4XMDW0FV1YMWHSZZQ4TX", +# # "sampleId": "PTC_SCMM1pc2", +# # "externalSampleId": "SSq-CompMM-1pc-10646259ilm", +# # "source": "cfDNA" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES4XKHKNQ1NF8EKKACZ032", +# # "subjectId": "CMM1pc-10646259ilm" +# # }, +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4XMWD0DH7MDRNER5TZS1", +# # "projectId": "Testing", +# # "name": null, +# # "description": null +# # } +# # ], +# # "bclconvertDataRows": [ +# # { +# # "sampleId": "L2400159", +# # "index": "GAGAATGGTT", +# # "index2": "TTGCTGCCGA", +# # "lane": 1, +# # "overrideCycles": "U7N1Y143;I10;I10;U7N1Y143" +# # } +# # ], +# # "fastqListRows": [ +# # { +# # "fastqListRowRgid": "GAGAATGGTT.TTGCTGCCGA.1.240424_A01052_0193_BH7JMMDRX5.L2400159" +# # } +# # ] +# # } +# # }, +# # { +# # "event_data": { +# # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", +# # "library": { +# # "orcabusId": "lib.01J8ES4XQG3MPBW94TTVT4STVG", +# # "libraryId": "L2400160", +# # "phenotype": "tumor", +# # "workflow": "manual", +# # "quality": "good", +# # "type": "ctDNA", +# # "assay": "ctTSOv2", +# # "coverage": 38.6 +# # }, +# # "sample": { +# # "orcabusId": "smp.01J8ES4XQ071BF3WZN111SNJ2B", +# # "sampleId": "PTC_SCMM1pc3", +# # "externalSampleId": "SSq-CompMM-1pc-10646259ilm", +# # "source": "cfDNA" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES4XKHKNQ1NF8EKKACZ032", +# # "subjectId": "CMM1pc-10646259ilm" +# # }, +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4XMWD0DH7MDRNER5TZS1", +# # "projectId": "Testing", +# # "name": null, +# # "description": null +# # } +# # ], +# # "bclconvertDataRows": [ +# # { +# # "sampleId": "L2400160", +# # "index": "AGAGGCAACC", +# # "index2": "CCATCATTAG", +# # "lane": 1, +# # "overrideCycles": "U7N1Y143;I10;I10;U7N1Y143" +# # } +# # ], +# # "fastqListRows": [ +# # { +# # "fastqListRowRgid": "AGAGGCAACC.CCATCATTAG.1.240424_A01052_0193_BH7JMMDRX5.L2400160" +# # } +# # ] +# # } +# # }, +# # { +# # "event_data": { +# # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", +# # "library": { +# # "orcabusId": "lib.01J8ES4XSS97XNRS8DH0B1RJRG", +# # "libraryId": "L2400161", +# # "phenotype": "tumor", +# # "workflow": "manual", +# # "quality": "good", +# # "type": "ctDNA", +# # "assay": "ctTSOv2", +# # "coverage": 38.6 +# # }, +# # "sample": { +# # "orcabusId": "smp.01J8ES4XRG9NB38N03688M2CCB", +# # "sampleId": "PTC_SCMM1pc4", +# # "externalSampleId": "SSq-CompMM-1pc-10646259ilm", +# # "source": "cfDNA" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES4XKHKNQ1NF8EKKACZ032", +# # "subjectId": "CMM1pc-10646259ilm" +# # }, +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4XMWD0DH7MDRNER5TZS1", +# # "projectId": "Testing", +# # "name": null, +# # "description": null +# # } +# # ], +# # "bclconvertDataRows": [ +# # { +# # "sampleId": "L2400161", +# # "index": "CCATCATTAG", +# # "index2": "AGAGGCAACC", +# # "lane": 1, +# # "overrideCycles": "U7N1Y143;I10;I10;U7N1Y143" +# # } +# # ], +# # "fastqListRows": [ +# # { +# # "fastqListRowRgid": "CCATCATTAG.AGAGGCAACC.1.240424_A01052_0193_BH7JMMDRX5.L2400161" +# # } +# # ] +# # } +# # }, +# # { +# # "event_data": { +# # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", +# # "library": { +# # "orcabusId": "lib.01J8ES4XXF6NMEJMM5M4GWS6KH", +# # "libraryId": "L2400162", +# # "phenotype": "tumor", +# # "workflow": "manual", +# # "quality": "good", +# # "type": "ctDNA", +# # "assay": "ctTSOv2", +# # "coverage": 38.6 +# # }, +# # "sample": { +# # "orcabusId": "smp.01J8ES4XWXFANT7P0T3AFXA85G", +# # "sampleId": "PTC_SCMM01pc20", +# # "externalSampleId": "SSq-CompMM-0.1pc-10624819 - 20ng", +# # "source": "cfDNA" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES4XW2TXGEJBQWCVMRZRTS", +# # "subjectId": "CMM0.1pc-10624819" +# # }, +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4XMWD0DH7MDRNER5TZS1", +# # "projectId": "Testing", +# # "name": null, +# # "description": null +# # } +# # ], +# # "bclconvertDataRows": [ +# # { +# # "sampleId": "L2400162", +# # "index": "GATAGGCCGA", +# # "index2": "GCCATGTGCG", +# # "lane": 1, +# # "overrideCycles": "U7N1Y143;I10;I10;U7N1Y143" +# # } +# # ], +# # "fastqListRows": [ +# # { +# # "fastqListRowRgid": "GATAGGCCGA.GCCATGTGCG.1.240424_A01052_0193_BH7JMMDRX5.L2400162" +# # } +# # ] +# # } +# # }, +# # { +# # "event_data": { +# # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", +# # "library": { +# # "orcabusId": "lib.01J8ES4XZD7T2VRPVQ1GSVZ11X", +# # "libraryId": "L2400163", +# # "phenotype": "tumor", +# # "workflow": "manual", +# # "quality": "good", +# # "type": "ctDNA", +# # "assay": "ctTSOv2", +# # "coverage": 38.6 +# # }, +# # "sample": { +# # "orcabusId": "smp.01J8ES4XYTSVQVRSBA9M26NSZY", +# # "sampleId": "PTC_SCMM01pc15", +# # "externalSampleId": "SSq-CompMM-0.1pc-10624819 - 15ng", +# # "source": "cfDNA" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES4XW2TXGEJBQWCVMRZRTS", +# # "subjectId": "CMM0.1pc-10624819" +# # }, +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4XMWD0DH7MDRNER5TZS1", +# # "projectId": "Testing", +# # "name": null, +# # "description": null +# # } +# # ], +# # "bclconvertDataRows": [ +# # { +# # "sampleId": "L2400163", +# # "index": "ATGGTTGACT", +# # "index2": "AGGACAGGCC", +# # "lane": 1, +# # "overrideCycles": "U7N1Y143;I10;I10;U7N1Y143" +# # } +# # ], +# # "fastqListRows": [ +# # { +# # "fastqListRowRgid": "ATGGTTGACT.AGGACAGGCC.1.240424_A01052_0193_BH7JMMDRX5.L2400163" +# # } +# # ] +# # } +# # }, +# # { +# # "event_data": { +# # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", +# # "library": { +# # "orcabusId": "lib.01J8ES4Y1AKAHYD9EW0TW4FBCP", +# # "libraryId": "L2400164", +# # "phenotype": "tumor", +# # "workflow": "manual", +# # "quality": "good", +# # "type": "ctDNA", +# # "assay": "ctTSOv2", +# # "coverage": 38.6 +# # }, +# # "sample": { +# # "orcabusId": "smp.01J8ES4Y0V0ZBKAE91TDSY0BBB", +# # "sampleId": "PTC_SCMM01pc10", +# # "externalSampleId": "SSq-CompMM-0.1pc-10624819 - 10ng", +# # "source": "cfDNA" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES4XW2TXGEJBQWCVMRZRTS", +# # "subjectId": "CMM0.1pc-10624819" +# # }, +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4XMWD0DH7MDRNER5TZS1", +# # "projectId": "Testing", +# # "name": null, +# # "description": null +# # } +# # ], +# # "bclconvertDataRows": [ +# # { +# # "sampleId": "L2400164", +# # "index": "TATTGCGCTC", +# # "index2": "CCTAACACAG", +# # "lane": 1, +# # "overrideCycles": "U7N1Y143;I10;I10;U7N1Y143" +# # } +# # ], +# # "fastqListRows": [ +# # { +# # "fastqListRowRgid": "TATTGCGCTC.CCTAACACAG.1.240424_A01052_0193_BH7JMMDRX5.L2400164" +# # } +# # ] +# # } +# # }, +# # { +# # "event_data": { +# # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", +# # "library": { +# # "orcabusId": "lib.01J8ES4Y3ZKRX3C5JAHA5NBXV1", +# # "libraryId": "L2400165", +# # "phenotype": "tumor", +# # "workflow": "manual", +# # "quality": "good", +# # "type": "ctDNA", +# # "assay": "ctTSOv2", +# # "coverage": 38.6 +# # }, +# # "sample": { +# # "orcabusId": "smp.01J8ES4Y37JTPEEJSED9BXH8N2", +# # "sampleId": "PTC_SCMM01pc5", +# # "externalSampleId": "SSq-CompMM-0.1pc-10624819 - 5ng", +# # "source": "cfDNA" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES4XW2TXGEJBQWCVMRZRTS", +# # "subjectId": "CMM0.1pc-10624819" +# # }, +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4XMWD0DH7MDRNER5TZS1", +# # "projectId": "Testing", +# # "name": null, +# # "description": null +# # } +# # ], +# # "bclconvertDataRows": [ +# # { +# # "sampleId": "L2400165", +# # "index": "ACGCCTTGTT", +# # "index2": "ACGTTCCTTA", +# # "lane": 4, +# # "overrideCycles": "U7N1Y143;I10;I10;U7N1Y143" +# # } +# # ], +# # "fastqListRows": [ +# # { +# # "fastqListRowRgid": "ACGCCTTGTT.ACGTTCCTTA.4.240424_A01052_0193_BH7JMMDRX5.L2400165" +# # } +# # ] +# # } +# # }, +# # { +# # "event_data": { +# # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", +# # "library": { +# # "orcabusId": "lib.01J8ES4Y5D52202JVBXHJ9Q9WF", +# # "libraryId": "L2400166", +# # "phenotype": "negative-control", +# # "workflow": "manual", +# # "quality": "good", +# # "type": "ctDNA", +# # "assay": "ctTSOv2", +# # "coverage": 0.1 +# # }, +# # "sample": { +# # "orcabusId": "smp.01J8ES4Y4XK1WX4WCPD6XY8KNM", +# # "sampleId": "NTC_v2ctTSO240207", +# # "externalSampleId": "negative control", +# # "source": "water" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES4DFMNF0SX6P8P8Y9J6K1", +# # "subjectId": "negative control" +# # }, +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4XMWD0DH7MDRNER5TZS1", +# # "projectId": "Testing", +# # "name": null, +# # "description": null +# # } +# # ], +# # "bclconvertDataRows": [ +# # { +# # "sampleId": "L2400166", +# # "index": "TTCTACATAC", +# # "index2": "TTACAGTTAG", +# # "lane": 1, +# # "overrideCycles": "U7N1Y143;I10;I10;U7N1Y143" +# # } +# # ], +# # "fastqListRows": [ +# # { +# # "fastqListRowRgid": "TTCTACATAC.TTACAGTTAG.1.240424_A01052_0193_BH7JMMDRX5.L2400166" +# # } +# # ] +# # } +# # }, +# # { +# # "event_data": { +# # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", +# # "library": { +# # "orcabusId": "lib.01J8ES4ZDRQAP2BN3SDYYV5PKW", +# # "libraryId": "L2400191", +# # "phenotype": "normal", +# # "workflow": "research", +# # "quality": "good", +# # "type": "WGS", +# # "assay": "TsqNano", +# # "coverage": 40.0 +# # }, +# # "sample": { +# # "orcabusId": "smp.01J8ES4ZDAFRK3K3PY33F8XS0W", +# # "sampleId": "PRJ240169", +# # "externalSampleId": "AUS-006-DRW_C1D1PRE", +# # "source": "blood" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES4ZCKNW6QKP006SYNZ5RA", +# # "subjectId": "AUS-006-DRW" +# # }, +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4ZAWHH3FKYA2CFHSMZ4B", +# # "projectId": "CAVATAK", +# # "name": null, +# # "description": null +# # } +# # ], +# # "bclconvertDataRows": [ +# # { +# # "sampleId": "L2400191", +# # "index": "GCACGGAC", +# # "index2": "TGCGAGAC", +# # "lane": 4, +# # "overrideCycles": "Y151;I8N2;I8N2;Y151" +# # } +# # ], +# # "fastqListRows": [ +# # { +# # "fastqListRowRgid": "GCACGGAC.TGCGAGAC.4.240424_A01052_0193_BH7JMMDRX5.L2400191" +# # } +# # ] +# # } +# # }, +# # { +# # "event_data": { +# # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", +# # "library": { +# # "orcabusId": "lib.01J8ES4ZMY0G1H9MDN7K2TH9Y6", +# # "libraryId": "L2400195", +# # "phenotype": "tumor", +# # "workflow": "research", +# # "quality": "good", +# # "type": "WGS", +# # "assay": "TsqNano", +# # "coverage": 80.0 +# # }, +# # "sample": { +# # "orcabusId": "smp.01J8ES4ZMETZP255WMFC8TSCYT", +# # "sampleId": "PRJ240180", +# # "externalSampleId": "AUS-006-DRW_Day0", +# # "source": "tissue" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES4ZCKNW6QKP006SYNZ5RA", +# # "subjectId": "AUS-006-DRW" +# # }, +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4ZAWHH3FKYA2CFHSMZ4B", +# # "projectId": "CAVATAK", +# # "name": null, +# # "description": null +# # } +# # ], +# # "bclconvertDataRows": [ +# # { +# # "sampleId": "L2400195", +# # "index": "ATGAGGCC", +# # "index2": "CAATTAAC", +# # "lane": 2, +# # "overrideCycles": "Y151;I8N2;I8N2;Y151" +# # }, +# # { +# # "sampleId": "L2400195", +# # "index": "ATGAGGCC", +# # "index2": "CAATTAAC", +# # "lane": 3, +# # "overrideCycles": "Y151;I8N2;I8N2;Y151" +# # } +# # ], +# # "fastqListRows": [ +# # { +# # "fastqListRowRgid": "ATGAGGCC.CAATTAAC.2.240424_A01052_0193_BH7JMMDRX5.L2400195" +# # }, +# # { +# # "fastqListRowRgid": "ATGAGGCC.CAATTAAC.3.240424_A01052_0193_BH7JMMDRX5.L2400195" +# # } +# # ] +# # } +# # }, +# # { +# # "event_data": { +# # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", +# # "library": { +# # "orcabusId": "lib.01J8ES4ZP88X2E17X5X1FRMTPK", +# # "libraryId": "L2400196", +# # "phenotype": "tumor", +# # "workflow": "research", +# # "quality": "good", +# # "type": "WGS", +# # "assay": "TsqNano", +# # "coverage": 80.0 +# # }, +# # "sample": { +# # "orcabusId": "smp.01J8ES4ZNT47EM37QKMT12JPPJ", +# # "sampleId": "PRJ240181", +# # "externalSampleId": "AUS-006-DRW_Day33", +# # "source": "tissue" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES4ZCKNW6QKP006SYNZ5RA", +# # "subjectId": "AUS-006-DRW" +# # }, +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4ZAWHH3FKYA2CFHSMZ4B", +# # "projectId": "CAVATAK", +# # "name": null, +# # "description": null +# # } +# # ], +# # "bclconvertDataRows": [ +# # { +# # "sampleId": "L2400196", +# # "index": "ACTAAGAT", +# # "index2": "CCGCGGTT", +# # "lane": 2, +# # "overrideCycles": "Y151;I8N2;I8N2;Y151" +# # }, +# # { +# # "sampleId": "L2400196", +# # "index": "ACTAAGAT", +# # "index2": "CCGCGGTT", +# # "lane": 3, +# # "overrideCycles": "Y151;I8N2;I8N2;Y151" +# # } +# # ], +# # "fastqListRows": [ +# # { +# # "fastqListRowRgid": "ACTAAGAT.CCGCGGTT.2.240424_A01052_0193_BH7JMMDRX5.L2400196" +# # }, +# # { +# # "fastqListRowRgid": "ACTAAGAT.CCGCGGTT.3.240424_A01052_0193_BH7JMMDRX5.L2400196" +# # } +# # ] +# # } +# # }, +# # { +# # "event_data": { +# # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", +# # "library": { +# # "orcabusId": "lib.01J8ES4ZST489C712CG3R9NQSQ", +# # "libraryId": "L2400197", +# # "phenotype": "tumor", +# # "workflow": "research", +# # "quality": "good", +# # "type": "WGS", +# # "assay": "TsqNano", +# # "coverage": 80.0 +# # }, +# # "sample": { +# # "orcabusId": "smp.01J8ES4ZQ76H8P0Q7S618F3BMA", +# # "sampleId": "PRJ240182", +# # "externalSampleId": "AUS-007-JMA_Day0", +# # "source": "tissue" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES4ZEQ3FVD6DDVEG8MW60Q", +# # "subjectId": "AUS-007-JMA" +# # }, +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4ZAWHH3FKYA2CFHSMZ4B", +# # "projectId": "CAVATAK", +# # "name": null, +# # "description": null +# # } +# # ], +# # "bclconvertDataRows": [ +# # { +# # "sampleId": "L2400197", +# # "index": "GTCGGAGC", +# # "index2": "TTATAACC", +# # "lane": 2, +# # "overrideCycles": "Y151;I8N2;I8N2;Y151" +# # }, +# # { +# # "sampleId": "L2400197", +# # "index": "GTCGGAGC", +# # "index2": "TTATAACC", +# # "lane": 3, +# # "overrideCycles": "Y151;I8N2;I8N2;Y151" +# # }, +# # { +# # "sampleId": "L2400197", +# # "index": "GTCGGAGC", +# # "index2": "TTATAACC", +# # "lane": 4, +# # "overrideCycles": "Y151;I8N2;I8N2;Y151" +# # } +# # ], +# # "fastqListRows": [ +# # { +# # "fastqListRowRgid": "GTCGGAGC.TTATAACC.2.240424_A01052_0193_BH7JMMDRX5.L2400197" +# # }, +# # { +# # "fastqListRowRgid": "GTCGGAGC.TTATAACC.3.240424_A01052_0193_BH7JMMDRX5.L2400197" +# # }, +# # { +# # "fastqListRowRgid": "GTCGGAGC.TTATAACC.4.240424_A01052_0193_BH7JMMDRX5.L2400197" +# # } +# # ] +# # } +# # }, +# # { +# # "event_data": { +# # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", +# # "library": { +# # "orcabusId": "lib.01J8ES4ZVWA2CGBHJVKAS3Y0G9", +# # "libraryId": "L2400198", +# # "phenotype": "tumor", +# # "workflow": "research", +# # "quality": "good", +# # "type": "WGS", +# # "assay": "TsqNano", +# # "coverage": 80.0 +# # }, +# # "sample": { +# # "orcabusId": "smp.01J8ES4ZVAR9NQM55Z2TXCDY9V", +# # "sampleId": "PRJ240183", +# # "externalSampleId": "AUS-007-JMA_Day15", +# # "source": "tissue" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES4ZEQ3FVD6DDVEG8MW60Q", +# # "subjectId": "AUS-007-JMA" +# # }, +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4ZAWHH3FKYA2CFHSMZ4B", +# # "projectId": "CAVATAK", +# # "name": null, +# # "description": null +# # } +# # ], +# # "bclconvertDataRows": [ +# # { +# # "sampleId": "L2400198", +# # "index": "CTTGGTAT", +# # "index2": "GGACTTGG", +# # "lane": 4, +# # "overrideCycles": "Y151;I8N2;I8N2;Y151" +# # } +# # ], +# # "fastqListRows": [ +# # { +# # "fastqListRowRgid": "CTTGGTAT.GGACTTGG.4.240424_A01052_0193_BH7JMMDRX5.L2400198" +# # } +# # ] +# # } +# # }, +# # { +# # "event_data": { +# # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", +# # "library": { +# # "orcabusId": "lib.01J8ES51V0RSVT6C7WQR72QQED", +# # "libraryId": "L2400231", +# # "phenotype": "tumor", +# # "workflow": "clinical", +# # "quality": "poor", +# # "type": "WGS", +# # "assay": "TsqNano", +# # "coverage": 100.0 +# # }, +# # "sample": { +# # "orcabusId": "smp.01J8ES51T84KVVVSEPYQFGW0EV", +# # "sampleId": "PRJ240199", +# # "externalSampleId": "DNA188239", +# # "source": "FFPE" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES51S87R4EJ61QJ0DMDYWZ", +# # "subjectId": "SN_PMC-141" +# # }, +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4EZAA5YMHX82664GJQB3", +# # "projectId": "CUP", +# # "name": null, +# # "description": null +# # } +# # ], +# # "bclconvertDataRows": [ +# # { +# # "sampleId": "L2400231", +# # "index": "TCGTAGTG", +# # "index2": "CCAAGTCT", +# # "lane": 2, +# # "overrideCycles": "Y151;I8N2;I8N2;Y151" +# # }, +# # { +# # "sampleId": "L2400231", +# # "index": "TCGTAGTG", +# # "index2": "CCAAGTCT", +# # "lane": 3, +# # "overrideCycles": "Y151;I8N2;I8N2;Y151" +# # } +# # ], +# # "fastqListRows": [ +# # { +# # "fastqListRowRgid": "TCGTAGTG.CCAAGTCT.2.240424_A01052_0193_BH7JMMDRX5.L2400231" +# # }, +# # { +# # "fastqListRowRgid": "TCGTAGTG.CCAAGTCT.3.240424_A01052_0193_BH7JMMDRX5.L2400231" +# # } +# # ] +# # } +# # }, +# # { +# # "event_data": { +# # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", +# # "library": { +# # "orcabusId": "lib.01J8ES52889Q8826P5SH9HDPP0", +# # "libraryId": "L2400238", +# # "phenotype": "normal", +# # "workflow": "clinical", +# # "quality": "good", +# # "type": "WGS", +# # "assay": "TsqNano", +# # "coverage": 40.0 +# # }, +# # "sample": { +# # "orcabusId": "smp.01J8ES527QKB5Y5RVZWZ8HQX0H", +# # "sampleId": "PRJ240643", +# # "externalSampleId": "DNA188378", +# # "source": "blood" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES51S87R4EJ61QJ0DMDYWZ", +# # "subjectId": "SN_PMC-141" +# # }, +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4EZAA5YMHX82664GJQB3", +# # "projectId": "CUP", +# # "name": null, +# # "description": null +# # } +# # ], +# # "bclconvertDataRows": [ +# # { +# # "sampleId": "L2400238", +# # "index": "GGAGCGTC", +# # "index2": "GCACGGAC", +# # "lane": 2, +# # "overrideCycles": "Y151;I8N2;I8N2;Y151" +# # }, +# # { +# # "sampleId": "L2400238", +# # "index": "GGAGCGTC", +# # "index2": "GCACGGAC", +# # "lane": 3, +# # "overrideCycles": "Y151;I8N2;I8N2;Y151" +# # } +# # ], +# # "fastqListRows": [ +# # { +# # "fastqListRowRgid": "GGAGCGTC.GCACGGAC.2.240424_A01052_0193_BH7JMMDRX5.L2400238" +# # }, +# # { +# # "fastqListRowRgid": "GGAGCGTC.GCACGGAC.3.240424_A01052_0193_BH7JMMDRX5.L2400238" +# # } +# # ] +# # } +# # }, +# # { +# # "event_data": { +# # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", +# # "library": { +# # "orcabusId": "lib.01J8ES52ANMRT3B7Y96T1Y3RY8", +# # "libraryId": "L2400239", +# # "phenotype": "normal", +# # "workflow": "clinical", +# # "quality": "good", +# # "type": "WGS", +# # "assay": "TsqNano", +# # "coverage": 40.0 +# # }, +# # "sample": { +# # "orcabusId": "smp.01J8ES52A5QX0GQ6RB78Z8DGYQ", +# # "sampleId": "PRJ240646", +# # "externalSampleId": "DNA189922", +# # "source": "blood" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES529GSPBV64SESK9SWD76", +# # "subjectId": "SN_PMC-145" +# # }, +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4EZAA5YMHX82664GJQB3", +# # "projectId": "CUP", +# # "name": null, +# # "description": null +# # } +# # ], +# # "bclconvertDataRows": [ +# # { +# # "sampleId": "L2400239", +# # "index": "ATGGCATG", +# # "index2": "GGTACCTT", +# # "lane": 2, +# # "overrideCycles": "Y151;I8N2;I8N2;Y151" +# # }, +# # { +# # "sampleId": "L2400239", +# # "index": "ATGGCATG", +# # "index2": "GGTACCTT", +# # "lane": 3, +# # "overrideCycles": "Y151;I8N2;I8N2;Y151" +# # } +# # ], +# # "fastqListRows": [ +# # { +# # "fastqListRowRgid": "ATGGCATG.GGTACCTT.2.240424_A01052_0193_BH7JMMDRX5.L2400239" +# # }, +# # { +# # "fastqListRowRgid": "ATGGCATG.GGTACCTT.3.240424_A01052_0193_BH7JMMDRX5.L2400239" +# # } +# # ] +# # } +# # }, +# # { +# # "event_data": { +# # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", +# # "library": { +# # "orcabusId": "lib.01J8ES52C3N585BGGY4VNXHC83", +# # "libraryId": "L2400240", +# # "phenotype": "tumor", +# # "workflow": "clinical", +# # "quality": "poor", +# # "type": "WGS", +# # "assay": "TsqNano", +# # "coverage": 100.0 +# # }, +# # "sample": { +# # "orcabusId": "smp.01J8ES52BM8BVS3PX47E6FM7D5", +# # "sampleId": "PRJ240647", +# # "externalSampleId": "DNA189848", +# # "source": "FFPE" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES529GSPBV64SESK9SWD76", +# # "subjectId": "SN_PMC-145" +# # }, +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4EZAA5YMHX82664GJQB3", +# # "projectId": "CUP", +# # "name": null, +# # "description": null +# # } +# # ], +# # "bclconvertDataRows": [ +# # { +# # "sampleId": "L2400240", +# # "index": "GCAATGCA", +# # "index2": "AACGTTCC", +# # "lane": 2, +# # "overrideCycles": "Y151;I8N2;I8N2;Y151" +# # }, +# # { +# # "sampleId": "L2400240", +# # "index": "GCAATGCA", +# # "index2": "AACGTTCC", +# # "lane": 3, +# # "overrideCycles": "Y151;I8N2;I8N2;Y151" +# # } +# # ], +# # "fastqListRows": [ +# # { +# # "fastqListRowRgid": "GCAATGCA.AACGTTCC.2.240424_A01052_0193_BH7JMMDRX5.L2400240" +# # }, +# # { +# # "fastqListRowRgid": "GCAATGCA.AACGTTCC.3.240424_A01052_0193_BH7JMMDRX5.L2400240" +# # } +# # ] +# # } +# # }, +# # { +# # "event_data": { +# # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", +# # "library": { +# # "orcabusId": "lib.01J8ES52DHAPZM6FZ0VZK89PRT", +# # "libraryId": "L2400241", +# # "phenotype": "negative-control", +# # "workflow": "control", +# # "quality": "good", +# # "type": "WGS", +# # "assay": "TsqNano", +# # "coverage": 0.1 +# # }, +# # "sample": { +# # "orcabusId": "smp.01J8ES52D076FQM5K8128AQ593", +# # "sampleId": "NTC_TSqN240226", +# # "externalSampleId": "NTC_TSqN240226", +# # "source": "water" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES4DFMNF0SX6P8P8Y9J6K1", +# # "subjectId": "negative control" +# # }, +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4FC6DVW20AR33FBX2SA8", +# # "projectId": "Control", +# # "name": null, +# # "description": null +# # } +# # ], +# # "bclconvertDataRows": [ +# # { +# # "sampleId": "L2400241", +# # "index": "GTTCCAAT", +# # "index2": "GCAGAATT", +# # "lane": 4, +# # "overrideCycles": "Y151;I8N2;I8N2;Y151" +# # } +# # ], +# # "fastqListRows": [ +# # { +# # "fastqListRowRgid": "GTTCCAAT.GCAGAATT.4.240424_A01052_0193_BH7JMMDRX5.L2400241" +# # } +# # ] +# # } +# # }, +# # { +# # "event_data": { +# # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", +# # "library": { +# # "orcabusId": "lib.01J8ES52F2ZHRXQY1AT1N1F81F", +# # "libraryId": "L2400242", +# # "phenotype": "normal", +# # "workflow": "control", +# # "quality": "good", +# # "type": "WGS", +# # "assay": "TsqNano", +# # "coverage": 15.0 +# # }, +# # "sample": { +# # "orcabusId": "smp.01J8ES52EEX67YRYAJS3F5GMJ5", +# # "sampleId": "PTC_TSqN240226", +# # "externalSampleId": "NA24385-3", +# # "source": "cell-line" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES4DRJ31Z2H1GJQZGVDXZR", +# # "subjectId": "NA24385" +# # }, +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4FC6DVW20AR33FBX2SA8", +# # "projectId": "Control", +# # "name": null, +# # "description": null +# # } +# # ], +# # "bclconvertDataRows": [ +# # { +# # "sampleId": "L2400242", +# # "index": "ACCTTGGC", +# # "index2": "ATGAGGCC", +# # "lane": 4, +# # "overrideCycles": "Y151;I8N2;I8N2;Y151" +# # } +# # ], +# # "fastqListRows": [ +# # { +# # "fastqListRowRgid": "ACCTTGGC.ATGAGGCC.4.240424_A01052_0193_BH7JMMDRX5.L2400242" +# # } +# # ] +# # } +# # }, +# # { +# # "event_data": { +# # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", +# # "library": { +# # "orcabusId": "lib.01J8ES52XYMVGRB1Q458THNG4T", +# # "libraryId": "L2400249", +# # "phenotype": "tumor", +# # "workflow": "control", +# # "quality": "good", +# # "type": "WTS", +# # "assay": "NebRNA", +# # "coverage": 1.0 +# # }, +# # "sample": { +# # "orcabusId": "smp.01J8ES52XE661E8V8XTWD02QCK", +# # "sampleId": "PTC_NebRNA240226", +# # "externalSampleId": "Colo829", +# # "source": "cell-line" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES4GNVGZSJVTHGVKS9VW7F", +# # "subjectId": "Colo829" +# # }, +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4FC6DVW20AR33FBX2SA8", +# # "projectId": "Control", +# # "name": null, +# # "description": null +# # } +# # ], +# # "bclconvertDataRows": [ +# # { +# # "sampleId": "L2400249", +# # "index": "AGTTTCGA", +# # "index2": "CCTACGAT", +# # "lane": 4, +# # "overrideCycles": "Y151;I8N2;I8N2;Y151" +# # } +# # ], +# # "fastqListRows": [ +# # { +# # "fastqListRowRgid": "AGTTTCGA.CCTACGAT.4.240424_A01052_0193_BH7JMMDRX5.L2400249" +# # } +# # ] +# # } +# # }, +# # { +# # "event_data": { +# # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", +# # "library": { +# # "orcabusId": "lib.01J8ES52Z2KTVVKZ2ZGVQ6YC10", +# # "libraryId": "L2400250", +# # "phenotype": "tumor", +# # "workflow": "research", +# # "quality": "good", +# # "type": "WTS", +# # "assay": "NebRNA", +# # "coverage": 6.0 +# # }, +# # "sample": { +# # "orcabusId": "smp.01J8ES4FP9WTFBDNGKVG3D9BD4", +# # "sampleId": "PRJ240003", +# # "externalSampleId": "3-23BCRL057T", +# # "source": "tissue" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES4FNJ2FCAK0RJST0428X0", +# # "subjectId": "23BCRL057T" +# # }, +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4FH3XMPZQNDJ9J000BXX", +# # "projectId": "BPOP-retro", +# # "name": null, +# # "description": null +# # } +# # ], +# # "bclconvertDataRows": [ +# # { +# # "sampleId": "L2400250", +# # "index": "GAACCTCT", +# # "index2": "GTCTGCGC", +# # "lane": 4, +# # "overrideCycles": "Y151;I8N2;I8N2;Y151" +# # } +# # ], +# # "fastqListRows": [ +# # { +# # "fastqListRowRgid": "GAACCTCT.GTCTGCGC.4.240424_A01052_0193_BH7JMMDRX5.L2400250" +# # } +# # ] +# # } +# # }, +# # { +# # "event_data": { +# # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", +# # "library": { +# # "orcabusId": "lib.01J8ES530H895X4WA3NQ6CY2QV", +# # "libraryId": "L2400251", +# # "phenotype": "tumor", +# # "workflow": "research", +# # "quality": "good", +# # "type": "WTS", +# # "assay": "NebRNA", +# # "coverage": 6.0 +# # }, +# # "sample": { +# # "orcabusId": "smp.01J8ES530355YNZ3VHQQQ204PF", +# # "sampleId": "PRJ240561", +# # "externalSampleId": "4-218-004_Bx", +# # "source": "tissue" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES51WC4GV5YDJNTMAK2YY1", +# # "subjectId": "218-004" +# # }, +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4FH3XMPZQNDJ9J000BXX", +# # "projectId": "BPOP-retro", +# # "name": null, +# # "description": null +# # } +# # ], +# # "bclconvertDataRows": [ +# # { +# # "sampleId": "L2400251", +# # "index": "GCCCAGTG", +# # "index2": "CCGCAATT", +# # "lane": 4, +# # "overrideCycles": "Y151;I8N2;I8N2;Y151" +# # } +# # ], +# # "fastqListRows": [ +# # { +# # "fastqListRowRgid": "GCCCAGTG.CCGCAATT.4.240424_A01052_0193_BH7JMMDRX5.L2400251" +# # } +# # ] +# # } +# # }, +# # { +# # "event_data": { +# # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", +# # "library": { +# # "orcabusId": "lib.01J8ES5320EWBNNYDGXF2SYJBD", +# # "libraryId": "L2400252", +# # "phenotype": "tumor", +# # "workflow": "research", +# # "quality": "good", +# # "type": "WTS", +# # "assay": "NebRNA", +# # "coverage": 6.0 +# # }, +# # "sample": { +# # "orcabusId": "smp.01J8ES531H420JM9MG5R4AE1AZ", +# # "sampleId": "PRJ240562", +# # "externalSampleId": "5-218-004_04", +# # "source": "tissue" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES51WC4GV5YDJNTMAK2YY1", +# # "subjectId": "218-004" +# # }, +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4FH3XMPZQNDJ9J000BXX", +# # "projectId": "BPOP-retro", +# # "name": null, +# # "description": null +# # } +# # ], +# # "bclconvertDataRows": [ +# # { +# # "sampleId": "L2400252", +# # "index": "TGACAGCT", +# # "index2": "CCCGTAGG", +# # "lane": 4, +# # "overrideCycles": "Y151;I8N2;I8N2;Y151" +# # } +# # ], +# # "fastqListRows": [ +# # { +# # "fastqListRowRgid": "TGACAGCT.CCCGTAGG.4.240424_A01052_0193_BH7JMMDRX5.L2400252" +# # } +# # ] +# # } +# # }, +# # { +# # "event_data": { +# # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", +# # "library": { +# # "orcabusId": "lib.01J8ES533DJZZNPP9MXYR5TRC0", +# # "libraryId": "L2400253", +# # "phenotype": "tumor", +# # "workflow": "research", +# # "quality": "good", +# # "type": "WTS", +# # "assay": "NebRNA", +# # "coverage": 6.0 +# # }, +# # "sample": { +# # "orcabusId": "smp.01J8ES532ZBHWY3DWY0DWQ223R", +# # "sampleId": "PRJ240566", +# # "externalSampleId": "9-218-007_Bx", +# # "source": "tissue" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES522WN7YPZS1Z9NGSPNDA", +# # "subjectId": "218-007" +# # }, +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4FH3XMPZQNDJ9J000BXX", +# # "projectId": "BPOP-retro", +# # "name": null, +# # "description": null +# # } +# # ], +# # "bclconvertDataRows": [ +# # { +# # "sampleId": "L2400253", +# # "index": "CATCACCC", +# # "index2": "ATATAGCA", +# # "lane": 4, +# # "overrideCycles": "Y151;I8N2;I8N2;Y151" +# # } +# # ], +# # "fastqListRows": [ +# # { +# # "fastqListRowRgid": "CATCACCC.ATATAGCA.4.240424_A01052_0193_BH7JMMDRX5.L2400253" +# # } +# # ] +# # } +# # }, +# # { +# # "event_data": { +# # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", +# # "library": { +# # "orcabusId": "lib.01J8ES534XGBFYDVYV8ZG6SYS0", +# # "libraryId": "L2400254", +# # "phenotype": "tumor", +# # "workflow": "research", +# # "quality": "borderline", +# # "type": "WTS", +# # "assay": "NebRNA", +# # "coverage": 6.0 +# # }, +# # "sample": { +# # "orcabusId": "smp.01J8ES534BX7B89X5EKSCFRDDZ", +# # "sampleId": "PRJ240567", +# # "externalSampleId": "10-218-007_04", +# # "source": "tissue" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES522WN7YPZS1Z9NGSPNDA", +# # "subjectId": "218-007" +# # }, +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4FH3XMPZQNDJ9J000BXX", +# # "projectId": "BPOP-retro", +# # "name": null, +# # "description": null +# # } +# # ], +# # "bclconvertDataRows": [ +# # { +# # "sampleId": "L2400254", +# # "index": "CTGGAGTA", +# # "index2": "GTTCGGTT", +# # "lane": 4, +# # "overrideCycles": "Y151;I8N2;I8N2;Y151" +# # } +# # ], +# # "fastqListRows": [ +# # { +# # "fastqListRowRgid": "CTGGAGTA.GTTCGGTT.4.240424_A01052_0193_BH7JMMDRX5.L2400254" +# # } +# # ] +# # } +# # }, +# # { +# # "event_data": { +# # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", +# # "library": { +# # "orcabusId": "lib.01J8ES536AB5A5PBJ8S45SZP7Q", +# # "libraryId": "L2400255", +# # "phenotype": "tumor", +# # "workflow": "clinical", +# # "quality": "very-poor", +# # "type": "WTS", +# # "assay": "NebRNA", +# # "coverage": 6.0 +# # }, +# # "sample": { +# # "orcabusId": "smp.01J8ES535VGG93023KWAFMWGH4", +# # "sampleId": "PRJ240200", +# # "externalSampleId": "RNA036747", +# # "source": "FFPE" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES51S87R4EJ61QJ0DMDYWZ", +# # "subjectId": "SN_PMC-141" +# # }, +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4EZAA5YMHX82664GJQB3", +# # "projectId": "CUP", +# # "name": null, +# # "description": null +# # } +# # ], +# # "bclconvertDataRows": [ +# # { +# # "sampleId": "L2400255", +# # "index": "GATCCGGG", +# # "index2": "AAGCAGGT", +# # "lane": 4, +# # "overrideCycles": "Y151;I8N2;I8N2;Y151" +# # } +# # ], +# # "fastqListRows": [ +# # { +# # "fastqListRowRgid": "GATCCGGG.AAGCAGGT.4.240424_A01052_0193_BH7JMMDRX5.L2400255" +# # } +# # ] +# # } +# # }, +# # { +# # "event_data": { +# # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", +# # "library": { +# # "orcabusId": "lib.01J8ES537S0W1AX9PQPST13GM9", +# # "libraryId": "L2400256", +# # "phenotype": "tumor", +# # "workflow": "clinical", +# # "quality": "very-poor", +# # "type": "WTS", +# # "assay": "NebRNA", +# # "coverage": 6.0 +# # }, +# # "sample": { +# # "orcabusId": "smp.01J8ES5379C40K08YG3JDMZJN7", +# # "sampleId": "PRJ240648", +# # "externalSampleId": "RNA037080", +# # "source": "FFPE" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES529GSPBV64SESK9SWD76", +# # "subjectId": "SN_PMC-145" +# # }, +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4EZAA5YMHX82664GJQB3", +# # "projectId": "CUP", +# # "name": null, +# # "description": null +# # } +# # ], +# # "bclconvertDataRows": [ +# # { +# # "sampleId": "L2400256", +# # "index": "AACACCTG", +# # "index2": "CGCATGGG", +# # "lane": 4, +# # "overrideCycles": "Y151;I8N2;I8N2;Y151" +# # } +# # ], +# # "fastqListRows": [ +# # { +# # "fastqListRowRgid": "AACACCTG.CGCATGGG.4.240424_A01052_0193_BH7JMMDRX5.L2400256" +# # } +# # ] +# # } +# # }, +# # { +# # "event_data": { +# # "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5", +# # "library": { +# # "orcabusId": "lib.01J8ES5395KETT9T2NJSVNDKNP", +# # "libraryId": "L2400257", +# # "phenotype": "negative-control", +# # "workflow": "control", +# # "quality": "good", +# # "type": "WTS", +# # "assay": "NebRNA", +# # "coverage": 0.1 +# # }, +# # "sample": { +# # "orcabusId": "smp.01J8ES538PFF6MQQ35PTC00JAY", +# # "sampleId": "NTC_NebRNA240226", +# # "externalSampleId": "NTC_NebRNA240226", +# # "source": "water" +# # }, +# # "subject": { +# # "orcabusId": "sbj.01J8ES4DFMNF0SX6P8P8Y9J6K1", +# # "subjectId": "negative control" +# # }, +# # "projectSet": [ +# # { +# # "orcabusId": "prj.01J8ES4FC6DVW20AR33FBX2SA8", +# # "projectId": "Control", +# # "name": null, +# # "description": null +# # } +# # ], +# # "bclconvertDataRows": [ +# # { +# # "sampleId": "L2400257", +# # "index": "GTGACGTT", +# # "index2": "TCCCAGAT", +# # "lane": 4, +# # "overrideCycles": "Y151;I8N2;I8N2;Y151" +# # } +# # ], +# # "fastqListRows": [ +# # { +# # "fastqListRowRgid": "GTGACGTT.TCCCAGAT.4.240424_A01052_0193_BH7JMMDRX5.L2400257" +# # } +# # ] +# # } +# # } +# # ], +# # "library_set": [ +# # "lib.01J8ES4XXF6NMEJMM5M4GWS6KH", +# # "lib.01J8ES530H895X4WA3NQ6CY2QV", +# # "lib.01J8ES534XGBFYDVYV8ZG6SYS0", +# # "lib.01J8ES4MPZ5B201R50K42XXM4M", +# # "lib.01J8ES4ZP88X2E17X5X1FRMTPK", +# # "lib.01J8ES4Y5D52202JVBXHJ9Q9WF", +# # "lib.01J8ES4ZVWA2CGBHJVKAS3Y0G9", +# # "lib.01J8ES533DJZZNPP9MXYR5TRC0", +# # "lib.01J8ES52XYMVGRB1Q458THNG4T", +# # "lib.01J8ES4ZDRQAP2BN3SDYYV5PKW", +# # "lib.01J8ES52DHAPZM6FZ0VZK89PRT", +# # "lib.01J8ES4XSS97XNRS8DH0B1RJRG", +# # "lib.01J8ES4ZST489C712CG3R9NQSQ", +# # "lib.01J8ES4ZMY0G1H9MDN7K2TH9Y6", +# # "lib.01J8ES537S0W1AX9PQPST13GM9", +# # "lib.01J8ES52C3N585BGGY4VNXHC83", +# # "lib.01J8ES52F2ZHRXQY1AT1N1F81F", +# # "lib.01J8ES536AB5A5PBJ8S45SZP7Q", +# # "lib.01J8ES5320EWBNNYDGXF2SYJBD", +# # "lib.01J8ES51V0RSVT6C7WQR72QQED", +# # "lib.01J8ES52889Q8826P5SH9HDPP0", +# # "lib.01J8ES5395KETT9T2NJSVNDKNP", +# # "lib.01J8ES4Y3ZKRX3C5JAHA5NBXV1", +# # "lib.01J8ES4XNYFP38JMDV7GMV0V3V", +# # "lib.01J8ES52ANMRT3B7Y96T1Y3RY8", +# # "lib.01J8ES4XZD7T2VRPVQ1GSVZ11X", +# # "lib.01J8ES4Y1AKAHYD9EW0TW4FBCP", +# # "lib.01J8ES52Z2KTVVKZ2ZGVQ6YC10", +# # "lib.01J8ES4XQG3MPBW94TTVT4STVG" +# # ], +# # "subject_set": [ +# # "sbj.01J8ES4XKHKNQ1NF8EKKACZ032", +# # "sbj.01J8ES4ZEQ3FVD6DDVEG8MW60Q", +# # "sbj.01J8ES51S87R4EJ61QJ0DMDYWZ", +# # "sbj.01J8ES4DRJ31Z2H1GJQZGVDXZR", +# # "sbj.01J8ES4MNXJSDRR406DAXFZP2N", +# # "sbj.01J8ES4GNVGZSJVTHGVKS9VW7F", +# # "sbj.01J8ES51WC4GV5YDJNTMAK2YY1", +# # "sbj.01J8ES4ZCKNW6QKP006SYNZ5RA", +# # "sbj.01J8ES522WN7YPZS1Z9NGSPNDA", +# # "sbj.01J8ES4DFMNF0SX6P8P8Y9J6K1", +# # "sbj.01J8ES4XW2TXGEJBQWCVMRZRTS", +# # "sbj.01J8ES529GSPBV64SESK9SWD76", +# # "sbj.01J8ES4FNJ2FCAK0RJST0428X0" +# # ], +# # "project_set": [ +# # "prj.01J8ES4FC6DVW20AR33FBX2SA8", +# # "prj.01J8ES4FH3XMPZQNDJ9J000BXX", +# # "prj.01J8ES4EZAA5YMHX82664GJQB3", +# # "prj.01J8ES4EBXK08WDWB97BSCX1C9", +# # "prj.01J8ES4ZAWHH3FKYA2CFHSMZ4B", +# # "prj.01J8ES4XMWD0DH7MDRNER5TZS1" +# # ], +# # "sample_set": [ +# # "smp.01J8ES5379C40K08YG3JDMZJN7", +# # "smp.01J8ES4XWXFANT7P0T3AFXA85G", +# # "smp.01J8ES4Y0V0ZBKAE91TDSY0BBB", +# # "smp.01J8ES535VGG93023KWAFMWGH4", +# # "smp.01J8ES4FP9WTFBDNGKVG3D9BD4", +# # "smp.01J8ES52EEX67YRYAJS3F5GMJ5", +# # "smp.01J8ES4XYTSVQVRSBA9M26NSZY", +# # "smp.01J8ES4ZMETZP255WMFC8TSCYT", +# # "smp.01J8ES51T84KVVVSEPYQFGW0EV", +# # "smp.01J8ES538PFF6MQQ35PTC00JAY", +# # "smp.01J8ES4XMDW0FV1YMWHSZZQ4TX", +# # "smp.01J8ES52D076FQM5K8128AQ593", +# # "smp.01J8ES4Y37JTPEEJSED9BXH8N2", +# # "smp.01J8ES4ZQ76H8P0Q7S618F3BMA", +# # "smp.01J8ES4Y4XK1WX4WCPD6XY8KNM", +# # "smp.01J8ES531H420JM9MG5R4AE1AZ", +# # "smp.01J8ES534BX7B89X5EKSCFRDDZ", +# # "smp.01J8ES4ZNT47EM37QKMT12JPPJ", +# # "smp.01J8ES4MPHSX7MRCTTFWJBYTT7", +# # "smp.01J8ES52A5QX0GQ6RB78Z8DGYQ", +# # "smp.01J8ES532ZBHWY3DWY0DWQ223R", +# # "smp.01J8ES4ZVAR9NQM55Z2TXCDY9V", +# # "smp.01J8ES4XRG9NB38N03688M2CCB", +# # "smp.01J8ES527QKB5Y5RVZWZ8HQX0H", +# # "smp.01J8ES530355YNZ3VHQQQ204PF", +# # "smp.01J8ES4ZDAFRK3K3PY33F8XS0W", +# # "smp.01J8ES52BM8BVS3PX47E6FM7D5", +# # "smp.01J8ES52XE661E8V8XTWD02QCK", +# # "smp.01J8ES4XQ071BF3WZN111SNJ2B" +# # ] +# # } \ No newline at end of file diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/clag/part_1/samplesheet-event-shower/lambdas/generate_event_data_objects_py/requirements.txt b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/clag/part_1/samplesheet-event-shower/lambdas/generate_event_data_objects_py/requirements.txt new file mode 100644 index 000000000..44c70391c --- /dev/null +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/clag/part_1/samplesheet-event-shower/lambdas/generate_event_data_objects_py/requirements.txt @@ -0,0 +1,2 @@ +more-itertools>=10.5.0 +pandas>=2.2.3 \ No newline at end of file diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/clag/part_1/samplesheet-event-shower/step_functions_templates/samplesheet_event_shower_sfn_template.asl.json b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/clag/part_1/samplesheet-event-shower/step_functions_templates/samplesheet_event_shower_sfn_template.asl.json index 43c6d8341..ff4bf0d96 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/clag/part_1/samplesheet-event-shower/step_functions_templates/samplesheet_event_shower_sfn_template.asl.json +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/clag/part_1/samplesheet-event-shower/step_functions_templates/samplesheet_event_shower_sfn_template.asl.json @@ -37,9 +37,9 @@ "ResultSelector": { "samplesheet.$": "$.Payload.decompressed_dict" }, - "Next": "Get Library / Subject Map from SampleSheet" + "Next": "Get Library Map from SampleSheet" }, - "Get Library / Subject Map from SampleSheet": { + "Get Library Map from SampleSheet": { "Type": "Task", "Resource": "arn:aws:states:::lambda:invoke", "Parameters": { @@ -63,9 +63,7 @@ ], "ResultPath": "$.get_subject_library_map_from_samplesheet", "ResultSelector": { - "library_obj_list.$": "$.Payload.library_obj_list", - "specimen_obj_list.$": "$.Payload.specimen_obj_list", - "subject_obj_list.$": "$.Payload.subject_obj_list" + "library_obj_list.$": "$.Payload.library_obj_list" }, "Next": "Save SampleSheet and Get Subject / Library Map Objects" }, @@ -84,9 +82,7 @@ "Payload": { "instrument_run_id.$": "$.inputs.payload.data.instrumentRunId", "samplesheet.$": "$.decompress_samplesheet_step.samplesheet", - "library_obj_list.$": "$.get_subject_library_map_from_samplesheet.library_obj_list", - "specimen_obj_list.$": "$.get_subject_library_map_from_samplesheet.specimen_obj_list", - "subject_obj_list.$": "$.get_subject_library_map_from_samplesheet.subject_obj_list" + "library_obj_list.$": "$.get_subject_library_map_from_samplesheet.library_obj_list" } }, "Retry": [ @@ -105,35 +101,51 @@ "ResultSelector": { "start_samplesheet_shower_event_data.$": "$.Payload.start_samplesheet_shower_event_data", "complete_samplesheet_shower_event_data.$": "$.Payload.complete_samplesheet_shower_event_data", - "subject_event_data_list.$": "$.Payload.subject_event_data_list", "project_event_data_list.$": "$.Payload.project_event_data_list", - "library_event_data_list.$": "$.Payload.library_event_data_list" + "library_event_data_list.$": "$.Payload.library_event_data_list", + "library_set.$": "$.Payload.library_set", + "subject_set.$": "$.Payload.subject_set", + "project_set.$": "$.Payload.project_set", + "sample_set.$": "$.Payload.sample_set" }, "ResultPath": "$.generate_event_objects_step", - "End": true - } - } - }, - { - "StartAt": "Register SampleSheet by RunID (Instrument Run DB)", - "States": { - "Register SampleSheet by RunID (Instrument Run DB)": { + "Next": "Register Instrument Run DB" + }, + "Register Instrument Run DB": { "Type": "Task", "Resource": "arn:aws:states:::dynamodb:putItem", "Parameters": { "TableName": "${__table_name__}", "Item": { "id.$": "$.inputs.payload.data.instrumentRunId", - "id_type": "${__samplesheet_table_partition_name__}", + "id_type": "${__instrument_run_table_partition_name__}", "samplesheet_dict": { "S.$": "States.JsonToString($.decompress_samplesheet_step.samplesheet)" }, - "library_ids_set": { - "SS.$": "$.get_subject_library_map_from_samplesheet.library_obj_list[*].libraryId" + "library_set": { + "SS.$": "$.generate_event_objects_step.library_set" + }, + "subject_set": { + "SS.$": "$.generate_event_objects_step.sample_set" + }, + "project_set": { + "SS.$": "$.generate_event_objects_step.project_set" + }, + "sample_set": { + "SS.$": "$.generate_event_objects_step.sample_set" } } }, - "ResultPath": null, + "End": true, + "ResultPath": null + } + } + }, + { + "StartAt": "Pass", + "States": { + "Pass": { + "Type": "Pass", "End": true } } @@ -142,7 +154,6 @@ "ResultSelector": { "start_samplesheet_shower_event_data.$": "$.[0].generate_event_objects_step.start_samplesheet_shower_event_data", "project_event_data_list.$": "$.[0].generate_event_objects_step.project_event_data_list", - "subject_event_data_list.$": "$.[0].generate_event_objects_step.subject_event_data_list", "library_event_data_list.$": "$.[0].generate_event_objects_step.library_event_data_list", "complete_samplesheet_shower_event_data.$": "$.[0].generate_event_objects_step.complete_samplesheet_shower_event_data" }, @@ -181,6 +192,7 @@ "ItemsPath": "$.generate_event_objects_step.project_event_data_list", "ItemSelector": { "project_event_data.$": "$$.Map.Item.Value.event_data", + "project_orcabus_id.$": "$$.Map.Item.Value.event_data.project.orcabusId", "instrument_run_id.$": "$.inputs.payload.data.instrumentRunId" }, "ItemProcessor": { @@ -195,7 +207,7 @@ "Parameters": { "TableName": "${__table_name__}", "Key": { - "id.$": "States.Format('{}__{}', $.project_event_data.projectName, $.project_event_data.projectOwner)", + "id.$": "$.project_orcabus_id", "id_type": "${__project_table_partition_name__}" } }, @@ -225,17 +237,16 @@ "Parameters": { "TableName": "${__table_name__}", "Key": { - "id": { - "S.$": "States.Format('{}__{}', $.project_event_data.projectName, $.project_event_data.projectOwner)" - }, - "id_type": { - "S": "${__project_table_partition_name__}" - } + "id.$": "$.project_orcabus_id", + "id_type": "${__project_table_partition_name__}" }, - "UpdateExpression": "ADD instrument_run_id_set :instrument_run_id_set", + "UpdateExpression": "ADD instrument_run_id_set :instrument_run_id_set, library_set :library_set", "ExpressionAttributeValues": { ":instrument_run_id_set": { "SS.$": "States.Array($.instrument_run_id)" + }, + ":library_set": { + "SS.$": "$.project_event_data.librarySet[*].orcabusId" } } }, @@ -248,20 +259,16 @@ "Parameters": { "TableName": "${__table_name__}", "Item": { - "id": { - "S.$": "States.Format('{}__{}', $.project_event_data.projectName, $.project_event_data.projectOwner)" - }, - "id_type": { - "S": "${__project_table_partition_name__}" - }, - "project_name": { - "S.$": "$.project_event_data.projectName" - }, - "project_owner": { - "S.$": "$.project_event_data.projectOwner" + "id.$": "$.project_orcabus_id", + "id_type": "${__project_table_partition_name__}", + "project_obj": { + "S.$": "States.JsonToString($.project_event_data.project)" }, "instrument_run_id_set": { "SS.$": "States.Array($.instrument_run_id)" + }, + "library_set": { + "SS.$": "$.project_event_data.librarySet[*].orcabusId" } } }, @@ -298,129 +305,6 @@ } }, "ResultPath": null, - "Next": "Push Subject Events" - }, - "Push Subject Events": { - "Type": "Map", - "ItemsPath": "$.generate_event_objects_step.subject_event_data_list", - "ItemSelector": { - "subject_id.$": "$$.Map.Item.Value.event_data.subject.subjectId", - "subject_orcabus_id.$": "$$.Map.Item.Value.event_data.subject.orcabusId", - "subject_event_data.$": "$$.Map.Item.Value.event_data", - "instrument_run_id.$": "$$.Map.Item.Value.event_data.instrumentRunId" - }, - "ItemProcessor": { - "ProcessorConfig": { - "Mode": "INLINE" - }, - "StartAt": "Get Subject (Instrument Run DB)", - "States": { - "Get Subject (Instrument Run DB)": { - "Type": "Task", - "Resource": "arn:aws:states:::dynamodb:getItem", - "Parameters": { - "TableName": "${__table_name__}", - "Key": { - "id.$": "$.subject_id", - "id_type": "${__subject_table_partition_name__}" - } - }, - "Next": "Check if Subject in Instrument Run DB", - "ResultPath": "$.get_subject_from_db_step", - "ResultSelector": { - "db_response.$": "$" - } - }, - "Check if Subject in Instrument Run DB": { - "Type": "Choice", - "Choices": [ - { - "Not": { - "Variable": "$.get_subject_from_db_step.db_response.Item", - "IsPresent": true - }, - "Next": "Add Instrument Run ID to Subject (Instrument Run DB)", - "Comment": "New Subject" - } - ], - "Default": "Append Instrument Run ID to Subject (Instrument Run DB)" - }, - "Add Instrument Run ID to Subject (Instrument Run DB)": { - "Type": "Task", - "Resource": "arn:aws:states:::dynamodb:putItem", - "Parameters": { - "TableName": "${__table_name__}", - "Item": { - "id": { - "S.$": "$.subject_id" - }, - "id_type": { - "S": "${__subject_table_partition_name__}" - }, - "orcabus_id": { - "S.$": "$.subject_orcabus_id" - }, - "subject_id": { - "S.$": "$.subject_id" - }, - "instrument_run_id_set": { - "SS.$": "States.Array($.instrument_run_id)" - } - } - }, - "Next": "Wait Subject DB Sync", - "ResultPath": null - }, - "Append Instrument Run ID to Subject (Instrument Run DB)": { - "Type": "Task", - "Resource": "arn:aws:states:::dynamodb:updateItem", - "Parameters": { - "TableName": "${__table_name__}", - "Key": { - "id.$": "$.subject_id", - "id_type": "${__subject_table_partition_name__}" - }, - "UpdateExpression": "ADD instrument_run_id_set :instrument_run_id_set", - "ExpressionAttributeValues": { - ":instrument_run_id_set": { - "SS.$": "States.Array($.instrument_run_id)" - } - } - }, - "Next": "Wait Subject DB Sync", - "ResultPath": null - }, - "Wait Subject DB Sync": { - "Type": "Wait", - "Seconds": 1, - "Next": "Subject In SampleSheet" - }, - "Subject In SampleSheet": { - "Type": "Task", - "Resource": "arn:aws:states:::events:putEvents", - "Parameters": { - "Entries": [ - { - "Detail": { - "timestamp.$": "$$.Execution.StartTime", - "status": "${__subject_in_samplesheet_status__}", - "payload": { - "version": "${__subject_in_samplesheet_payload_version__}", - "data.$": "$.subject_event_data" - } - }, - "DetailType": "${__subject_in_samplesheet_detail_type__}", - "EventBusName": "${__event_bus_name__}", - "Source": "${__event_source__}" - } - ] - }, - "End": true, - "ResultPath": null - } - } - }, - "ResultPath": null, "Next": "Push Library Events" }, "Push Library Events": { @@ -444,7 +328,7 @@ "Parameters": { "TableName": "${__table_name__}", "Key": { - "id.$": "$.library_id", + "id.$": "$.library_orcabus_id", "id_type": "${__library_table_partition_name__}" } }, @@ -493,23 +377,13 @@ "Parameters": { "TableName": "${__table_name__}", "Item": { - "id": { - "S.$": "$.library_id" - }, - "id_type": { - "S": "${__library_table_partition_name__}" - }, - "orcabus_id": { - "S.$": "$.library_orcabus_id" - }, + "id.$": "$.library_orcabus_id", + "id_type": "${__library_table_partition_name__}", "library_id": { - "S.$": "$.library_event_data.library.libraryId" - }, - "project_name": { - "S.$": "$.library_event_data.library.projectName" + "S.$": "$.library_id" }, - "project_owner": { - "S.$": "$.library_event_data.library.projectOwner" + "library_obj": { + "S.$": "States.JsonToString($.library_event_data.library)" }, "instrument_run_id_set": { "SS.$": "States.Array($.instrument_run_id)" diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/clag/part_2/fastq-list-rows-event-shower/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/clag/part_2/fastq-list-rows-event-shower/index.ts index 01cc523e2..2b1b03bbf 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/clag/part_2/fastq-list-rows-event-shower/index.ts +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/clag/part_2/fastq-list-rows-event-shower/index.ts @@ -30,7 +30,7 @@ export class NewFastqListRowsEventShowerConstruct extends Construct { // Tables tablePartition: { fastqListRowsByInstrumentRun: 'fastqlistrows_by_instrument_run', - samplesheetByInstrumentRun: 'samplesheet_by_instrument_run', + instrumentRun: 'instrument_run', subject: 'subject', library: 'library', project: 'project', @@ -143,8 +143,10 @@ export class NewFastqListRowsEventShowerConstruct extends Construct { this.newFastqListRowsEventShowerMap.tablePartition.fastqListRowsByInstrumentRun, __library_table_partition_name__: this.newFastqListRowsEventShowerMap.tablePartition.library, - __samplesheet_table_partition_name__: - this.newFastqListRowsEventShowerMap.tablePartition.samplesheetByInstrumentRun, + __instrument_run_table_partition_name__: + this.newFastqListRowsEventShowerMap.tablePartition.instrumentRun, + __project_table_partition_name__: + this.newFastqListRowsEventShowerMap.tablePartition.project, /* Lambda functions */ __decompress_fastq_list_rows_lambda_function_arn__: diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/clag/part_2/fastq-list-rows-event-shower/lambdas/generate_event_data_objects_py/generate_event_data_objects.py b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/clag/part_2/fastq-list-rows-event-shower/lambdas/generate_event_data_objects_py/generate_event_data_objects.py index c90f48121..67be21985 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/clag/part_2/fastq-list-rows-event-shower/lambdas/generate_event_data_objects_py/generate_event_data_objects.py +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/clag/part_2/fastq-list-rows-event-shower/lambdas/generate_event_data_objects_py/generate_event_data_objects.py @@ -4,6 +4,8 @@ Given a set of fastq list rows, generate a set of event maps for each fastq file, along with a create event shower as well """ + +# Imports from typing import Dict @@ -54,12 +56,12 @@ def generate_fastq_list_row_event(fastq_list_row: Dict, library: Dict, instrumen new_fastq_list_row_dict["rgid"] = fastq_list_row_rgid return { - "fastqListRow": new_fastq_list_row_dict, "instrumentRunId": instrument_run_id, "library": { - "libraryId": library.get("library_id"), - "orcabusId": library.get("orcabus_id") - } + "libraryId": library.get("libraryId"), + "orcabusId": library.get("orcabusId") + }, + "fastqListRow": new_fastq_list_row_dict, } @@ -74,6 +76,7 @@ def handler(event, context): # Get the fastq list rows and instrument run id fastq_list_rows = event['fastq_list_rows'] library_obj_list = event['library_objs'] + project_obj_list = event['project_objs'] instrument_run_id = event['instrument_run_id'] # Generate the fastq list row events @@ -83,7 +86,7 @@ def handler(event, context): fastq_list_row_iter, next( filter( - lambda library_iter: library_iter['library_id'] == fastq_list_row_iter['RGSM'], + lambda library_iter: library_iter['libraryId'] == fastq_list_row_iter['RGSM'], library_obj_list ) ), @@ -102,52 +105,61 @@ def handler(event, context): "instrumentRunId": instrument_run_id, } - # Generate project data level events - project_list = list( - set( + project_event_data_list = [] + for project_obj_iter_ in project_obj_list: + library_objects_in_project = list( map( - lambda library_iter: ( - library_iter.get("project_owner"), library_iter.get("project_name") - ), - library_obj_list + lambda library_obj_iter: { + "library": { + "orcabusId": library_obj_iter.get("orcabusId"), + "libraryId": library_obj_iter.get("libraryId"), + }, + "fastqPairs": list( + map( + lambda fastq_list_row_event_data_iter: ( + fastq_list_row_event_data_iter.get("fastqListRow") + ), + filter( + lambda fastq_list_row_event_data_iter: ( + fastq_list_row_event_data_iter.get("library").get("libraryId") == + library_obj_iter.get("libraryId") + ), + fastq_list_row_event_data_list + ) + ) + ) + }, + filter( + lambda library_obj_iter_: ( + any( + map( + lambda project_obj_iter_: ( + library_obj_iter_.get("orcabusId") == + project_obj_iter_ + ), + project_obj_iter_.get("librarySet") + ) + ) + ), + library_obj_list + ) + ) + ) + + # Get project object + project_obj = dict( + filter( + lambda kv: not kv[0] == 'librarySet', + project_obj_iter_.items() ) ) - ) - project_event_data_list = [] - for project_owner, project_name in project_list: project_event_data_list.append( { "event_data": { "instrumentRunId": instrument_run_id, - "projectOwner": project_owner, - "projectName": project_name, - "libraries": list( - map( - lambda library_obj_iter: { - "orcabusId": library_obj_iter.get("orcabus_id"), - "libraryId": library_obj_iter.get("library_id"), - "fastqPairs": list( - map( - lambda fastq_list_row_event_data_iter: fastq_list_row_event_data_iter.get("fastqListRow"), - filter( - lambda fastq_list_row_event_data_iter: ( - fastq_list_row_event_data_iter.get("library").get("libraryId") == library_obj_iter.get("library_id") - ), - fastq_list_row_event_data_list - ) - ) - ) - }, - filter( - lambda library_obj_iter: ( - library_obj_iter.get("project_owner") == project_owner and - library_obj_iter.get("project_name") == project_name - ), - library_obj_list - ) - ) - ) + "project": project_obj, + "libraryFastqSet": library_objects_in_project } } ) @@ -161,7 +173,7 @@ def handler(event, context): } -# Test the function +# # Test the function # if __name__ == "__main__": # import json # @@ -171,178 +183,373 @@ def handler(event, context): # { # "library_objs": [ # { -# "library_id": "L2400102", -# "project_owner": "VCCC", -# "orcabus_id": "lib.01J5S9C4VMJ6PZ8GJ2G189AMXX", -# "project_name": "PO" -# }, -# { -# "library_id": "L2400159", -# "project_owner": "UMCCR", -# "orcabus_id": "lib.01J5S9CBG0NF8QBNVKM6ESCD60", -# "project_name": "Testing" -# }, -# { -# "library_id": "L2400160", -# "project_owner": "UMCCR", -# "orcabus_id": "lib.01J5S9CBHP6NSB42RVFAP9PGJP", -# "project_name": "Testing" -# }, -# { -# "library_id": "L2400161", -# "project_owner": "UMCCR", -# "orcabus_id": "lib.01J5S9CBKCATYSFY40BRX6WJWX", -# "project_name": "Testing" -# }, -# { -# "library_id": "L2400162", -# "project_owner": "UMCCR", -# "orcabus_id": "lib.01J5S9CBN6EAXW4AXG7TQ1H6NC", -# "project_name": "Testing" -# }, -# { -# "library_id": "L2400163", -# "project_owner": "UMCCR", -# "orcabus_id": "lib.01J5S9CBQFX8V1QRW7KAV3MD1W", -# "project_name": "Testing" -# }, -# { -# "library_id": "L2400164", -# "project_owner": "UMCCR", -# "orcabus_id": "lib.01J5S9CBS64DNTHK6CE850CCNZ", -# "project_name": "Testing" -# }, -# { -# "library_id": "L2400165", -# "project_owner": "UMCCR", -# "orcabus_id": "lib.01J5S9CBTZRYQNTGAHPC2T601D", -# "project_name": "Testing" -# }, -# { -# "library_id": "L2400166", -# "project_owner": "UMCCR", -# "orcabus_id": "lib.01J5S9CBX10204CK7EKGTH9TMB", -# "project_name": "Testing" -# }, -# { -# "library_id": "L2400191", -# "project_owner": "TJohn", -# "orcabus_id": "lib.01J5S9CDF8HHG5PJE3ECJMKMY7", -# "project_name": "CAVATAK" -# }, -# { -# "library_id": "L2400195", -# "project_owner": "TJohn", -# "orcabus_id": "lib.01J5S9CDQSSAG1WYCRWMD82Z1S", -# "project_name": "CAVATAK" -# }, -# { -# "library_id": "L2400196", -# "project_owner": "TJohn", -# "orcabus_id": "lib.01J5S9CDSJ2BGEYM8FTXGKVGV8", -# "project_name": "CAVATAK" -# }, -# { -# "library_id": "L2400197", -# "project_owner": "TJohn", -# "orcabus_id": "lib.01J5S9CDVEHDZHZR3BZTQ7WNJQ", -# "project_name": "CAVATAK" -# }, -# { -# "library_id": "L2400198", -# "project_owner": "TJohn", -# "orcabus_id": "lib.01J5S9CDXCR7Q5K6A8VJRSMM4Q", -# "project_name": "CAVATAK" -# }, -# { -# "library_id": "L2400231", -# "project_owner": "Tothill", -# "orcabus_id": "lib.01J5S9CFX5P69S4KZRQGDFKV1N", -# "project_name": "CUP" -# }, -# { -# "library_id": "L2400238", -# "project_owner": "Tothill", -# "orcabus_id": "lib.01J5S9CGCAKQWHD9RBM9VXENY9", -# "project_name": "CUP" -# }, -# { -# "library_id": "L2400239", -# "project_owner": "Tothill", -# "orcabus_id": "lib.01J5S9CGEM1DHRQP72EP09B2TA", -# "project_name": "CUP" -# }, -# { -# "library_id": "L2400240", -# "project_owner": "Tothill", -# "orcabus_id": "lib.01J5S9CGG9N9GH5879SY6A6BJB", -# "project_name": "CUP" -# }, -# { -# "library_id": "L2400241", -# "project_owner": "UMCCR", -# "orcabus_id": "lib.01J5S9CGJ6G09YQ9KFHPSXMMVD", -# "project_name": "Control" -# }, -# { -# "library_id": "L2400242", -# "project_owner": "UMCCR", -# "orcabus_id": "lib.01J5S9CGKWDN7STKZKQM3KH9XR", -# "project_name": "Control" -# }, -# { -# "library_id": "L2400249", -# "project_owner": "UMCCR", -# "orcabus_id": "lib.01J5S9CH2SQ0P1SF7WAT5H4DSE", -# "project_name": "Control" -# }, -# { -# "library_id": "L2400250", -# "project_owner": "Whittle", -# "orcabus_id": "lib.01J5S9CH4CYPA4SP05H8KRX4W9", -# "project_name": "BPOP-retro" -# }, -# { -# "library_id": "L2400251", -# "project_owner": "Whittle", -# "orcabus_id": "lib.01J5S9CH65E4EE5QJEJ1C60GGG", -# "project_name": "BPOP-retro" -# }, -# { -# "library_id": "L2400252", -# "project_owner": "Whittle", -# "orcabus_id": "lib.01J5S9CH7TGZMV39Z59WJ8H5GP", -# "project_name": "BPOP-retro" -# }, -# { -# "library_id": "L2400253", -# "project_owner": "Whittle", -# "orcabus_id": "lib.01J5S9CH9TGMT2TJGBZX5VXHJY", -# "project_name": "BPOP-retro" -# }, -# { -# "library_id": "L2400254", -# "project_owner": "Whittle", -# "orcabus_id": "lib.01J5S9CHBGAP2XSN4TG8SAMRYY", -# "project_name": "BPOP-retro" -# }, -# { -# "library_id": "L2400255", -# "project_owner": "Tothill", -# "orcabus_id": "lib.01J5S9CHE4ERQ4H209DH397W8A", -# "project_name": "CUP" -# }, -# { -# "library_id": "L2400256", -# "project_owner": "Tothill", -# "orcabus_id": "lib.01J5S9CHFXPDGYQ8TXHRWQR3PY", -# "project_name": "CUP" -# }, -# { -# "library_id": "L2400257", -# "project_owner": "UMCCR", -# "orcabus_id": "lib.01J5S9CHHNGFJN73NPRQMSYGN9", -# "project_name": "Control" +# "orcabusId": "lib.01J8ES4MPZ5B201R50K42XXM4M", +# "libraryId": "L2400102", +# "phenotype": "tumor", +# "workflow": "research", +# "quality": "borderline", +# "type": "WGS", +# "assay": "ctTSO", +# "coverage": 50 +# }, +# { +# "orcabusId": "lib.01J8ES4XNYFP38JMDV7GMV0V3V", +# "libraryId": "L2400159", +# "phenotype": "tumor", +# "workflow": "manual", +# "quality": "good", +# "type": "ctDNA", +# "assay": "ctTSOv2", +# "coverage": 38.6 +# }, +# { +# "orcabusId": "lib.01J8ES4XQG3MPBW94TTVT4STVG", +# "libraryId": "L2400160", +# "phenotype": "tumor", +# "workflow": "manual", +# "quality": "good", +# "type": "ctDNA", +# "assay": "ctTSOv2", +# "coverage": 38.6 +# }, +# { +# "orcabusId": "lib.01J8ES4XSS97XNRS8DH0B1RJRG", +# "libraryId": "L2400161", +# "phenotype": "tumor", +# "workflow": "manual", +# "quality": "good", +# "type": "ctDNA", +# "assay": "ctTSOv2", +# "coverage": 38.6 +# }, +# { +# "orcabusId": "lib.01J8ES4XXF6NMEJMM5M4GWS6KH", +# "libraryId": "L2400162", +# "phenotype": "tumor", +# "workflow": "manual", +# "quality": "good", +# "type": "ctDNA", +# "assay": "ctTSOv2", +# "coverage": 38.6 +# }, +# { +# "orcabusId": "lib.01J8ES4XZD7T2VRPVQ1GSVZ11X", +# "libraryId": "L2400163", +# "phenotype": "tumor", +# "workflow": "manual", +# "quality": "good", +# "type": "ctDNA", +# "assay": "ctTSOv2", +# "coverage": 38.6 +# }, +# { +# "orcabusId": "lib.01J8ES4Y1AKAHYD9EW0TW4FBCP", +# "libraryId": "L2400164", +# "phenotype": "tumor", +# "workflow": "manual", +# "quality": "good", +# "type": "ctDNA", +# "assay": "ctTSOv2", +# "coverage": 38.6 +# }, +# { +# "orcabusId": "lib.01J8ES4Y3ZKRX3C5JAHA5NBXV1", +# "libraryId": "L2400165", +# "phenotype": "tumor", +# "workflow": "manual", +# "quality": "good", +# "type": "ctDNA", +# "assay": "ctTSOv2", +# "coverage": 38.6 +# }, +# { +# "orcabusId": "lib.01J8ES4Y5D52202JVBXHJ9Q9WF", +# "libraryId": "L2400166", +# "phenotype": "negative-control", +# "workflow": "manual", +# "quality": "good", +# "type": "ctDNA", +# "assay": "ctTSOv2", +# "coverage": 0.1 +# }, +# { +# "orcabusId": "lib.01J8ES4ZDRQAP2BN3SDYYV5PKW", +# "libraryId": "L2400191", +# "phenotype": "normal", +# "workflow": "research", +# "quality": "good", +# "type": "WGS", +# "assay": "TsqNano", +# "coverage": 40 +# }, +# { +# "orcabusId": "lib.01J8ES4ZMY0G1H9MDN7K2TH9Y6", +# "libraryId": "L2400195", +# "phenotype": "tumor", +# "workflow": "research", +# "quality": "good", +# "type": "WGS", +# "assay": "TsqNano", +# "coverage": 80 +# }, +# { +# "orcabusId": "lib.01J8ES4ZP88X2E17X5X1FRMTPK", +# "libraryId": "L2400196", +# "phenotype": "tumor", +# "workflow": "research", +# "quality": "good", +# "type": "WGS", +# "assay": "TsqNano", +# "coverage": 80 +# }, +# { +# "orcabusId": "lib.01J8ES4ZST489C712CG3R9NQSQ", +# "libraryId": "L2400197", +# "phenotype": "tumor", +# "workflow": "research", +# "quality": "good", +# "type": "WGS", +# "assay": "TsqNano", +# "coverage": 80 +# }, +# { +# "orcabusId": "lib.01J8ES4ZVWA2CGBHJVKAS3Y0G9", +# "libraryId": "L2400198", +# "phenotype": "tumor", +# "workflow": "research", +# "quality": "good", +# "type": "WGS", +# "assay": "TsqNano", +# "coverage": 80 +# }, +# { +# "orcabusId": "lib.01J8ES51V0RSVT6C7WQR72QQED", +# "libraryId": "L2400231", +# "phenotype": "tumor", +# "workflow": "clinical", +# "quality": "poor", +# "type": "WGS", +# "assay": "TsqNano", +# "coverage": 100 +# }, +# { +# "orcabusId": "lib.01J8ES52889Q8826P5SH9HDPP0", +# "libraryId": "L2400238", +# "phenotype": "normal", +# "workflow": "clinical", +# "quality": "good", +# "type": "WGS", +# "assay": "TsqNano", +# "coverage": 40 +# }, +# { +# "orcabusId": "lib.01J8ES52ANMRT3B7Y96T1Y3RY8", +# "libraryId": "L2400239", +# "phenotype": "normal", +# "workflow": "clinical", +# "quality": "good", +# "type": "WGS", +# "assay": "TsqNano", +# "coverage": 40 +# }, +# { +# "orcabusId": "lib.01J8ES52C3N585BGGY4VNXHC83", +# "libraryId": "L2400240", +# "phenotype": "tumor", +# "workflow": "clinical", +# "quality": "poor", +# "type": "WGS", +# "assay": "TsqNano", +# "coverage": 100 +# }, +# { +# "orcabusId": "lib.01J8ES52DHAPZM6FZ0VZK89PRT", +# "libraryId": "L2400241", +# "phenotype": "negative-control", +# "workflow": "control", +# "quality": "good", +# "type": "WGS", +# "assay": "TsqNano", +# "coverage": 0.1 +# }, +# { +# "orcabusId": "lib.01J8ES52F2ZHRXQY1AT1N1F81F", +# "libraryId": "L2400242", +# "phenotype": "normal", +# "workflow": "control", +# "quality": "good", +# "type": "WGS", +# "assay": "TsqNano", +# "coverage": 15 +# }, +# { +# "orcabusId": "lib.01J8ES52XYMVGRB1Q458THNG4T", +# "libraryId": "L2400249", +# "phenotype": "tumor", +# "workflow": "control", +# "quality": "good", +# "type": "WTS", +# "assay": "NebRNA", +# "coverage": 1 +# }, +# { +# "orcabusId": "lib.01J8ES52Z2KTVVKZ2ZGVQ6YC10", +# "libraryId": "L2400250", +# "phenotype": "tumor", +# "workflow": "research", +# "quality": "good", +# "type": "WTS", +# "assay": "NebRNA", +# "coverage": 6 +# }, +# { +# "orcabusId": "lib.01J8ES530H895X4WA3NQ6CY2QV", +# "libraryId": "L2400251", +# "phenotype": "tumor", +# "workflow": "research", +# "quality": "good", +# "type": "WTS", +# "assay": "NebRNA", +# "coverage": 6 +# }, +# { +# "orcabusId": "lib.01J8ES5320EWBNNYDGXF2SYJBD", +# "libraryId": "L2400252", +# "phenotype": "tumor", +# "workflow": "research", +# "quality": "good", +# "type": "WTS", +# "assay": "NebRNA", +# "coverage": 6 +# }, +# { +# "orcabusId": "lib.01J8ES533DJZZNPP9MXYR5TRC0", +# "libraryId": "L2400253", +# "phenotype": "tumor", +# "workflow": "research", +# "quality": "good", +# "type": "WTS", +# "assay": "NebRNA", +# "coverage": 6 +# }, +# { +# "orcabusId": "lib.01J8ES534XGBFYDVYV8ZG6SYS0", +# "libraryId": "L2400254", +# "phenotype": "tumor", +# "workflow": "research", +# "quality": "borderline", +# "type": "WTS", +# "assay": "NebRNA", +# "coverage": 6 +# }, +# { +# "orcabusId": "lib.01J8ES536AB5A5PBJ8S45SZP7Q", +# "libraryId": "L2400255", +# "phenotype": "tumor", +# "workflow": "clinical", +# "quality": "very-poor", +# "type": "WTS", +# "assay": "NebRNA", +# "coverage": 6 +# }, +# { +# "orcabusId": "lib.01J8ES537S0W1AX9PQPST13GM9", +# "libraryId": "L2400256", +# "phenotype": "tumor", +# "workflow": "clinical", +# "quality": "very-poor", +# "type": "WTS", +# "assay": "NebRNA", +# "coverage": 6 +# }, +# { +# "orcabusId": "lib.01J8ES5395KETT9T2NJSVNDKNP", +# "libraryId": "L2400257", +# "phenotype": "negative-control", +# "workflow": "control", +# "quality": "good", +# "type": "WTS", +# "assay": "NebRNA", +# "coverage": 0.1 +# } +# ], +# "project_objs": [ +# { +# "name": None, +# "description": None, +# "librarySet": [ +# "lib.01J8ES4MPZ5B201R50K42XXM4M" +# ], +# "projectId": "PO", +# "orcabusId": "prj.01J8ES4EBXK08WDWB97BSCX1C9" +# }, +# { +# "name": None, +# "description": None, +# "librarySet": [ +# "lib.01J8ES51V0RSVT6C7WQR72QQED", +# "lib.01J8ES52889Q8826P5SH9HDPP0", +# "lib.01J8ES52ANMRT3B7Y96T1Y3RY8", +# "lib.01J8ES52C3N585BGGY4VNXHC83", +# "lib.01J8ES536AB5A5PBJ8S45SZP7Q", +# "lib.01J8ES537S0W1AX9PQPST13GM9" +# ], +# "projectId": "CUP", +# "orcabusId": "prj.01J8ES4EZAA5YMHX82664GJQB3" +# }, +# { +# "name": None, +# "description": None, +# "librarySet": [ +# "lib.01J8ES52DHAPZM6FZ0VZK89PRT", +# "lib.01J8ES52F2ZHRXQY1AT1N1F81F", +# "lib.01J8ES52XYMVGRB1Q458THNG4T", +# "lib.01J8ES5395KETT9T2NJSVNDKNP" +# ], +# "projectId": "Control", +# "orcabusId": "prj.01J8ES4FC6DVW20AR33FBX2SA8" +# }, +# { +# "name": None, +# "description": None, +# "librarySet": [ +# "lib.01J8ES52Z2KTVVKZ2ZGVQ6YC10", +# "lib.01J8ES530H895X4WA3NQ6CY2QV", +# "lib.01J8ES5320EWBNNYDGXF2SYJBD", +# "lib.01J8ES533DJZZNPP9MXYR5TRC0", +# "lib.01J8ES534XGBFYDVYV8ZG6SYS0" +# ], +# "projectId": "BPOP-retro", +# "orcabusId": "prj.01J8ES4FH3XMPZQNDJ9J000BXX" +# }, +# { +# "name": None, +# "description": None, +# "librarySet": [ +# "lib.01J8ES4XNYFP38JMDV7GMV0V3V", +# "lib.01J8ES4XQG3MPBW94TTVT4STVG", +# "lib.01J8ES4XSS97XNRS8DH0B1RJRG", +# "lib.01J8ES4XXF6NMEJMM5M4GWS6KH", +# "lib.01J8ES4XZD7T2VRPVQ1GSVZ11X", +# "lib.01J8ES4Y1AKAHYD9EW0TW4FBCP", +# "lib.01J8ES4Y3ZKRX3C5JAHA5NBXV1", +# "lib.01J8ES4Y5D52202JVBXHJ9Q9WF" +# ], +# "projectId": "Testing", +# "orcabusId": "prj.01J8ES4XMWD0DH7MDRNER5TZS1" +# }, +# { +# "name": None, +# "description": None, +# "librarySet": [ +# "lib.01J8ES4ZDRQAP2BN3SDYYV5PKW", +# "lib.01J8ES4ZMY0G1H9MDN7K2TH9Y6", +# "lib.01J8ES4ZP88X2E17X5X1FRMTPK", +# "lib.01J8ES4ZST489C712CG3R9NQSQ", +# "lib.01J8ES4ZVWA2CGBHJVKAS3Y0G9" +# ], +# "projectId": "CAVATAK", +# "orcabusId": "prj.01J8ES4ZAWHH3FKYA2CFHSMZ4B" # } # ], # "fastq_list_rows": [ @@ -351,300 +558,301 @@ def handler(event, context): # "RGSM": "L2400102", # "RGLB": "L2400102", # "Lane": 1, -# "Read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_1/L2400102/L2400102_S1_L001_R1_001.fastq.gz", -# "Read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_1/L2400102/L2400102_S1_L001_R2_001.fastq.gz" +# "Read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_1/L2400102/L2400102_S1_L001_R1_001.fastq.gz", +# "Read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_1/L2400102/L2400102_S1_L001_R2_001.fastq.gz" # }, # { # "RGID": "GAGAATGGTT.TTGCTGCCGA.1", # "RGSM": "L2400159", # "RGLB": "L2400159", # "Lane": 1, -# "Read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_1/L2400159/L2400159_S2_L001_R1_001.fastq.gz", -# "Read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_1/L2400159/L2400159_S2_L001_R2_001.fastq.gz" +# "Read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_1/L2400159/L2400159_S2_L001_R1_001.fastq.gz", +# "Read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_1/L2400159/L2400159_S2_L001_R2_001.fastq.gz" # }, # { # "RGID": "AGAGGCAACC.CCATCATTAG.1", # "RGSM": "L2400160", # "RGLB": "L2400160", # "Lane": 1, -# "Read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_1/L2400160/L2400160_S3_L001_R1_001.fastq.gz", -# "Read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_1/L2400160/L2400160_S3_L001_R2_001.fastq.gz" +# "Read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_1/L2400160/L2400160_S3_L001_R1_001.fastq.gz", +# "Read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_1/L2400160/L2400160_S3_L001_R2_001.fastq.gz" # }, # { # "RGID": "CCATCATTAG.AGAGGCAACC.1", # "RGSM": "L2400161", # "RGLB": "L2400161", # "Lane": 1, -# "Read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_1/L2400161/L2400161_S4_L001_R1_001.fastq.gz", -# "Read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_1/L2400161/L2400161_S4_L001_R2_001.fastq.gz" +# "Read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_1/L2400161/L2400161_S4_L001_R1_001.fastq.gz", +# "Read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_1/L2400161/L2400161_S4_L001_R2_001.fastq.gz" # }, # { # "RGID": "GATAGGCCGA.GCCATGTGCG.1", # "RGSM": "L2400162", # "RGLB": "L2400162", # "Lane": 1, -# "Read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_1/L2400162/L2400162_S5_L001_R1_001.fastq.gz", -# "Read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_1/L2400162/L2400162_S5_L001_R2_001.fastq.gz" +# "Read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_1/L2400162/L2400162_S5_L001_R1_001.fastq.gz", +# "Read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_1/L2400162/L2400162_S5_L001_R2_001.fastq.gz" # }, # { # "RGID": "ATGGTTGACT.AGGACAGGCC.1", # "RGSM": "L2400163", # "RGLB": "L2400163", # "Lane": 1, -# "Read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_1/L2400163/L2400163_S6_L001_R1_001.fastq.gz", -# "Read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_1/L2400163/L2400163_S6_L001_R2_001.fastq.gz" +# "Read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_1/L2400163/L2400163_S6_L001_R1_001.fastq.gz", +# "Read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_1/L2400163/L2400163_S6_L001_R2_001.fastq.gz" # }, # { # "RGID": "TATTGCGCTC.CCTAACACAG.1", # "RGSM": "L2400164", # "RGLB": "L2400164", # "Lane": 1, -# "Read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_1/L2400164/L2400164_S7_L001_R1_001.fastq.gz", -# "Read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_1/L2400164/L2400164_S7_L001_R2_001.fastq.gz" +# "Read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_1/L2400164/L2400164_S7_L001_R1_001.fastq.gz", +# "Read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_1/L2400164/L2400164_S7_L001_R2_001.fastq.gz" # }, # { # "RGID": "TTCTACATAC.TTACAGTTAG.1", # "RGSM": "L2400166", # "RGLB": "L2400166", # "Lane": 1, -# "Read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_1/L2400166/L2400166_S8_L001_R1_001.fastq.gz", -# "Read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_1/L2400166/L2400166_S8_L001_R2_001.fastq.gz" +# "Read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_1/L2400166/L2400166_S8_L001_R1_001.fastq.gz", +# "Read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_1/L2400166/L2400166_S8_L001_R2_001.fastq.gz" # }, # { # "RGID": "ATGAGGCC.CAATTAAC.2", # "RGSM": "L2400195", # "RGLB": "L2400195", # "Lane": 2, -# "Read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_2/L2400195/L2400195_S9_L002_R1_001.fastq.gz", -# "Read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_2/L2400195/L2400195_S9_L002_R2_001.fastq.gz" +# "Read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_2/L2400195/L2400195_S9_L002_R1_001.fastq.gz", +# "Read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_2/L2400195/L2400195_S9_L002_R2_001.fastq.gz" # }, # { # "RGID": "ACTAAGAT.CCGCGGTT.2", # "RGSM": "L2400196", # "RGLB": "L2400196", # "Lane": 2, -# "Read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_2/L2400196/L2400196_S10_L002_R1_001.fastq.gz", -# "Read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_2/L2400196/L2400196_S10_L002_R2_001.fastq.gz" +# "Read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_2/L2400196/L2400196_S10_L002_R1_001.fastq.gz", +# "Read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_2/L2400196/L2400196_S10_L002_R2_001.fastq.gz" # }, # { # "RGID": "GTCGGAGC.TTATAACC.2", # "RGSM": "L2400197", # "RGLB": "L2400197", # "Lane": 2, -# "Read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_2/L2400197/L2400197_S11_L002_R1_001.fastq.gz", -# "Read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_2/L2400197/L2400197_S11_L002_R2_001.fastq.gz" +# "Read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_2/L2400197/L2400197_S11_L002_R1_001.fastq.gz", +# "Read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_2/L2400197/L2400197_S11_L002_R2_001.fastq.gz" # }, # { # "RGID": "TCGTAGTG.CCAAGTCT.2", # "RGSM": "L2400231", # "RGLB": "L2400231", # "Lane": 2, -# "Read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_2/L2400231/L2400231_S12_L002_R1_001.fastq.gz", -# "Read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_2/L2400231/L2400231_S12_L002_R2_001.fastq.gz" +# "Read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_2/L2400231/L2400231_S12_L002_R1_001.fastq.gz", +# "Read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_2/L2400231/L2400231_S12_L002_R2_001.fastq.gz" # }, # { # "RGID": "GGAGCGTC.GCACGGAC.2", # "RGSM": "L2400238", # "RGLB": "L2400238", # "Lane": 2, -# "Read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_2/L2400238/L2400238_S13_L002_R1_001.fastq.gz", -# "Read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_2/L2400238/L2400238_S13_L002_R2_001.fastq.gz" +# "Read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_2/L2400238/L2400238_S13_L002_R1_001.fastq.gz", +# "Read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_2/L2400238/L2400238_S13_L002_R2_001.fastq.gz" # }, # { # "RGID": "ATGGCATG.GGTACCTT.2", # "RGSM": "L2400239", # "RGLB": "L2400239", # "Lane": 2, -# "Read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_2/L2400239/L2400239_S14_L002_R1_001.fastq.gz", -# "Read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_2/L2400239/L2400239_S14_L002_R2_001.fastq.gz" +# "Read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_2/L2400239/L2400239_S14_L002_R1_001.fastq.gz", +# "Read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_2/L2400239/L2400239_S14_L002_R2_001.fastq.gz" # }, # { # "RGID": "GCAATGCA.AACGTTCC.2", # "RGSM": "L2400240", # "RGLB": "L2400240", # "Lane": 2, -# "Read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_2/L2400240/L2400240_S15_L002_R1_001.fastq.gz", -# "Read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_2/L2400240/L2400240_S15_L002_R2_001.fastq.gz" +# "Read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_2/L2400240/L2400240_S15_L002_R1_001.fastq.gz", +# "Read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_2/L2400240/L2400240_S15_L002_R2_001.fastq.gz" # }, # { # "RGID": "ATGAGGCC.CAATTAAC.3", # "RGSM": "L2400195", # "RGLB": "L2400195", # "Lane": 3, -# "Read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_3/L2400195/L2400195_S9_L003_R1_001.fastq.gz", -# "Read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_3/L2400195/L2400195_S9_L003_R2_001.fastq.gz" +# "Read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_3/L2400195/L2400195_S9_L003_R1_001.fastq.gz", +# "Read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_3/L2400195/L2400195_S9_L003_R2_001.fastq.gz" # }, # { # "RGID": "ACTAAGAT.CCGCGGTT.3", # "RGSM": "L2400196", # "RGLB": "L2400196", # "Lane": 3, -# "Read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_3/L2400196/L2400196_S10_L003_R1_001.fastq.gz", -# "Read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_3/L2400196/L2400196_S10_L003_R2_001.fastq.gz" +# "Read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_3/L2400196/L2400196_S10_L003_R1_001.fastq.gz", +# "Read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_3/L2400196/L2400196_S10_L003_R2_001.fastq.gz" # }, # { # "RGID": "GTCGGAGC.TTATAACC.3", # "RGSM": "L2400197", # "RGLB": "L2400197", # "Lane": 3, -# "Read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_3/L2400197/L2400197_S11_L003_R1_001.fastq.gz", -# "Read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_3/L2400197/L2400197_S11_L003_R2_001.fastq.gz" +# "Read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_3/L2400197/L2400197_S11_L003_R1_001.fastq.gz", +# "Read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_3/L2400197/L2400197_S11_L003_R2_001.fastq.gz" # }, # { # "RGID": "TCGTAGTG.CCAAGTCT.3", # "RGSM": "L2400231", # "RGLB": "L2400231", # "Lane": 3, -# "Read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_3/L2400231/L2400231_S12_L003_R1_001.fastq.gz", -# "Read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_3/L2400231/L2400231_S12_L003_R2_001.fastq.gz" +# "Read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_3/L2400231/L2400231_S12_L003_R1_001.fastq.gz", +# "Read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_3/L2400231/L2400231_S12_L003_R2_001.fastq.gz" # }, # { # "RGID": "GGAGCGTC.GCACGGAC.3", # "RGSM": "L2400238", # "RGLB": "L2400238", # "Lane": 3, -# "Read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_3/L2400238/L2400238_S13_L003_R1_001.fastq.gz", -# "Read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_3/L2400238/L2400238_S13_L003_R2_001.fastq.gz" +# "Read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_3/L2400238/L2400238_S13_L003_R1_001.fastq.gz", +# "Read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_3/L2400238/L2400238_S13_L003_R2_001.fastq.gz" # }, # { # "RGID": "ATGGCATG.GGTACCTT.3", # "RGSM": "L2400239", # "RGLB": "L2400239", # "Lane": 3, -# "Read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_3/L2400239/L2400239_S14_L003_R1_001.fastq.gz", -# "Read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_3/L2400239/L2400239_S14_L003_R2_001.fastq.gz" +# "Read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_3/L2400239/L2400239_S14_L003_R1_001.fastq.gz", +# "Read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_3/L2400239/L2400239_S14_L003_R2_001.fastq.gz" # }, # { # "RGID": "GCAATGCA.AACGTTCC.3", # "RGSM": "L2400240", # "RGLB": "L2400240", # "Lane": 3, -# "Read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_3/L2400240/L2400240_S15_L003_R1_001.fastq.gz", -# "Read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_3/L2400240/L2400240_S15_L003_R2_001.fastq.gz" +# "Read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_3/L2400240/L2400240_S15_L003_R1_001.fastq.gz", +# "Read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_3/L2400240/L2400240_S15_L003_R2_001.fastq.gz" # }, # { # "RGID": "ACGCCTTGTT.ACGTTCCTTA.4", # "RGSM": "L2400165", # "RGLB": "L2400165", # "Lane": 4, -# "Read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400165/L2400165_S16_L004_R1_001.fastq.gz", -# "Read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400165/L2400165_S16_L004_R2_001.fastq.gz" +# "Read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400165/L2400165_S16_L004_R1_001.fastq.gz", +# "Read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400165/L2400165_S16_L004_R2_001.fastq.gz" # }, # { # "RGID": "GCACGGAC.TGCGAGAC.4", # "RGSM": "L2400191", # "RGLB": "L2400191", # "Lane": 4, -# "Read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400191/L2400191_S17_L004_R1_001.fastq.gz", -# "Read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400191/L2400191_S17_L004_R2_001.fastq.gz" +# "Read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400191/L2400191_S17_L004_R1_001.fastq.gz", +# "Read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400191/L2400191_S17_L004_R2_001.fastq.gz" # }, # { # "RGID": "GTCGGAGC.TTATAACC.4", # "RGSM": "L2400197", # "RGLB": "L2400197", # "Lane": 4, -# "Read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400197/L2400197_S11_L004_R1_001.fastq.gz", -# "Read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400197/L2400197_S11_L004_R2_001.fastq.gz" +# "Read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400197/L2400197_S11_L004_R1_001.fastq.gz", +# "Read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400197/L2400197_S11_L004_R2_001.fastq.gz" # }, # { # "RGID": "CTTGGTAT.GGACTTGG.4", # "RGSM": "L2400198", # "RGLB": "L2400198", # "Lane": 4, -# "Read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400198/L2400198_S18_L004_R1_001.fastq.gz", -# "Read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400198/L2400198_S18_L004_R2_001.fastq.gz" +# "Read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400198/L2400198_S18_L004_R1_001.fastq.gz", +# "Read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400198/L2400198_S18_L004_R2_001.fastq.gz" # }, # { # "RGID": "GTTCCAAT.GCAGAATT.4", # "RGSM": "L2400241", # "RGLB": "L2400241", # "Lane": 4, -# "Read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400241/L2400241_S19_L004_R1_001.fastq.gz", -# "Read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400241/L2400241_S19_L004_R2_001.fastq.gz" +# "Read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400241/L2400241_S19_L004_R1_001.fastq.gz", +# "Read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400241/L2400241_S19_L004_R2_001.fastq.gz" # }, # { # "RGID": "ACCTTGGC.ATGAGGCC.4", # "RGSM": "L2400242", # "RGLB": "L2400242", # "Lane": 4, -# "Read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400242/L2400242_S20_L004_R1_001.fastq.gz", -# "Read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400242/L2400242_S20_L004_R2_001.fastq.gz" +# "Read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400242/L2400242_S20_L004_R1_001.fastq.gz", +# "Read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400242/L2400242_S20_L004_R2_001.fastq.gz" # }, # { # "RGID": "AGTTTCGA.CCTACGAT.4", # "RGSM": "L2400249", # "RGLB": "L2400249", # "Lane": 4, -# "Read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400249/L2400249_S21_L004_R1_001.fastq.gz", -# "Read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400249/L2400249_S21_L004_R2_001.fastq.gz" +# "Read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400249/L2400249_S21_L004_R1_001.fastq.gz", +# "Read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400249/L2400249_S21_L004_R2_001.fastq.gz" # }, # { # "RGID": "GAACCTCT.GTCTGCGC.4", # "RGSM": "L2400250", # "RGLB": "L2400250", # "Lane": 4, -# "Read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400250/L2400250_S22_L004_R1_001.fastq.gz", -# "Read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400250/L2400250_S22_L004_R2_001.fastq.gz" +# "Read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400250/L2400250_S22_L004_R1_001.fastq.gz", +# "Read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400250/L2400250_S22_L004_R2_001.fastq.gz" # }, # { # "RGID": "GCCCAGTG.CCGCAATT.4", # "RGSM": "L2400251", # "RGLB": "L2400251", # "Lane": 4, -# "Read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400251/L2400251_S23_L004_R1_001.fastq.gz", -# "Read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400251/L2400251_S23_L004_R2_001.fastq.gz" +# "Read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400251/L2400251_S23_L004_R1_001.fastq.gz", +# "Read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400251/L2400251_S23_L004_R2_001.fastq.gz" # }, # { # "RGID": "TGACAGCT.CCCGTAGG.4", # "RGSM": "L2400252", # "RGLB": "L2400252", # "Lane": 4, -# "Read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400252/L2400252_S24_L004_R1_001.fastq.gz", -# "Read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400252/L2400252_S24_L004_R2_001.fastq.gz" +# "Read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400252/L2400252_S24_L004_R1_001.fastq.gz", +# "Read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400252/L2400252_S24_L004_R2_001.fastq.gz" # }, # { # "RGID": "CATCACCC.ATATAGCA.4", # "RGSM": "L2400253", # "RGLB": "L2400253", # "Lane": 4, -# "Read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400253/L2400253_S25_L004_R1_001.fastq.gz", -# "Read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400253/L2400253_S25_L004_R2_001.fastq.gz" +# "Read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400253/L2400253_S25_L004_R1_001.fastq.gz", +# "Read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400253/L2400253_S25_L004_R2_001.fastq.gz" # }, # { # "RGID": "CTGGAGTA.GTTCGGTT.4", # "RGSM": "L2400254", # "RGLB": "L2400254", # "Lane": 4, -# "Read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400254/L2400254_S26_L004_R1_001.fastq.gz", -# "Read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400254/L2400254_S26_L004_R2_001.fastq.gz" +# "Read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400254/L2400254_S26_L004_R1_001.fastq.gz", +# "Read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400254/L2400254_S26_L004_R2_001.fastq.gz" # }, # { # "RGID": "GATCCGGG.AAGCAGGT.4", # "RGSM": "L2400255", # "RGLB": "L2400255", # "Lane": 4, -# "Read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400255/L2400255_S27_L004_R1_001.fastq.gz", -# "Read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400255/L2400255_S27_L004_R2_001.fastq.gz" +# "Read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400255/L2400255_S27_L004_R1_001.fastq.gz", +# "Read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400255/L2400255_S27_L004_R2_001.fastq.gz" # }, # { # "RGID": "AACACCTG.CGCATGGG.4", # "RGSM": "L2400256", # "RGLB": "L2400256", # "Lane": 4, -# "Read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400256/L2400256_S28_L004_R1_001.fastq.gz", -# "Read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400256/L2400256_S28_L004_R2_001.fastq.gz" +# "Read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400256/L2400256_S28_L004_R1_001.fastq.gz", +# "Read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400256/L2400256_S28_L004_R2_001.fastq.gz" # }, # { # "RGID": "GTGACGTT.TCCCAGAT.4", # "RGSM": "L2400257", # "RGLB": "L2400257", # "Lane": 4, -# "Read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400257/L2400257_S29_L004_R1_001.fastq.gz", -# "Read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400257/L2400257_S29_L004_R2_001.fastq.gz" +# "Read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400257/L2400257_S29_L004_R1_001.fastq.gz", +# "Read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400257/L2400257_S29_L004_R2_001.fastq.gz" # } # ], # "instrument_run_id": "240229_A00130_0288_BH5HM2DSXC" # } +# # , # None # ) @@ -661,141 +869,167 @@ def handler(event, context): # # { # # "event_data": { # # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", -# # "projectOwner": "UMCCR", -# # "projectName": "Testing", -# # "libraries": [ +# # "project": { +# # "name": null, +# # "description": null, +# # "projectId": "PO", +# # "orcabusId": "prj.01J8ES4EBXK08WDWB97BSCX1C9" +# # }, +# # "libraryFastqSet": [ # # { -# # "orcabusId": "lib.01J5S9CBG0NF8QBNVKM6ESCD60", -# # "libraryId": "L2400159", +# # "library": { +# # "orcabusId": "lib.01J8ES4MPZ5B201R50K42XXM4M", +# # "libraryId": "L2400102" +# # }, # # "fastqPairs": [ # # { -# # "rgid": "GAGAATGGTT.TTGCTGCCGA.1.240229_A00130_0288_BH5HM2DSXC.L2400159", -# # "rgsm": "L2400159", -# # "rglb": "L2400159", +# # "rgid": "GAATTCGT.TTATGAGT.1.240229_A00130_0288_BH5HM2DSXC.L2400102", +# # "rgsm": "L2400102", +# # "rglb": "L2400102", # # "lane": 1, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_1/L2400159/L2400159_S2_L001_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_1/L2400159/L2400159_S2_L001_R2_001.fastq.gz" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_1/L2400102/L2400102_S1_L001_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_1/L2400102/L2400102_S1_L001_R2_001.fastq.gz" # # } # # ] -# # }, +# # } +# # ] +# # } +# # }, +# # { +# # "event_data": { +# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", +# # "project": { +# # "name": null, +# # "description": null, +# # "projectId": "CUP", +# # "orcabusId": "prj.01J8ES4EZAA5YMHX82664GJQB3" +# # }, +# # "libraryFastqSet": [ # # { -# # "orcabusId": "lib.01J5S9CBHP6NSB42RVFAP9PGJP", -# # "libraryId": "L2400160", +# # "library": { +# # "orcabusId": "lib.01J8ES51V0RSVT6C7WQR72QQED", +# # "libraryId": "L2400231" +# # }, # # "fastqPairs": [ # # { -# # "rgid": "AGAGGCAACC.CCATCATTAG.1.240229_A00130_0288_BH5HM2DSXC.L2400160", -# # "rgsm": "L2400160", -# # "rglb": "L2400160", -# # "lane": 1, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_1/L2400160/L2400160_S3_L001_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_1/L2400160/L2400160_S3_L001_R2_001.fastq.gz" -# # } -# # ] -# # }, -# # { -# # "orcabusId": "lib.01J5S9CBKCATYSFY40BRX6WJWX", -# # "libraryId": "L2400161", -# # "fastqPairs": [ +# # "rgid": "TCGTAGTG.CCAAGTCT.2.240229_A00130_0288_BH5HM2DSXC.L2400231", +# # "rgsm": "L2400231", +# # "rglb": "L2400231", +# # "lane": 2, +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_2/L2400231/L2400231_S12_L002_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_2/L2400231/L2400231_S12_L002_R2_001.fastq.gz" +# # }, # # { -# # "rgid": "CCATCATTAG.AGAGGCAACC.1.240229_A00130_0288_BH5HM2DSXC.L2400161", -# # "rgsm": "L2400161", -# # "rglb": "L2400161", -# # "lane": 1, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_1/L2400161/L2400161_S4_L001_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_1/L2400161/L2400161_S4_L001_R2_001.fastq.gz" +# # "rgid": "TCGTAGTG.CCAAGTCT.3.240229_A00130_0288_BH5HM2DSXC.L2400231", +# # "rgsm": "L2400231", +# # "rglb": "L2400231", +# # "lane": 3, +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_3/L2400231/L2400231_S12_L003_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_3/L2400231/L2400231_S12_L003_R2_001.fastq.gz" # # } # # ] # # }, # # { -# # "orcabusId": "lib.01J5S9CBN6EAXW4AXG7TQ1H6NC", -# # "libraryId": "L2400162", +# # "library": { +# # "orcabusId": "lib.01J8ES52889Q8826P5SH9HDPP0", +# # "libraryId": "L2400238" +# # }, # # "fastqPairs": [ # # { -# # "rgid": "GATAGGCCGA.GCCATGTGCG.1.240229_A00130_0288_BH5HM2DSXC.L2400162", -# # "rgsm": "L2400162", -# # "rglb": "L2400162", -# # "lane": 1, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_1/L2400162/L2400162_S5_L001_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_1/L2400162/L2400162_S5_L001_R2_001.fastq.gz" +# # "rgid": "GGAGCGTC.GCACGGAC.2.240229_A00130_0288_BH5HM2DSXC.L2400238", +# # "rgsm": "L2400238", +# # "rglb": "L2400238", +# # "lane": 2, +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_2/L2400238/L2400238_S13_L002_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_2/L2400238/L2400238_S13_L002_R2_001.fastq.gz" +# # }, +# # { +# # "rgid": "GGAGCGTC.GCACGGAC.3.240229_A00130_0288_BH5HM2DSXC.L2400238", +# # "rgsm": "L2400238", +# # "rglb": "L2400238", +# # "lane": 3, +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_3/L2400238/L2400238_S13_L003_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_3/L2400238/L2400238_S13_L003_R2_001.fastq.gz" # # } # # ] # # }, # # { -# # "orcabusId": "lib.01J5S9CBQFX8V1QRW7KAV3MD1W", -# # "libraryId": "L2400163", +# # "library": { +# # "orcabusId": "lib.01J8ES52ANMRT3B7Y96T1Y3RY8", +# # "libraryId": "L2400239" +# # }, # # "fastqPairs": [ # # { -# # "rgid": "ATGGTTGACT.AGGACAGGCC.1.240229_A00130_0288_BH5HM2DSXC.L2400163", -# # "rgsm": "L2400163", -# # "rglb": "L2400163", -# # "lane": 1, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_1/L2400163/L2400163_S6_L001_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_1/L2400163/L2400163_S6_L001_R2_001.fastq.gz" +# # "rgid": "ATGGCATG.GGTACCTT.2.240229_A00130_0288_BH5HM2DSXC.L2400239", +# # "rgsm": "L2400239", +# # "rglb": "L2400239", +# # "lane": 2, +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_2/L2400239/L2400239_S14_L002_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_2/L2400239/L2400239_S14_L002_R2_001.fastq.gz" +# # }, +# # { +# # "rgid": "ATGGCATG.GGTACCTT.3.240229_A00130_0288_BH5HM2DSXC.L2400239", +# # "rgsm": "L2400239", +# # "rglb": "L2400239", +# # "lane": 3, +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_3/L2400239/L2400239_S14_L003_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_3/L2400239/L2400239_S14_L003_R2_001.fastq.gz" # # } # # ] # # }, # # { -# # "orcabusId": "lib.01J5S9CBS64DNTHK6CE850CCNZ", -# # "libraryId": "L2400164", +# # "library": { +# # "orcabusId": "lib.01J8ES52C3N585BGGY4VNXHC83", +# # "libraryId": "L2400240" +# # }, # # "fastqPairs": [ # # { -# # "rgid": "TATTGCGCTC.CCTAACACAG.1.240229_A00130_0288_BH5HM2DSXC.L2400164", -# # "rgsm": "L2400164", -# # "rglb": "L2400164", -# # "lane": 1, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_1/L2400164/L2400164_S7_L001_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_1/L2400164/L2400164_S7_L001_R2_001.fastq.gz" +# # "rgid": "GCAATGCA.AACGTTCC.2.240229_A00130_0288_BH5HM2DSXC.L2400240", +# # "rgsm": "L2400240", +# # "rglb": "L2400240", +# # "lane": 2, +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_2/L2400240/L2400240_S15_L002_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_2/L2400240/L2400240_S15_L002_R2_001.fastq.gz" +# # }, +# # { +# # "rgid": "GCAATGCA.AACGTTCC.3.240229_A00130_0288_BH5HM2DSXC.L2400240", +# # "rgsm": "L2400240", +# # "rglb": "L2400240", +# # "lane": 3, +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_3/L2400240/L2400240_S15_L003_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_3/L2400240/L2400240_S15_L003_R2_001.fastq.gz" # # } # # ] # # }, # # { -# # "orcabusId": "lib.01J5S9CBTZRYQNTGAHPC2T601D", -# # "libraryId": "L2400165", +# # "library": { +# # "orcabusId": "lib.01J8ES536AB5A5PBJ8S45SZP7Q", +# # "libraryId": "L2400255" +# # }, # # "fastqPairs": [ # # { -# # "rgid": "ACGCCTTGTT.ACGTTCCTTA.4.240229_A00130_0288_BH5HM2DSXC.L2400165", -# # "rgsm": "L2400165", -# # "rglb": "L2400165", +# # "rgid": "GATCCGGG.AAGCAGGT.4.240229_A00130_0288_BH5HM2DSXC.L2400255", +# # "rgsm": "L2400255", +# # "rglb": "L2400255", # # "lane": 4, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400165/L2400165_S16_L004_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400165/L2400165_S16_L004_R2_001.fastq.gz" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400255/L2400255_S27_L004_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400255/L2400255_S27_L004_R2_001.fastq.gz" # # } # # ] # # }, # # { -# # "orcabusId": "lib.01J5S9CBX10204CK7EKGTH9TMB", -# # "libraryId": "L2400166", -# # "fastqPairs": [ -# # { -# # "rgid": "TTCTACATAC.TTACAGTTAG.1.240229_A00130_0288_BH5HM2DSXC.L2400166", -# # "rgsm": "L2400166", -# # "rglb": "L2400166", -# # "lane": 1, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_1/L2400166/L2400166_S8_L001_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_1/L2400166/L2400166_S8_L001_R2_001.fastq.gz" -# # } -# # ] -# # } -# # ] -# # } -# # }, -# # { -# # "event_data": { -# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", -# # "projectOwner": "VCCC", -# # "projectName": "PO", -# # "libraries": [ -# # { -# # "orcabusId": "lib.01J5S9C4VMJ6PZ8GJ2G189AMXX", -# # "libraryId": "L2400102", +# # "library": { +# # "orcabusId": "lib.01J8ES537S0W1AX9PQPST13GM9", +# # "libraryId": "L2400256" +# # }, # # "fastqPairs": [ # # { -# # "rgid": "GAATTCGT.TTATGAGT.1.240229_A00130_0288_BH5HM2DSXC.L2400102", -# # "rgsm": "L2400102", -# # "rglb": "L2400102", -# # "lane": 1, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_1/L2400102/L2400102_S1_L001_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_1/L2400102/L2400102_S1_L001_R2_001.fastq.gz" +# # "rgid": "AACACCTG.CGCATGGG.4.240229_A00130_0288_BH5HM2DSXC.L2400256", +# # "rgsm": "L2400256", +# # "rglb": "L2400256", +# # "lane": 4, +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400256/L2400256_S28_L004_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400256/L2400256_S28_L004_R2_001.fastq.gz" # # } # # ] # # } @@ -805,62 +1039,74 @@ def handler(event, context): # # { # # "event_data": { # # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", -# # "projectOwner": "UMCCR", -# # "projectName": "Control", -# # "libraries": [ +# # "project": { +# # "name": null, +# # "description": null, +# # "projectId": "Control", +# # "orcabusId": "prj.01J8ES4FC6DVW20AR33FBX2SA8" +# # }, +# # "libraryFastqSet": [ # # { -# # "orcabusId": "lib.01J5S9CGJ6G09YQ9KFHPSXMMVD", -# # "libraryId": "L2400241", +# # "library": { +# # "orcabusId": "lib.01J8ES52DHAPZM6FZ0VZK89PRT", +# # "libraryId": "L2400241" +# # }, # # "fastqPairs": [ # # { # # "rgid": "GTTCCAAT.GCAGAATT.4.240229_A00130_0288_BH5HM2DSXC.L2400241", # # "rgsm": "L2400241", # # "rglb": "L2400241", # # "lane": 4, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400241/L2400241_S19_L004_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400241/L2400241_S19_L004_R2_001.fastq.gz" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400241/L2400241_S19_L004_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400241/L2400241_S19_L004_R2_001.fastq.gz" # # } # # ] # # }, # # { -# # "orcabusId": "lib.01J5S9CGKWDN7STKZKQM3KH9XR", -# # "libraryId": "L2400242", +# # "library": { +# # "orcabusId": "lib.01J8ES52F2ZHRXQY1AT1N1F81F", +# # "libraryId": "L2400242" +# # }, # # "fastqPairs": [ # # { # # "rgid": "ACCTTGGC.ATGAGGCC.4.240229_A00130_0288_BH5HM2DSXC.L2400242", # # "rgsm": "L2400242", # # "rglb": "L2400242", # # "lane": 4, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400242/L2400242_S20_L004_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400242/L2400242_S20_L004_R2_001.fastq.gz" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400242/L2400242_S20_L004_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400242/L2400242_S20_L004_R2_001.fastq.gz" # # } # # ] # # }, # # { -# # "orcabusId": "lib.01J5S9CH2SQ0P1SF7WAT5H4DSE", -# # "libraryId": "L2400249", +# # "library": { +# # "orcabusId": "lib.01J8ES52XYMVGRB1Q458THNG4T", +# # "libraryId": "L2400249" +# # }, # # "fastqPairs": [ # # { # # "rgid": "AGTTTCGA.CCTACGAT.4.240229_A00130_0288_BH5HM2DSXC.L2400249", # # "rgsm": "L2400249", # # "rglb": "L2400249", # # "lane": 4, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400249/L2400249_S21_L004_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400249/L2400249_S21_L004_R2_001.fastq.gz" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400249/L2400249_S21_L004_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400249/L2400249_S21_L004_R2_001.fastq.gz" # # } # # ] # # }, # # { -# # "orcabusId": "lib.01J5S9CHHNGFJN73NPRQMSYGN9", -# # "libraryId": "L2400257", +# # "library": { +# # "orcabusId": "lib.01J8ES5395KETT9T2NJSVNDKNP", +# # "libraryId": "L2400257" +# # }, # # "fastqPairs": [ # # { # # "rgid": "GTGACGTT.TCCCAGAT.4.240229_A00130_0288_BH5HM2DSXC.L2400257", # # "rgsm": "L2400257", # # "rglb": "L2400257", # # "lane": 4, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400257/L2400257_S29_L004_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400257/L2400257_S29_L004_R2_001.fastq.gz" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400257/L2400257_S29_L004_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400257/L2400257_S29_L004_R2_001.fastq.gz" # # } # # ] # # } @@ -870,108 +1116,90 @@ def handler(event, context): # # { # # "event_data": { # # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", -# # "projectOwner": "TJohn", -# # "projectName": "CAVATAK", -# # "libraries": [ +# # "project": { +# # "name": null, +# # "description": null, +# # "projectId": "BPOP-retro", +# # "orcabusId": "prj.01J8ES4FH3XMPZQNDJ9J000BXX" +# # }, +# # "libraryFastqSet": [ # # { -# # "orcabusId": "lib.01J5S9CDF8HHG5PJE3ECJMKMY7", -# # "libraryId": "L2400191", +# # "library": { +# # "orcabusId": "lib.01J8ES52Z2KTVVKZ2ZGVQ6YC10", +# # "libraryId": "L2400250" +# # }, # # "fastqPairs": [ # # { -# # "rgid": "GCACGGAC.TGCGAGAC.4.240229_A00130_0288_BH5HM2DSXC.L2400191", -# # "rgsm": "L2400191", -# # "rglb": "L2400191", +# # "rgid": "GAACCTCT.GTCTGCGC.4.240229_A00130_0288_BH5HM2DSXC.L2400250", +# # "rgsm": "L2400250", +# # "rglb": "L2400250", # # "lane": 4, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400191/L2400191_S17_L004_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400191/L2400191_S17_L004_R2_001.fastq.gz" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400250/L2400250_S22_L004_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400250/L2400250_S22_L004_R2_001.fastq.gz" # # } # # ] # # }, # # { -# # "orcabusId": "lib.01J5S9CDQSSAG1WYCRWMD82Z1S", -# # "libraryId": "L2400195", +# # "library": { +# # "orcabusId": "lib.01J8ES530H895X4WA3NQ6CY2QV", +# # "libraryId": "L2400251" +# # }, # # "fastqPairs": [ # # { -# # "rgid": "ATGAGGCC.CAATTAAC.2.240229_A00130_0288_BH5HM2DSXC.L2400195", -# # "rgsm": "L2400195", -# # "rglb": "L2400195", -# # "lane": 2, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_2/L2400195/L2400195_S9_L002_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_2/L2400195/L2400195_S9_L002_R2_001.fastq.gz" -# # }, -# # { -# # "rgid": "ATGAGGCC.CAATTAAC.3.240229_A00130_0288_BH5HM2DSXC.L2400195", -# # "rgsm": "L2400195", -# # "rglb": "L2400195", -# # "lane": 3, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_3/L2400195/L2400195_S9_L003_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_3/L2400195/L2400195_S9_L003_R2_001.fastq.gz" +# # "rgid": "GCCCAGTG.CCGCAATT.4.240229_A00130_0288_BH5HM2DSXC.L2400251", +# # "rgsm": "L2400251", +# # "rglb": "L2400251", +# # "lane": 4, +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400251/L2400251_S23_L004_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400251/L2400251_S23_L004_R2_001.fastq.gz" # # } # # ] # # }, # # { -# # "orcabusId": "lib.01J5S9CDSJ2BGEYM8FTXGKVGV8", -# # "libraryId": "L2400196", +# # "library": { +# # "orcabusId": "lib.01J8ES5320EWBNNYDGXF2SYJBD", +# # "libraryId": "L2400252" +# # }, # # "fastqPairs": [ # # { -# # "rgid": "ACTAAGAT.CCGCGGTT.2.240229_A00130_0288_BH5HM2DSXC.L2400196", -# # "rgsm": "L2400196", -# # "rglb": "L2400196", -# # "lane": 2, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_2/L2400196/L2400196_S10_L002_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_2/L2400196/L2400196_S10_L002_R2_001.fastq.gz" -# # }, -# # { -# # "rgid": "ACTAAGAT.CCGCGGTT.3.240229_A00130_0288_BH5HM2DSXC.L2400196", -# # "rgsm": "L2400196", -# # "rglb": "L2400196", -# # "lane": 3, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_3/L2400196/L2400196_S10_L003_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_3/L2400196/L2400196_S10_L003_R2_001.fastq.gz" +# # "rgid": "TGACAGCT.CCCGTAGG.4.240229_A00130_0288_BH5HM2DSXC.L2400252", +# # "rgsm": "L2400252", +# # "rglb": "L2400252", +# # "lane": 4, +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400252/L2400252_S24_L004_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400252/L2400252_S24_L004_R2_001.fastq.gz" # # } # # ] # # }, # # { -# # "orcabusId": "lib.01J5S9CDVEHDZHZR3BZTQ7WNJQ", -# # "libraryId": "L2400197", +# # "library": { +# # "orcabusId": "lib.01J8ES533DJZZNPP9MXYR5TRC0", +# # "libraryId": "L2400253" +# # }, # # "fastqPairs": [ # # { -# # "rgid": "GTCGGAGC.TTATAACC.2.240229_A00130_0288_BH5HM2DSXC.L2400197", -# # "rgsm": "L2400197", -# # "rglb": "L2400197", -# # "lane": 2, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_2/L2400197/L2400197_S11_L002_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_2/L2400197/L2400197_S11_L002_R2_001.fastq.gz" -# # }, -# # { -# # "rgid": "GTCGGAGC.TTATAACC.3.240229_A00130_0288_BH5HM2DSXC.L2400197", -# # "rgsm": "L2400197", -# # "rglb": "L2400197", -# # "lane": 3, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_3/L2400197/L2400197_S11_L003_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_3/L2400197/L2400197_S11_L003_R2_001.fastq.gz" -# # }, -# # { -# # "rgid": "GTCGGAGC.TTATAACC.4.240229_A00130_0288_BH5HM2DSXC.L2400197", -# # "rgsm": "L2400197", -# # "rglb": "L2400197", +# # "rgid": "CATCACCC.ATATAGCA.4.240229_A00130_0288_BH5HM2DSXC.L2400253", +# # "rgsm": "L2400253", +# # "rglb": "L2400253", # # "lane": 4, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400197/L2400197_S11_L004_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400197/L2400197_S11_L004_R2_001.fastq.gz" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400253/L2400253_S25_L004_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400253/L2400253_S25_L004_R2_001.fastq.gz" # # } # # ] # # }, # # { -# # "orcabusId": "lib.01J5S9CDXCR7Q5K6A8VJRSMM4Q", -# # "libraryId": "L2400198", +# # "library": { +# # "orcabusId": "lib.01J8ES534XGBFYDVYV8ZG6SYS0", +# # "libraryId": "L2400254" +# # }, # # "fastqPairs": [ # # { -# # "rgid": "CTTGGTAT.GGACTTGG.4.240229_A00130_0288_BH5HM2DSXC.L2400198", -# # "rgsm": "L2400198", -# # "rglb": "L2400198", +# # "rgid": "CTGGAGTA.GTTCGGTT.4.240229_A00130_0288_BH5HM2DSXC.L2400254", +# # "rgsm": "L2400254", +# # "rglb": "L2400254", # # "lane": 4, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400198/L2400198_S18_L004_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400198/L2400198_S18_L004_R2_001.fastq.gz" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400254/L2400254_S26_L004_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400254/L2400254_S26_L004_R2_001.fastq.gz" # # } # # ] # # } @@ -981,122 +1209,138 @@ def handler(event, context): # # { # # "event_data": { # # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", -# # "projectOwner": "Tothill", -# # "projectName": "CUP", -# # "libraries": [ +# # "project": { +# # "name": null, +# # "description": null, +# # "projectId": "Testing", +# # "orcabusId": "prj.01J8ES4XMWD0DH7MDRNER5TZS1" +# # }, +# # "libraryFastqSet": [ # # { -# # "orcabusId": "lib.01J5S9CFX5P69S4KZRQGDFKV1N", -# # "libraryId": "L2400231", +# # "library": { +# # "orcabusId": "lib.01J8ES4XNYFP38JMDV7GMV0V3V", +# # "libraryId": "L2400159" +# # }, # # "fastqPairs": [ # # { -# # "rgid": "TCGTAGTG.CCAAGTCT.2.240229_A00130_0288_BH5HM2DSXC.L2400231", -# # "rgsm": "L2400231", -# # "rglb": "L2400231", -# # "lane": 2, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_2/L2400231/L2400231_S12_L002_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_2/L2400231/L2400231_S12_L002_R2_001.fastq.gz" -# # }, -# # { -# # "rgid": "TCGTAGTG.CCAAGTCT.3.240229_A00130_0288_BH5HM2DSXC.L2400231", -# # "rgsm": "L2400231", -# # "rglb": "L2400231", -# # "lane": 3, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_3/L2400231/L2400231_S12_L003_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_3/L2400231/L2400231_S12_L003_R2_001.fastq.gz" +# # "rgid": "GAGAATGGTT.TTGCTGCCGA.1.240229_A00130_0288_BH5HM2DSXC.L2400159", +# # "rgsm": "L2400159", +# # "rglb": "L2400159", +# # "lane": 1, +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_1/L2400159/L2400159_S2_L001_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_1/L2400159/L2400159_S2_L001_R2_001.fastq.gz" # # } # # ] # # }, # # { -# # "orcabusId": "lib.01J5S9CGCAKQWHD9RBM9VXENY9", -# # "libraryId": "L2400238", +# # "library": { +# # "orcabusId": "lib.01J8ES4XQG3MPBW94TTVT4STVG", +# # "libraryId": "L2400160" +# # }, # # "fastqPairs": [ # # { -# # "rgid": "GGAGCGTC.GCACGGAC.2.240229_A00130_0288_BH5HM2DSXC.L2400238", -# # "rgsm": "L2400238", -# # "rglb": "L2400238", -# # "lane": 2, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_2/L2400238/L2400238_S13_L002_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_2/L2400238/L2400238_S13_L002_R2_001.fastq.gz" -# # }, -# # { -# # "rgid": "GGAGCGTC.GCACGGAC.3.240229_A00130_0288_BH5HM2DSXC.L2400238", -# # "rgsm": "L2400238", -# # "rglb": "L2400238", -# # "lane": 3, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_3/L2400238/L2400238_S13_L003_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_3/L2400238/L2400238_S13_L003_R2_001.fastq.gz" +# # "rgid": "AGAGGCAACC.CCATCATTAG.1.240229_A00130_0288_BH5HM2DSXC.L2400160", +# # "rgsm": "L2400160", +# # "rglb": "L2400160", +# # "lane": 1, +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_1/L2400160/L2400160_S3_L001_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_1/L2400160/L2400160_S3_L001_R2_001.fastq.gz" # # } # # ] # # }, # # { -# # "orcabusId": "lib.01J5S9CGEM1DHRQP72EP09B2TA", -# # "libraryId": "L2400239", +# # "library": { +# # "orcabusId": "lib.01J8ES4XSS97XNRS8DH0B1RJRG", +# # "libraryId": "L2400161" +# # }, # # "fastqPairs": [ # # { -# # "rgid": "ATGGCATG.GGTACCTT.2.240229_A00130_0288_BH5HM2DSXC.L2400239", -# # "rgsm": "L2400239", -# # "rglb": "L2400239", -# # "lane": 2, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_2/L2400239/L2400239_S14_L002_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_2/L2400239/L2400239_S14_L002_R2_001.fastq.gz" -# # }, +# # "rgid": "CCATCATTAG.AGAGGCAACC.1.240229_A00130_0288_BH5HM2DSXC.L2400161", +# # "rgsm": "L2400161", +# # "rglb": "L2400161", +# # "lane": 1, +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_1/L2400161/L2400161_S4_L001_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_1/L2400161/L2400161_S4_L001_R2_001.fastq.gz" +# # } +# # ] +# # }, +# # { +# # "library": { +# # "orcabusId": "lib.01J8ES4XXF6NMEJMM5M4GWS6KH", +# # "libraryId": "L2400162" +# # }, +# # "fastqPairs": [ # # { -# # "rgid": "ATGGCATG.GGTACCTT.3.240229_A00130_0288_BH5HM2DSXC.L2400239", -# # "rgsm": "L2400239", -# # "rglb": "L2400239", -# # "lane": 3, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_3/L2400239/L2400239_S14_L003_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_3/L2400239/L2400239_S14_L003_R2_001.fastq.gz" +# # "rgid": "GATAGGCCGA.GCCATGTGCG.1.240229_A00130_0288_BH5HM2DSXC.L2400162", +# # "rgsm": "L2400162", +# # "rglb": "L2400162", +# # "lane": 1, +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_1/L2400162/L2400162_S5_L001_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_1/L2400162/L2400162_S5_L001_R2_001.fastq.gz" # # } # # ] # # }, # # { -# # "orcabusId": "lib.01J5S9CGG9N9GH5879SY6A6BJB", -# # "libraryId": "L2400240", +# # "library": { +# # "orcabusId": "lib.01J8ES4XZD7T2VRPVQ1GSVZ11X", +# # "libraryId": "L2400163" +# # }, # # "fastqPairs": [ # # { -# # "rgid": "GCAATGCA.AACGTTCC.2.240229_A00130_0288_BH5HM2DSXC.L2400240", -# # "rgsm": "L2400240", -# # "rglb": "L2400240", -# # "lane": 2, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_2/L2400240/L2400240_S15_L002_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_2/L2400240/L2400240_S15_L002_R2_001.fastq.gz" -# # }, +# # "rgid": "ATGGTTGACT.AGGACAGGCC.1.240229_A00130_0288_BH5HM2DSXC.L2400163", +# # "rgsm": "L2400163", +# # "rglb": "L2400163", +# # "lane": 1, +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_1/L2400163/L2400163_S6_L001_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_1/L2400163/L2400163_S6_L001_R2_001.fastq.gz" +# # } +# # ] +# # }, +# # { +# # "library": { +# # "orcabusId": "lib.01J8ES4Y1AKAHYD9EW0TW4FBCP", +# # "libraryId": "L2400164" +# # }, +# # "fastqPairs": [ # # { -# # "rgid": "GCAATGCA.AACGTTCC.3.240229_A00130_0288_BH5HM2DSXC.L2400240", -# # "rgsm": "L2400240", -# # "rglb": "L2400240", -# # "lane": 3, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_3/L2400240/L2400240_S15_L003_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_3/L2400240/L2400240_S15_L003_R2_001.fastq.gz" +# # "rgid": "TATTGCGCTC.CCTAACACAG.1.240229_A00130_0288_BH5HM2DSXC.L2400164", +# # "rgsm": "L2400164", +# # "rglb": "L2400164", +# # "lane": 1, +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_1/L2400164/L2400164_S7_L001_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_1/L2400164/L2400164_S7_L001_R2_001.fastq.gz" # # } # # ] # # }, # # { -# # "orcabusId": "lib.01J5S9CHE4ERQ4H209DH397W8A", -# # "libraryId": "L2400255", +# # "library": { +# # "orcabusId": "lib.01J8ES4Y3ZKRX3C5JAHA5NBXV1", +# # "libraryId": "L2400165" +# # }, # # "fastqPairs": [ # # { -# # "rgid": "GATCCGGG.AAGCAGGT.4.240229_A00130_0288_BH5HM2DSXC.L2400255", -# # "rgsm": "L2400255", -# # "rglb": "L2400255", +# # "rgid": "ACGCCTTGTT.ACGTTCCTTA.4.240229_A00130_0288_BH5HM2DSXC.L2400165", +# # "rgsm": "L2400165", +# # "rglb": "L2400165", # # "lane": 4, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400255/L2400255_S27_L004_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400255/L2400255_S27_L004_R2_001.fastq.gz" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400165/L2400165_S16_L004_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400165/L2400165_S16_L004_R2_001.fastq.gz" # # } # # ] # # }, # # { -# # "orcabusId": "lib.01J5S9CHFXPDGYQ8TXHRWQR3PY", -# # "libraryId": "L2400256", +# # "library": { +# # "orcabusId": "lib.01J8ES4Y5D52202JVBXHJ9Q9WF", +# # "libraryId": "L2400166" +# # }, # # "fastqPairs": [ # # { -# # "rgid": "AACACCTG.CGCATGGG.4.240229_A00130_0288_BH5HM2DSXC.L2400256", -# # "rgsm": "L2400256", -# # "rglb": "L2400256", -# # "lane": 4, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400256/L2400256_S28_L004_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400256/L2400256_S28_L004_R2_001.fastq.gz" +# # "rgid": "TTCTACATAC.TTACAGTTAG.1.240229_A00130_0288_BH5HM2DSXC.L2400166", +# # "rgsm": "L2400166", +# # "rglb": "L2400166", +# # "lane": 1, +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_1/L2400166/L2400166_S8_L001_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_1/L2400166/L2400166_S8_L001_R2_001.fastq.gz" # # } # # ] # # } @@ -1106,76 +1350,122 @@ def handler(event, context): # # { # # "event_data": { # # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", -# # "projectOwner": "Whittle", -# # "projectName": "BPOP-retro", -# # "libraries": [ +# # "project": { +# # "name": null, +# # "description": null, +# # "projectId": "CAVATAK", +# # "orcabusId": "prj.01J8ES4ZAWHH3FKYA2CFHSMZ4B" +# # }, +# # "libraryFastqSet": [ # # { -# # "orcabusId": "lib.01J5S9CH4CYPA4SP05H8KRX4W9", -# # "libraryId": "L2400250", +# # "library": { +# # "orcabusId": "lib.01J8ES4ZDRQAP2BN3SDYYV5PKW", +# # "libraryId": "L2400191" +# # }, # # "fastqPairs": [ # # { -# # "rgid": "GAACCTCT.GTCTGCGC.4.240229_A00130_0288_BH5HM2DSXC.L2400250", -# # "rgsm": "L2400250", -# # "rglb": "L2400250", +# # "rgid": "GCACGGAC.TGCGAGAC.4.240229_A00130_0288_BH5HM2DSXC.L2400191", +# # "rgsm": "L2400191", +# # "rglb": "L2400191", # # "lane": 4, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400250/L2400250_S22_L004_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400250/L2400250_S22_L004_R2_001.fastq.gz" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400191/L2400191_S17_L004_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400191/L2400191_S17_L004_R2_001.fastq.gz" # # } # # ] # # }, # # { -# # "orcabusId": "lib.01J5S9CH65E4EE5QJEJ1C60GGG", -# # "libraryId": "L2400251", +# # "library": { +# # "orcabusId": "lib.01J8ES4ZMY0G1H9MDN7K2TH9Y6", +# # "libraryId": "L2400195" +# # }, # # "fastqPairs": [ # # { -# # "rgid": "GCCCAGTG.CCGCAATT.4.240229_A00130_0288_BH5HM2DSXC.L2400251", -# # "rgsm": "L2400251", -# # "rglb": "L2400251", -# # "lane": 4, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400251/L2400251_S23_L004_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400251/L2400251_S23_L004_R2_001.fastq.gz" +# # "rgid": "ATGAGGCC.CAATTAAC.2.240229_A00130_0288_BH5HM2DSXC.L2400195", +# # "rgsm": "L2400195", +# # "rglb": "L2400195", +# # "lane": 2, +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_2/L2400195/L2400195_S9_L002_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_2/L2400195/L2400195_S9_L002_R2_001.fastq.gz" +# # }, +# # { +# # "rgid": "ATGAGGCC.CAATTAAC.3.240229_A00130_0288_BH5HM2DSXC.L2400195", +# # "rgsm": "L2400195", +# # "rglb": "L2400195", +# # "lane": 3, +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_3/L2400195/L2400195_S9_L003_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_3/L2400195/L2400195_S9_L003_R2_001.fastq.gz" # # } # # ] # # }, # # { -# # "orcabusId": "lib.01J5S9CH7TGZMV39Z59WJ8H5GP", -# # "libraryId": "L2400252", +# # "library": { +# # "orcabusId": "lib.01J8ES4ZP88X2E17X5X1FRMTPK", +# # "libraryId": "L2400196" +# # }, # # "fastqPairs": [ # # { -# # "rgid": "TGACAGCT.CCCGTAGG.4.240229_A00130_0288_BH5HM2DSXC.L2400252", -# # "rgsm": "L2400252", -# # "rglb": "L2400252", -# # "lane": 4, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400252/L2400252_S24_L004_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400252/L2400252_S24_L004_R2_001.fastq.gz" +# # "rgid": "ACTAAGAT.CCGCGGTT.2.240229_A00130_0288_BH5HM2DSXC.L2400196", +# # "rgsm": "L2400196", +# # "rglb": "L2400196", +# # "lane": 2, +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_2/L2400196/L2400196_S10_L002_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_2/L2400196/L2400196_S10_L002_R2_001.fastq.gz" +# # }, +# # { +# # "rgid": "ACTAAGAT.CCGCGGTT.3.240229_A00130_0288_BH5HM2DSXC.L2400196", +# # "rgsm": "L2400196", +# # "rglb": "L2400196", +# # "lane": 3, +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_3/L2400196/L2400196_S10_L003_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_3/L2400196/L2400196_S10_L003_R2_001.fastq.gz" # # } # # ] # # }, # # { -# # "orcabusId": "lib.01J5S9CH9TGMT2TJGBZX5VXHJY", -# # "libraryId": "L2400253", +# # "library": { +# # "orcabusId": "lib.01J8ES4ZST489C712CG3R9NQSQ", +# # "libraryId": "L2400197" +# # }, # # "fastqPairs": [ # # { -# # "rgid": "CATCACCC.ATATAGCA.4.240229_A00130_0288_BH5HM2DSXC.L2400253", -# # "rgsm": "L2400253", -# # "rglb": "L2400253", +# # "rgid": "GTCGGAGC.TTATAACC.2.240229_A00130_0288_BH5HM2DSXC.L2400197", +# # "rgsm": "L2400197", +# # "rglb": "L2400197", +# # "lane": 2, +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_2/L2400197/L2400197_S11_L002_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_2/L2400197/L2400197_S11_L002_R2_001.fastq.gz" +# # }, +# # { +# # "rgid": "GTCGGAGC.TTATAACC.3.240229_A00130_0288_BH5HM2DSXC.L2400197", +# # "rgsm": "L2400197", +# # "rglb": "L2400197", +# # "lane": 3, +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_3/L2400197/L2400197_S11_L003_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_3/L2400197/L2400197_S11_L003_R2_001.fastq.gz" +# # }, +# # { +# # "rgid": "GTCGGAGC.TTATAACC.4.240229_A00130_0288_BH5HM2DSXC.L2400197", +# # "rgsm": "L2400197", +# # "rglb": "L2400197", # # "lane": 4, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400253/L2400253_S25_L004_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400253/L2400253_S25_L004_R2_001.fastq.gz" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400197/L2400197_S11_L004_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400197/L2400197_S11_L004_R2_001.fastq.gz" # # } # # ] # # }, # # { -# # "orcabusId": "lib.01J5S9CHBGAP2XSN4TG8SAMRYY", -# # "libraryId": "L2400254", +# # "library": { +# # "orcabusId": "lib.01J8ES4ZVWA2CGBHJVKAS3Y0G9", +# # "libraryId": "L2400198" +# # }, # # "fastqPairs": [ # # { -# # "rgid": "CTGGAGTA.GTTCGGTT.4.240229_A00130_0288_BH5HM2DSXC.L2400254", -# # "rgsm": "L2400254", -# # "rglb": "L2400254", +# # "rgid": "CTTGGTAT.GGACTTGG.4.240229_A00130_0288_BH5HM2DSXC.L2400198", +# # "rgsm": "L2400198", +# # "rglb": "L2400198", # # "lane": 4, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400254/L2400254_S26_L004_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400254/L2400254_S26_L004_R2_001.fastq.gz" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400198/L2400198_S18_L004_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400198/L2400198_S18_L004_R2_001.fastq.gz" # # } # # ] # # } @@ -1185,558 +1475,558 @@ def handler(event, context): # # ], # # "fastq_list_rows_event_data_list": [ # # { +# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", +# # "library": { +# # "libraryId": "L2400102", +# # "orcabusId": "lib.01J8ES4MPZ5B201R50K42XXM4M" +# # }, # # "fastqListRow": { # # "rgid": "GAATTCGT.TTATGAGT.1.240229_A00130_0288_BH5HM2DSXC.L2400102", # # "rgsm": "L2400102", # # "rglb": "L2400102", # # "lane": 1, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_1/L2400102/L2400102_S1_L001_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_1/L2400102/L2400102_S1_L001_R2_001.fastq.gz" -# # }, -# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", -# # "library": { -# # "libraryId": "L2400102", -# # "orcabusId": "lib.01J5S9C4VMJ6PZ8GJ2G189AMXX" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_1/L2400102/L2400102_S1_L001_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_1/L2400102/L2400102_S1_L001_R2_001.fastq.gz" # # } # # }, # # { +# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", +# # "library": { +# # "libraryId": "L2400159", +# # "orcabusId": "lib.01J8ES4XNYFP38JMDV7GMV0V3V" +# # }, # # "fastqListRow": { # # "rgid": "GAGAATGGTT.TTGCTGCCGA.1.240229_A00130_0288_BH5HM2DSXC.L2400159", # # "rgsm": "L2400159", # # "rglb": "L2400159", # # "lane": 1, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_1/L2400159/L2400159_S2_L001_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_1/L2400159/L2400159_S2_L001_R2_001.fastq.gz" -# # }, -# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", -# # "library": { -# # "libraryId": "L2400159", -# # "orcabusId": "lib.01J5S9CBG0NF8QBNVKM6ESCD60" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_1/L2400159/L2400159_S2_L001_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_1/L2400159/L2400159_S2_L001_R2_001.fastq.gz" # # } # # }, # # { +# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", +# # "library": { +# # "libraryId": "L2400160", +# # "orcabusId": "lib.01J8ES4XQG3MPBW94TTVT4STVG" +# # }, # # "fastqListRow": { # # "rgid": "AGAGGCAACC.CCATCATTAG.1.240229_A00130_0288_BH5HM2DSXC.L2400160", # # "rgsm": "L2400160", # # "rglb": "L2400160", # # "lane": 1, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_1/L2400160/L2400160_S3_L001_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_1/L2400160/L2400160_S3_L001_R2_001.fastq.gz" -# # }, -# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", -# # "library": { -# # "libraryId": "L2400160", -# # "orcabusId": "lib.01J5S9CBHP6NSB42RVFAP9PGJP" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_1/L2400160/L2400160_S3_L001_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_1/L2400160/L2400160_S3_L001_R2_001.fastq.gz" # # } # # }, # # { +# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", +# # "library": { +# # "libraryId": "L2400161", +# # "orcabusId": "lib.01J8ES4XSS97XNRS8DH0B1RJRG" +# # }, # # "fastqListRow": { # # "rgid": "CCATCATTAG.AGAGGCAACC.1.240229_A00130_0288_BH5HM2DSXC.L2400161", # # "rgsm": "L2400161", # # "rglb": "L2400161", # # "lane": 1, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_1/L2400161/L2400161_S4_L001_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_1/L2400161/L2400161_S4_L001_R2_001.fastq.gz" -# # }, -# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", -# # "library": { -# # "libraryId": "L2400161", -# # "orcabusId": "lib.01J5S9CBKCATYSFY40BRX6WJWX" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_1/L2400161/L2400161_S4_L001_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_1/L2400161/L2400161_S4_L001_R2_001.fastq.gz" # # } # # }, # # { +# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", +# # "library": { +# # "libraryId": "L2400162", +# # "orcabusId": "lib.01J8ES4XXF6NMEJMM5M4GWS6KH" +# # }, # # "fastqListRow": { # # "rgid": "GATAGGCCGA.GCCATGTGCG.1.240229_A00130_0288_BH5HM2DSXC.L2400162", # # "rgsm": "L2400162", # # "rglb": "L2400162", # # "lane": 1, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_1/L2400162/L2400162_S5_L001_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_1/L2400162/L2400162_S5_L001_R2_001.fastq.gz" -# # }, -# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", -# # "library": { -# # "libraryId": "L2400162", -# # "orcabusId": "lib.01J5S9CBN6EAXW4AXG7TQ1H6NC" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_1/L2400162/L2400162_S5_L001_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_1/L2400162/L2400162_S5_L001_R2_001.fastq.gz" # # } # # }, # # { +# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", +# # "library": { +# # "libraryId": "L2400163", +# # "orcabusId": "lib.01J8ES4XZD7T2VRPVQ1GSVZ11X" +# # }, # # "fastqListRow": { # # "rgid": "ATGGTTGACT.AGGACAGGCC.1.240229_A00130_0288_BH5HM2DSXC.L2400163", # # "rgsm": "L2400163", # # "rglb": "L2400163", # # "lane": 1, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_1/L2400163/L2400163_S6_L001_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_1/L2400163/L2400163_S6_L001_R2_001.fastq.gz" -# # }, -# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", -# # "library": { -# # "libraryId": "L2400163", -# # "orcabusId": "lib.01J5S9CBQFX8V1QRW7KAV3MD1W" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_1/L2400163/L2400163_S6_L001_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_1/L2400163/L2400163_S6_L001_R2_001.fastq.gz" # # } # # }, # # { +# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", +# # "library": { +# # "libraryId": "L2400164", +# # "orcabusId": "lib.01J8ES4Y1AKAHYD9EW0TW4FBCP" +# # }, # # "fastqListRow": { # # "rgid": "TATTGCGCTC.CCTAACACAG.1.240229_A00130_0288_BH5HM2DSXC.L2400164", # # "rgsm": "L2400164", # # "rglb": "L2400164", # # "lane": 1, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_1/L2400164/L2400164_S7_L001_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_1/L2400164/L2400164_S7_L001_R2_001.fastq.gz" -# # }, -# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", -# # "library": { -# # "libraryId": "L2400164", -# # "orcabusId": "lib.01J5S9CBS64DNTHK6CE850CCNZ" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_1/L2400164/L2400164_S7_L001_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_1/L2400164/L2400164_S7_L001_R2_001.fastq.gz" # # } # # }, # # { +# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", +# # "library": { +# # "libraryId": "L2400166", +# # "orcabusId": "lib.01J8ES4Y5D52202JVBXHJ9Q9WF" +# # }, # # "fastqListRow": { # # "rgid": "TTCTACATAC.TTACAGTTAG.1.240229_A00130_0288_BH5HM2DSXC.L2400166", # # "rgsm": "L2400166", # # "rglb": "L2400166", # # "lane": 1, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_1/L2400166/L2400166_S8_L001_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_1/L2400166/L2400166_S8_L001_R2_001.fastq.gz" -# # }, -# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", -# # "library": { -# # "libraryId": "L2400166", -# # "orcabusId": "lib.01J5S9CBX10204CK7EKGTH9TMB" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_1/L2400166/L2400166_S8_L001_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_1/L2400166/L2400166_S8_L001_R2_001.fastq.gz" # # } # # }, # # { +# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", +# # "library": { +# # "libraryId": "L2400195", +# # "orcabusId": "lib.01J8ES4ZMY0G1H9MDN7K2TH9Y6" +# # }, # # "fastqListRow": { # # "rgid": "ATGAGGCC.CAATTAAC.2.240229_A00130_0288_BH5HM2DSXC.L2400195", # # "rgsm": "L2400195", # # "rglb": "L2400195", # # "lane": 2, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_2/L2400195/L2400195_S9_L002_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_2/L2400195/L2400195_S9_L002_R2_001.fastq.gz" -# # }, -# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", -# # "library": { -# # "libraryId": "L2400195", -# # "orcabusId": "lib.01J5S9CDQSSAG1WYCRWMD82Z1S" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_2/L2400195/L2400195_S9_L002_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_2/L2400195/L2400195_S9_L002_R2_001.fastq.gz" # # } # # }, # # { +# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", +# # "library": { +# # "libraryId": "L2400196", +# # "orcabusId": "lib.01J8ES4ZP88X2E17X5X1FRMTPK" +# # }, # # "fastqListRow": { # # "rgid": "ACTAAGAT.CCGCGGTT.2.240229_A00130_0288_BH5HM2DSXC.L2400196", # # "rgsm": "L2400196", # # "rglb": "L2400196", # # "lane": 2, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_2/L2400196/L2400196_S10_L002_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_2/L2400196/L2400196_S10_L002_R2_001.fastq.gz" -# # }, -# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", -# # "library": { -# # "libraryId": "L2400196", -# # "orcabusId": "lib.01J5S9CDSJ2BGEYM8FTXGKVGV8" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_2/L2400196/L2400196_S10_L002_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_2/L2400196/L2400196_S10_L002_R2_001.fastq.gz" # # } # # }, # # { +# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", +# # "library": { +# # "libraryId": "L2400197", +# # "orcabusId": "lib.01J8ES4ZST489C712CG3R9NQSQ" +# # }, # # "fastqListRow": { # # "rgid": "GTCGGAGC.TTATAACC.2.240229_A00130_0288_BH5HM2DSXC.L2400197", # # "rgsm": "L2400197", # # "rglb": "L2400197", # # "lane": 2, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_2/L2400197/L2400197_S11_L002_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_2/L2400197/L2400197_S11_L002_R2_001.fastq.gz" -# # }, -# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", -# # "library": { -# # "libraryId": "L2400197", -# # "orcabusId": "lib.01J5S9CDVEHDZHZR3BZTQ7WNJQ" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_2/L2400197/L2400197_S11_L002_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_2/L2400197/L2400197_S11_L002_R2_001.fastq.gz" # # } # # }, # # { +# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", +# # "library": { +# # "libraryId": "L2400231", +# # "orcabusId": "lib.01J8ES51V0RSVT6C7WQR72QQED" +# # }, # # "fastqListRow": { # # "rgid": "TCGTAGTG.CCAAGTCT.2.240229_A00130_0288_BH5HM2DSXC.L2400231", # # "rgsm": "L2400231", # # "rglb": "L2400231", # # "lane": 2, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_2/L2400231/L2400231_S12_L002_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_2/L2400231/L2400231_S12_L002_R2_001.fastq.gz" -# # }, -# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", -# # "library": { -# # "libraryId": "L2400231", -# # "orcabusId": "lib.01J5S9CFX5P69S4KZRQGDFKV1N" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_2/L2400231/L2400231_S12_L002_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_2/L2400231/L2400231_S12_L002_R2_001.fastq.gz" # # } # # }, # # { +# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", +# # "library": { +# # "libraryId": "L2400238", +# # "orcabusId": "lib.01J8ES52889Q8826P5SH9HDPP0" +# # }, # # "fastqListRow": { # # "rgid": "GGAGCGTC.GCACGGAC.2.240229_A00130_0288_BH5HM2DSXC.L2400238", # # "rgsm": "L2400238", # # "rglb": "L2400238", # # "lane": 2, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_2/L2400238/L2400238_S13_L002_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_2/L2400238/L2400238_S13_L002_R2_001.fastq.gz" -# # }, -# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", -# # "library": { -# # "libraryId": "L2400238", -# # "orcabusId": "lib.01J5S9CGCAKQWHD9RBM9VXENY9" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_2/L2400238/L2400238_S13_L002_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_2/L2400238/L2400238_S13_L002_R2_001.fastq.gz" # # } # # }, # # { +# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", +# # "library": { +# # "libraryId": "L2400239", +# # "orcabusId": "lib.01J8ES52ANMRT3B7Y96T1Y3RY8" +# # }, # # "fastqListRow": { # # "rgid": "ATGGCATG.GGTACCTT.2.240229_A00130_0288_BH5HM2DSXC.L2400239", # # "rgsm": "L2400239", # # "rglb": "L2400239", # # "lane": 2, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_2/L2400239/L2400239_S14_L002_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_2/L2400239/L2400239_S14_L002_R2_001.fastq.gz" -# # }, -# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", -# # "library": { -# # "libraryId": "L2400239", -# # "orcabusId": "lib.01J5S9CGEM1DHRQP72EP09B2TA" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_2/L2400239/L2400239_S14_L002_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_2/L2400239/L2400239_S14_L002_R2_001.fastq.gz" # # } # # }, # # { +# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", +# # "library": { +# # "libraryId": "L2400240", +# # "orcabusId": "lib.01J8ES52C3N585BGGY4VNXHC83" +# # }, # # "fastqListRow": { # # "rgid": "GCAATGCA.AACGTTCC.2.240229_A00130_0288_BH5HM2DSXC.L2400240", # # "rgsm": "L2400240", # # "rglb": "L2400240", # # "lane": 2, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_2/L2400240/L2400240_S15_L002_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_2/L2400240/L2400240_S15_L002_R2_001.fastq.gz" -# # }, -# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", -# # "library": { -# # "libraryId": "L2400240", -# # "orcabusId": "lib.01J5S9CGG9N9GH5879SY6A6BJB" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_2/L2400240/L2400240_S15_L002_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_2/L2400240/L2400240_S15_L002_R2_001.fastq.gz" # # } # # }, # # { +# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", +# # "library": { +# # "libraryId": "L2400195", +# # "orcabusId": "lib.01J8ES4ZMY0G1H9MDN7K2TH9Y6" +# # }, # # "fastqListRow": { # # "rgid": "ATGAGGCC.CAATTAAC.3.240229_A00130_0288_BH5HM2DSXC.L2400195", # # "rgsm": "L2400195", # # "rglb": "L2400195", # # "lane": 3, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_3/L2400195/L2400195_S9_L003_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_3/L2400195/L2400195_S9_L003_R2_001.fastq.gz" -# # }, -# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", -# # "library": { -# # "libraryId": "L2400195", -# # "orcabusId": "lib.01J5S9CDQSSAG1WYCRWMD82Z1S" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_3/L2400195/L2400195_S9_L003_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_3/L2400195/L2400195_S9_L003_R2_001.fastq.gz" # # } # # }, # # { +# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", +# # "library": { +# # "libraryId": "L2400196", +# # "orcabusId": "lib.01J8ES4ZP88X2E17X5X1FRMTPK" +# # }, # # "fastqListRow": { # # "rgid": "ACTAAGAT.CCGCGGTT.3.240229_A00130_0288_BH5HM2DSXC.L2400196", # # "rgsm": "L2400196", # # "rglb": "L2400196", # # "lane": 3, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_3/L2400196/L2400196_S10_L003_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_3/L2400196/L2400196_S10_L003_R2_001.fastq.gz" -# # }, -# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", -# # "library": { -# # "libraryId": "L2400196", -# # "orcabusId": "lib.01J5S9CDSJ2BGEYM8FTXGKVGV8" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_3/L2400196/L2400196_S10_L003_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_3/L2400196/L2400196_S10_L003_R2_001.fastq.gz" # # } # # }, # # { +# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", +# # "library": { +# # "libraryId": "L2400197", +# # "orcabusId": "lib.01J8ES4ZST489C712CG3R9NQSQ" +# # }, # # "fastqListRow": { # # "rgid": "GTCGGAGC.TTATAACC.3.240229_A00130_0288_BH5HM2DSXC.L2400197", # # "rgsm": "L2400197", # # "rglb": "L2400197", # # "lane": 3, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_3/L2400197/L2400197_S11_L003_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_3/L2400197/L2400197_S11_L003_R2_001.fastq.gz" -# # }, -# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", -# # "library": { -# # "libraryId": "L2400197", -# # "orcabusId": "lib.01J5S9CDVEHDZHZR3BZTQ7WNJQ" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_3/L2400197/L2400197_S11_L003_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_3/L2400197/L2400197_S11_L003_R2_001.fastq.gz" # # } # # }, # # { +# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", +# # "library": { +# # "libraryId": "L2400231", +# # "orcabusId": "lib.01J8ES51V0RSVT6C7WQR72QQED" +# # }, # # "fastqListRow": { # # "rgid": "TCGTAGTG.CCAAGTCT.3.240229_A00130_0288_BH5HM2DSXC.L2400231", # # "rgsm": "L2400231", # # "rglb": "L2400231", # # "lane": 3, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_3/L2400231/L2400231_S12_L003_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_3/L2400231/L2400231_S12_L003_R2_001.fastq.gz" -# # }, -# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", -# # "library": { -# # "libraryId": "L2400231", -# # "orcabusId": "lib.01J5S9CFX5P69S4KZRQGDFKV1N" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_3/L2400231/L2400231_S12_L003_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_3/L2400231/L2400231_S12_L003_R2_001.fastq.gz" # # } # # }, # # { +# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", +# # "library": { +# # "libraryId": "L2400238", +# # "orcabusId": "lib.01J8ES52889Q8826P5SH9HDPP0" +# # }, # # "fastqListRow": { # # "rgid": "GGAGCGTC.GCACGGAC.3.240229_A00130_0288_BH5HM2DSXC.L2400238", # # "rgsm": "L2400238", # # "rglb": "L2400238", # # "lane": 3, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_3/L2400238/L2400238_S13_L003_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_3/L2400238/L2400238_S13_L003_R2_001.fastq.gz" -# # }, -# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", -# # "library": { -# # "libraryId": "L2400238", -# # "orcabusId": "lib.01J5S9CGCAKQWHD9RBM9VXENY9" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_3/L2400238/L2400238_S13_L003_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_3/L2400238/L2400238_S13_L003_R2_001.fastq.gz" # # } # # }, # # { +# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", +# # "library": { +# # "libraryId": "L2400239", +# # "orcabusId": "lib.01J8ES52ANMRT3B7Y96T1Y3RY8" +# # }, # # "fastqListRow": { # # "rgid": "ATGGCATG.GGTACCTT.3.240229_A00130_0288_BH5HM2DSXC.L2400239", # # "rgsm": "L2400239", # # "rglb": "L2400239", # # "lane": 3, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_3/L2400239/L2400239_S14_L003_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_3/L2400239/L2400239_S14_L003_R2_001.fastq.gz" -# # }, -# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", -# # "library": { -# # "libraryId": "L2400239", -# # "orcabusId": "lib.01J5S9CGEM1DHRQP72EP09B2TA" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_3/L2400239/L2400239_S14_L003_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_3/L2400239/L2400239_S14_L003_R2_001.fastq.gz" # # } # # }, # # { +# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", +# # "library": { +# # "libraryId": "L2400240", +# # "orcabusId": "lib.01J8ES52C3N585BGGY4VNXHC83" +# # }, # # "fastqListRow": { # # "rgid": "GCAATGCA.AACGTTCC.3.240229_A00130_0288_BH5HM2DSXC.L2400240", # # "rgsm": "L2400240", # # "rglb": "L2400240", # # "lane": 3, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_3/L2400240/L2400240_S15_L003_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_3/L2400240/L2400240_S15_L003_R2_001.fastq.gz" -# # }, -# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", -# # "library": { -# # "libraryId": "L2400240", -# # "orcabusId": "lib.01J5S9CGG9N9GH5879SY6A6BJB" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_3/L2400240/L2400240_S15_L003_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_3/L2400240/L2400240_S15_L003_R2_001.fastq.gz" # # } # # }, # # { +# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", +# # "library": { +# # "libraryId": "L2400165", +# # "orcabusId": "lib.01J8ES4Y3ZKRX3C5JAHA5NBXV1" +# # }, # # "fastqListRow": { # # "rgid": "ACGCCTTGTT.ACGTTCCTTA.4.240229_A00130_0288_BH5HM2DSXC.L2400165", # # "rgsm": "L2400165", # # "rglb": "L2400165", # # "lane": 4, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400165/L2400165_S16_L004_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400165/L2400165_S16_L004_R2_001.fastq.gz" -# # }, -# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", -# # "library": { -# # "libraryId": "L2400165", -# # "orcabusId": "lib.01J5S9CBTZRYQNTGAHPC2T601D" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400165/L2400165_S16_L004_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400165/L2400165_S16_L004_R2_001.fastq.gz" # # } # # }, # # { +# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", +# # "library": { +# # "libraryId": "L2400191", +# # "orcabusId": "lib.01J8ES4ZDRQAP2BN3SDYYV5PKW" +# # }, # # "fastqListRow": { # # "rgid": "GCACGGAC.TGCGAGAC.4.240229_A00130_0288_BH5HM2DSXC.L2400191", # # "rgsm": "L2400191", # # "rglb": "L2400191", # # "lane": 4, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400191/L2400191_S17_L004_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400191/L2400191_S17_L004_R2_001.fastq.gz" -# # }, -# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", -# # "library": { -# # "libraryId": "L2400191", -# # "orcabusId": "lib.01J5S9CDF8HHG5PJE3ECJMKMY7" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400191/L2400191_S17_L004_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400191/L2400191_S17_L004_R2_001.fastq.gz" # # } # # }, # # { +# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", +# # "library": { +# # "libraryId": "L2400197", +# # "orcabusId": "lib.01J8ES4ZST489C712CG3R9NQSQ" +# # }, # # "fastqListRow": { # # "rgid": "GTCGGAGC.TTATAACC.4.240229_A00130_0288_BH5HM2DSXC.L2400197", # # "rgsm": "L2400197", # # "rglb": "L2400197", # # "lane": 4, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400197/L2400197_S11_L004_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400197/L2400197_S11_L004_R2_001.fastq.gz" -# # }, -# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", -# # "library": { -# # "libraryId": "L2400197", -# # "orcabusId": "lib.01J5S9CDVEHDZHZR3BZTQ7WNJQ" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400197/L2400197_S11_L004_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400197/L2400197_S11_L004_R2_001.fastq.gz" # # } # # }, # # { +# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", +# # "library": { +# # "libraryId": "L2400198", +# # "orcabusId": "lib.01J8ES4ZVWA2CGBHJVKAS3Y0G9" +# # }, # # "fastqListRow": { # # "rgid": "CTTGGTAT.GGACTTGG.4.240229_A00130_0288_BH5HM2DSXC.L2400198", # # "rgsm": "L2400198", # # "rglb": "L2400198", # # "lane": 4, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400198/L2400198_S18_L004_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400198/L2400198_S18_L004_R2_001.fastq.gz" -# # }, -# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", -# # "library": { -# # "libraryId": "L2400198", -# # "orcabusId": "lib.01J5S9CDXCR7Q5K6A8VJRSMM4Q" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400198/L2400198_S18_L004_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400198/L2400198_S18_L004_R2_001.fastq.gz" # # } # # }, # # { +# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", +# # "library": { +# # "libraryId": "L2400241", +# # "orcabusId": "lib.01J8ES52DHAPZM6FZ0VZK89PRT" +# # }, # # "fastqListRow": { # # "rgid": "GTTCCAAT.GCAGAATT.4.240229_A00130_0288_BH5HM2DSXC.L2400241", # # "rgsm": "L2400241", # # "rglb": "L2400241", # # "lane": 4, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400241/L2400241_S19_L004_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400241/L2400241_S19_L004_R2_001.fastq.gz" -# # }, -# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", -# # "library": { -# # "libraryId": "L2400241", -# # "orcabusId": "lib.01J5S9CGJ6G09YQ9KFHPSXMMVD" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400241/L2400241_S19_L004_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400241/L2400241_S19_L004_R2_001.fastq.gz" # # } # # }, # # { +# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", +# # "library": { +# # "libraryId": "L2400242", +# # "orcabusId": "lib.01J8ES52F2ZHRXQY1AT1N1F81F" +# # }, # # "fastqListRow": { # # "rgid": "ACCTTGGC.ATGAGGCC.4.240229_A00130_0288_BH5HM2DSXC.L2400242", # # "rgsm": "L2400242", # # "rglb": "L2400242", # # "lane": 4, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400242/L2400242_S20_L004_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400242/L2400242_S20_L004_R2_001.fastq.gz" -# # }, -# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", -# # "library": { -# # "libraryId": "L2400242", -# # "orcabusId": "lib.01J5S9CGKWDN7STKZKQM3KH9XR" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400242/L2400242_S20_L004_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400242/L2400242_S20_L004_R2_001.fastq.gz" # # } # # }, # # { +# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", +# # "library": { +# # "libraryId": "L2400249", +# # "orcabusId": "lib.01J8ES52XYMVGRB1Q458THNG4T" +# # }, # # "fastqListRow": { # # "rgid": "AGTTTCGA.CCTACGAT.4.240229_A00130_0288_BH5HM2DSXC.L2400249", # # "rgsm": "L2400249", # # "rglb": "L2400249", # # "lane": 4, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400249/L2400249_S21_L004_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400249/L2400249_S21_L004_R2_001.fastq.gz" -# # }, -# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", -# # "library": { -# # "libraryId": "L2400249", -# # "orcabusId": "lib.01J5S9CH2SQ0P1SF7WAT5H4DSE" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400249/L2400249_S21_L004_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400249/L2400249_S21_L004_R2_001.fastq.gz" # # } # # }, # # { +# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", +# # "library": { +# # "libraryId": "L2400250", +# # "orcabusId": "lib.01J8ES52Z2KTVVKZ2ZGVQ6YC10" +# # }, # # "fastqListRow": { # # "rgid": "GAACCTCT.GTCTGCGC.4.240229_A00130_0288_BH5HM2DSXC.L2400250", # # "rgsm": "L2400250", # # "rglb": "L2400250", # # "lane": 4, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400250/L2400250_S22_L004_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400250/L2400250_S22_L004_R2_001.fastq.gz" -# # }, -# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", -# # "library": { -# # "libraryId": "L2400250", -# # "orcabusId": "lib.01J5S9CH4CYPA4SP05H8KRX4W9" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400250/L2400250_S22_L004_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400250/L2400250_S22_L004_R2_001.fastq.gz" # # } # # }, # # { +# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", +# # "library": { +# # "libraryId": "L2400251", +# # "orcabusId": "lib.01J8ES530H895X4WA3NQ6CY2QV" +# # }, # # "fastqListRow": { # # "rgid": "GCCCAGTG.CCGCAATT.4.240229_A00130_0288_BH5HM2DSXC.L2400251", # # "rgsm": "L2400251", # # "rglb": "L2400251", # # "lane": 4, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400251/L2400251_S23_L004_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400251/L2400251_S23_L004_R2_001.fastq.gz" -# # }, -# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", -# # "library": { -# # "libraryId": "L2400251", -# # "orcabusId": "lib.01J5S9CH65E4EE5QJEJ1C60GGG" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400251/L2400251_S23_L004_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400251/L2400251_S23_L004_R2_001.fastq.gz" # # } # # }, # # { +# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", +# # "library": { +# # "libraryId": "L2400252", +# # "orcabusId": "lib.01J8ES5320EWBNNYDGXF2SYJBD" +# # }, # # "fastqListRow": { # # "rgid": "TGACAGCT.CCCGTAGG.4.240229_A00130_0288_BH5HM2DSXC.L2400252", # # "rgsm": "L2400252", # # "rglb": "L2400252", # # "lane": 4, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400252/L2400252_S24_L004_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400252/L2400252_S24_L004_R2_001.fastq.gz" -# # }, -# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", -# # "library": { -# # "libraryId": "L2400252", -# # "orcabusId": "lib.01J5S9CH7TGZMV39Z59WJ8H5GP" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400252/L2400252_S24_L004_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400252/L2400252_S24_L004_R2_001.fastq.gz" # # } # # }, # # { +# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", +# # "library": { +# # "libraryId": "L2400253", +# # "orcabusId": "lib.01J8ES533DJZZNPP9MXYR5TRC0" +# # }, # # "fastqListRow": { # # "rgid": "CATCACCC.ATATAGCA.4.240229_A00130_0288_BH5HM2DSXC.L2400253", # # "rgsm": "L2400253", # # "rglb": "L2400253", # # "lane": 4, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400253/L2400253_S25_L004_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400253/L2400253_S25_L004_R2_001.fastq.gz" -# # }, -# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", -# # "library": { -# # "libraryId": "L2400253", -# # "orcabusId": "lib.01J5S9CH9TGMT2TJGBZX5VXHJY" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400253/L2400253_S25_L004_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400253/L2400253_S25_L004_R2_001.fastq.gz" # # } # # }, # # { +# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", +# # "library": { +# # "libraryId": "L2400254", +# # "orcabusId": "lib.01J8ES534XGBFYDVYV8ZG6SYS0" +# # }, # # "fastqListRow": { # # "rgid": "CTGGAGTA.GTTCGGTT.4.240229_A00130_0288_BH5HM2DSXC.L2400254", # # "rgsm": "L2400254", # # "rglb": "L2400254", # # "lane": 4, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400254/L2400254_S26_L004_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400254/L2400254_S26_L004_R2_001.fastq.gz" -# # }, -# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", -# # "library": { -# # "libraryId": "L2400254", -# # "orcabusId": "lib.01J5S9CHBGAP2XSN4TG8SAMRYY" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400254/L2400254_S26_L004_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400254/L2400254_S26_L004_R2_001.fastq.gz" # # } # # }, # # { +# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", +# # "library": { +# # "libraryId": "L2400255", +# # "orcabusId": "lib.01J8ES536AB5A5PBJ8S45SZP7Q" +# # }, # # "fastqListRow": { # # "rgid": "GATCCGGG.AAGCAGGT.4.240229_A00130_0288_BH5HM2DSXC.L2400255", # # "rgsm": "L2400255", # # "rglb": "L2400255", # # "lane": 4, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400255/L2400255_S27_L004_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400255/L2400255_S27_L004_R2_001.fastq.gz" -# # }, -# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", -# # "library": { -# # "libraryId": "L2400255", -# # "orcabusId": "lib.01J5S9CHE4ERQ4H209DH397W8A" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400255/L2400255_S27_L004_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400255/L2400255_S27_L004_R2_001.fastq.gz" # # } # # }, # # { +# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", +# # "library": { +# # "libraryId": "L2400256", +# # "orcabusId": "lib.01J8ES537S0W1AX9PQPST13GM9" +# # }, # # "fastqListRow": { # # "rgid": "AACACCTG.CGCATGGG.4.240229_A00130_0288_BH5HM2DSXC.L2400256", # # "rgsm": "L2400256", # # "rglb": "L2400256", # # "lane": 4, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400256/L2400256_S28_L004_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400256/L2400256_S28_L004_R2_001.fastq.gz" -# # }, -# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", -# # "library": { -# # "libraryId": "L2400256", -# # "orcabusId": "lib.01J5S9CHFXPDGYQ8TXHRWQR3PY" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400256/L2400256_S28_L004_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400256/L2400256_S28_L004_R2_001.fastq.gz" # # } # # }, # # { +# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", +# # "library": { +# # "libraryId": "L2400257", +# # "orcabusId": "lib.01J8ES5395KETT9T2NJSVNDKNP" +# # }, # # "fastqListRow": { # # "rgid": "GTGACGTT.TCCCAGAT.4.240229_A00130_0288_BH5HM2DSXC.L2400257", # # "rgsm": "L2400257", # # "rglb": "L2400257", # # "lane": 4, -# # "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400257/L2400257_S29_L004_R1_001.fastq.gz", -# # "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400257/L2400257_S29_L004_R2_001.fastq.gz" -# # }, -# # "instrumentRunId": "240229_A00130_0288_BH5HM2DSXC", -# # "library": { -# # "libraryId": "L2400257", -# # "orcabusId": "lib.01J5S9CHHNGFJN73NPRQMSYGN9" +# # "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400257/L2400257_S29_L004_R1_001.fastq.gz", +# # "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/240229_A00130_0288_BH5HM2DSXC/202409108ed29dcc/Samples/Lane_4/L2400257/L2400257_S29_L004_R2_001.fastq.gz" # # } # # } # # ], diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/clag/part_2/fastq-list-rows-event-shower/step_functions_templates/fastq_list_row_event_shower_sfn_template.asl.json b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/clag/part_2/fastq-list-rows-event-shower/step_functions_templates/fastq_list_row_event_shower_sfn_template.asl.json index b34aa9377..15d8646d2 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/clag/part_2/fastq-list-rows-event-shower/step_functions_templates/fastq_list_row_event_shower_sfn_template.asl.json +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/clag/part_2/fastq-list-rows-event-shower/step_functions_templates/fastq_list_row_event_shower_sfn_template.asl.json @@ -78,20 +78,20 @@ "TableName": "${__table_name__}", "Key": { "id.$": "$.inputs.payload.data.outputs.instrumentRunId", - "id_type": "${__samplesheet_table_partition_name__}" + "id_type": "${__instrument_run_table_partition_name__}" } }, "ResultSelector": { - "library_ids.$": "$.Item.library_ids_set.SS" + "library_orcabus_ids.$": "$.Item.library_set.SS" }, "ResultPath": "$.get_libraries_step", "Next": "Get Library Objects Map" }, "Get Library Objects Map": { "Type": "Map", - "ItemsPath": "$.get_libraries_step.library_ids", + "ItemsPath": "$.get_libraries_step.library_orcabus_ids", "ItemSelector": { - "library_id.$": "$$.Map.Item.Value" + "library_orcabus_id.$": "$$.Map.Item.Value" }, "ItemProcessor": { "ProcessorConfig": { @@ -105,17 +105,12 @@ "Parameters": { "TableName": "${__table_name__}", "Key": { - "id.$": "$.library_id", + "id.$": "$.library_orcabus_id", "id_type": "${__library_table_partition_name__}" } }, "ResultSelector": { - "library_obj": { - "library_id.$": "$.Item.library_id.S", - "orcabus_id.$": "$.Item.orcabus_id.S", - "project_name.$": "$.Item.project_name.S", - "project_owner.$": "$.Item.project_owner.S" - } + "library_obj.$": "States.StringToJson($.Item.library_obj.S)" }, "End": true } @@ -127,11 +122,75 @@ "End": true } } + }, + { + "StartAt": "Get Projects in Instrument Run", + "States": { + "Get Projects in Instrument Run": { + "Type": "Task", + "Resource": "arn:aws:states:::dynamodb:getItem", + "Parameters": { + "TableName": "${__table_name__}", + "Key": { + "id.$": "$.inputs.payload.data.outputs.instrumentRunId", + "id_type": "${__instrument_run_table_partition_name__}" + } + }, + "ResultSelector": { + "project_orcabus_ids.$": "$.Item.project_set.SS" + }, + "ResultPath": "$.get_projects_step", + "Next": "Get Project Objects Map" + }, + "Get Project Objects Map": { + "Type": "Map", + "ItemsPath": "$.get_projects_step.project_orcabus_ids", + "ItemSelector": { + "project_orcabus_id.$": "$$.Map.Item.Value" + }, + "ItemProcessor": { + "ProcessorConfig": { + "Mode": "INLINE" + }, + "StartAt": "Get Project", + "States": { + "Get Project": { + "Type": "Task", + "Resource": "arn:aws:states:::dynamodb:getItem", + "Parameters": { + "TableName": "${__table_name__}", + "Key": { + "id.$": "$.project_orcabus_id", + "id_type": "${__project_table_partition_name__}" + } + }, + "ResultSelector": { + "project_obj.$": "States.StringToJson($.Item.project_obj.S)", + "library_orcabus_ids_set.$": "$.Item.library_set.SS" + }, + "Next": "Append library set to project object" + }, + "Append library set to project object": { + "Type": "Pass", + "End": true, + "Parameters": { + "project_obj.$": "States.JsonMerge($.project_obj, States.StringToJson(States.Format('\\{\"librarySet\":{}\\}', $.library_orcabus_ids_set)), false)" + } + } + } + }, + "ResultSelector": { + "project_objects_list.$": "$.[*].project_obj" + }, + "End": true + } + } } ], "ResultSelector": { "fastq_list_rows.$": "$.[0].decompress_fastq_list_rows_step.fastq_list_rows", - "library_objects_list.$": "$.[1].library_objects_list" + "library_objects_list.$": "$.[1].library_objects_list", + "project_objects_list.$": "$.[2].project_objects_list" }, "ResultPath": "$.get_inputs_step" }, @@ -143,6 +202,7 @@ "Payload": { "fastq_list_rows.$": "$.get_inputs_step.fastq_list_rows", "library_objs.$": "$.get_inputs_step.library_objects_list", + "project_objs.$": "$.get_inputs_step.project_objects_list", "instrument_run_id.$": "$.inputs.payload.data.outputs.instrumentRunId" } }, diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/elmer/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/elmer/index.ts index 29d86adac..329062673 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/elmer/index.ts +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/elmer/index.ts @@ -3,8 +3,7 @@ import * as events from 'aws-cdk-lib/aws-events'; import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; import * as ssm from 'aws-cdk-lib/aws-ssm'; import * as secretsManager from 'aws-cdk-lib/aws-secretsmanager'; -import { BsshFastqCopyManagerDraftMakerConstruct } from './part_1/bclconvert-succeeded-to-bssh-fastq-copy-draft'; -import { BsshFastqCopyManagerDraftToReadyMakerConstruct } from './part_2/bssh-fastq-copy-manager-draft-to-ready'; +import { BsshFastqCopyManagerReadyMakerConstruct } from './part_1/bclconvert-succeeded-to-bssh-fastq-copy'; /* Provide the glue to get from the bclconvertmanager success event @@ -12,9 +11,13 @@ To triggering the bsshFastqCopyManager */ export interface BclconvertToBsshFastqCopyEventHandlerConstructProps { + /* Event Bus */ eventBusObj: events.IEventBus; - inputMakerTableObj: dynamodb.ITableV2; + + /* SSM Parameter */ bsshOutputFastqCopyUriSsmParameterObj: ssm.IStringParameter; + + /* Secrets */ icav2AccessTokenSecretObj: secretsManager.ISecret; } @@ -39,36 +42,16 @@ export class BclconvertToBsshFastqCopyEventHandlerConstruct extends Construct { * Subscribes to the BCLConvertManagerEventHandler Stack outputs and creates the input for the BSSHFastqCopyManager * Generates the portal run id and submits a draft event */ - const bclconvertToBsshFastqCopyDraftMaker = new BsshFastqCopyManagerDraftMakerConstruct( + const bclconvertToBsshFastqCopyDraftMaker = new BsshFastqCopyManagerReadyMakerConstruct( this, 'bclconvert_to_bssh_fastq_copy_draft_maker', { + /* Event Bus handler */ eventBusObj: props.eventBusObj, - tableObj: props.inputMakerTableObj, - } - ); - - /* - Part 2 - - Input Event Source: `orcabus.bclconvertmanagerinputeventglue` - Input Event DetailType: `WorkflowDraftRunStateChange` - Input Event status: `draft` - - Output Event source: `orcabus.bclconvertmanagerinputeventglue` - Output Event DetailType: `WorkflowDraftRunStateChange` - Output Event status: `ready` - - * Pushes an event payload of the input for the BsshFastqCopyManagerReadyEventSubmitter - */ - const bsshFastqCopyManagerInputMaker = new BsshFastqCopyManagerDraftToReadyMakerConstruct( - this, - 'bssh_fastq_copy_input_maker', - { - eventBusObj: props.eventBusObj, - outputUriSsmParameterObj: props.bsshOutputFastqCopyUriSsmParameterObj, - tableObj: props.inputMakerTableObj, + /* ICAv2 Secret */ icav2AccessTokenSecretObj: props.icav2AccessTokenSecretObj, + /* Output URI SSM Configuration Obj */ + outputUriSsmParameterObj: props.bsshOutputFastqCopyUriSsmParameterObj, } ); } diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/elmer/part_1/bclconvert-succeeded-to-bssh-fastq-copy-draft/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/elmer/part_1/bclconvert-succeeded-to-bssh-fastq-copy/index.ts similarity index 69% rename from lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/elmer/part_1/bclconvert-succeeded-to-bssh-fastq-copy-draft/index.ts rename to lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/elmer/part_1/bclconvert-succeeded-to-bssh-fastq-copy/index.ts index f48315a9f..ba3e2f3ad 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/elmer/part_1/bclconvert-succeeded-to-bssh-fastq-copy-draft/index.ts +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/elmer/part_1/bclconvert-succeeded-to-bssh-fastq-copy/index.ts @@ -7,10 +7,11 @@ import * as eventsTargets from 'aws-cdk-lib/aws-events-targets'; import { WorkflowDraftRunStateChangeCommonPreambleConstruct } from '../../../../../../../components/sfn-workflowdraftrunstatechange-common-preamble'; import * as iam from 'aws-cdk-lib/aws-iam'; import * as cdk from 'aws-cdk-lib'; +import * as ssm from 'aws-cdk-lib/aws-ssm'; +import * as secretsManager from 'aws-cdk-lib/aws-secretsmanager'; import { LambdaB64GzTranslatorConstruct } from '../../../../../../../components/python-lambda-b64gz-translator'; -import { PythonFunction } from '@aws-cdk/aws-lambda-python-alpha'; -import { Architecture, Runtime } from 'aws-cdk-lib/aws-lambda'; import { GetLibraryObjectsFromSamplesheetConstruct } from '../../../../../../../components/python-lambda-get-metadata-objects-from-samplesheet'; +import { GenerateWorkflowRunStateChangeReadyConstruct } from '../../../../../../../components/sfn-generate-workflowrunstatechange-ready-event'; /* Part 1 @@ -22,8 +23,8 @@ Part 1 * Output Event source: `orcabus.bsshfastqcopyinputeventglue` -* Output Event DetailType: `WorkflowDraftRunStateChange` -* Output Event status: `draft` +* Output Event DetailType: `WorkflowRunStateChange` +* Output Event status: `READY` * The BCLConvertSucceededToBsshFastqCopyDraft Construct @@ -33,21 +34,23 @@ Part 1 */ export interface bsshFastqCopyManagerDraftMakerConstructProps { - tableObj: dynamodb.ITableV2; + /* Event bus object handler */ eventBusObj: events.IEventBus; + /* SSM Parameter for the output uri */ + outputUriSsmParameterObj: ssm.IStringParameter; + /* Secret for icav2 access token */ + icav2AccessTokenSecretObj: secretsManager.ISecret; } -export class BsshFastqCopyManagerDraftMakerConstruct extends Construct { +export class BsshFastqCopyManagerReadyMakerConstruct extends Construct { public readonly bsshFastqCopyManagerDraftMakerEventMap = { - prefix: 'elmer-bclconv-2-bssh-fq-copy-draft', + prefix: 'elmer-bclconv-2-bssh-fq-copy', portalRunPartitionName: 'portal_run', triggerSource: 'orcabus.workflowmanager', triggerStatus: 'succeeded', triggerDetailType: 'WorkflowRunStateChange', triggerWorkflowName: 'bclconvert', outputSource: 'orcabus.bsshfastqcopyinputeventglue', - outputDetailType: 'WorkflowDraftRunStateChange', - outputStatus: 'DRAFT', payloadVersion: '2024.05.24', workflowName: 'bsshFastqCopy', workflowVersion: '2024.05.24', @@ -59,14 +62,11 @@ export class BsshFastqCopyManagerDraftMakerConstruct extends Construct { /* Part 1: Generate the preamble (sfn to generate the portal run id and the workflow run name) */ - const sfn_preamble = new WorkflowDraftRunStateChangeCommonPreambleConstruct( + const sfnPreamble = new WorkflowDraftRunStateChangeCommonPreambleConstruct( this, `${this.bsshFastqCopyManagerDraftMakerEventMap.prefix}_sfn_preamble`, { - portalRunTablePartitionName: - this.bsshFastqCopyManagerDraftMakerEventMap.portalRunPartitionName, stateMachinePrefix: this.bsshFastqCopyManagerDraftMakerEventMap.prefix, - tableObj: props.tableObj, workflowName: this.bsshFastqCopyManagerDraftMakerEventMap.workflowName, workflowVersion: this.bsshFastqCopyManagerDraftMakerEventMap.workflowVersion, } @@ -94,9 +94,35 @@ export class BsshFastqCopyManagerDraftMakerConstruct extends Construct { ).lambdaObj; /* - Part 2: Build the sfn + Part 2: Build the engine parameters sfn through the construct */ - const draftMakerSfn = new sfn.StateMachine(this, 'bclconvert_succeeded_to_bssh_draft', { + const engineParametersAndReadyLaunchSfn = new GenerateWorkflowRunStateChangeReadyConstruct( + this, + 'bclconvert_succeeded_to_bssh_ready_submitter', + { + /* Event Placeholders */ + eventBusObj: props.eventBusObj, + outputSource: this.bsshFastqCopyManagerDraftMakerEventMap.outputSource, + payloadVersion: this.bsshFastqCopyManagerDraftMakerEventMap.payloadVersion, + workflowName: this.bsshFastqCopyManagerDraftMakerEventMap.workflowName, + workflowVersion: this.bsshFastqCopyManagerDraftMakerEventMap.workflowVersion, + + /* SSM Parameters */ + outputUriSsmParameterObj: props.outputUriSsmParameterObj, + + /* Secrets */ + icav2AccessTokenSecretObj: props.icav2AccessTokenSecretObj, + + /* Prefixes */ + lambdaPrefix: this.bsshFastqCopyManagerDraftMakerEventMap.prefix, + stateMachinePrefix: this.bsshFastqCopyManagerDraftMakerEventMap.prefix, + } + ).stepFunctionObj; + + /* + Part 2: Build the inputs sfn + */ + const inputsMakerSfn = new sfn.StateMachine(this, 'bclconvert_succeeded_to_bssh_draft', { stateMachineName: `${this.bsshFastqCopyManagerDraftMakerEventMap.prefix}-sfn`, definitionBody: sfn.DefinitionBody.fromFile( path.join( @@ -106,16 +132,9 @@ export class BsshFastqCopyManagerDraftMakerConstruct extends Construct { ) ), definitionSubstitutions: { - // Event stuff - __event_bus_name__: props.eventBusObj.eventBusName, - __event_source__: this.bsshFastqCopyManagerDraftMakerEventMap.outputSource, - __detail_type__: this.bsshFastqCopyManagerDraftMakerEventMap.outputDetailType, - // Workflow stuff - __workflow_name__: this.bsshFastqCopyManagerDraftMakerEventMap.workflowName, - __workflow_version__: this.bsshFastqCopyManagerDraftMakerEventMap.workflowVersion, - __payload_version__: this.bsshFastqCopyManagerDraftMakerEventMap.payloadVersion, // Subfunctions - __sfn_preamble_state_machine_arn__: sfn_preamble.stateMachineArn, + __sfn_preamble_state_machine_arn__: sfnPreamble.stateMachineArn, + __launch_ready_event_sfn_arn__: engineParametersAndReadyLaunchSfn.stateMachineArn, // Lambda __decompression_samplesheet_lambda_function_arn__: decompressSamplesheetLambda.currentVersion.functionArn, @@ -125,22 +144,17 @@ export class BsshFastqCopyManagerDraftMakerConstruct extends Construct { }); /* - Part 2: Grant the sfn permissions + Part 2a: Grant the sfn permissions */ - // Read/write to the table - props.tableObj.grantReadWriteData(draftMakerSfn); - - // Allow the step function to submit events - props.eventBusObj.grantPutEventsTo(draftMakerSfn); // Allow the step function to launch the lambdas [decompressSamplesheetLambda, getLibrarySubjectMapLambda].forEach((lambda) => { - lambda.currentVersion.grantInvoke(draftMakerSfn); + lambda.currentVersion.grantInvoke(inputsMakerSfn); }); // Because we run a nested state machine, we need to add the permissions to the state machine role // See https://stackoverflow.com/questions/60612853/nested-step-function-in-a-step-function-unknown-error-not-authorized-to-cr - draftMakerSfn.addToRolePolicy( + inputsMakerSfn.addToRolePolicy( new iam.PolicyStatement({ resources: [ `arn:aws:events:${cdk.Aws.REGION}:${cdk.Aws.ACCOUNT_ID}:rule/StepFunctionsGetEventsForStepFunctionsExecutionRule`, @@ -150,7 +164,8 @@ export class BsshFastqCopyManagerDraftMakerConstruct extends Construct { ); // Add state machine execution permissions to stateMachine role - sfn_preamble.grantStartExecution(draftMakerSfn.role); + sfnPreamble.grantStartExecution(inputsMakerSfn); + engineParametersAndReadyLaunchSfn.grantStartExecution(inputsMakerSfn); const eventRule = new events.Rule(this, 'update_database_on_new_samplesheet_event_rule', { ruleName: `stacky-${this.bsshFastqCopyManagerDraftMakerEventMap.prefix}-event-rule`, @@ -173,7 +188,7 @@ export class BsshFastqCopyManagerDraftMakerConstruct extends Construct { // Add target to event rule eventRule.addTarget( - new eventsTargets.SfnStateMachine(draftMakerSfn, { + new eventsTargets.SfnStateMachine(inputsMakerSfn, { input: events.RuleTargetInput.fromEventPath('$.detail'), }) ); diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/elmer/part_1/bclconvert-succeeded-to-bssh-fastq-copy-draft/step_functions_templates/bclconvert_succeeded_to_bssh_fastq_copy_draft_template.asl.json b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/elmer/part_1/bclconvert-succeeded-to-bssh-fastq-copy/step_functions_templates/bclconvert_succeeded_to_bssh_fastq_copy_draft_template.asl.json similarity index 78% rename from lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/elmer/part_1/bclconvert-succeeded-to-bssh-fastq-copy-draft/step_functions_templates/bclconvert_succeeded_to_bssh_fastq_copy_draft_template.asl.json rename to lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/elmer/part_1/bclconvert-succeeded-to-bssh-fastq-copy/step_functions_templates/bclconvert_succeeded_to_bssh_fastq_copy_draft_template.asl.json index 261247c8e..49b0f3679 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/elmer/part_1/bclconvert-succeeded-to-bssh-fastq-copy-draft/step_functions_templates/bclconvert_succeeded_to_bssh_fastq_copy_draft_template.asl.json +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/elmer/part_1/bclconvert-succeeded-to-bssh-fastq-copy/step_functions_templates/bclconvert_succeeded_to_bssh_fastq_copy_draft_template.asl.json @@ -11,7 +11,7 @@ }, "Get Workflow Inputs": { "Type": "Parallel", - "Next": "Push Draft Event", + "Next": "Launch Ready Event", "Branches": [ { "StartAt": "Get Portal Run ID and Workflow Run Name", @@ -130,42 +130,29 @@ }, "ResultPath": "$.get_workflow_inputs_step" }, - "Push Draft Event": { + "Launch Ready Event": { "Type": "Task", - "Resource": "arn:aws:states:::events:putEvents", + "Resource": "arn:aws:states:::states:startExecution.sync:2", "Parameters": { - "Entries": [ - { - "EventBusName": "${__event_bus_name__}", - "Source": "${__event_source__}", - "DetailType": "${__detail_type__}", - "Detail": { - "portalRunId.$": "$.get_workflow_inputs_step.portal_run_id", - "timestamp.$": "$$.State.EnteredTime", - "status": "DRAFT", - "workflowName": "${__workflow_name__}", - "workflowVersion": "${__workflow_version__}", - "workflowRunName.$": "$.get_workflow_inputs_step.workflow_run_name", - "linkedLibraries.$": "$.get_workflow_inputs_step.linked_libraries_list", - "payload": { - "version": "${__payload_version__}", - "data": { - "inputs": { - "instrumentRunId.$": "$.inputs.payload.data.instrumentRunId", - "bsshProjectId.$": "$.inputs.payload.data.projectId", - "bsshAnalysisId.$": "$.inputs.payload.data.analysisId" - }, - "tags": { - "instrumentRunId.$": "$.inputs.payload.data.instrumentRunId" - } - } - } + "StateMachineArn": "${__launch_ready_event_sfn_arn__}", + "Input": { + "StatePayload": { + "portal_run_id.$": "$.get_workflow_inputs_step.portal_run_id", + "workflow_run_name.$": "$.get_workflow_inputs_step.workflow_run_name", + "linked_libraries.$": "$.get_workflow_inputs_step.linked_libraries_list", + "data_inputs": { + "instrumentRunId.$": "$.inputs.payload.data.instrumentRunId", + "bsshProjectId.$": "$.inputs.payload.data.projectId", + "bsshAnalysisId.$": "$.inputs.payload.data.analysisId" + }, + "data_tags": { + "instrumentRunId.$": "$.inputs.payload.data.instrumentRunId" } } - ] + } }, - "End": true, - "ResultPath": null + "ResultPath": null, + "End": true } } } diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/elmer/part_2/bssh-fastq-copy-manager-draft-to-ready/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/elmer/part_2/bssh-fastq-copy-manager-draft-to-ready/index.ts deleted file mode 100644 index 35827cd76..000000000 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/elmer/part_2/bssh-fastq-copy-manager-draft-to-ready/index.ts +++ /dev/null @@ -1,99 +0,0 @@ -import { Construct } from 'constructs'; -import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; -import * as ssm from 'aws-cdk-lib/aws-ssm'; -import * as events from 'aws-cdk-lib/aws-events'; -import * as secretsManager from 'aws-cdk-lib/aws-secretsmanager'; -import { WorkflowDraftRunStateChangeToWorkflowRunStateChangeReadyConstruct } from '../../../../../../../components/event-workflowdraftrunstatechange-to-workflowrunstatechange-ready'; - -/* -Part 2 - -* Input Event source: `orcabus.bsshfastqcopyinputeventglue` -* Input Event DetailType: `WorkflowDraftRunStateChange` -* Input Event status: `draft` - -* Output Event source: `orcabus.bsshfastqcopyinputeventglue` -* Output Event DetailType: `WorkflowRunStateChange` -* Output Event status: `ready` - -* The BsshFastqCopyManagerDraftToReadyMaker Construct - * Subscribes to the draft events generated by this stack - * Pushes an event payload of the input for the BSSH Fastq Copy Object -*/ - -export interface bsshFastqCopyManagerDraftToReadyMakerConstructProps { - tableObj: dynamodb.ITableV2; - outputUriSsmParameterObj: ssm.IStringParameter; - eventBusObj: events.IEventBus; - icav2AccessTokenSecretObj: secretsManager.ISecret; -} - -export class BsshFastqCopyManagerDraftToReadyMakerConstruct extends Construct { - public readonly bsshFastqCopyManagerDraftToReadyMakerEventMap = { - prefix: 'elmer-bssh-fastq-copy', - tablePartition: 'bssh_fastq_copy', - triggerSource: 'orcabus.bsshfastqcopyinputeventglue', - triggerStatus: 'DRAFT', - triggerDetailType: 'WorkflowDraftRunStateChange', - outputSource: 'orcabus.bsshfastqcopyinputeventglue', - outputStatus: 'READY', - payloadVersion: '2024.05.24', - workflowName: 'bsshFastqCopy', - workflowVersion: '2024.05.24', - }; - - constructor( - scope: Construct, - id: string, - props: bsshFastqCopyManagerDraftToReadyMakerConstructProps - ) { - super(scope, id); - - /* - Part 1: Initialise the workflow draft run state change to workflow run state change construct - */ - new WorkflowDraftRunStateChangeToWorkflowRunStateChangeReadyConstruct( - this, - 'bssh_fastq_copy_manager_input_maker_external', - { - /* - Set Input StateMachine Object - */ - lambdaPrefix: this.bsshFastqCopyManagerDraftToReadyMakerEventMap.prefix, - payloadVersion: this.bsshFastqCopyManagerDraftToReadyMakerEventMap.payloadVersion, - stateMachinePrefix: this.bsshFastqCopyManagerDraftToReadyMakerEventMap.prefix, - rulePrefix: `stacky-${this.bsshFastqCopyManagerDraftToReadyMakerEventMap.prefix}`, - - /* - Table objects - */ - tableObj: props.tableObj, - tablePartitionName: this.bsshFastqCopyManagerDraftToReadyMakerEventMap.tablePartition, - - /* - Event Triggers - */ - eventBusObj: props.eventBusObj, - triggerSource: this.bsshFastqCopyManagerDraftToReadyMakerEventMap.triggerSource, - triggerStatus: this.bsshFastqCopyManagerDraftToReadyMakerEventMap.triggerStatus, - - /* - Event Outputs - */ - workflowName: this.bsshFastqCopyManagerDraftToReadyMakerEventMap.workflowName, - workflowVersion: this.bsshFastqCopyManagerDraftToReadyMakerEventMap.workflowVersion, - outputSource: this.bsshFastqCopyManagerDraftToReadyMakerEventMap.outputSource, - - /* - Set the output uri - */ - outputUriSsmParameterObj: props.outputUriSsmParameterObj, - - /* - ICAv2 Secret Construct - */ - icav2AccessTokenSecretObj: props.icav2AccessTokenSecretObj, - } - ); - } -} diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/gorilla/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/gorilla/index.ts index a6addec6b..02cc6be96 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/gorilla/index.ts +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/gorilla/index.ts @@ -3,9 +3,7 @@ import * as events from 'aws-cdk-lib/aws-events'; import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; import * as ssm from 'aws-cdk-lib/aws-ssm'; import * as secretsManager from 'aws-cdk-lib/aws-secretsmanager'; -import { BsshFastqCopyManagerDraftMakerConstruct } from '../elmer/part_1/bclconvert-succeeded-to-bssh-fastq-copy-draft'; import { BclconvertInteropQcDraftMakerConstruct } from './part_1/bclconvert-interop-qc-draft-event-maker'; -import { BclconvertInteropQcDraftToReadyMakerConstruct } from './part_2/bclconvert-interop-qc-input-maker'; /* Provide the glue to get from the bclconvertmanager success event @@ -15,8 +13,6 @@ To triggering the bsshFastqCopyManager export interface BsshFastqCopyToBclconvertInteropQcConstructProps { /* Event Objects */ eventBusObj: events.IEventBus; - /* Table Objects */ - inputMakerTableObj: dynamodb.ITableV2; /* SSM Parameter Ojbects */ analysisLogsUriSsmParameterObj: ssm.IStringParameter; analysisOutputUriSsmParameterObj: ssm.IStringParameter; @@ -42,7 +38,7 @@ export class BsshFastqCopyToBclconvertInteropQcConstruct extends Construct { Output Event source: `orcabus.bclconvertinteropqcinputeventglue` Output Event DetailType: `WorkflowRunStateChange` - Output Event status: `complete` + Output Event status: `READY` * The BCLConvertInteropQCInputMaker Construct * Subscribes to the BSSHFastqCopyManagerEventHandler Construct outputs and creates the input for the BCLConvertInteropQC @@ -53,22 +49,15 @@ export class BsshFastqCopyToBclconvertInteropQcConstruct extends Construct { this, 'bssh_fastq_copy_complete_to_bclconvert_interop_qc_draft_maker', { + /* Event Bus */ eventBusObj: props.eventBusObj, - tableObj: props.inputMakerTableObj, + /* SSM Parameter Objects */ + logsUriSsmParameterObj: props.analysisLogsUriSsmParameterObj, + outputUriSsmParameterObj: props.analysisOutputUriSsmParameterObj, + icav2ProjectIdSsmParameterObj: props.icav2ProjectIdSsmParameterObj, + /* Secrets Manager */ + icav2AccessTokenSecretObj: props.icav2AccessTokenSecretObj, } ); - - const bclconvertInteropqcInputMaker = new BclconvertInteropQcDraftToReadyMakerConstruct( - this, - 'bclconvert_interopqc_input_maker', - { - logsUriSsmParameterObj: props.analysisLogsUriSsmParameterObj, - outputUriSsmParameterObj: props.analysisOutputUriSsmParameterObj, - eventBusObj: props.eventBusObj, - tableObj: props.inputMakerTableObj, - icav2ProjectIdSsmParameterObj: props.icav2ProjectIdSsmParameterObj, - icav2AccessTokenSecretObj: props.icav2AccessTokenSecretObj, - } - ); } } diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/gorilla/part_1/bclconvert-interop-qc-draft-event-maker/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/gorilla/part_1/bclconvert-interop-qc-draft-event-maker/index.ts index 92f2a4645..7d24da1e6 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/gorilla/part_1/bclconvert-interop-qc-draft-event-maker/index.ts +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/gorilla/part_1/bclconvert-interop-qc-draft-event-maker/index.ts @@ -1,5 +1,4 @@ import { Construct } from 'constructs'; -import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; import path from 'path'; import * as sfn from 'aws-cdk-lib/aws-stepfunctions'; import * as events from 'aws-cdk-lib/aws-events'; @@ -7,8 +6,9 @@ import * as eventsTargets from 'aws-cdk-lib/aws-events-targets'; import { WorkflowDraftRunStateChangeCommonPreambleConstruct } from '../../../../../../../components/sfn-workflowdraftrunstatechange-common-preamble'; import * as iam from 'aws-cdk-lib/aws-iam'; import * as cdk from 'aws-cdk-lib'; -import { LambdaB64GzTranslatorConstruct } from '../../../../../../../components/python-lambda-b64gz-translator'; -import { GetLibraryObjectsFromSamplesheetConstruct } from '../../../../../../../components/python-lambda-get-metadata-objects-from-samplesheet'; +import * as ssm from 'aws-cdk-lib/aws-ssm'; +import * as secretsManager from 'aws-cdk-lib/aws-secretsmanager'; +import { GenerateWorkflowRunStateChangeReadyConstruct } from '../../../../../../../components/sfn-generate-workflowrunstatechange-ready-event'; /* Part 1 @@ -32,8 +32,14 @@ Part 1 */ export interface bclconvertInteropQcDraftMakerConstructProps { - tableObj: dynamodb.ITableV2; + /* Event Bus */ eventBusObj: events.IEventBus; + /* SSM Parameter Objects */ + outputUriSsmParameterObj: ssm.IStringParameter; + logsUriSsmParameterObj: ssm.IStringParameter; + icav2ProjectIdSsmParameterObj: ssm.IStringParameter; + /* Secrets */ + icav2AccessTokenSecretObj: secretsManager.ISecret; } export class BclconvertInteropQcDraftMakerConstruct extends Construct { @@ -45,8 +51,6 @@ export class BclconvertInteropQcDraftMakerConstruct extends Construct { triggerDetailType: 'WorkflowRunStateChange', triggerWorkflowName: 'bsshFastqCopy', outputSource: 'orcabus.bclconvertinteropqcinputeventglue', - outputDetailType: 'WorkflowDraftRunStateChange', - outputStatus: 'DRAFT', payloadVersion: '2024.05.24', workflowName: 'bclconvert-interop-qc', workflowVersion: '2024.05.24', @@ -62,19 +66,44 @@ export class BclconvertInteropQcDraftMakerConstruct extends Construct { this, `${this.bclconvertInteropQcDraftMakerEventMap.prefix}_sfn_preamble`, { - portalRunTablePartitionName: - this.bclconvertInteropQcDraftMakerEventMap.portalRunPartitionName, stateMachinePrefix: this.bclconvertInteropQcDraftMakerEventMap.prefix, - tableObj: props.tableObj, workflowName: this.bclconvertInteropQcDraftMakerEventMap.workflowName, workflowVersion: this.bclconvertInteropQcDraftMakerEventMap.workflowVersion, } ).stepFunctionObj; + /* + Part 2: Build the engine parameters sfn + */ + const engineParametersAndReadyLaunchSfn = new GenerateWorkflowRunStateChangeReadyConstruct( + this, + 'bssh_copy_complete_to_bclconvert_interop_qc_ready_ep_sfn', + { + /* Event Placeholders */ + eventBusObj: props.eventBusObj, + outputSource: this.bclconvertInteropQcDraftMakerEventMap.outputSource, + payloadVersion: this.bclconvertInteropQcDraftMakerEventMap.payloadVersion, + workflowName: this.bclconvertInteropQcDraftMakerEventMap.workflowName, + workflowVersion: this.bclconvertInteropQcDraftMakerEventMap.workflowVersion, + + /* SSM Parameters */ + outputUriSsmParameterObj: props.outputUriSsmParameterObj, + icav2ProjectIdSsmParameterObj: props.icav2ProjectIdSsmParameterObj, + logsUriSsmParameterObj: props.logsUriSsmParameterObj, + + /* Secrets */ + icav2AccessTokenSecretObj: props.icav2AccessTokenSecretObj, + + /* Prefixes */ + lambdaPrefix: this.bclconvertInteropQcDraftMakerEventMap.prefix, + stateMachinePrefix: this.bclconvertInteropQcDraftMakerEventMap.prefix, + } + ).stepFunctionObj; + /* Part 2: Build the sfn */ - const draftMakerSfn = new sfn.StateMachine(this, 'bssh_complete_to_bclconvert_sfn', { + const inputsMakerSfn = new sfn.StateMachine(this, 'bssh_complete_to_bclconvert_sfn', { stateMachineName: `${this.bclconvertInteropQcDraftMakerEventMap.prefix}-sfn`, definitionBody: sfn.DefinitionBody.fromFile( path.join( @@ -84,31 +113,23 @@ export class BclconvertInteropQcDraftMakerConstruct extends Construct { ) ), definitionSubstitutions: { - // Event stuff - __event_bus_name__: props.eventBusObj.eventBusName, - __event_source__: this.bclconvertInteropQcDraftMakerEventMap.outputSource, - __detail_type__: this.bclconvertInteropQcDraftMakerEventMap.outputDetailType, // Workflow stuff __workflow_name__: this.bclconvertInteropQcDraftMakerEventMap.workflowName, __workflow_version__: this.bclconvertInteropQcDraftMakerEventMap.workflowVersion, __payload_version__: this.bclconvertInteropQcDraftMakerEventMap.payloadVersion, // Subfunctions __sfn_preamble_state_machine_arn__: sfn_preamble.stateMachineArn, + __launch_ready_event_sfn_arn__: engineParametersAndReadyLaunchSfn.stateMachineArn, }, }); /* Part 2: Grant the sfn permissions */ - // Read/write to the table - props.tableObj.grantReadWriteData(draftMakerSfn); - - // Allow the step function to submit events - props.eventBusObj.grantPutEventsTo(draftMakerSfn); // Because we run a nested state machine, we need to add the permissions to the state machine role // See https://stackoverflow.com/questions/60612853/nested-step-function-in-a-step-function-unknown-error-not-authorized-to-cr - draftMakerSfn.addToRolePolicy( + inputsMakerSfn.addToRolePolicy( new iam.PolicyStatement({ resources: [ `arn:aws:events:${cdk.Aws.REGION}:${cdk.Aws.ACCOUNT_ID}:rule/StepFunctionsGetEventsForStepFunctionsExecutionRule`, @@ -118,7 +139,8 @@ export class BclconvertInteropQcDraftMakerConstruct extends Construct { ); // Add state machine execution permissions to stateMachine role - sfn_preamble.grantStartExecution(draftMakerSfn.role); + sfn_preamble.grantStartExecution(inputsMakerSfn); + engineParametersAndReadyLaunchSfn.grantStartExecution(inputsMakerSfn); /* Part 3: Subscribe to the event bus for this event type @@ -146,7 +168,7 @@ export class BclconvertInteropQcDraftMakerConstruct extends Construct { // Add target of event to be the state machine rule.addTarget( - new eventsTargets.SfnStateMachine(draftMakerSfn, { + new eventsTargets.SfnStateMachine(inputsMakerSfn, { input: events.RuleTargetInput.fromEventPath('$.detail'), }) ); diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/gorilla/part_1/bclconvert-interop-qc-draft-event-maker/step_functions_templates/generate_bclconvert_interop_qc_draft_event_sfn_template.asl.json b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/gorilla/part_1/bclconvert-interop-qc-draft-event-maker/step_functions_templates/generate_bclconvert_interop_qc_draft_event_sfn_template.asl.json index afdda37a5..6062e1842 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/gorilla/part_1/bclconvert-interop-qc-draft-event-maker/step_functions_templates/generate_bclconvert_interop_qc_draft_event_sfn_template.asl.json +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/gorilla/part_1/bclconvert-interop-qc-draft-event-maker/step_functions_templates/generate_bclconvert_interop_qc_draft_event_sfn_template.asl.json @@ -16,49 +16,36 @@ "StateMachineArn": "${__sfn_preamble_state_machine_arn__}", "Input": {} }, - "Next": "Push Draft Event", - "ResultPath": "$.get_sfn_preamble_outputs_step", "ResultSelector": { "portal_run_id.$": "$.Output.portal_run_id", "workflow_run_name.$": "$.Output.workflow_run_name" - } + }, + "ResultPath": "$.get_sfn_preamble_outputs_step", + "Next": "Launch Ready Event" }, - "Push Draft Event": { + "Launch Ready Event": { "Type": "Task", - "Resource": "arn:aws:states:::events:putEvents", + "Resource": "arn:aws:states:::states:startExecution.sync:2", "Parameters": { - "Entries": [ - { - "EventBusName": "${__event_bus_name__}", - "Source": "${__event_source__}", - "DetailType": "${__detail_type__}", - "Detail": { - "portalRunId.$": "$.get_sfn_preamble_outputs_step.portal_run_id", - "timestamp.$": "$$.State.EnteredTime", - "status": "DRAFT", - "workflowName": "${__workflow_name__}", - "workflowVersion": "${__workflow_version__}", - "workflowRunName.$": "$.get_sfn_preamble_outputs_step.workflow_run_name", - "linkedLibraries.$": "$.inputs.linkedLibraries", - "payload": { - "version": "${__payload_version__}", - "data": { - "inputs": { - "bclconvertReportDirectory.$": "States.Format('{}Reports/', $.inputs.payload.data.outputs.outputUri)", - "interopDirectory.$": "States.Format('{}InterOp/', $.inputs.payload.data.outputs.outputUri)", - "instrumentRunId.$": "$.inputs.payload.data.outputs.instrumentRunId" - }, - "tags": { - "instrumentRunId.$": "$.inputs.payload.data.outputs.instrumentRunId" - } - } - } + "StateMachineArn": "${__launch_ready_event_sfn_arn__}", + "Input": { + "StatePayload": { + "portal_run_id.$": "$.get_workflow_inputs_step.portal_run_id", + "workflow_run_name.$": "$.get_workflow_inputs_step.workflow_run_name", + "linked_libraries.$": "$.get_workflow_inputs_step.linked_libraries_list", + "data_inputs": { + "bclconvertReportDirectory.$": "States.Format('{}Reports/', $.inputs.payload.data.outputs.outputUri)", + "interopDirectory.$": "States.Format('{}InterOp/', $.inputs.payload.data.outputs.outputUri)", + "instrumentRunId.$": "$.inputs.payload.data.outputs.instrumentRunId" + }, + "data_tags": { + "instrumentRunId.$": "$.inputs.payload.data.outputs.instrumentRunId" } } - ] + } }, - "End": true, - "ResultPath": null + "ResultPath": null, + "End": true } } } diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/gorilla/part_2/bclconvert-interop-qc-input-maker/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/gorilla/part_2/bclconvert-interop-qc-input-maker/index.ts deleted file mode 100644 index 28964b963..000000000 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/gorilla/part_2/bclconvert-interop-qc-input-maker/index.ts +++ /dev/null @@ -1,104 +0,0 @@ -import { Construct } from 'constructs'; -import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; -import * as ssm from 'aws-cdk-lib/aws-ssm'; -import * as events from 'aws-cdk-lib/aws-events'; -import * as secretsManager from 'aws-cdk-lib/aws-secretsmanager'; -import { WorkflowDraftRunStateChangeToWorkflowRunStateChangeReadyConstruct } from '../../../../../../../components/event-workflowdraftrunstatechange-to-workflowrunstatechange-ready'; - -/* -Part 1 - -* Input Event Source: `orcabus.bclconvertinteropqcinputeventglue` -* Input Event DetailType: `WorkflowDraftRunStateChange` -* Input Event WorkflowName: bclconvertInteropQc -* Input Event status: `draft` - -* Output Event source: `orcabus.bclconvertinteropqcinputeventglue` -* Output Event DetailType: `WorkflowRunStateChange` -* Output Event status: `ready` - - -* The bclconvertInteropQcDraftToReady Construct - * Subscribes to the bclconvert draft maker Stack outputs and creates the input for the BCLConvert Interop QC Pipeline -*/ - -export interface bclconvertInteropQcDraftToReadyMakerConstructProps { - tableObj: dynamodb.ITableV2; - outputUriSsmParameterObj: ssm.IStringParameter; - logsUriSsmParameterObj: ssm.IStringParameter; - icav2ProjectIdSsmParameterObj: ssm.IStringParameter; - icav2AccessTokenSecretObj: secretsManager.ISecret; - eventBusObj: events.IEventBus; -} - -export class BclconvertInteropQcDraftToReadyMakerConstruct extends Construct { - public readonly bclconvertInteropQcDraftToReadyMakerEventMap = { - prefix: 'gorilla-interop-qc', - tablePartition: 'bclconvert_interop_qc', - triggerSource: 'orcabus.bclconvertinteropqcinputeventglue', - triggerStatus: 'DRAFT', - triggerDetailType: 'WorkflowDraftRunStateChange', - outputSource: 'orcabus.bclconvertinteropqcinputeventglue', - outputStatus: 'READY', - payloadVersion: '2024.05.24', - workflowName: 'bclconvert-interop-qc', - workflowVersion: '2024.05.24', - }; - - constructor( - scope: Construct, - id: string, - props: bclconvertInteropQcDraftToReadyMakerConstructProps - ) { - super(scope, id); - - /* - Part 1: Initialise the workflow draft run state change to workflow run state change construct - */ - new WorkflowDraftRunStateChangeToWorkflowRunStateChangeReadyConstruct( - this, - 'bclconvert_interop_qc_draft_to_ready_sfn', - { - /* - Set Input StateMachine Object - */ - lambdaPrefix: this.bclconvertInteropQcDraftToReadyMakerEventMap.prefix, - payloadVersion: this.bclconvertInteropQcDraftToReadyMakerEventMap.payloadVersion, - stateMachinePrefix: this.bclconvertInteropQcDraftToReadyMakerEventMap.prefix, - rulePrefix: `stacky-${this.bclconvertInteropQcDraftToReadyMakerEventMap.prefix}-rule`, - - /* - Table objects - */ - tableObj: props.tableObj, - tablePartitionName: this.bclconvertInteropQcDraftToReadyMakerEventMap.tablePartition, - - /* - Event Triggers - */ - eventBusObj: props.eventBusObj, - triggerSource: this.bclconvertInteropQcDraftToReadyMakerEventMap.triggerSource, - triggerStatus: this.bclconvertInteropQcDraftToReadyMakerEventMap.triggerStatus, - - /* - Event Outputs - */ - workflowName: this.bclconvertInteropQcDraftToReadyMakerEventMap.workflowName, - workflowVersion: this.bclconvertInteropQcDraftToReadyMakerEventMap.workflowVersion, - outputSource: this.bclconvertInteropQcDraftToReadyMakerEventMap.outputSource, - - /* - Set the output uri - */ - outputUriSsmParameterObj: props.outputUriSsmParameterObj, - logsUriSsmParameterObj: props.logsUriSsmParameterObj, - icav2ProjectIdSsmParameterObj: props.icav2ProjectIdSsmParameterObj, - - /* - Set the secrets - */ - icav2AccessTokenSecretObj: props.icav2AccessTokenSecretObj, - } - ); - } -} diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/index.ts index fdee87309..8db1bf7aa 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/index.ts +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/index.ts @@ -4,6 +4,7 @@ import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; import * as ssm from 'aws-cdk-lib/aws-ssm'; import * as cdk from 'aws-cdk-lib'; import * as secretsManager from 'aws-cdk-lib/aws-secretsmanager'; +import * as lambda from 'aws-cdk-lib/aws-lambda'; import { showerGlueHandlerConstruct } from './clag'; import { BclconvertToBsshFastqCopyEventHandlerConstruct } from './elmer'; import { BsshFastqCopyToBclconvertInteropQcConstruct } from './gorilla'; @@ -13,6 +14,7 @@ import { TnGlueHandlerConstruct } from './loctite'; import { WtsGlueHandlerConstruct } from './mod-podge'; import { UmccriseGlueHandlerConstruct } from './pva'; import { RnasumGlueHandlerConstruct } from './roket'; +import { PieriandxGlueHandlerConstruct } from './nails/'; /* Provide the glue to get from the bclconvertmanager success event @@ -22,6 +24,7 @@ To triggering the bsshFastqCopyManager export interface GlueConstructProps { /* Event Bus */ eventBusObj: events.IEventBus; + /* Tables */ instrumentRunTableObj: dynamodb.ITableV2; inputMakerTableObj: dynamodb.ITableV2; @@ -32,14 +35,23 @@ export interface GlueConstructProps { wtsGlueTableObj: dynamodb.ITableV2; umccriseGlueTableObj: dynamodb.ITableV2; rnasumGlueTableObj: dynamodb.ITableV2; - /* SSM Parameters */ + pieriandxGlueTableObj: dynamodb.ITableV2; + + /* Standard SSM Parameters */ icav2ProjectIdSsmParameterObj: ssm.IStringParameter; - bsshOutputFastqCopyOutputUriSsmParameterObj: ssm.IStringParameter; analysisOutputUriSsmParameterObj: ssm.IStringParameter; analysisCacheUriSsmParameterObj: ssm.IStringParameter; analysisLogsUriSsmParameterObj: ssm.IStringParameter; - /* Secrests */ + + /* Secrets */ icav2AccessTokenSecretObj: secretsManager.ISecret; + + /* BSSH SSM Parameters */ + bsshOutputFastqCopyOutputUriSsmParameterObj: ssm.IStringParameter; + + /* PierianDX SSM Parameters */ + pieriandxProjectInfoSsmParameterObj: ssm.IStringParameter; + redcapLambdaObj: lambda.IFunction; } export class GlueConstruct extends Construct { @@ -62,8 +74,6 @@ export class GlueConstruct extends Construct { const elmer = new BclconvertToBsshFastqCopyEventHandlerConstruct(this, 'elmer', { /* Event Bus */ eventBusObj: props.eventBusObj, - /* Tables */ - inputMakerTableObj: props.inputMakerTableObj, /* SSM Parameters */ bsshOutputFastqCopyUriSsmParameterObj: props.bsshOutputFastqCopyOutputUriSsmParameterObj, /* Secrets */ @@ -76,8 +86,6 @@ export class GlueConstruct extends Construct { const gorilla = new BsshFastqCopyToBclconvertInteropQcConstruct(this, 'gorilla', { /* Event Objects */ eventBusObj: props.eventBusObj, - /* Table Objects */ - inputMakerTableObj: props.inputMakerTableObj, /* SSM Parameter Ojbects */ analysisLogsUriSsmParameterObj: props.analysisLogsUriSsmParameterObj, analysisOutputUriSsmParameterObj: props.analysisOutputUriSsmParameterObj, @@ -93,7 +101,6 @@ export class GlueConstruct extends Construct { /* Event Bus */ eventBusObj: props.eventBusObj, /* Tables */ - inputMakerTableObj: props.inputMakerTableObj, cttsov2GlueTableObj: props.cttsov2GlueTableObj, /* SSM Parameters */ icav2ProjectIdSsmParameterObj: props.icav2ProjectIdSsmParameterObj, @@ -111,7 +118,6 @@ export class GlueConstruct extends Construct { /* Event Bus */ eventBusObj: props.eventBusObj, /* Tables */ - inputMakerTableObj: props.inputMakerTableObj, wgtsQcGlueTableObj: props.wgtsQcGlueTableObj, /* SSM Parameters */ icav2ProjectIdSsmParameterObj: props.icav2ProjectIdSsmParameterObj, @@ -129,7 +135,6 @@ export class GlueConstruct extends Construct { /* Event Bus */ eventBusObj: props.eventBusObj, /* Tables */ - inputMakerTableObj: props.inputMakerTableObj, tnGlueTableObj: props.tnGlueTableObj, /* SSM Parameters */ icav2ProjectIdSsmParameterObj: props.icav2ProjectIdSsmParameterObj, @@ -147,7 +152,6 @@ export class GlueConstruct extends Construct { /* Event Bus */ eventBusObj: props.eventBusObj, /* Tables */ - inputMakerTableObj: props.inputMakerTableObj, wtsGlueTableObj: props.wtsGlueTableObj, /* SSM Parameters */ icav2ProjectIdSsmParameterObj: props.icav2ProjectIdSsmParameterObj, @@ -159,13 +163,30 @@ export class GlueConstruct extends Construct { }); /* - Part H: Plumber-up the UMCCRise Execution Service to the shower services + Part H: Plumber-up the cttsov2 to pieriandx services + */ + const nails = new PieriandxGlueHandlerConstruct(this, 'nails', { + /* Event Bus */ + eventBusObj: props.eventBusObj, + + /* Tables */ + pieriandxGlueTableObj: props.pieriandxGlueTableObj, + + /* Secrets */ + icav2AccessTokenSecretObj: props.icav2AccessTokenSecretObj, + + /* Extras */ + pieriandxProjectInfoSsmParameterObj: props.pieriandxProjectInfoSsmParameterObj, + redcapLambdaObj: props.redcapLambdaObj, + }); + + /* + Part I: Plumber-up the UMCCRise Execution Service to the shower services */ const pva = new UmccriseGlueHandlerConstruct(this, 'pva', { /* Event Bus */ eventBusObj: props.eventBusObj, /* Tables */ - inputMakerTableObj: props.inputMakerTableObj, umccriseGlueTableObj: props.umccriseGlueTableObj, /* SSM Parameters */ icav2ProjectIdSsmParameterObj: props.icav2ProjectIdSsmParameterObj, @@ -177,13 +198,12 @@ export class GlueConstruct extends Construct { }); /* - Part I: Plumber-up the RNASum Exection Service to the shower services + Part J: Plumber-up the RNASum Execution Service to the shower services */ const roket = new RnasumGlueHandlerConstruct(this, 'roket', { /* Event Bus */ eventBusObj: props.eventBusObj, /* Tables */ - inputMakerTableObj: props.inputMakerTableObj, rnasumGlueTableObj: props.rnasumGlueTableObj, /* SSM Parameters */ icav2ProjectIdSsmParameterObj: props.icav2ProjectIdSsmParameterObj, @@ -199,6 +219,7 @@ export class GlueConstruct extends Construct { export interface GlueStackConfig { /* Event Bus */ eventBusName: string; + /* Tables */ instrumentRunTableName: string; inputMakerTableName: string; @@ -209,14 +230,25 @@ export interface GlueStackConfig { wtsGlueTableName: string; umccriseGlueTableName: string; rnasumGlueTableName: string; + pieriandxGlueTableName: string; + /* SSM Parameters */ icav2ProjectIdSsmParameterName: string; - bsshOutputFastqCopyUriSsmParameterName: string; analysisCacheUriSsmParameterName: string; analysisOutputUriSsmParameterName: string; analysisLogsUriSsmParameterName: string; + /* Secrets */ icav2AccessTokenSecretName: string; + + /* BSSH SSM Parameters */ + bsshOutputFastqCopyUriSsmParameterName: string; + + /* PierianDX SSM Parameters */ + pieriandxProjectInfoSsmParameterPath: string; + + /* PierianDX External Functions */ + redcapLambdaFunctionName: string; } export type GlueStackProps = GlueStackConfig & cdk.StackProps; @@ -226,17 +258,17 @@ export class GlueStack extends cdk.Stack { super(scope, id, props); /* - Part 0: Get the inputs as objects - */ + Part 0: Get the inputs as objects + */ /* - Get the event bus - */ + Get the event bus + */ const eventBusObj = events.EventBus.fromEventBusName(this, 'eventBusObj', props.eventBusName); /* - Get the tables - */ + Get the tables + */ const workflowManagerTableObj = dynamodb.Table.fromTableName( this, 'workflowManagerTableObj', @@ -282,6 +314,11 @@ export class GlueStack extends cdk.Stack { 'rnasumGlueTableObj', props.rnasumGlueTableName ); + const pieriandxGlueTableObj = dynamodb.Table.fromTableName( + this, + 'pieriandxGlueTableObj', + props.pieriandxGlueTableName + ); /* Get the SSM Parameters @@ -291,11 +328,6 @@ export class GlueStack extends cdk.Stack { 'icav2ProjectIdSsmParameterObj', props.icav2ProjectIdSsmParameterName ); - const bsshOutputFastqCopyUriSsmParameterObj = ssm.StringParameter.fromStringParameterName( - this, - 'bsshOutputFastqCopyUriPrefixSsmParameterObj', - props.bsshOutputFastqCopyUriSsmParameterName - ); const analysisCacheUriSsmParameterObj = ssm.StringParameter.fromStringParameterName( this, @@ -315,20 +347,48 @@ export class GlueStack extends cdk.Stack { ); /* - Secrets - */ + Secrets + */ const icav2AccessTokenSecretObj = secretsManager.Secret.fromSecretNameV2( this, 'icav2AccessTokenSecretObj', props.icav2AccessTokenSecretName ); + /* + BSSH SSM Parameters + */ + const bsshOutputFastqCopyUriSsmParameterObj = ssm.StringParameter.fromStringParameterName( + this, + 'bsshOutputFastqCopyUriPrefixSsmParameterObj', + props.bsshOutputFastqCopyUriSsmParameterName + ); + + /* + PierianDx SSM Parameters + */ + const pieriandxProjectInfoSsmParameterObj = ssm.StringParameter.fromStringParameterName( + this, + 'pieriandxProjectInfoSsmParameterObj', + props.pieriandxProjectInfoSsmParameterPath + ); + + /* + PierianDx External Functions + */ + const redcapLambdaObj = lambda.Function.fromFunctionName( + this, + 'redcapLambdaObj', + props.redcapLambdaFunctionName + ); + /* Call the construct */ new GlueConstruct(this, 'stacky_glue', { /* Event stuff */ eventBusObj: eventBusObj, + /* Tables */ workflowManagerTableObj: workflowManagerTableObj, inputMakerTableObj: inputMakerTableObj, @@ -339,14 +399,25 @@ export class GlueStack extends cdk.Stack { wtsGlueTableObj: wtsGlueTableObj, umccriseGlueTableObj: umccriseGlueTableObj, rnasumGlueTableObj: rnasumGlueTableObj, + pieriandxGlueTableObj: pieriandxGlueTableObj, + /* SSM Parameters */ icav2ProjectIdSsmParameterObj: icav2ProjectIdSsmParameterObj, - bsshOutputFastqCopyOutputUriSsmParameterObj: bsshOutputFastqCopyUriSsmParameterObj, analysisOutputUriSsmParameterObj: analysisOutputUriSsmParameterObj, analysisCacheUriSsmParameterObj: analysisCacheUriSsmParameterObj, analysisLogsUriSsmParameterObj: analysisLogsUriSsmParameterObj, + /* Secrets */ icav2AccessTokenSecretObj: icav2AccessTokenSecretObj, + + /* BSSH SSM Parameters */ + bsshOutputFastqCopyOutputUriSsmParameterObj: bsshOutputFastqCopyUriSsmParameterObj, + + /* PierianDx SSM Parameters */ + pieriandxProjectInfoSsmParameterObj: pieriandxProjectInfoSsmParameterObj, + + /* PierianDx External Functions */ + redcapLambdaObj: redcapLambdaObj, }); } } diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/index.ts index f2e91a446..8b91b42eb 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/index.ts +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/index.ts @@ -3,11 +3,9 @@ import * as events from 'aws-cdk-lib/aws-events'; import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; import * as ssm from 'aws-cdk-lib/aws-ssm'; import * as secretsManager from 'aws-cdk-lib/aws-secretsmanager'; -import { Cttsov2InitialiseInstrumentRunDbRowConstruct } from './part_1/initialise-cttsov2-instrument-dbs'; -import { Cttsov2InitialiseLibraryAndFastqListRowConstruct } from './part_2/initialise-cttsov2-library-dbs'; -import { Cttsov2PopulateFastqListRowConstruct } from './part_3/populate-fastq-list-row-dbs'; -import { Cttsov2FastqListRowShowerCompleteToWorkflowDraftConstruct } from './part_4/fastq-list-row-event-shower-complete-to-cttsov2-draft'; -import { Cttsov2InputMakerConstruct } from './part_5/cttsov2-draft-to-ready'; +import { Cttsov2InitialiseLibraryAndFastqListRowConstruct } from './part_1/initialise-cttsov2-library-dbs'; +import { Cttsov2PopulateFastqListRowConstruct } from './part_2/populate-fastq-list-row-dbs'; +import { Cttsov2FastqListRowShowerCompleteToWorkflowDraftConstruct } from './part_3/fastq-list-row-event-shower-complete-to-cttsov2-ready'; /* Provide the glue to get from the bssh fastq copy manager to submitting cttsov2 analyses @@ -18,7 +16,6 @@ export interface cttsov2GlueHandlerConstructProps { eventBusObj: events.IEventBus; /* Tables */ cttsov2GlueTableObj: dynamodb.ITableV2; - inputMakerTableObj: dynamodb.ITableV2; /* SSM Parameters */ analysisOutputUriSsmParameterObj: ssm.IStringParameter; analysisLogsUriSsmParameterObj: ssm.IStringParameter; @@ -33,32 +30,14 @@ export class Cttsov2GlueHandlerConstruct extends Construct { super(scope, id); /* - Part 1 - Input Event Source: `orcabus.instrumentrunmanager` - Input Event DetailType: `SamplesheetShowerStateChange` - Input Event status: `SamplesheetRegisteredEventShowerStarting` + Part 1 - * Initialise cttsov2 instrument db construct - */ - const cttsov2_initialise_instrument_run_db_row = - new Cttsov2InitialiseInstrumentRunDbRowConstruct( - this, - 'initialise_cttsov2_instrument_run_db_row', - { - eventBusObj: props.eventBusObj, - tableObj: props.cttsov2GlueTableObj, - } - ); - - /* - Part 2 - - Input Event Source: `orcabus.instrumentrunmanager` - Input Event DetailType: `SamplesheetMetadataUnion` - Input Event status: `LibraryInSamplesheet` + Input Event Source: `orcabus.instrumentrunmanager` + Input Event DetailType: `SamplesheetMetadataUnion` + Input Event status: `LibraryInSamplesheet` - * Initialise cttsov2 instrument db construct - */ + * Initialise cttsov2 instrument db construct + */ const cttsov2_initialise_library_and_fastq_list_row = new Cttsov2InitialiseLibraryAndFastqListRowConstruct( this, @@ -70,14 +49,14 @@ export class Cttsov2GlueHandlerConstruct extends Construct { ); /* - Part 3 + Part 2 - Input Event Source: `orcabus.instrumentrunmanager` - Input Event DetailType: `FastqListRowStateChange` - Input Event status: `newFastqListRow` + Input Event Source: `orcabus.instrumentrunmanager` + Input Event DetailType: `FastqListRowStateChange` + Input Event status: `newFastqListRow` - * Populate the fastq list row attributes for the rgid for this workflow - */ + * Populate the fastq list row attributes for the rgid for this workflow + */ const cttsov2_populate_fastq_list_row = new Cttsov2PopulateFastqListRowConstruct( this, 'populate_cttsov2_fastq_list_row', @@ -88,56 +67,36 @@ export class Cttsov2GlueHandlerConstruct extends Construct { ); /* - Part 4 + Part 3 - Input Event Source: `orcabus.instrumentrunmanager` - Input Event DetailType: `FastqListRowStateChange` - Input Event status: `FastqListRowEventShowerComplete` + Input Event Source: `orcabus.instrumentrunmanager` + Input Event DetailType: `FastqListRowStateChange` + Input Event status: `FastqListRowEventShowerComplete` - Output Event source: `orcabus.cttsov2inputeventglue` - Output Event DetailType: `WorkflowDraftRunStateChange` - Output Event status: `draft` + Output Event source: `orcabus.cttsov2inputeventglue` + Output Event DetailType: `WorkflowDraftRunStateChange` + Output Event status: `draft` - * Trigger cttsov2 events collecting all cttsov2 in the run + * Trigger cttsov2 events collecting all cttsov2 in the run - */ - const fastq_list_row_shower_complete_to_workflow_draft = + */ + const fastq_list_row_shower_complete_to_workflow_ready = new Cttsov2FastqListRowShowerCompleteToWorkflowDraftConstruct( this, 'fastq_list_row_shower_complete_to_workflow_draft', { - workflowsTableObj: props.inputMakerTableObj, + /* Events */ eventBusObj: props.eventBusObj, + /* Tables */ cttsov2GlueTableObj: props.cttsov2GlueTableObj, + /* SSM Param objects */ + icav2ProjectIdSsmParameterObj: props.icav2ProjectIdSsmParameterObj, + outputUriSsmParameterObj: props.analysisOutputUriSsmParameterObj, + cacheUriSsmParameterObj: props.analysisCacheUriSsmParameterObj, + logsUriSsmParameterObj: props.analysisLogsUriSsmParameterObj, + /* Secrets */ + icav2AccessTokenSecretObj: props.icav2AccessTokenSecretObj, } ); - - /* - Part 5 - Input Event source: `orcabus.cttsov2inputeventglue` - Input Event DetailType: `WorkflowDraftRunStateChange` - Input Event status: `draft` - - Output Event source: `orcabus.cttsov2inputeventglue` - Output Event DetailType: `WorkflowRunStateChange` - Output Event status: `ready` - */ - const fastqListRowsToctTSOv2InputMaker = new Cttsov2InputMakerConstruct( - this, - 'fastq_list_rows_to_cttso_v2_input_maker', - { - /* Event bus */ - eventBusObj: props.eventBusObj, - /* Tables */ - inputMakerTableObj: props.inputMakerTableObj, - /* SSM Param objects */ - icav2ProjectIdSsmParameterObj: props.icav2ProjectIdSsmParameterObj, - outputUriSsmParameterObj: props.analysisOutputUriSsmParameterObj, - cacheUriSsmParameterObj: props.analysisCacheUriSsmParameterObj, - logsUriSsmParameterObj: props.analysisLogsUriSsmParameterObj, - /* Secrets */ - icav2AccessTokenSecretObj: props.icav2AccessTokenSecretObj, - } - ); } } diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_1/initialise-cttsov2-instrument-dbs/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_1/initialise-cttsov2-instrument-dbs/index.ts deleted file mode 100644 index 78ea2066f..000000000 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_1/initialise-cttsov2-instrument-dbs/index.ts +++ /dev/null @@ -1,89 +0,0 @@ -import { Construct } from 'constructs'; -import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; -import * as ssm from 'aws-cdk-lib/aws-ssm'; -import path from 'path'; -import * as sfn from 'aws-cdk-lib/aws-stepfunctions'; -import * as events from 'aws-cdk-lib/aws-events'; -import * as eventsTargets from 'aws-cdk-lib/aws-events-targets'; - -/* -Part 1 - -Input Event Source: `orcabus.instrumentrunmanager` -Input Event DetailType: `SamplesheetShowerStateChange` -Input Event status: `SamplesheetRegisteredEventShowerStarting` - -* Initialise cttsov2 instrument db construct -*/ - -export interface Cttsov2InitialiseInstrumentRunDbRowConstructProps { - tableObj: dynamodb.ITableV2; - eventBusObj: events.IEventBus; -} - -export class Cttsov2InitialiseInstrumentRunDbRowConstruct extends Construct { - public readonly Cttsov2InitialiseInstrumentRunDbRowMap = { - prefix: 'jbweld-make-instrument-run-row', - tablePartition: 'instrument_run', - triggerSource: 'orcabus.instrumentrunmanager', - triggerStatus: 'SamplesheetRegisteredEventShowerStarting', - triggerDetailType: 'SamplesheetShowerStateChange', - }; - - constructor( - scope: Construct, - id: string, - props: Cttsov2InitialiseInstrumentRunDbRowConstructProps - ) { - super(scope, id); - - /* - Part 1: Build the internal sfn - */ - const inputMakerSfn = new sfn.StateMachine(this, 'initialise_instrument_run_db_row', { - stateMachineName: `${this.Cttsov2InitialiseInstrumentRunDbRowMap.prefix}-initialise-run-db-row`, - definitionBody: sfn.DefinitionBody.fromFile( - path.join( - __dirname, - 'step_functions_templates', - 'initialise_cttsov2_instrument_run_db_sfn_template.asl.json' - ) - ), - definitionSubstitutions: { - __table_name__: props.tableObj.tableName, - __instrument_run_partition_name__: - this.Cttsov2InitialiseInstrumentRunDbRowMap.tablePartition, - }, - }); - - /* - Part 2: Grant the internal sfn permissions - */ - // access the dynamodb table - props.tableObj.grantReadWriteData(inputMakerSfn.role); - - /* - Part 3: Subscribe to the event bus and trigger the internal sfn - */ - const rule = new events.Rule(this, 'cttsov2_subscribe_to_samplesheet_shower', { - ruleName: `stacky-${this.Cttsov2InitialiseInstrumentRunDbRowMap.prefix}-rule`, - eventBus: props.eventBusObj, - eventPattern: { - source: [this.Cttsov2InitialiseInstrumentRunDbRowMap.triggerSource], - detailType: [this.Cttsov2InitialiseInstrumentRunDbRowMap.triggerDetailType], - detail: { - status: [ - { 'equals-ignore-case': this.Cttsov2InitialiseInstrumentRunDbRowMap.triggerStatus }, - ], - }, - }, - }); - - // Add target of event to be the state machine - rule.addTarget( - new eventsTargets.SfnStateMachine(inputMakerSfn, { - input: events.RuleTargetInput.fromEventPath('$.detail'), - }) - ); - } -} diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_1/initialise-cttsov2-instrument-dbs/step_functions_templates/initialise_cttsov2_instrument_run_db_sfn_template.asl.json b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_1/initialise-cttsov2-instrument-dbs/step_functions_templates/initialise_cttsov2_instrument_run_db_sfn_template.asl.json deleted file mode 100644 index d56d8d202..000000000 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_1/initialise-cttsov2-instrument-dbs/step_functions_templates/initialise_cttsov2_instrument_run_db_sfn_template.asl.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "Comment": "A description of my state machine", - "StartAt": "Set DB Inputs", - "States": { - "Set DB Inputs": { - "Type": "Pass", - "Next": "Initialise Instrument Run Item", - "ResultPath": "$.db_inputs", - "Parameters": { - "instrument_run_id.$": "$.payload.data.instrumentRunId" - } - }, - "Initialise Instrument Run Item": { - "Type": "Task", - "Resource": "arn:aws:states:::dynamodb:putItem", - "Parameters": { - "TableName": "${__table_name__}", - "Item": { - "id": { - "S.$": "$.db_inputs.instrument_run_id" - }, - "id_type": { - "S": "${__instrument_run_partition_name__}" - } - } - }, - "End": true, - "ResultPath": null - } - } -} diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_2/initialise-cttsov2-library-dbs/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_1/initialise-cttsov2-library-dbs/index.ts similarity index 100% rename from lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_2/initialise-cttsov2-library-dbs/index.ts rename to lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_1/initialise-cttsov2-library-dbs/index.ts diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_2/initialise-cttsov2-library-dbs/step_functions_templates/initialise_cttsov2_library_db_sfn_template.asl.json b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_1/initialise-cttsov2-library-dbs/step_functions_templates/initialise_cttsov2_library_db_sfn_template.asl.json similarity index 73% rename from lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_2/initialise-cttsov2-library-dbs/step_functions_templates/initialise_cttsov2_library_db_sfn_template.asl.json rename to lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_1/initialise-cttsov2-library-dbs/step_functions_templates/initialise_cttsov2_library_db_sfn_template.asl.json index 0176936a8..f06d4e1c4 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_2/initialise-cttsov2-library-dbs/step_functions_templates/initialise_cttsov2_library_db_sfn_template.asl.json +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_1/initialise-cttsov2-library-dbs/step_functions_templates/initialise_cttsov2_library_db_sfn_template.asl.json @@ -50,32 +50,6 @@ "Add DataBase Inputs": { "Type": "Parallel", "Branches": [ - { - "StartAt": "Append Library and FastqListRowID to Instrument Run ID", - "States": { - "Append Library and FastqListRowID to Instrument Run ID": { - "Type": "Task", - "Resource": "arn:aws:states:::dynamodb:updateItem", - "Parameters": { - "TableName": "${__table_name__}", - "Key": { - "id.$": "$.input_payload_data.instrumentRunId", - "id_type": "${__instrument_run_partition_name__}" - }, - "UpdateExpression": "ADD library_set :library_set, fastq_list_row_id_set :fastq_list_row_id_set", - "ExpressionAttributeValues": { - ":library_set": { - "SS.$": "States.Array($.input_payload_data.library.libraryId)" - }, - ":fastq_list_row_id_set": { - "SS.$": "$.bclconvert_and_fastq_list_row_id_inputs[*].fastq_list_row_id" - } - } - }, - "End": true - } - } - }, { "StartAt": "Initialise Library ID", "States": { @@ -85,20 +59,16 @@ "Parameters": { "TableName": "${__table_name__}", "Item": { - "id": { - "S.$": "$.input_payload_data.library.libraryId" - }, - "id_type": { - "S": "${__library_partition_name__}" - }, - "orcabus_id": { - "S.$": "$.input_payload_data.library.orcabusId" - }, + "id.$": "$.input_payload_data.library.orcabusId", + "id_type": "${__library_partition_name__}", "library_id": { "S.$": "$.input_payload_data.library.libraryId" }, "fastq_list_row_id_set": { "SS.$": "$.bclconvert_and_fastq_list_row_id_inputs[*].fastq_list_row_id" + }, + "instrument_run_id": { + "S.$": "$.input_payload_data.instrumentRunId" } } }, @@ -145,12 +115,8 @@ "Parameters": { "TableName": "${__table_name__}", "Item": { - "id": { - "S.$": "$.fastq_list_row_id" - }, - "id_type": { - "S": "${__bclconvert_data_row_partition_name__}" - }, + "id.$": "$.fastq_list_row_id", + "id_type": "${__bclconvert_data_row_partition_name__}", "bclconvert_data_row": { "S.$": "States.JsonToString($.set_bclconvert_json)" } @@ -169,12 +135,8 @@ "Parameters": { "TableName": "${__table_name__}", "Item": { - "id": { - "S.$": "$.fastq_list_row_id" - }, - "id_type": { - "S": "${__fastq_list_row_partition_name__}" - } + "id.$": "$.fastq_list_row_id", + "id_type": "${__fastq_list_row_partition_name__}" } }, "End": true diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_3/populate-fastq-list-row-dbs/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_2/populate-fastq-list-row-dbs/index.ts similarity index 100% rename from lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_3/populate-fastq-list-row-dbs/index.ts rename to lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_2/populate-fastq-list-row-dbs/index.ts diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_3/populate-fastq-list-row-dbs/step_functions_templates/update_fastq_list_row_sfn_template.asl.json b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_2/populate-fastq-list-row-dbs/step_functions_templates/update_fastq_list_row_sfn_template.asl.json similarity index 100% rename from lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_3/populate-fastq-list-row-dbs/step_functions_templates/update_fastq_list_row_sfn_template.asl.json rename to lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_2/populate-fastq-list-row-dbs/step_functions_templates/update_fastq_list_row_sfn_template.asl.json diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_4/fastq-list-row-event-shower-complete-to-cttsov2-draft/Readme.md b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_3/fastq-list-row-event-shower-complete-to-cttsov2-ready/Readme.md similarity index 100% rename from lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_4/fastq-list-row-event-shower-complete-to-cttsov2-draft/Readme.md rename to lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_3/fastq-list-row-event-shower-complete-to-cttsov2-ready/Readme.md diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_4/fastq-list-row-event-shower-complete-to-cttsov2-draft/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_3/fastq-list-row-event-shower-complete-to-cttsov2-ready/index.ts similarity index 73% rename from lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_4/fastq-list-row-event-shower-complete-to-cttsov2-draft/index.ts rename to lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_3/fastq-list-row-event-shower-complete-to-cttsov2-ready/index.ts index e7b31e764..9bcf6e5e1 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_4/fastq-list-row-event-shower-complete-to-cttsov2-draft/index.ts +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_3/fastq-list-row-event-shower-complete-to-cttsov2-ready/index.ts @@ -9,6 +9,9 @@ import * as lambda from 'aws-cdk-lib/aws-lambda'; import { WorkflowDraftRunStateChangeCommonPreambleConstruct } from '../../../../../../../components/sfn-workflowdraftrunstatechange-common-preamble'; import * as iam from 'aws-cdk-lib/aws-iam'; import * as cdk from 'aws-cdk-lib'; +import * as ssm from 'aws-cdk-lib/aws-ssm'; +import * as secretsManager from 'aws-cdk-lib/aws-secretsmanager'; +import { GenerateWorkflowRunStateChangeReadyConstruct } from '../../../../../../../components/sfn-generate-workflowrunstatechange-ready-event'; /* Part 4 @@ -19,15 +22,26 @@ Input Event status: `FastqListRowEventShowerComplete` Output Event source: `orcabus.cttsov2inputeventglue` Output Event DetailType: `WorkflowDraftRunStateChange` -Output Event status: `draft` +Output Event status: `READY` * Trigger cttsov2 events collecting all cttsov2 in the run */ export interface Cttsov2FastqListRowShowerCompleteToWorkflowDraftRunDbRowConstructProps { - cttsov2GlueTableObj: dynamodb.ITableV2; - workflowsTableObj: dynamodb.ITableV2; + /* Event Obj */ eventBusObj: events.IEventBus; + + /* Tables */ + cttsov2GlueTableObj: dynamodb.ITableV2; + + /* SSM Parameters */ + icav2ProjectIdSsmParameterObj: ssm.IStringParameter; + outputUriSsmParameterObj: ssm.IStringParameter; + cacheUriSsmParameterObj: ssm.IStringParameter; + logsUriSsmParameterObj: ssm.IStringParameter; + + /* Secrets */ + icav2AccessTokenSecretObj: secretsManager.ISecret; } export class Cttsov2FastqListRowShowerCompleteToWorkflowDraftConstruct extends Construct { @@ -36,7 +50,6 @@ export class Cttsov2FastqListRowShowerCompleteToWorkflowDraftConstruct extends C prefix: 'jbweld-fqlr-shower-to-cttsov2', /* Table Partition Settings */ cttsov2GlueTablePartition: { - instrumentRun: 'instrument_run', library: 'library', bclconvertData: 'bclconvert_data', fastqListRow: 'fastq_list_row', @@ -79,14 +92,11 @@ export class Cttsov2FastqListRowShowerCompleteToWorkflowDraftConstruct extends C /* Part 1: Generate the preamble (sfn to generate the portal run id and the workflow run name) */ - const sfn_preamble = new WorkflowDraftRunStateChangeCommonPreambleConstruct( + const sfnPreamble = new WorkflowDraftRunStateChangeCommonPreambleConstruct( this, `${this.Cttsov2FastqListRowShowerCompleteToWorkflowDraftRunDbRowMap.prefix}_sfn_preamble`, { - portalRunTablePartitionName: - this.Cttsov2FastqListRowShowerCompleteToWorkflowDraftRunDbRowMap.portalRunPartitionName, stateMachinePrefix: this.Cttsov2FastqListRowShowerCompleteToWorkflowDraftRunDbRowMap.prefix, - tableObj: props.workflowsTableObj, workflowName: this.Cttsov2FastqListRowShowerCompleteToWorkflowDraftRunDbRowMap.workflowName, workflowVersion: this.Cttsov2FastqListRowShowerCompleteToWorkflowDraftRunDbRowMap.workflowVersion, @@ -94,9 +104,40 @@ export class Cttsov2FastqListRowShowerCompleteToWorkflowDraftConstruct extends C ).stepFunctionObj; /* - Part 2: Build the sfn + Part 2: Build the engine parameters sfn */ - const draftMakerSfn = new sfn.StateMachine( + const engineParameterAndReadyEventMakerSfn = new GenerateWorkflowRunStateChangeReadyConstruct( + this, + 'fastqlistrow_complete_to_cttsov2_ready_submitter', + { + /* Event Placeholders */ + eventBusObj: props.eventBusObj, + outputSource: this.Cttsov2FastqListRowShowerCompleteToWorkflowDraftRunDbRowMap.outputSource, + payloadVersion: + this.Cttsov2FastqListRowShowerCompleteToWorkflowDraftRunDbRowMap.payloadVersion, + workflowName: this.Cttsov2FastqListRowShowerCompleteToWorkflowDraftRunDbRowMap.workflowName, + workflowVersion: + this.Cttsov2FastqListRowShowerCompleteToWorkflowDraftRunDbRowMap.workflowVersion, + + /* SSM Parameters */ + outputUriSsmParameterObj: props.outputUriSsmParameterObj, + icav2ProjectIdSsmParameterObj: props.icav2ProjectIdSsmParameterObj, + logsUriSsmParameterObj: props.logsUriSsmParameterObj, + cacheUriSsmParameterObj: props.cacheUriSsmParameterObj, + + /* Secrets */ + icav2AccessTokenSecretObj: props.icav2AccessTokenSecretObj, + + /* Prefixes */ + lambdaPrefix: this.Cttsov2FastqListRowShowerCompleteToWorkflowDraftRunDbRowMap.prefix, + stateMachinePrefix: this.Cttsov2FastqListRowShowerCompleteToWorkflowDraftRunDbRowMap.prefix, + } + ).stepFunctionObj; + + /* + Part 3: Build the sfn + */ + const inputMakerSfn = new sfn.StateMachine( this, 'fastq_list_row_complete_to_workflow_draft_run_events', { @@ -120,9 +161,6 @@ export class Cttsov2FastqListRowShowerCompleteToWorkflowDraftConstruct extends C __fastq_list_row_partition_name__: this.Cttsov2FastqListRowShowerCompleteToWorkflowDraftRunDbRowMap .cttsov2GlueTablePartition.fastqListRow, - __instrument_run_partition_name__: - this.Cttsov2FastqListRowShowerCompleteToWorkflowDraftRunDbRowMap - .cttsov2GlueTablePartition.instrumentRun, __library_partition_name__: this.Cttsov2FastqListRowShowerCompleteToWorkflowDraftRunDbRowMap .cttsov2GlueTablePartition.library, @@ -148,7 +186,8 @@ export class Cttsov2FastqListRowShowerCompleteToWorkflowDraftConstruct extends C buildCttsoV2Samplesheet.currentVersion.functionArn, /* Subfunctions */ - __sfn_preamble_state_machine_arn__: sfn_preamble.stateMachineArn, + __sfn_preamble_state_machine_arn__: sfnPreamble.stateMachineArn, + __launch_ready_event_sfn_arn__: engineParameterAndReadyEventMakerSfn.stateMachineArn, }, } ); @@ -157,18 +196,15 @@ export class Cttsov2FastqListRowShowerCompleteToWorkflowDraftConstruct extends C Part 3: Grant the internal sfn permissions */ // access the dynamodb table - props.cttsov2GlueTableObj.grantReadWriteData(draftMakerSfn.role); + props.cttsov2GlueTableObj.grantReadWriteData(inputMakerSfn); // Allow the sfn to invoke the lambda - buildCttsoV2Samplesheet.currentVersion.grantInvoke(draftMakerSfn.role); - - // Allow the sfn to submit events to the event bus - props.eventBusObj.grantPutEventsTo(draftMakerSfn.role); + buildCttsoV2Samplesheet.currentVersion.grantInvoke(inputMakerSfn); /* Allow step function to call nested state machine */ // Because we run a nested state machine, we need to add the permissions to the state machine role // See https://stackoverflow.com/questions/60612853/nested-step-function-in-a-step-function-unknown-error-not-authorized-to-cr - draftMakerSfn.addToRolePolicy( + inputMakerSfn.addToRolePolicy( new iam.PolicyStatement({ resources: [ `arn:aws:events:${cdk.Aws.REGION}:${cdk.Aws.ACCOUNT_ID}:rule/StepFunctionsGetEventsForStepFunctionsExecutionRule`, @@ -177,7 +213,8 @@ export class Cttsov2FastqListRowShowerCompleteToWorkflowDraftConstruct extends C }) ); // Allow the state machine to be able to invoke the preamble sfn - sfn_preamble.grantStartExecution(draftMakerSfn.role); + sfnPreamble.grantStartExecution(inputMakerSfn); + engineParameterAndReadyEventMakerSfn.grantStartExecution(inputMakerSfn); /* Part 4: Subscribe to the event bus for this event type @@ -203,7 +240,7 @@ export class Cttsov2FastqListRowShowerCompleteToWorkflowDraftConstruct extends C // Add target of event to be the state machine rule.addTarget( - new eventsTargets.SfnStateMachine(draftMakerSfn, { + new eventsTargets.SfnStateMachine(inputMakerSfn, { input: events.RuleTargetInput.fromEventPath('$.detail'), }) ); diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_4/fastq-list-row-event-shower-complete-to-cttsov2-draft/lambdas/build_cttsov2_samplesheet_py/build_cttso_v2_samplesheet.py b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_3/fastq-list-row-event-shower-complete-to-cttsov2-ready/lambdas/build_cttsov2_samplesheet_py/build_cttso_v2_samplesheet.py similarity index 100% rename from lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_4/fastq-list-row-event-shower-complete-to-cttsov2-draft/lambdas/build_cttsov2_samplesheet_py/build_cttso_v2_samplesheet.py rename to lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_3/fastq-list-row-event-shower-complete-to-cttsov2-ready/lambdas/build_cttsov2_samplesheet_py/build_cttso_v2_samplesheet.py diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_4/fastq-list-row-event-shower-complete-to-cttsov2-draft/step_functions_templates/fastq_list_row_shower_complete_event_to_cttsov2_draft_events_sfn_template.asl.json b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_3/fastq-list-row-event-shower-complete-to-cttsov2-ready/step_functions_templates/fastq_list_row_shower_complete_event_to_cttsov2_draft_events_sfn_template.asl.json similarity index 76% rename from lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_4/fastq-list-row-event-shower-complete-to-cttsov2-draft/step_functions_templates/fastq_list_row_shower_complete_event_to_cttsov2_draft_events_sfn_template.asl.json rename to lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_3/fastq-list-row-event-shower-complete-to-cttsov2-ready/step_functions_templates/fastq_list_row_shower_complete_event_to_cttsov2_draft_events_sfn_template.asl.json index 9a940345a..539e41f0d 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_4/fastq-list-row-event-shower-complete-to-cttsov2-draft/step_functions_templates/fastq_list_row_shower_complete_event_to_cttsov2_draft_events_sfn_template.asl.json +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_3/fastq-list-row-event-shower-complete-to-cttsov2-ready/step_functions_templates/fastq_list_row_shower_complete_event_to_cttsov2_draft_events_sfn_template.asl.json @@ -11,25 +11,51 @@ }, "Get Libraries from Instrument Run": { "Type": "Task", - "Resource": "arn:aws:states:::dynamodb:getItem", + "Resource": "arn:aws:states:::aws-sdk:dynamodb:scan", "Parameters": { "TableName": "${__table_name__}", - "Key": { - "id.$": "$.fastq_list_row_shower_complete_event_data.instrumentRunId", - "id_type": "${__instrument_run_partition_name__}" - } + "ExpressionAttributeValues": { + ":instrument_run_id": { + "S.$": "$.fastq_list_row_shower_complete_event_data.instrumentRunId" + }, + ":id_type": { + "S": "${__library_partition_name__}" + } + }, + "ExpressionAttributeNames": { + "#instrument_run_id": "instrument_run_id", + "#id_type": "id_type" + }, + "FilterExpression": "#instrument_run_id = :instrument_run_id AND #id_type = :id_type" }, - "ResultSelector": { - "library_list.$": "$.Item.library_set.SS" + "ResultPath": "$.get_libraries_on_instrument_run_step", + "Next": "Check library items is not empty" + }, + "Check library items is not empty": { + "Type": "Choice", + "Choices": [ + { + "Variable": "$.get_libraries_on_instrument_run_step.Items", + "IsPresent": true, + "Comment": "Items list exists", + "Next": "Collect Library Orcabus Ids" + } + ], + "Default": "Pass" + }, + "Collect Library Orcabus Ids": { + "Type": "Pass", + "Next": "Iterate over each library", + "Parameters": { + "library_orcabus_ids_list.$": "$.get_libraries_on_instrument_run_step.Items[*].id" }, - "ResultPath": "$.get_libraries_step", - "Next": "Iterate over each library" + "ResultPath": "$.collect_library_orcabus_ids_step" }, "Iterate over each library": { "Type": "Map", - "ItemsPath": "$.get_libraries_step.library_list", + "ItemsPath": "$.collect_library_orcabus_ids_step.library_orcabus_ids_list", "ItemSelector": { - "library_id.$": "$$.Map.Item.Value", + "library_orcabus_id.$": "$$.Map.Item.Value", "instrument_run_id.$": "$.fastq_list_row_shower_complete_event_data.instrumentRunId" }, "ItemProcessor": { @@ -50,13 +76,13 @@ "Parameters": { "TableName": "${__table_name__}", "Key": { - "id.$": "$.library_id", + "id.$": "$.library_orcabus_id", "id_type": "${__library_partition_name__}" } }, "ResultSelector": { "library_id.$": "$.Item.library_id.S", - "orcabus_id.$": "$.Item.orcabus_id.S", + "orcabus_id.$": "$.Item.id.S", "fastq_list_row_id_list.$": "$.Item.fastq_list_row_id_set.SS" }, "ResultPath": "$.get_library_obj_step", @@ -201,48 +227,40 @@ ] }, "ResultPath": "$.get_draft_inputs", - "Next": "Push cttsov2 draft event" + "Next": "Launch Ready Event" }, - "Push cttsov2 draft event": { + "Launch Ready Event": { "Type": "Task", - "Resource": "arn:aws:states:::events:putEvents", + "Resource": "arn:aws:states:::states:startExecution.sync:2", "Parameters": { - "Entries": [ - { - "Detail": { - "portalRunId.$": "$.get_draft_inputs.portal_run_id", - "timestamp.$": "$$.State.EnteredTime", - "status": "${__output_status__}", - "workflowName": "${__workflow_name__}", - "workflowVersion": "${__workflow_version__}", - "workflowRunName.$": "$.get_draft_inputs.workflow_run_name", - "linkedLibraries.$": "$.get_draft_inputs.linked_libraries", - "payload": { - "version": "${__payload_version__}", - "data": { - "inputs": { - "sampleId.$": "$.get_draft_inputs.sample_id", - "instrumentRunId.$": "$.instrument_run_id", - "samplesheet.$": "$.get_draft_inputs.samplesheet", - "fastqListRows.$": "$.get_draft_inputs.fastq_list_rows" - }, - "tags": { - "libraryId.$": "$.get_draft_inputs.sample_id", - "fastqListRowIds.$": "$.get_draft_inputs.fastq_list_row_ids" - } - } - } + "StateMachineArn": "${__launch_ready_event_sfn_arn__}", + "Input": { + "StatePayload": { + "portal_run_id.$": "$.get_draft_inputs.portal_run_id", + "workflow_run_name.$": "$.get_draft_inputs.workflow_run_name", + "linked_libraries.$": "$.get_draft_inputs.linked_libraries", + "data_inputs": { + "sampleId.$": "$.get_draft_inputs.sample_id", + "instrumentRunId.$": "$.instrument_run_id", + "samplesheet.$": "$.get_draft_inputs.samplesheet", + "fastqListRows.$": "$.get_draft_inputs.fastq_list_rows" }, - "DetailType": "${__detail_type__}", - "EventBusName": "${__event_bus_name__}", - "Source": "${__event_source__}" + "data_tags": { + "libraryId.$": "$.get_draft_inputs.sample_id", + "fastqListRowIds.$": "$.get_draft_inputs.fastq_list_row_ids" + } } - ] + } }, + "ResultPath": null, "End": true } } }, + "Next": "Pass" + }, + "Pass": { + "Type": "Pass", "End": true } } diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_5/cttsov2-draft-to-ready/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_5/cttsov2-draft-to-ready/index.ts deleted file mode 100644 index 4a8eae4cb..000000000 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/jb-weld/part_5/cttsov2-draft-to-ready/index.ts +++ /dev/null @@ -1,110 +0,0 @@ -import { Construct } from 'constructs'; -import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; -import * as ssm from 'aws-cdk-lib/aws-ssm'; -import path from 'path'; -import * as sfn from 'aws-cdk-lib/aws-stepfunctions'; -import * as events from 'aws-cdk-lib/aws-events'; -import * as secretsManager from 'aws-cdk-lib/aws-secretsmanager'; -import { WorkflowDraftRunStateChangeToWorkflowRunStateChangeReadyConstruct } from '../../../../../../../components/event-workflowdraftrunstatechange-to-workflowrunstatechange-ready'; - -/* -Part 5 - -Input Event source: `orcabus.cttsov2inputeventglue` -Input Event DetailType: `WorkflowDraftRunStateChange` -Input Event status: `draft` - -Output Event source: `orcabus.cttsov2inputeventglue` -Output Event DetailType: `WorkflowRunStateChange` -Output Event status: `ready` - -* The ctTSOv2InputMaker, subscribes to the cttsov2 input event glue (itself) and generates a ready event for the ctTSOv2ReadySfn - * For the cttso v2 workflow we require a samplesheet, a set of fastq list rows (provided in the last step) - * However, in order to be 'READY' we need to use a few more variables such as - * icaLogsUri, - * analysisOutputUri - * cacheUri - * projectId - * userReference -*/ - -export interface Cttsov2InputMakerConstructProps { - /* Event bus object */ - eventBusObj: events.IEventBus; - /* Tables */ - inputMakerTableObj: dynamodb.ITableV2; - /* SSM Parameter Objects */ - icav2ProjectIdSsmParameterObj: ssm.IStringParameter; - outputUriSsmParameterObj: ssm.IStringParameter; - logsUriSsmParameterObj: ssm.IStringParameter; - cacheUriSsmParameterObj: ssm.IStringParameter; - /* Secrets Objects */ - icav2AccessTokenSecretObj: secretsManager.ISecret; -} - -export class Cttsov2InputMakerConstruct extends Construct { - public readonly cttsov2InputMakerEventMap = { - prefix: 'jbweld-cttso-v2', - tablePartition: 'cttso_v2', - triggerSource: 'orcabus.cttsov2inputeventglue', - triggerStatus: 'DRAFT', - triggerDetailType: 'WorkflowDraftRunStateChange', - triggerWorkflowName: 'cttsov2', - outputSource: 'orcabus.cttsov2inputeventglue', - outputStatus: 'READY', - payloadVersion: '2024.05.24', - workflowName: 'cttsov2', - workflowVersion: '2.6.0', - }; - - constructor(scope: Construct, id: string, props: Cttsov2InputMakerConstructProps) { - super(scope, id); - - /* - Part 1: Build the draft to ready maker - */ - new WorkflowDraftRunStateChangeToWorkflowRunStateChangeReadyConstruct( - this, - 'cttso_v2_draft_to_ready_sfn', - { - /* - Set Input StateMachine Object - */ - lambdaPrefix: this.cttsov2InputMakerEventMap.prefix, - payloadVersion: this.cttsov2InputMakerEventMap.payloadVersion, - stateMachinePrefix: this.cttsov2InputMakerEventMap.prefix, - rulePrefix: this.cttsov2InputMakerEventMap.prefix, - - /* - Table objects - */ - tableObj: props.inputMakerTableObj, - tablePartitionName: this.cttsov2InputMakerEventMap.tablePartition, - - /* - Event Triggers - */ - eventBusObj: props.eventBusObj, - triggerDetailType: this.cttsov2InputMakerEventMap.triggerDetailType, - triggerSource: this.cttsov2InputMakerEventMap.triggerSource, - triggerStatus: this.cttsov2InputMakerEventMap.triggerStatus, - outputSource: this.cttsov2InputMakerEventMap.outputSource, - workflowName: this.cttsov2InputMakerEventMap.workflowName, - workflowVersion: this.cttsov2InputMakerEventMap.workflowVersion, - - /* - SSM Parameters - */ - icav2ProjectIdSsmParameterObj: props.icav2ProjectIdSsmParameterObj, - outputUriSsmParameterObj: props.outputUriSsmParameterObj, - logsUriSsmParameterObj: props.logsUriSsmParameterObj, - cacheUriSsmParameterObj: props.cacheUriSsmParameterObj, - - /* - Secrets - */ - icav2AccessTokenSecretObj: props.icav2AccessTokenSecretObj, - } - ); - } -} diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/index.ts index 816eb6387..b22ceb18a 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/index.ts +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/index.ts @@ -3,13 +3,11 @@ import * as events from 'aws-cdk-lib/aws-events'; import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; import * as ssm from 'aws-cdk-lib/aws-ssm'; import * as secretsManager from 'aws-cdk-lib/aws-secretsmanager'; -import { WgtsQcInitialiseInstrumentRunDbRowConstruct } from './part_1/initialise-wgts-instrument-run-db'; -import { WgtsQcInitialiseLibraryAndFastqListRowConstruct } from './part_2/initialise-wgts-library-dbs'; -import { WgtsQcPopulateFastqListRowConstruct } from './part_3/populate-fastq-list-row-dbs'; -import { WgtsQcFastqListRowShowerCompleteToWorkflowDraftConstruct } from './part_4/fastq-list-rows-shower-complete-to-wgts-qc-draft'; -import { WgtsQcInputMakerConstruct } from './part_5/wgts-qc-draft-to-ready'; -import { FastqListRowQcCompleteConstruct } from './part_6/push-fastq-list-row-qc-complete-event'; -import { WgtsQcLibraryQcCompleteConstruct } from './part_7/library-qc-complete-event'; +import { WgtsQcInitialiseLibraryAndFastqListRowConstruct } from './part_1/initialise-wgts-library-dbs'; +import { WgtsQcPopulateFastqListRowConstruct } from './part_2/populate-fastq-list-row-dbs'; +import { WgtsQcFastqListRowShowerCompleteToWorkflowReadyConstruct } from './part_3/fastq-list-rows-shower-complete-to-wgts-qc'; +import { FastqListRowQcCompleteConstruct } from './part_4/push-fastq-list-row-qc-complete-event'; +import { WgtsQcLibraryQcCompleteConstruct } from './part_5/library-qc-complete-event'; /* Provide the glue to get from the bssh fastq copy manager to submitting wgts qc analyses @@ -20,7 +18,6 @@ export interface wgtsQcGlueHandlerConstructProps { eventBusObj: events.IEventBus; /* Tables */ wgtsQcGlueTableObj: dynamodb.ITableV2; - inputMakerTableObj: dynamodb.ITableV2; /* SSM Parameters */ analysisOutputUriSsmParameterObj: ssm.IStringParameter; analysisLogsUriSsmParameterObj: ssm.IStringParameter; @@ -36,24 +33,6 @@ export class WgtsQcGlueHandlerConstruct extends Construct { /* Part 1 - Input Event Source: `orcabus.instrumentrunmanager` - Input Event DetailType: `SamplesheetShowerStateChange` - Input Event status: `SamplesheetRegisteredEventShowerStarting` - - * Initialise wgts qc instrument db construct - */ - const wgts_qc_initialise_instrument_run_db_row = - new WgtsQcInitialiseInstrumentRunDbRowConstruct( - this, - 'initialise_wgts_qc_instrument_run_db_row', - { - eventBusObj: props.eventBusObj, - tableObj: props.wgtsQcGlueTableObj, - } - ); - - /* - Part 2 Input Event Source: `orcabus.instrumentrunmanager` Input Event DetailType: `SamplesheetMetadataUnion` @@ -72,7 +51,7 @@ export class WgtsQcGlueHandlerConstruct extends Construct { ); /* - Part 3 + Part 2 Input Event Source: `orcabus.instrumentrunmanager` Input Event DetailType: `FastqListRowStateChange` @@ -90,7 +69,7 @@ export class WgtsQcGlueHandlerConstruct extends Construct { ); /* - Part 4 + Part 3 Input Event Source: `orcabus.instrumentrunmanager` Input Event DetailType: `FastqListRowStateChange` @@ -98,52 +77,35 @@ export class WgtsQcGlueHandlerConstruct extends Construct { Output Event source: `orcabus.wgtsqcinputeventglue` Output Event DetailType: `WorkflowDraftRunStateChange` - Output Event status: `draft` + Output Event status: `READY` * Trigger wgts qc events collecting all wgts qc libraries in the run */ const fastq_list_row_shower_complete_to_workflow_draft = - new WgtsQcFastqListRowShowerCompleteToWorkflowDraftConstruct( + new WgtsQcFastqListRowShowerCompleteToWorkflowReadyConstruct( this, 'fastq_list_row_shower_complete_to_workflow_draft', { - workflowsTableObj: props.inputMakerTableObj, + /* Events */ eventBusObj: props.eventBusObj, + + /* Tables */ wgtsQcGlueTableObj: props.wgtsQcGlueTableObj, - } - ); - /* - Part 5 - Input Event source: `orcabus.wgtsqcinputeventglue` - Input Event DetailType: `WorkflowDraftRunStateChange` - Input Event status: `draft` + /* SSM Parameters */ + cacheUriSsmParameterObj: props.analysisCacheUriSsmParameterObj, + icav2ProjectIdSsmParameterObj: props.icav2ProjectIdSsmParameterObj, + logsUriSsmParameterObj: props.analysisLogsUriSsmParameterObj, + outputUriSsmParameterObj: props.analysisOutputUriSsmParameterObj, - Output Event source: `orcabus.wgtsqcinputeventglue` - Output Event DetailType: `WorkflowRunStateChange` - Output Event status: `ready` - */ - const fastqListRowsToWgtsQcInputMaker = new WgtsQcInputMakerConstruct( - this, - 'fastq_list_rows_to_wgts_qc_input_maker', - { - /* Event bus */ - eventBusObj: props.eventBusObj, - /* Tables */ - inputMakerTableObj: props.inputMakerTableObj, - /* SSM Param objects */ - icav2ProjectIdSsmParameterObj: props.icav2ProjectIdSsmParameterObj, - outputUriSsmParameterObj: props.analysisOutputUriSsmParameterObj, - cacheUriSsmParameterObj: props.analysisCacheUriSsmParameterObj, - logsUriSsmParameterObj: props.analysisLogsUriSsmParameterObj, - /* Secrets */ - icav2AccessTokenSecretObj: props.icav2AccessTokenSecretObj, - } - ); + /* Secrets */ + icav2AccessTokenSecretObj: props.icav2AccessTokenSecretObj, + } + ); /* - Part 6 + Part 4 Input Event Source: `orcabus.workflowmanager` Input Event DetailType: `WorkflowRunStateChange` @@ -168,7 +130,7 @@ export class WgtsQcGlueHandlerConstruct extends Construct { ); /* - Part 7 + Part 5 Input Event Source: `orcabus.wgtsqcinputeventglue` Input Event DetailType: `FastqListRowStateChange` diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_1/initialise-wgts-instrument-run-db/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_1/initialise-wgts-instrument-run-db/index.ts deleted file mode 100644 index 5de6224dc..000000000 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_1/initialise-wgts-instrument-run-db/index.ts +++ /dev/null @@ -1,88 +0,0 @@ -import { Construct } from 'constructs'; -import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; -import path from 'path'; -import * as sfn from 'aws-cdk-lib/aws-stepfunctions'; -import * as events from 'aws-cdk-lib/aws-events'; -import * as eventsTargets from 'aws-cdk-lib/aws-events-targets'; - -/* -Part 1 - -Input Event Source: `orcabus.instrumentrunmanager` -Input Event DetailType: `SamplesheetShowerStateChange` -Input Event status: `SamplesheetRegisteredEventShowerStarting` - -* Initialise wgts instrument db construct -*/ - -export interface WgtsQcInitialiseInstrumentRunDbRowConstructProps { - tableObj: dynamodb.ITableV2; - eventBusObj: events.IEventBus; -} - -export class WgtsQcInitialiseInstrumentRunDbRowConstruct extends Construct { - public readonly WgtsQcInitialiseInstrumentRunDbRowMap = { - prefix: 'kwik-make-instrument-run-row', - tablePartition: 'instrument_run', - triggerSource: 'orcabus.instrumentrunmanager', - triggerStatus: 'SamplesheetRegisteredEventShowerStarting', - triggerDetailType: 'SamplesheetShowerStateChange', - }; - - constructor( - scope: Construct, - id: string, - props: WgtsQcInitialiseInstrumentRunDbRowConstructProps - ) { - super(scope, id); - - /* - Part 1: Build the internal sfn - */ - const inputMakerSfn = new sfn.StateMachine(this, 'initialise_instrument_run_db_row', { - stateMachineName: `${this.WgtsQcInitialiseInstrumentRunDbRowMap.prefix}-initialise-run-db-row`, - definitionBody: sfn.DefinitionBody.fromFile( - path.join( - __dirname, - 'step_functions_templates', - 'initialise_wgts_instrument_run_db_sfn_template.asl.json' - ) - ), - definitionSubstitutions: { - __table_name__: props.tableObj.tableName, - __instrument_run_partition_name__: - this.WgtsQcInitialiseInstrumentRunDbRowMap.tablePartition, - }, - }); - - /* - Part 2: Grant the internal sfn permissions - */ - // access the dynamodb table - props.tableObj.grantReadWriteData(inputMakerSfn.role); - - /* - Part 3: Subscribe to the event bus and trigger the internal sfn - */ - const rule = new events.Rule(this, 'wgts_subscribe_to_samplesheet_shower', { - ruleName: `stacky-${this.WgtsQcInitialiseInstrumentRunDbRowMap.prefix}-rule`, - eventBus: props.eventBusObj, - eventPattern: { - source: [this.WgtsQcInitialiseInstrumentRunDbRowMap.triggerSource], - detailType: [this.WgtsQcInitialiseInstrumentRunDbRowMap.triggerDetailType], - detail: { - status: [ - { 'equals-ignore-case': this.WgtsQcInitialiseInstrumentRunDbRowMap.triggerStatus }, - ], - }, - }, - }); - - // Add target of event to be the state machine - rule.addTarget( - new eventsTargets.SfnStateMachine(inputMakerSfn, { - input: events.RuleTargetInput.fromEventPath('$.detail'), - }) - ); - } -} diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_1/initialise-wgts-instrument-run-db/step_functions_templates/initialise_wgts_instrument_run_db_sfn_template.asl.json b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_1/initialise-wgts-instrument-run-db/step_functions_templates/initialise_wgts_instrument_run_db_sfn_template.asl.json deleted file mode 100644 index d56d8d202..000000000 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_1/initialise-wgts-instrument-run-db/step_functions_templates/initialise_wgts_instrument_run_db_sfn_template.asl.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "Comment": "A description of my state machine", - "StartAt": "Set DB Inputs", - "States": { - "Set DB Inputs": { - "Type": "Pass", - "Next": "Initialise Instrument Run Item", - "ResultPath": "$.db_inputs", - "Parameters": { - "instrument_run_id.$": "$.payload.data.instrumentRunId" - } - }, - "Initialise Instrument Run Item": { - "Type": "Task", - "Resource": "arn:aws:states:::dynamodb:putItem", - "Parameters": { - "TableName": "${__table_name__}", - "Item": { - "id": { - "S.$": "$.db_inputs.instrument_run_id" - }, - "id_type": { - "S": "${__instrument_run_partition_name__}" - } - } - }, - "End": true, - "ResultPath": null - } - } -} diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_2/initialise-wgts-library-dbs/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_1/initialise-wgts-library-dbs/index.ts similarity index 100% rename from lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_2/initialise-wgts-library-dbs/index.ts rename to lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_1/initialise-wgts-library-dbs/index.ts diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_2/initialise-wgts-library-dbs/step_functions_templates/initialise_wgts_library_db_sfn_template.asl.json b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_1/initialise-wgts-library-dbs/step_functions_templates/initialise_wgts_library_db_sfn_template.asl.json similarity index 78% rename from lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_2/initialise-wgts-library-dbs/step_functions_templates/initialise_wgts_library_db_sfn_template.asl.json rename to lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_1/initialise-wgts-library-dbs/step_functions_templates/initialise_wgts_library_db_sfn_template.asl.json index 1f95225bc..9c0f702e3 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_2/initialise-wgts-library-dbs/step_functions_templates/initialise_wgts_library_db_sfn_template.asl.json +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_1/initialise-wgts-library-dbs/step_functions_templates/initialise_wgts_library_db_sfn_template.asl.json @@ -18,7 +18,8 @@ "index.$": "$$.Map.Item.Index", "fastq_list_row_objs.$": "$.input_payload_data.fastqListRows", "instrument_run_id.$": "$.input_payload_data.instrumentRunId", - "library_id.$": "$.input_payload_data.library.libraryId" + "library_id.$": "$.input_payload_data.library.libraryId", + "library_orcabus_id.$": "$.input_payload_data.library.orcabusId" }, "ItemProcessor": { "ProcessorConfig": { @@ -49,32 +50,6 @@ "Add DataBase Inputs": { "Type": "Parallel", "Branches": [ - { - "StartAt": "Append Library and FastqListRowID to Instrument Run ID", - "States": { - "Append Library and FastqListRowID to Instrument Run ID": { - "Type": "Task", - "Resource": "arn:aws:states:::dynamodb:updateItem", - "Parameters": { - "TableName": "${__table_name__}", - "Key": { - "id.$": "$.input_payload_data.instrumentRunId", - "id_type": "${__instrument_run_partition_name__}" - }, - "UpdateExpression": "ADD library_set :library_set, fastq_list_row_id_set :fastq_list_row_id_set", - "ExpressionAttributeValues": { - ":library_set": { - "SS.$": "States.Array($.input_payload_data.library.libraryId)" - }, - ":fastq_list_row_id_set": { - "SS.$": "$.bclconvert_and_fastq_list_row_id_inputs[*].fastq_list_row_id" - } - } - }, - "End": true - } - } - }, { "StartAt": "Initialise Library ID", "States": { @@ -85,14 +60,11 @@ "TableName": "${__table_name__}", "Item": { "id": { - "S.$": "$.input_payload_data.library.libraryId" + "S.$": "$.input_payload_data.library.orcabusId" }, "id_type": { "S": "${__library_partition_name__}" }, - "orcabus_id": { - "S.$": "$.input_payload_data.library.orcabusId" - }, "library_id": { "S.$": "$.input_payload_data.library.libraryId" }, @@ -101,6 +73,9 @@ }, "fastq_list_row_id_set": { "SS.$": "$.bclconvert_and_fastq_list_row_id_inputs[*].fastq_list_row_id" + }, + "instrument_run_id": { + "S.$": "$.input_payload_data.instrumentRunId" } } }, @@ -117,6 +92,7 @@ "ItemSelector": { "fastq_list_row_id.$": "$$.Map.Item.Value.fastq_list_row_id", "library_id.$": "$.input_payload_data.library.libraryId", + "library_orcabus_id.$": "$.input_payload_data.library.orcabusId", "instrument_run_id.$": "$.input_payload_data.instrumentRunId" }, "ItemProcessor": { @@ -140,6 +116,9 @@ "library_id": { "S.$": "$.library_id" }, + "library_orcabus_id": { + "S.$": "$.library_orcabus_id" + }, "instrument_run_id": { "S.$": "$.instrument_run_id" } diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_3/populate-fastq-list-row-dbs/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_2/populate-fastq-list-row-dbs/index.ts similarity index 100% rename from lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_3/populate-fastq-list-row-dbs/index.ts rename to lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_2/populate-fastq-list-row-dbs/index.ts diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_3/populate-fastq-list-row-dbs/step_functions_templates/update_fastq_list_row_sfn_template.asl.json b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_2/populate-fastq-list-row-dbs/step_functions_templates/update_fastq_list_row_sfn_template.asl.json similarity index 96% rename from lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_3/populate-fastq-list-row-dbs/step_functions_templates/update_fastq_list_row_sfn_template.asl.json rename to lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_2/populate-fastq-list-row-dbs/step_functions_templates/update_fastq_list_row_sfn_template.asl.json index 718abf09d..c86d99a6d 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_3/populate-fastq-list-row-dbs/step_functions_templates/update_fastq_list_row_sfn_template.asl.json +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_2/populate-fastq-list-row-dbs/step_functions_templates/update_fastq_list_row_sfn_template.asl.json @@ -7,8 +7,7 @@ "Next": "DynamoDB GetItem", "Parameters": { "fastq_list_row_id.$": "$.payload.data.fastqListRow.rgid", - "fastq_list_row_event_data.$": "$.payload.data.fastqListRow", - "instrument_run_id.$": "$.payload.data.instrumentRunId" + "fastq_list_row_event_data.$": "$.payload.data.fastqListRow" } }, "DynamoDB GetItem": { diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_4/fastq-list-rows-shower-complete-to-wgts-qc-draft/Readme.md b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_3/fastq-list-rows-shower-complete-to-wgts-qc/Readme.md similarity index 100% rename from lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_4/fastq-list-rows-shower-complete-to-wgts-qc-draft/Readme.md rename to lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_3/fastq-list-rows-shower-complete-to-wgts-qc/Readme.md diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_4/fastq-list-rows-shower-complete-to-wgts-qc-draft/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_3/fastq-list-rows-shower-complete-to-wgts-qc/index.ts similarity index 72% rename from lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_4/fastq-list-rows-shower-complete-to-wgts-qc-draft/index.ts rename to lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_3/fastq-list-rows-shower-complete-to-wgts-qc/index.ts index 8bed7d003..4d6f679db 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_4/fastq-list-rows-shower-complete-to-wgts-qc-draft/index.ts +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_3/fastq-list-rows-shower-complete-to-wgts-qc/index.ts @@ -9,9 +9,12 @@ import * as lambda from 'aws-cdk-lib/aws-lambda'; import { WorkflowDraftRunStateChangeCommonPreambleConstruct } from '../../../../../../../components/sfn-workflowdraftrunstatechange-common-preamble'; import * as iam from 'aws-cdk-lib/aws-iam'; import * as cdk from 'aws-cdk-lib'; +import * as ssm from 'aws-cdk-lib/aws-ssm'; +import * as secretsManager from 'aws-cdk-lib/aws-secretsmanager'; +import { GenerateWorkflowRunStateChangeReadyConstruct } from '../../../../../../../components/sfn-generate-workflowrunstatechange-ready-event'; /* -Part 4 +Part 3 Input Event Source: `orcabus.instrumentrunmanager` Input Event DetailType: `FastqListRowStateChange` @@ -19,24 +22,33 @@ Input Event status: `FastqListRowEventShowerComplete` Output Event source: `orcabus.wgtsinputeventglue` Output Event DetailType: `WorkflowDraftRunStateChange` -Output Event status: `draft` +Output Event status: `READY` * Trigger wgts events collecting all wgts in the run */ export interface WgtsQcFastqListRowShowerCompleteToWorkflowDraftRunDbRowConstructProps { - workflowsTableObj: dynamodb.ITableV2; - wgtsQcGlueTableObj: dynamodb.ITableV2; + /* Event Bus Object */ eventBusObj: events.IEventBus; + + /* DynamoDB Table */ + wgtsQcGlueTableObj: dynamodb.ITableV2; + + /* SSM Param objects */ + icav2ProjectIdSsmParameterObj: ssm.IStringParameter; + outputUriSsmParameterObj: ssm.IStringParameter; + cacheUriSsmParameterObj: ssm.IStringParameter; + logsUriSsmParameterObj: ssm.IStringParameter; + /* Secrets */ + icav2AccessTokenSecretObj: secretsManager.ISecret; } -export class WgtsQcFastqListRowShowerCompleteToWorkflowDraftConstruct extends Construct { +export class WgtsQcFastqListRowShowerCompleteToWorkflowReadyConstruct extends Construct { public readonly WgtsQcFastqListRowShowerCompleteToWorkflowDraftRunDbRowMap = { /* General settings */ prefix: 'kwik-fqlr-shower-to-wgts-qc', /* Table Partition Settings */ wgtsGlueTablePartition: { - instrumentRun: 'instrument_run', library: 'library', fastqListRow: 'fastq_list_row', }, @@ -45,14 +57,12 @@ export class WgtsQcFastqListRowShowerCompleteToWorkflowDraftConstruct extends Co triggerSource: 'orcabus.instrumentrunmanager', triggerStatus: 'FastqListRowEventShowerComplete', triggerDetailType: 'FastqListRowShowerStateChange', - /* Output Event Settings */ + /* Output Source */ outputSource: 'orcabus.wgtsqcinputeventglue', - outputStatus: 'DRAFT', - outputDetailType: 'WorkflowDraftRunStateChange', /* Payload version */ payloadVersion: '0.1.0', /* Workflow settings */ - workflowName: 'wgtsQc', + workflowName: 'wgts-qc', workflowVersion: '4.2.4', }; @@ -66,14 +76,11 @@ export class WgtsQcFastqListRowShowerCompleteToWorkflowDraftConstruct extends Co /* Part 1: Generate the preamble (sfn to generate the portal run id and the workflow run name) */ - const sfn_preamble = new WorkflowDraftRunStateChangeCommonPreambleConstruct( + const sfnPreamble = new WorkflowDraftRunStateChangeCommonPreambleConstruct( this, `${this.WgtsQcFastqListRowShowerCompleteToWorkflowDraftRunDbRowMap.prefix}_sfn_preamble`, { - portalRunTablePartitionName: - this.WgtsQcFastqListRowShowerCompleteToWorkflowDraftRunDbRowMap.portalRunPartitionName, stateMachinePrefix: this.WgtsQcFastqListRowShowerCompleteToWorkflowDraftRunDbRowMap.prefix, - tableObj: props.workflowsTableObj, workflowName: this.WgtsQcFastqListRowShowerCompleteToWorkflowDraftRunDbRowMap.workflowName, workflowVersion: this.WgtsQcFastqListRowShowerCompleteToWorkflowDraftRunDbRowMap.workflowVersion, @@ -81,7 +88,7 @@ export class WgtsQcFastqListRowShowerCompleteToWorkflowDraftConstruct extends Co ).stepFunctionObj; /* - Part 1: Build the lambdas + Part 2: Build the lambdas */ const generateEventDataLambdaObj = new PythonFunction(this, 'generate_event_data', { entry: path.join(__dirname, 'lambdas', 'generate_event_data_py'), @@ -93,7 +100,38 @@ export class WgtsQcFastqListRowShowerCompleteToWorkflowDraftConstruct extends Co }); /* - Part 2: Build the sfn + Part 3: Build the engine parameters sfn + */ + const engineParameterAndReadyEventMakerSfn = new GenerateWorkflowRunStateChangeReadyConstruct( + this, + 'fastqlistrow_complete_to_wgtsqc_ready_submitter', + { + /* Event Placeholders */ + eventBusObj: props.eventBusObj, + outputSource: this.WgtsQcFastqListRowShowerCompleteToWorkflowDraftRunDbRowMap.outputSource, + payloadVersion: + this.WgtsQcFastqListRowShowerCompleteToWorkflowDraftRunDbRowMap.payloadVersion, + workflowName: this.WgtsQcFastqListRowShowerCompleteToWorkflowDraftRunDbRowMap.workflowName, + workflowVersion: + this.WgtsQcFastqListRowShowerCompleteToWorkflowDraftRunDbRowMap.workflowVersion, + + /* SSM Parameters */ + outputUriSsmParameterObj: props.outputUriSsmParameterObj, + icav2ProjectIdSsmParameterObj: props.icav2ProjectIdSsmParameterObj, + logsUriSsmParameterObj: props.logsUriSsmParameterObj, + cacheUriSsmParameterObj: props.cacheUriSsmParameterObj, + + /* Secrets */ + icav2AccessTokenSecretObj: props.icav2AccessTokenSecretObj, + + /* Prefixes */ + lambdaPrefix: this.WgtsQcFastqListRowShowerCompleteToWorkflowDraftRunDbRowMap.prefix, + stateMachinePrefix: this.WgtsQcFastqListRowShowerCompleteToWorkflowDraftRunDbRowMap.prefix, + } + ).stepFunctionObj; + + /* + Part 4: Build the inputs sfn */ const inputMakerSfn = new sfn.StateMachine( this, @@ -116,24 +154,12 @@ export class WgtsQcFastqListRowShowerCompleteToWorkflowDraftConstruct extends Co __fastq_list_row_partition_name__: this.WgtsQcFastqListRowShowerCompleteToWorkflowDraftRunDbRowMap.wgtsGlueTablePartition .fastqListRow, - __instrument_run_partition_name__: - this.WgtsQcFastqListRowShowerCompleteToWorkflowDraftRunDbRowMap.wgtsGlueTablePartition - .instrumentRun, __library_partition_name__: this.WgtsQcFastqListRowShowerCompleteToWorkflowDraftRunDbRowMap.wgtsGlueTablePartition .library, __portal_run_partition_name__: this.WgtsQcFastqListRowShowerCompleteToWorkflowDraftRunDbRowMap.portalRunPartitionName, /* Output event settings */ - // Event detail - __event_source__: - this.WgtsQcFastqListRowShowerCompleteToWorkflowDraftRunDbRowMap.outputSource, - __detail_type__: - this.WgtsQcFastqListRowShowerCompleteToWorkflowDraftRunDbRowMap.outputDetailType, - __output_status__: - this.WgtsQcFastqListRowShowerCompleteToWorkflowDraftRunDbRowMap.outputStatus, - __payload_version__: - this.WgtsQcFastqListRowShowerCompleteToWorkflowDraftRunDbRowMap.payloadVersion, // Workflow detail __workflow_name__: this.WgtsQcFastqListRowShowerCompleteToWorkflowDraftRunDbRowMap.workflowName, @@ -145,7 +171,8 @@ export class WgtsQcFastqListRowShowerCompleteToWorkflowDraftConstruct extends Co generateEventDataLambdaObj.currentVersion.functionArn, /* Nested sfn */ - __sfn_preamble_state_machine_arn__: sfn_preamble.stateMachineArn, + __sfn_preamble_state_machine_arn__: sfnPreamble.stateMachineArn, + __launch_ready_event_sfn_arn__: engineParameterAndReadyEventMakerSfn.stateMachineArn, }, } ); @@ -154,13 +181,10 @@ export class WgtsQcFastqListRowShowerCompleteToWorkflowDraftConstruct extends Co Part 3: Grant the internal sfn permissions */ // access the dynamodb table - props.wgtsQcGlueTableObj.grantReadWriteData(inputMakerSfn.role); + props.wgtsQcGlueTableObj.grantReadWriteData(inputMakerSfn); // Allow the sfn to invoke the lambda - generateEventDataLambdaObj.currentVersion.grantInvoke(inputMakerSfn.role); - - // Allow the sfn to submit events to the event bus - props.eventBusObj.grantPutEventsTo(inputMakerSfn.role); + generateEventDataLambdaObj.currentVersion.grantInvoke(inputMakerSfn); /* Allow step function to call nested state machine */ // Because we run a nested state machine, we need to add the permissions to the state machine role @@ -173,8 +197,10 @@ export class WgtsQcFastqListRowShowerCompleteToWorkflowDraftConstruct extends Co actions: ['events:PutTargets', 'events:PutRule', 'events:DescribeRule'], }) ); + // Allow the state machine to be able to invoke the preamble sfn - sfn_preamble.grantStartExecution(inputMakerSfn.role); + sfnPreamble.grantStartExecution(inputMakerSfn); + engineParameterAndReadyEventMakerSfn.grantStartExecution(inputMakerSfn); /* Part 4: Subscribe to the event bus for this event type diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_4/fastq-list-rows-shower-complete-to-wgts-qc-draft/lambdas/generate_event_data_py/generate_event_data.py b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_3/fastq-list-rows-shower-complete-to-wgts-qc/lambdas/generate_event_data_py/generate_event_data.py similarity index 100% rename from lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_4/fastq-list-rows-shower-complete-to-wgts-qc-draft/lambdas/generate_event_data_py/generate_event_data.py rename to lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_3/fastq-list-rows-shower-complete-to-wgts-qc/lambdas/generate_event_data_py/generate_event_data.py diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_4/fastq-list-rows-shower-complete-to-wgts-qc-draft/step_functions_templates/fastq_list_rows_shower_complete_to_wgts_qc_draft_sfn_template.asl.json b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_3/fastq-list-rows-shower-complete-to-wgts-qc/step_functions_templates/fastq_list_rows_shower_complete_to_wgts_qc_draft_sfn_template.asl.json similarity index 77% rename from lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_4/fastq-list-rows-shower-complete-to-wgts-qc-draft/step_functions_templates/fastq_list_rows_shower_complete_to_wgts_qc_draft_sfn_template.asl.json rename to lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_3/fastq-list-rows-shower-complete-to-wgts-qc/step_functions_templates/fastq_list_rows_shower_complete_to_wgts_qc_draft_sfn_template.asl.json index 4737ca0b2..041c3cad1 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_4/fastq-list-rows-shower-complete-to-wgts-qc-draft/step_functions_templates/fastq_list_rows_shower_complete_to_wgts_qc_draft_sfn_template.asl.json +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_3/fastq-list-rows-shower-complete-to-wgts-qc/step_functions_templates/fastq_list_rows_shower_complete_to_wgts_qc_draft_sfn_template.asl.json @@ -11,25 +11,55 @@ }, "Get Libraries from Instrument Run": { "Type": "Task", - "Resource": "arn:aws:states:::dynamodb:getItem", + "Resource": "arn:aws:states:::aws-sdk:dynamodb:scan", "Parameters": { "TableName": "${__table_name__}", - "Key": { - "id.$": "$.fastq_list_row_shower_complete_event_data.instrumentRunId", - "id_type": "${__instrument_run_partition_name__}" - } - }, - "ResultSelector": { - "library_list.$": "$.Item.library_set.SS" + "ExpressionAttributeValues": { + ":instrument_run_id": { + "S.$": "$.fastq_list_row_shower_complete_event_data.instrumentRunId" + }, + ":id_type": { + "S": "${__library_partition_name__}" + } + }, + "ExpressionAttributeNames": { + "#instrument_run_id": "instrument_run_id", + "#id_type": "id_type" + }, + "FilterExpression": "#instrument_run_id = :instrument_run_id AND #id_type = :id_type" }, + "ResultPath": "$.get_libraries_on_instrument_run_step", + "Next": "Check library items is not empty" + }, + "Check library items is not empty": { + "Type": "Choice", + "Choices": [ + { + "Variable": "$.get_libraries_on_instrument_run_step.Items", + "IsPresent": true, + "Comment": "Items list exists", + "Next": "Collect Library Orcabus Ids" + } + ], + "Default": "Pass" + }, + "Pass": { + "Type": "Pass", + "End": true + }, + "Collect Library Orcabus Ids": { + "Type": "Pass", "Next": "Iterate over each library", - "ResultPath": "$.get_libraries_step" + "Parameters": { + "library_orcabus_ids_list.$": "$.get_libraries_on_instrument_run_step.Items[*].id" + }, + "ResultPath": "$.collect_library_orcabus_ids_step" }, "Iterate over each library": { "Type": "Map", - "ItemsPath": "$.get_libraries_step.library_list", + "ItemsPath": "$.collect_library_orcabus_ids_step.library_orcabus_ids_list", "ItemSelector": { - "library_id.$": "$$.Map.Item.Value", + "library_orcabus_id.$": "$$.Map.Item.Value", "instrument_run_id.$": "$.fastq_list_row_shower_complete_event_data.instrumentRunId" }, "ItemProcessor": { @@ -44,14 +74,14 @@ "Parameters": { "TableName": "${__table_name__}", "Key": { - "id.$": "$.library_id", + "id.$": "$.library_orcabus_id", "id_type": "${__library_partition_name__}" } }, "Next": "For each FastqListRowID", "ResultSelector": { "library_id.$": "$.Item.library_id.S", - "orcabus_id.$": "$.Item.orcabus_id.S", + "library_orcabus_id.$": "$.Item.id.S", "sample_type.$": "$.Item.sample_type.S", "fastq_list_row_id_list.$": "$.Item.fastq_list_row_id_set.SS" }, @@ -64,7 +94,7 @@ "fastq_list_row_id.$": "$$.Map.Item.Value", "sample_type.$": "$.get_fastq_list_row_ids_step.sample_type", "library_id.$": "$.get_fastq_list_row_ids_step.library_id", - "orcabus_id.$": "$.get_fastq_list_row_ids_step.orcabus_id", + "library_orcabus_id.$": "$.get_fastq_list_row_ids_step.library_orcabus_id", "instrument_run_id.$": "$.instrument_run_id" }, "ItemProcessor": { @@ -94,9 +124,9 @@ "fastq_list_row.$": "States.StringToJson($.Item.fastq_list_row_json.S)" }, "ResultPath": "$.get_fastq_list_row_step", - "Next": "Generate Wgts Draft Event Data" + "Next": "Generate Wgts Input and Tags Event Data" }, - "Generate Wgts Draft Event Data": { + "Generate Wgts Input and Tags Event Data": { "Type": "Task", "Resource": "arn:aws:states:::lambda:invoke", "Parameters": { @@ -214,41 +244,29 @@ "Wait 1 Second": { "Type": "Wait", "Seconds": 1, - "Next": "Push wgts draft event" + "Next": "Push wgts ready event" }, - "Push wgts draft event": { + "Push wgts ready event": { "Type": "Task", - "Resource": "arn:aws:states:::events:putEvents", + "Resource": "arn:aws:states:::states:startExecution.sync:2", "Parameters": { - "Entries": [ - { - "Detail": { - "portalRunId.$": "$.get_per_workflow_run_inputs_step.portal_run_id", - "timestamp.$": "$$.State.EnteredTime", - "status": "${__output_status__}", - "workflowName": "${__workflow_name__}", - "workflowVersion": "${__workflow_version__}", - "workflowRunName.$": "$.get_per_workflow_run_inputs_step.workflow_run_name", - "linkedLibraries": [ - { - "libraryId.$": "$.library_id", - "orcabusId.$": "$.orcabus_id" - } - ], - "payload": { - "version": "${__payload_version__}", - "data": { - "inputs.$": "$.get_per_workflow_run_inputs_step.event_data", - "tags.$": "$.get_per_workflow_run_inputs_step.event_tags" - } + "StateMachineArn": "${__launch_ready_event_sfn_arn__}", + "Input": { + "StatePayload": { + "portal_run_id.$": "$.get_per_workflow_run_inputs_step.portal_run_id", + "workflow_run_name.$": "$.get_per_workflow_run_inputs_step.workflow_run_name", + "linked_libraries": [ + { + "libraryId.$": "$.library_id", + "orcabusId.$": "$.library_orcabus_id" } - }, - "DetailType": "${__detail_type__}", - "EventBusName": "${__event_bus_name__}", - "Source": "${__event_source__}" + ], + "data_inputs.$": "$.get_per_workflow_run_inputs_step.event_data", + "data_tags.$": "$.get_per_workflow_run_inputs_step.event_tags" } - ] + } }, + "ResultPath": null, "End": true } } diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_6/push-fastq-list-row-qc-complete-event/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_4/push-fastq-list-row-qc-complete-event/index.ts similarity index 99% rename from lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_6/push-fastq-list-row-qc-complete-event/index.ts rename to lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_4/push-fastq-list-row-qc-complete-event/index.ts index 0539d55ad..c1eefcd67 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_6/push-fastq-list-row-qc-complete-event/index.ts +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_4/push-fastq-list-row-qc-complete-event/index.ts @@ -40,7 +40,7 @@ export class FastqListRowQcCompleteConstruct extends Construct { triggerSource: 'orcabus.workflowmanager', triggerStatus: 'succeeded', triggerDetailType: 'WorkflowRunStateChange', - triggerWorkflowName: 'wgtsQc', + triggerWorkflowName: 'wgts-qc', outputSource: 'orcabus.wgtsqcinputeventglue', outputDetailType: 'FastqListRowStateChange', outputStatus: 'QC_COMPLETE', diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_6/push-fastq-list-row-qc-complete-event/lambdas/collect_qc_metrics_from_alignment_directory_py/collect_qc_metrics_from_alignment_directory.py b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_4/push-fastq-list-row-qc-complete-event/lambdas/collect_qc_metrics_from_alignment_directory_py/collect_qc_metrics_from_alignment_directory.py similarity index 100% rename from lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_6/push-fastq-list-row-qc-complete-event/lambdas/collect_qc_metrics_from_alignment_directory_py/collect_qc_metrics_from_alignment_directory.py rename to lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_4/push-fastq-list-row-qc-complete-event/lambdas/collect_qc_metrics_from_alignment_directory_py/collect_qc_metrics_from_alignment_directory.py diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_6/push-fastq-list-row-qc-complete-event/lambdas/collect_qc_metrics_from_alignment_directory_py/requirements.txt b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_4/push-fastq-list-row-qc-complete-event/lambdas/collect_qc_metrics_from_alignment_directory_py/requirements.txt similarity index 100% rename from lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_6/push-fastq-list-row-qc-complete-event/lambdas/collect_qc_metrics_from_alignment_directory_py/requirements.txt rename to lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_4/push-fastq-list-row-qc-complete-event/lambdas/collect_qc_metrics_from_alignment_directory_py/requirements.txt diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_6/push-fastq-list-row-qc-complete-event/lambdas/generate_event_data_objects_py/generate_event_data_objects.py b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_4/push-fastq-list-row-qc-complete-event/lambdas/generate_event_data_objects_py/generate_event_data_objects.py similarity index 90% rename from lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_6/push-fastq-list-row-qc-complete-event/lambdas/generate_event_data_objects_py/generate_event_data_objects.py rename to lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_4/push-fastq-list-row-qc-complete-event/lambdas/generate_event_data_objects_py/generate_event_data_objects.py index 36bb89117..bf281a42b 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_6/push-fastq-list-row-qc-complete-event/lambdas/generate_event_data_objects_py/generate_event_data_objects.py +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_4/push-fastq-list-row-qc-complete-event/lambdas/generate_event_data_objects_py/generate_event_data_objects.py @@ -43,13 +43,17 @@ def handler(event, context) -> Dict[str, Dict[str, str]]: fastq_list_row_id = event['fastq_list_row_id'] sample_type = event['sample_type'] qc_metrics = event['qc_metrics'] + library_orcabus_id = event['library_orcabus_id'] library_id = event['library_id'] # Initialise output dict event_output_dict = { "fastqListRowId": fastq_list_row_id, "sampleType": sample_type, - "libraryId": library_id + "library": { + "libraryId": library_id, + "orcabusId": library_orcabus_id + } } # Update dict per sample type diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_6/push-fastq-list-row-qc-complete-event/step_functions_templates/wgts_qc_complete_sfn_template.asl.json b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_4/push-fastq-list-row-qc-complete-event/step_functions_templates/wgts_qc_complete_sfn_template.asl.json similarity index 95% rename from lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_6/push-fastq-list-row-qc-complete-event/step_functions_templates/wgts_qc_complete_sfn_template.asl.json rename to lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_4/push-fastq-list-row-qc-complete-event/step_functions_templates/wgts_qc_complete_sfn_template.asl.json index 7f2fe162c..572fdf7b7 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_6/push-fastq-list-row-qc-complete-event/step_functions_templates/wgts_qc_complete_sfn_template.asl.json +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_4/push-fastq-list-row-qc-complete-event/step_functions_templates/wgts_qc_complete_sfn_template.asl.json @@ -36,8 +36,7 @@ } }, "ResultSelector": { - "library_id.$": "$.Item.library_id", - "instrument_run_id.$": "$.Item.instrument_run_id" + "library_orcabus_id.$": "$.Item.library_orcabus_id.S" }, "ResultPath": "$.get_library_id_step", "Next": "GetLibraryInfo" @@ -48,13 +47,13 @@ "Parameters": { "TableName": "${__table_name__}", "Key": { - "id.$": "$.get_library_id_step.library_id", + "id.$": "$.get_library_id_step.library_orcabus_id", "id_type": "${__library_partition__}" } }, "ResultSelector": { "sample_type.$": "$.Item.sample_type.S", - "library_id.$": "$.Item.id.S" + "library_id.$": "$.Item.library_id.S" }, "ResultPath": "$.get_library_info_step", "Next": "Collect QC Metrics from alignment directory" @@ -121,6 +120,7 @@ "fastq_list_row_id.$": "$.get_fastq_list_row_id_from_portal_run_id.fastq_list_row_id", "sample_type.$": "$.get_library_info_step.sample_type", "qc_metrics.$": "$.get_qc_metrics_step.qc_metrics", + "library_orcabus_id.$": "$.get_library_id_step.library_orcabus_id", "library_id.$": "$.get_library_info_step.library_id" } }, diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_7/library-qc-complete-event/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_5/library-qc-complete-event/index.ts similarity index 100% rename from lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_7/library-qc-complete-event/index.ts rename to lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_5/library-qc-complete-event/index.ts diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_7/library-qc-complete-event/lambdas/sum_coverages_for_rgids_py/sum_coverages_for_rgids.py b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_5/library-qc-complete-event/lambdas/sum_coverages_for_rgids_py/sum_coverages_for_rgids.py similarity index 100% rename from lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_7/library-qc-complete-event/lambdas/sum_coverages_for_rgids_py/sum_coverages_for_rgids.py rename to lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_5/library-qc-complete-event/lambdas/sum_coverages_for_rgids_py/sum_coverages_for_rgids.py diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_7/library-qc-complete-event/step_functions_templates/wgts_library_qc_complete_event_template.asl.json b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_5/library-qc-complete-event/step_functions_templates/wgts_library_qc_complete_event_template.asl.json similarity index 96% rename from lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_7/library-qc-complete-event/step_functions_templates/wgts_library_qc_complete_event_template.asl.json rename to lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_5/library-qc-complete-event/step_functions_templates/wgts_library_qc_complete_event_template.asl.json index f3445f348..94af46139 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_7/library-qc-complete-event/step_functions_templates/wgts_library_qc_complete_event_template.asl.json +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_5/library-qc-complete-event/step_functions_templates/wgts_library_qc_complete_event_template.asl.json @@ -15,7 +15,7 @@ "Parameters": { "TableName": "${__table_name__}", "Key": { - "id.$": "$.payload_data.libraryId", + "id.$": "$.payload_data.library.orcabusId", "id_type": "${__library_partition__}" } }, @@ -31,7 +31,7 @@ "ItemsPath": "$.get_library_info_from_library_step.fastq_list_row_ids", "ItemSelector": { "fastq_list_row_id.$": "$$.Map.Item.Value", - "library_id.$": "$.payload_data.libraryId", + "library_id.$": "$.payload_data.library.libraryId", "sample_type.$": "$.get_library_info_from_library_step.sample_type" }, "ItemProcessor": { @@ -139,7 +139,7 @@ "Parameters": { "TableName": "${__table_name__}", "Key": { - "id.$": "$.payload_data.libraryId", + "id.$": "$.payload_data.library.orcabusId", "id_type": "${__library_partition__}" }, "UpdateExpression": "SET qc_metrics = :qc_metrics", @@ -172,7 +172,7 @@ "payload": { "version": "${__payload_version__}", "data": { - "libraryId.$": "$.payload_data.libraryId", + "library.$": "$.payload_data.library", "fastqListRowIds.$": "$.get_library_info_from_library_step.fastq_list_row_ids", "qcMetrics.$": "$.collate_metrics_step.library_qc_metrics" } diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_5/wgts-qc-draft-to-ready/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_5/wgts-qc-draft-to-ready/index.ts deleted file mode 100644 index 46b3984bd..000000000 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/kwik/part_5/wgts-qc-draft-to-ready/index.ts +++ /dev/null @@ -1,106 +0,0 @@ -import { Construct } from 'constructs'; -import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; -import * as ssm from 'aws-cdk-lib/aws-ssm'; -import * as secretsManager from 'aws-cdk-lib/aws-secretsmanager'; -import * as events from 'aws-cdk-lib/aws-events'; -import { WorkflowDraftRunStateChangeToWorkflowRunStateChangeReadyConstruct } from '../../../../../../../components/event-workflowdraftrunstatechange-to-workflowrunstatechange-ready'; - -/* -Part 5 - -Input Event source: `orcabus.wgtsqcinputeventglue` -Input Event DetailType: `WorkflowDraftRunStateChange` -Input Event status: `draft` - -Output Event source: `orcabus.wgtsqcinputeventglue` -Output Event DetailType: `WorkflowRunStateChange` -Output Event status: `ready` - -* The wgtsQcInputMaker, subscribes to the wgtsqc input event glue (itself) and generates a ready event for the wgtsqcReadySfn - * For the cttso v2 workflow we require a samplesheet, a set of fastq list rows (provided in the last step) - * However, in order to be 'READY' we need to use a few more variables such as - * icaLogsUri, - * analysisOutputUri - * cacheUri - * projectId - * userReference -*/ - -export interface WgtsQcInputMakerConstructProps { - /* Event bus object */ - eventBusObj: events.IEventBus; - /* Tables */ - inputMakerTableObj: dynamodb.ITableV2; - /* SSM Parameter Objects */ - icav2ProjectIdSsmParameterObj: ssm.IStringParameter; - outputUriSsmParameterObj: ssm.IStringParameter; - logsUriSsmParameterObj: ssm.IStringParameter; - cacheUriSsmParameterObj: ssm.IStringParameter; - /* Secrets */ - icav2AccessTokenSecretObj: secretsManager.ISecret; -} - -export class WgtsQcInputMakerConstruct extends Construct { - public readonly wgtsQcInputMakerEventMap = { - prefix: 'kwik-wgtsqc', - tablePartition: 'wgts_qc', - triggerSource: 'orcabus.wgtsqcinputeventglue', - triggerStatus: 'DRAFT', - triggerDetailType: 'WorkflowDraftRunStateChange', - outputSource: 'orcabus.wgtsqcinputeventglue', - outputStatus: 'READY', - payloadVersion: '2024.05.24', - workflowName: 'wgtsQc', - workflowVersion: '4.2.4', - }; - - constructor(scope: Construct, id: string, props: WgtsQcInputMakerConstructProps) { - super(scope, id); - - /* - Part 3: Build the external sfn - */ - new WorkflowDraftRunStateChangeToWorkflowRunStateChangeReadyConstruct( - this, - 'wgts_qc_copy_manager_internal_input_maker', - { - /* - Set Input StateMachine Object - */ - lambdaPrefix: this.wgtsQcInputMakerEventMap.prefix, - payloadVersion: this.wgtsQcInputMakerEventMap.payloadVersion, - stateMachinePrefix: this.wgtsQcInputMakerEventMap.prefix, - rulePrefix: `stacky-${this.wgtsQcInputMakerEventMap.prefix}`, - - /* - Table objects - */ - tableObj: props.inputMakerTableObj, - tablePartitionName: this.wgtsQcInputMakerEventMap.tablePartition, - - /* - Event Triggers - */ - eventBusObj: props.eventBusObj, - triggerDetailType: this.wgtsQcInputMakerEventMap.triggerDetailType, - triggerSource: this.wgtsQcInputMakerEventMap.triggerSource, - triggerStatus: this.wgtsQcInputMakerEventMap.triggerStatus, - outputSource: this.wgtsQcInputMakerEventMap.outputSource, - workflowName: this.wgtsQcInputMakerEventMap.workflowName, - workflowVersion: this.wgtsQcInputMakerEventMap.workflowVersion, - - /* - SSM Parameter Objects - */ - icav2ProjectIdSsmParameterObj: props.icav2ProjectIdSsmParameterObj, - outputUriSsmParameterObj: props.outputUriSsmParameterObj, - logsUriSsmParameterObj: props.logsUriSsmParameterObj, - - /* - Secrets - */ - icav2AccessTokenSecretObj: props.icav2AccessTokenSecretObj, - } - ); - } -} diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/index.ts index 34a0f25f6..7100d2e1f 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/index.ts +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/index.ts @@ -3,12 +3,9 @@ import * as events from 'aws-cdk-lib/aws-events'; import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; import * as ssm from 'aws-cdk-lib/aws-ssm'; import * as secretsManager from 'aws-cdk-lib/aws-secretsmanager'; -import { TnInitialiseSubjectDbRowConstruct } from './part_1/initialise-tn-subject-dbs'; -import { TnInitialiseLibraryAndFastqListRowConstruct } from './part_2/initialise-tn-library-dbs'; -import { TnPopulateFastqListRowConstruct } from './part_3/update-fastq-list-rows-dbs'; -import { TnFastqListRowQcCompleteConstruct } from './part_4/update-fastq-list-row-qc-complete-dbs'; -import { LibraryQcCompleteToTnDraftConstruct } from './part_5/library-qc-complete-db-to-tn-draft'; -import { TnInputMakerConstruct } from './part_6/tn-draft-to-ready'; +import { TnInitialiseLibraryAndFastqListRowConstruct } from './part_1/initialise-tn-library-dbs'; +import { TnPopulateFastqListRowConstruct } from './part_2/update-fastq-list-rows-dbs'; +import { LibraryQcCompleteToTnReadyConstruct } from './part_3/library-qc-complete-db-to-tn-ready'; /* Provide the glue to get from the bssh fastq copy manager to submitting wgts qc analyses @@ -19,7 +16,6 @@ export interface tnGlueHandlerConstructProps { eventBusObj: events.IEventBus; /* Tables */ tnGlueTableObj: dynamodb.ITableV2; - inputMakerTableObj: dynamodb.ITableV2; /* SSM Parameters */ analysisOutputUriSsmParameterObj: ssm.IStringParameter; analysisLogsUriSsmParameterObj: ssm.IStringParameter; @@ -34,53 +30,17 @@ export class TnGlueHandlerConstruct extends Construct { super(scope, id); /* - Part 1 - Input Event Source: `orcabus.instrumentrunmanager` - Input Event DetailType: `SamplesheetMetadataUnion` - Input Event status: `SubjectInSamplesheet` + Part 1 - * Initialise tn subject db row construct - */ - const tn_initialise_subject_db_row = new TnInitialiseSubjectDbRowConstruct( - this, - 'tn_initialise_subject_db_row', - { - eventBusObj: props.eventBusObj, - tableObj: props.tnGlueTableObj, - } - ); - - /* - Part 2 - - Input Event Source: `orcabus.instrumentrunmanager` - Input Event DetailType: `SamplesheetMetadataUnion` - Input Event status: `LibraryInSamplesheet` - - * Initialise wgts qc library and fastq list row constructs - */ - const tn_initialise_library_and_fastq_list_row = - new TnInitialiseLibraryAndFastqListRowConstruct( - this, - 'tn_initialise_library_and_fastq_list_row', - { - eventBusObj: props.eventBusObj, - tableObj: props.tnGlueTableObj, - } - ); - - /* - Part 3 - - Input Event Source: `orcabus.instrumentrunmanager` - Input Event DetailType: `FastqListRowStateChange` - Input Event status: `newFastqListRow` + Input Event Source: `orcabus.instrumentrunmanager` + Input Event DetailType: `SamplesheetMetadataUnion` + Input Event status: `LibraryInSamplesheet` - * Populate the fastq list row attributes for the rgid for this workflow - */ - const tn_populate_fastq_list_row = new TnPopulateFastqListRowConstruct( + * Initialise wgts qc library and fastq list row constructs + */ + const tnInitialiseLibraryAndFastqListRow = new TnInitialiseLibraryAndFastqListRowConstruct( this, - 'tn_populate_fastq_list_row', + 'tn_initialise_library_and_fastq_list_row', { eventBusObj: props.eventBusObj, tableObj: props.tnGlueTableObj, @@ -88,19 +48,17 @@ export class TnGlueHandlerConstruct extends Construct { ); /* - Part 4 + Part 2 - Input Event Source: `orcabus.wgtsqcinputeventglue` - Input Event DetailType: `FastqListRowStateChange` - Input Event status: `QcComplete` + Input Event Source: `orcabus.instrumentrunmanager` + Input Event DetailType: `FastqListRowStateChange` + Input Event status: `newFastqListRow` - * Populate the fastq list row attributes with the qc metrics for this fastq list row id - * Currently not used by the glue service - - */ - const tn_fastq_list_row_qc_complete = new TnFastqListRowQcCompleteConstruct( + * Populate the fastq list row attributes for the rgid for this workflow + */ + const tnPopulateFastqListRow = new TnPopulateFastqListRowConstruct( this, - 'tn_fastq_list_row_qc_complete', + 'tn_populate_fastq_list_row', { eventBusObj: props.eventBusObj, tableObj: props.tnGlueTableObj, @@ -108,54 +66,31 @@ export class TnGlueHandlerConstruct extends Construct { ); /* - Part 5 - Input Event source: `orcabus.wgtsqcinputeventglue` - Input Event DetailType: `LibraryStateChange` - Input Event status: `QcComplete` - - Output Event source: `orcabus.tninputeventglue` - Output Event DetailType: `WorkflowDraftRunStateChange` - Output Event status: `draft` - */ - const libraryQcCompleteToTnDraft = new LibraryQcCompleteToTnDraftConstruct( + Part 3 + Input Event source: `orcabus.wgtsqcinputeventglue` + Input Event DetailType: `LibraryStateChange` + Input Event status: `QcComplete` + + Output Event source: `orcabus.tninputeventglue` + Output Event DetailType: `WorkflowDraftRunStateChange` + Output Event status: `draft` + */ + const libraryQcCompleteToTnDraft = new LibraryQcCompleteToTnReadyConstruct( this, 'library_qc_complete_to_tn_draft', { // Event bus eventBusObj: props.eventBusObj, - // SSM Param objects + // Table objects tableObj: props.tnGlueTableObj, - workflowsTableObj: props.inputMakerTableObj, + /* SSM Param objects */ + icav2ProjectIdSsmParameterObj: props.icav2ProjectIdSsmParameterObj, + outputUriSsmParameterObj: props.analysisOutputUriSsmParameterObj, + cacheUriSsmParameterObj: props.analysisCacheUriSsmParameterObj, + logsUriSsmParameterObj: props.analysisLogsUriSsmParameterObj, + /* Secrets */ + icav2AccessTokenSecretObj: props.icav2AccessTokenSecretObj, } ); - - /* - Part 6 - - Input Event Source: `orcabus.workflowmanager` - Input Event DetailType: `WorkflowRunStateChange` - Input Event status: `succeeded` - Input Event WorkflowName: `wgts_qc` - - Output Event Source: `orcabus.wgtsqcinputeventglue` - Output Event DetailType: `FastqListRowStateChange` - Output Event status: `QcComplete` - - * Subscribe to workflow run state change events, map the fastq list row id from the portal run id in the data base - * We output the fastq list row id to the event bus with the status `QcComplete` - */ - const tnInputMaker = new TnInputMakerConstruct(this, 'fastq_list_row_qc_complete', { - /* Event bus */ - eventBusObj: props.eventBusObj, - /* Tables */ - inputMakerTableObj: props.inputMakerTableObj, - /* SSM Param objects */ - icav2ProjectIdSsmParameterObj: props.icav2ProjectIdSsmParameterObj, - outputUriSsmParameterObj: props.analysisOutputUriSsmParameterObj, - cacheUriSsmParameterObj: props.analysisCacheUriSsmParameterObj, - logsUriSsmParameterObj: props.analysisLogsUriSsmParameterObj, - /* Secrets */ - icav2AccessTokenSecretObj: props.icav2AccessTokenSecretObj, - }); } } diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_2/initialise-tn-library-dbs/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_1/initialise-tn-library-dbs/index.ts similarity index 100% rename from lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_2/initialise-tn-library-dbs/index.ts rename to lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_1/initialise-tn-library-dbs/index.ts diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_2/initialise-tn-library-dbs/step_functions_templates/initialise_tn_library_db_sfn_template.asl.json b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_1/initialise-tn-library-dbs/step_functions_templates/initialise_tn_library_db_sfn_template.asl.json similarity index 83% rename from lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_2/initialise-tn-library-dbs/step_functions_templates/initialise_tn_library_db_sfn_template.asl.json rename to lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_1/initialise-tn-library-dbs/step_functions_templates/initialise_tn_library_db_sfn_template.asl.json index 5660a6a76..0122e12f3 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_2/initialise-tn-library-dbs/step_functions_templates/initialise_tn_library_db_sfn_template.asl.json +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_1/initialise-tn-library-dbs/step_functions_templates/initialise_tn_library_db_sfn_template.asl.json @@ -15,7 +15,7 @@ "Parameters": { "TableName": "${__table_name__}", "Key": { - "id.$": "$.payload_data.library.libraryId", + "id.$": "$.payload_data.library.orcabusId", "id_type": "${__library_partition_name__}" } }, @@ -26,25 +26,10 @@ "Type": "Parallel", "Branches": [ { - "StartAt": "Add Library to Subject", + "StartAt": "Pass", "States": { - "Add Library to Subject": { - "Type": "Task", - "Resource": "arn:aws:states:::dynamodb:updateItem", - "Parameters": { - "TableName": "${__table_name__}", - "Key": { - "id.$": "$.payload_data.library.subject.subjectId", - "id_type": "${__subject_partition_name__}" - }, - "UpdateExpression": "ADD library_set :library_set", - "ExpressionAttributeValues": { - ":library_set": { - "SS.$": "States.Array($.payload_data.library.libraryId)" - } - } - }, - "ResultPath": null, + "Pass": { + "Type": "Pass", "End": true } } @@ -70,10 +55,10 @@ "Parameters": { "TableName": "${__table_name__}", "Item": { - "id.$": "$.payload_data.library.libraryId", + "id.$": "$.payload_data.library.orcabusId", "id_type": "${__library_partition_name__}", - "orcabus_id": { - "S.$": "$.payload_data.library.orcabusId" + "library_id": { + "S.$": "$.payload_data.library.libraryId" }, "phenotype": { "S.$": "$.payload_data.library.phenotype" @@ -88,7 +73,10 @@ "S.$": "$.payload_data.library.assay" }, "subject_id": { - "S.$": "$.payload_data.library.subject.subjectId" + "S.$": "$.payload_data.subject.subjectId" + }, + "subject_orcabus_id": { + "S.$": "$.payload_data.subject.orcabusId" } } }, @@ -113,6 +101,7 @@ "bclconvert_data_row.$": "$$.Map.Item.Value", "instrument_run_id.$": "$.payload_data.instrumentRunId", "library_id.$": "$.payload_data.library.libraryId", + "library_orcabus_id.$": "$.payload_data.library.orcabusId", "index.$": "$$.Map.Item.Index", "fastq_list_row_objs.$": "$.payload_data.fastqListRows" }, @@ -146,6 +135,9 @@ "id_type": "${__fastq_list_row_partition_name__}", "library_id": { "S.$": "$.library_id" + }, + "library_orcabus_id": { + "S.$": "$.library_orcabus_id" } } }, @@ -163,7 +155,7 @@ "Parameters": { "TableName": "${__table_name__}", "Key": { - "id.$": "$.library_id", + "id.$": "$.library_orcabus_id", "id_type": "${__library_partition_name__}" }, "UpdateExpression": "ADD fastq_list_row_id_set :fastq_list_row_id_set", diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_1/initialise-tn-subject-dbs/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_1/initialise-tn-subject-dbs/index.ts deleted file mode 100644 index ed0edbbd1..000000000 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_1/initialise-tn-subject-dbs/index.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { Construct } from 'constructs'; -import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; -import path from 'path'; -import * as sfn from 'aws-cdk-lib/aws-stepfunctions'; -import * as events from 'aws-cdk-lib/aws-events'; -import * as eventsTargets from 'aws-cdk-lib/aws-events-targets'; - -/* -Part 1 - -Input Event Source: `orcabus.instrumentrunmanager` -Input Event DetailType: `SamplesheetMetadataUnion` -Input Event status: `SubjectInSamplesheet` - -* Initialise tn subject db construct -*/ - -export interface TnInitialiseSubjectDbRowConstructProps { - tableObj: dynamodb.ITableV2; - eventBusObj: events.IEventBus; -} - -export class TnInitialiseSubjectDbRowConstruct extends Construct { - public readonly TnInitialiseSubjectDbRowMap = { - prefix: 'loctite-make-subject-row', - tablePartition: 'subject', - triggerSource: 'orcabus.instrumentrunmanager', - triggerStatus: 'SubjectInSamplesheet', - triggerDetailType: 'SamplesheetMetadataUnion', - }; - - constructor(scope: Construct, id: string, props: TnInitialiseSubjectDbRowConstructProps) { - super(scope, id); - - /* - Part 1: Build the internal sfn - */ - const inputMakerSfn = new sfn.StateMachine(this, 'initialise_subject_db_row', { - stateMachineName: `${this.TnInitialiseSubjectDbRowMap.prefix}-initialise-subject`, - definitionBody: sfn.DefinitionBody.fromFile( - path.join( - __dirname, - 'step_functions_templates', - 'initialise_tn_subject_db_sfn_template.asl.json' - ) - ), - definitionSubstitutions: { - __table_name__: props.tableObj.tableName, - __subject_partition_name__: this.TnInitialiseSubjectDbRowMap.tablePartition, - }, - }); - - /* - Part 2: Grant the internal sfn permissions - */ - // access the dynamodb table - props.tableObj.grantReadWriteData(inputMakerSfn.role); - - /* - Part 3: Subscribe to the event bus and trigger the internal sfn - */ - const rule = new events.Rule(this, 'tn_subscribe_to_samplesheet_shower_subject', { - ruleName: `stacky-${this.TnInitialiseSubjectDbRowMap.prefix}-rule`, - eventBus: props.eventBusObj, - eventPattern: { - source: [this.TnInitialiseSubjectDbRowMap.triggerSource], - detailType: [this.TnInitialiseSubjectDbRowMap.triggerDetailType], - detail: { - status: [{ 'equals-ignore-case': this.TnInitialiseSubjectDbRowMap.triggerStatus }], - }, - }, - }); - - // Add target of event to be the state machine - rule.addTarget( - new eventsTargets.SfnStateMachine(inputMakerSfn, { - input: events.RuleTargetInput.fromEventPath('$.detail'), - }) - ); - } -} diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_1/initialise-tn-subject-dbs/step_functions_templates/initialise_tn_subject_db_sfn_template.asl.json b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_1/initialise-tn-subject-dbs/step_functions_templates/initialise_tn_subject_db_sfn_template.asl.json deleted file mode 100644 index e0c23b290..000000000 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_1/initialise-tn-subject-dbs/step_functions_templates/initialise_tn_subject_db_sfn_template.asl.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "Comment": "A description of my state machine", - "StartAt": "Move Inputs", - "States": { - "Move Inputs": { - "Type": "Pass", - "Parameters": { - "payload_data.$": "$.payload.data" - }, - "Next": "Get Subject Item" - }, - "Get Subject Item": { - "Type": "Task", - "Resource": "arn:aws:states:::dynamodb:getItem", - "Parameters": { - "TableName": "${__table_name__}", - "Key": { - "id.$": "$.payload_data.subject.subjectId", - "id_type": "${__subject_partition_name__}" - } - }, - "ResultPath": "$.get_subject_item_step", - "Next": "Subject in Database" - }, - "Subject in Database": { - "Type": "Choice", - "Choices": [ - { - "Variable": "$.get_subject_item_step.Item", - "IsPresent": false, - "Comment": "Subject Not In Database", - "Next": "Initialise Subject" - } - ], - "Default": "Pass" - }, - "Pass": { - "Type": "Pass", - "End": true - }, - "Initialise Subject": { - "Type": "Task", - "Resource": "arn:aws:states:::dynamodb:putItem", - "Parameters": { - "TableName": "${__table_name__}", - "Item": { - "id.$": "$.payload_data.subject.subjectId", - "id_type": "${__subject_partition_name__}", - "orcabus_id": { - "S.$": "$.payload_data.subject.orcabusId" - } - } - }, - "End": true - } - } -} diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_3/update-fastq-list-rows-dbs/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_2/update-fastq-list-rows-dbs/index.ts similarity index 100% rename from lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_3/update-fastq-list-rows-dbs/index.ts rename to lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_2/update-fastq-list-rows-dbs/index.ts diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_3/update-fastq-list-rows-dbs/step_functions_templates/add_fastq_list_rows_db_sfn_template.asl.json b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_2/update-fastq-list-rows-dbs/step_functions_templates/add_fastq_list_rows_db_sfn_template.asl.json similarity index 100% rename from lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_3/update-fastq-list-rows-dbs/step_functions_templates/add_fastq_list_rows_db_sfn_template.asl.json rename to lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_2/update-fastq-list-rows-dbs/step_functions_templates/add_fastq_list_rows_db_sfn_template.asl.json diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_5/library-qc-complete-db-to-tn-draft/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_3/library-qc-complete-db-to-tn-ready/index.ts similarity index 53% rename from lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_5/library-qc-complete-db-to-tn-draft/index.ts rename to lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_3/library-qc-complete-db-to-tn-ready/index.ts index 934af789d..bc5dc3521 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_5/library-qc-complete-db-to-tn-draft/index.ts +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_3/library-qc-complete-db-to-tn-ready/index.ts @@ -4,14 +4,18 @@ import * as iam from 'aws-cdk-lib/aws-iam'; import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; import path from 'path'; import * as sfn from 'aws-cdk-lib/aws-stepfunctions'; +import * as ssm from 'aws-cdk-lib/aws-ssm'; +import * as secretsManager from 'aws-cdk-lib/aws-secretsmanager'; import * as events from 'aws-cdk-lib/aws-events'; import * as eventsTargets from 'aws-cdk-lib/aws-events-targets'; import { PythonFunction } from '@aws-cdk/aws-lambda-python-alpha'; import * as lambda from 'aws-cdk-lib/aws-lambda'; import { WorkflowDraftRunStateChangeCommonPreambleConstruct } from '../../../../../../../components/sfn-workflowdraftrunstatechange-common-preamble'; +import { GenerateWorkflowRunStateChangeReadyConstruct } from '../../../../../../../components/sfn-generate-workflowrunstatechange-ready-event'; +import { GetMetadataLambdaConstruct } from '../../../../../../../components/python-lambda-metadata-mapper'; /* -Part 5 +Part 3 Input Event Source: `orcabus.wgtsqcinputeventglue` Input Event DetailType: `LibraryStateChange` @@ -26,28 +30,34 @@ Output Event status: `draft` */ export interface LibraryQcCompleteToTnDraftConstructProps { + /* Events */ eventBusObj: events.IEventBus; + /* Tables */ tableObj: dynamodb.ITableV2; - workflowsTableObj: dynamodb.ITableV2; + /* SSM Parameters */ + outputUriSsmParameterObj: ssm.IStringParameter; + icav2ProjectIdSsmParameterObj: ssm.IStringParameter; + logsUriSsmParameterObj: ssm.IStringParameter; + cacheUriSsmParameterObj: ssm.IStringParameter; + + /* Secrets */ + icav2AccessTokenSecretObj: secretsManager.ISecret; } -export class LibraryQcCompleteToTnDraftConstruct extends Construct { - public readonly TnDraftMap = { +export class LibraryQcCompleteToTnReadyConstruct extends Construct { + public readonly TnReadyMap = { prefix: 'loctite-qc-complete-to-tn', tablePartition: { subject: 'subject', library: 'library', fastq_list_row: 'fastq_list_row', }, - portalRunPartitionName: 'portal_run', triggerSource: 'orcabus.wgtsqcinputeventglue', triggerStatus: 'QC_COMPLETE', triggerDetailType: 'LibraryStateChange', outputSource: 'orcabus.tninputeventglue', - outputDetailType: 'WorkflowDraftRunStateChange', - outputStatus: 'DRAFT', payloadVersion: '2024.07.23', - workflowName: 'tumor_normal', + workflowName: 'tumor-normal', workflowVersion: '4.2.4', }; @@ -76,26 +86,67 @@ export class LibraryQcCompleteToTnDraftConstruct extends Construct { memorySize: 1024, }); + // Generate the lambda to collect the orcabus id from the subject id + const collectOrcaBusIdLambdaObj = new GetMetadataLambdaConstruct( + this, + 'get_orcabus_id_from_subject_id', + { + functionNamePrefix: this.TnReadyMap.prefix, + } + ).lambdaObj; + + // Add CONTEXT, FROM_ID and RETURN_OBJ environment variables to the lambda + collectOrcaBusIdLambdaObj.addEnvironment('CONTEXT', 'subject'); + collectOrcaBusIdLambdaObj.addEnvironment('FROM_ORCABUS', ''); + collectOrcaBusIdLambdaObj.addEnvironment('RETURN_OBJ', ''); + /* Part 1: Generate the preamble (sfn to generate the portal run id and the workflow run name) */ - const sfn_preamble = new WorkflowDraftRunStateChangeCommonPreambleConstruct( + const sfnPreamble = new WorkflowDraftRunStateChangeCommonPreambleConstruct( this, - `${this.TnDraftMap.prefix}_sfn_preamble`, + `${this.TnReadyMap.prefix}_sfn_preamble`, { - portalRunTablePartitionName: this.TnDraftMap.portalRunPartitionName, - stateMachinePrefix: this.TnDraftMap.prefix, - tableObj: props.workflowsTableObj, - workflowName: this.TnDraftMap.workflowName, - workflowVersion: this.TnDraftMap.workflowVersion, + stateMachinePrefix: this.TnReadyMap.prefix, + workflowName: this.TnReadyMap.workflowName, + workflowVersion: this.TnReadyMap.workflowVersion, + } + ).stepFunctionObj; + + /* + Part 2: Build the engineparameters event sfn + */ + const engineParameterAndReadyEventMakerSfn = new GenerateWorkflowRunStateChangeReadyConstruct( + this, + 'fastqlistrow_complete_to_wgtsqc_ready_submitter', + { + /* Event Placeholders */ + eventBusObj: props.eventBusObj, + outputSource: this.TnReadyMap.outputSource, + payloadVersion: this.TnReadyMap.payloadVersion, + workflowName: this.TnReadyMap.workflowName, + workflowVersion: this.TnReadyMap.workflowVersion, + + /* SSM Parameters */ + outputUriSsmParameterObj: props.outputUriSsmParameterObj, + icav2ProjectIdSsmParameterObj: props.icav2ProjectIdSsmParameterObj, + logsUriSsmParameterObj: props.logsUriSsmParameterObj, + cacheUriSsmParameterObj: props.cacheUriSsmParameterObj, + + /* Secrets */ + icav2AccessTokenSecretObj: props.icav2AccessTokenSecretObj, + + /* Prefixes */ + lambdaPrefix: this.TnReadyMap.prefix, + stateMachinePrefix: this.TnReadyMap.prefix, } ).stepFunctionObj; /* Part 2: Build the sfn */ - const qcCompleteToDraftSfn = new sfn.StateMachine(this, 'library_qc_complete_sfn_to_tn_draft', { - stateMachineName: `${this.TnDraftMap.prefix}-sfn`, + const inputMakerSfn = new sfn.StateMachine(this, 'library_qc_complete_sfn_to_tn_draft', { + stateMachineName: `${this.TnReadyMap.prefix}-sfn`, definitionBody: sfn.DefinitionBody.fromFile( path.join( __dirname, @@ -104,29 +155,23 @@ export class LibraryQcCompleteToTnDraftConstruct extends Construct { ) ), definitionSubstitutions: { - /* Events */ - __event_bus_name__: props.eventBusObj.eventBusName, - __event_source__: this.TnDraftMap.outputSource, - __detail_type__: this.TnDraftMap.outputDetailType, - __output_status__: this.TnDraftMap.outputStatus, - __payload_version__: this.TnDraftMap.payloadVersion, - __workflow_name__: this.TnDraftMap.workflowName, - __workflow_version__: this.TnDraftMap.workflowVersion, - /* Lambdas */ __generate_draft_event_payload_lambda_function_arn__: generateEventDataLambdaObj.currentVersion.functionArn, __get_complement_library_pair_lambda_function_arn__: findComplementLibraryPair.currentVersion.functionArn, + __get_orcabus_obj_from_subject_id_lambda_function_arn__: + collectOrcaBusIdLambdaObj.currentVersion.functionArn, /* Tables */ __table_name__: props.tableObj.tableName, - __subject_partition_name__: this.TnDraftMap.tablePartition.subject, - __library_partition_name__: this.TnDraftMap.tablePartition.library, - __fastq_list_row_partition_name__: this.TnDraftMap.tablePartition.fastq_list_row, + __subject_partition_name__: this.TnReadyMap.tablePartition.subject, + __library_partition_name__: this.TnReadyMap.tablePartition.library, + __fastq_list_row_partition_name__: this.TnReadyMap.tablePartition.fastq_list_row, // State Machines - __sfn_preamble_state_machine_arn__: sfn_preamble.stateMachineArn, + __sfn_preamble_state_machine_arn__: sfnPreamble.stateMachineArn, + __launch_ready_event_sfn_arn__: engineParameterAndReadyEventMakerSfn.stateMachineArn, }, }); @@ -134,20 +179,19 @@ export class LibraryQcCompleteToTnDraftConstruct extends Construct { Part 2: Grant the sfn permissions */ // access the dynamodb table - props.tableObj.grantReadWriteData(qcCompleteToDraftSfn); - - // allow the step function to submit events - props.eventBusObj.grantPutEventsTo(qcCompleteToDraftSfn.role); + props.tableObj.grantReadWriteData(inputMakerSfn); // allow the step function to invoke the lambdas - [generateEventDataLambdaObj, findComplementLibraryPair].forEach((lambdaObj) => { - lambdaObj.currentVersion.grantInvoke(qcCompleteToDraftSfn.role); - }); + [generateEventDataLambdaObj, findComplementLibraryPair, collectOrcaBusIdLambdaObj].forEach( + (lambdaObj) => { + lambdaObj.currentVersion.grantInvoke(inputMakerSfn); + } + ); /* Allow step function to call nested state machine */ // Because we run a nested state machine, we need to add the permissions to the state machine role // See https://stackoverflow.com/questions/60612853/nested-step-function-in-a-step-function-unknown-error-not-authorized-to-cr - qcCompleteToDraftSfn.addToRolePolicy( + inputMakerSfn.addToRolePolicy( new iam.PolicyStatement({ resources: [ `arn:aws:events:${cdk.Aws.REGION}:${cdk.Aws.ACCOUNT_ID}:rule/StepFunctionsGetEventsForStepFunctionsExecutionRule`, @@ -156,26 +200,27 @@ export class LibraryQcCompleteToTnDraftConstruct extends Construct { }) ); // Allow the state machine to be able to invoke the preamble sfn - sfn_preamble.grantStartExecution(qcCompleteToDraftSfn.role); + sfnPreamble.grantStartExecution(inputMakerSfn); + engineParameterAndReadyEventMakerSfn.grantStartExecution(inputMakerSfn); /* Part 3: Subscribe to the event bus and trigger the internal sfn */ const rule = new events.Rule(this, 'library_qc_complete_to_tn_draft', { - ruleName: `stacky-${this.TnDraftMap.prefix}-rule`, + ruleName: `stacky-${this.TnReadyMap.prefix}-rule`, eventBus: props.eventBusObj, eventPattern: { - source: [this.TnDraftMap.triggerSource], - detailType: [this.TnDraftMap.triggerDetailType], + source: [this.TnReadyMap.triggerSource], + detailType: [this.TnReadyMap.triggerDetailType], detail: { - status: [{ 'equals-ignore-case': this.TnDraftMap.triggerStatus }], + status: [{ 'equals-ignore-case': this.TnReadyMap.triggerStatus }], }, }, }); // Add target of event to be the state machine rule.addTarget( - new eventsTargets.SfnStateMachine(qcCompleteToDraftSfn, { + new eventsTargets.SfnStateMachine(inputMakerSfn, { input: events.RuleTargetInput.fromEventPath('$.detail'), }) ); diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_3/library-qc-complete-db-to-tn-ready/lambdas/find_complement_library_pair_py/find_complement_library_pair.py b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_3/library-qc-complete-db-to-tn-ready/lambdas/find_complement_library_pair_py/find_complement_library_pair.py new file mode 100644 index 000000000..0cc674246 --- /dev/null +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_3/library-qc-complete-db-to-tn-ready/lambdas/find_complement_library_pair_py/find_complement_library_pair.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 + +""" +Given a library object and a list of complementary library objects, find a matching pair for the library object. + +The library objects must match on 'workflow' and 'type' attributes, but must be the opposite phenotype. +""" + +from typing import Dict, List + + +def find_complement_library_pair(library: Dict, complement_libraries: List[Dict]): + """ + Given a library object and a list of complementary library objects, find a matching pair for the library object + within the complement library list. + :param library: + :param complement_libraries: + :return: + """ + for complement_library in complement_libraries: + # Need to both be of the same type + if not library['type'] == complement_library['type']: + continue + + # Need to be of different phenotypes + if library['phenotype'] == complement_library['phenotype']: + continue + + # Can be different workflows IF + # The 'research' workflow is the tumor and the 'clinical' workflow is the normal + # But do not allow clinical tumors to be matched with research normals + # Or if they are the same workflow, but different phenotypes + if ( + # Special case for research + ( + ( + library['workflow'] == 'research' and + complement_library['workflow'] == 'clinical' + ) and ( + library['phenotype'] == 'tumor' and + complement_library['phenotype'] == 'normal' + ) + ) or + # Complement case + ( + ( + library['workflow'] == 'clinical' and + complement_library['workflow'] == 'research' + ) and ( + library['phenotype'] == 'normal' and + complement_library['phenotype'] == 'tumor' + ) + ) or + # Standard clinical+clinical or research+research + ( + ( + library['workflow'] == complement_library['workflow'] + ) + ) + ): + return library, complement_library + + return None, None + + +def handler(event, context): + """ + Lambda handler function + :param event: + :param context: + :return: + """ + + library_obj: Dict = event['library_obj'] + complement_libraries: List[Dict] = event['complementary_library_obj_list'] + + # Filter out empty complement libraries + complement_libraries = list( + filter( + lambda comp_lib_iter_: ( + comp_lib_iter_ is not None and + not comp_lib_iter_['orcabus_id'] == library_obj['orcabus_id'] + ), + complement_libraries + ) + ) + + library, complement_library = find_complement_library_pair(library_obj, complement_libraries) + + if library is None: + return { + 'successful_pairing': False, + 'tumor_library': None, + 'normal_library': None + } + + if library['phenotype'] == 'tumor': + tumor_library = library + normal_library = complement_library + else: + tumor_library = complement_library + normal_library = library + + return { + 'successful_pairing': True, + 'tumor_library': tumor_library, + 'normal_library': normal_library + } diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_5/library-qc-complete-db-to-tn-draft/lambdas/generate_draft_event_payload_py/generate_draft_event_payload.py b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_3/library-qc-complete-db-to-tn-ready/lambdas/generate_draft_event_payload_py/generate_draft_event_payload.py similarity index 95% rename from lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_5/library-qc-complete-db-to-tn-draft/lambdas/generate_draft_event_payload_py/generate_draft_event_payload.py rename to lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_3/library-qc-complete-db-to-tn-ready/lambdas/generate_draft_event_payload_py/generate_draft_event_payload.py index 87b2b11b4..9a54672fb 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_5/library-qc-complete-db-to-tn-draft/lambdas/generate_draft_event_payload_py/generate_draft_event_payload.py +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_3/library-qc-complete-db-to-tn-ready/lambdas/generate_draft_event_payload_py/generate_draft_event_payload.py @@ -30,6 +30,7 @@ def handler(event, context) -> Dict: :return: draft event payload """ subject_id = event['subject_id'] + individual_id = event['individual_id'] tumor_library_id = event['tumor_library_id'] normal_library_id = event['normal_library_id'] @@ -54,6 +55,7 @@ def handler(event, context) -> Dict: }, "event_tags": { "subjectId": subject_id, + "individualId": individual_id, "tumorLibraryId": tumor_library_id, "normalLibraryId": normal_library_id, "tumorFastqListRowIds": tumor_fastq_list_row_ids, diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_5/library-qc-complete-db-to-tn-draft/step_functions_templates/add_library_qc_complete_to_tn_draft_sfn_template.asl.json b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_3/library-qc-complete-db-to-tn-ready/step_functions_templates/add_library_qc_complete_to_tn_draft_sfn_template.asl.json similarity index 80% rename from lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_5/library-qc-complete-db-to-tn-draft/step_functions_templates/add_library_qc_complete_to_tn_draft_sfn_template.asl.json rename to lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_3/library-qc-complete-db-to-tn-ready/step_functions_templates/add_library_qc_complete_to_tn_draft_sfn_template.asl.json index 7bc89329a..a072cad6c 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_5/library-qc-complete-db-to-tn-draft/step_functions_templates/add_library_qc_complete_to_tn_draft_sfn_template.asl.json +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_3/library-qc-complete-db-to-tn-ready/step_functions_templates/add_library_qc_complete_to_tn_draft_sfn_template.asl.json @@ -15,7 +15,7 @@ "Parameters": { "TableName": "${__table_name__}", "Key": { - "id.$": "$.payload_data.libraryId", + "id.$": "$.payload_data.library.orcabusId", "id_type": "${__library_partition_name__}" } }, @@ -40,7 +40,7 @@ "Parameters": { "TableName": "${__table_name__}", "Key": { - "id.$": "$.payload_data.libraryId", + "id.$": "$.payload_data.library.orcabusId", "id_type": "${__library_partition_name__}" }, "UpdateExpression": "SET qc_metrics_json = :qc_metrics_json", @@ -60,16 +60,25 @@ }, "Get Libraries From Subject": { "Type": "Task", - "Resource": "arn:aws:states:::dynamodb:getItem", + "Resource": "arn:aws:states:::aws-sdk:dynamodb:scan", "Parameters": { "TableName": "${__table_name__}", - "Key": { - "id.$": "$.get_library_item_step.Item.subject_id.S", - "id_type": "${__subject_partition_name__}" - } + "ExpressionAttributeValues": { + ":subject_orcabus_id": { + "S.$": "$.get_library_item_step.Item.subject_orcabus_id.S" + }, + ":id_type": { + "S": "${__library_partition_name__}" + } + }, + "ExpressionAttributeNames": { + "#subject_orcabus_id": "subject_orcabus_id", + "#id_type": "id_type" + }, + "FilterExpression": "#subject_orcabus_id = :subject_orcabus_id AND #id_type = :id_type" }, "ResultSelector": { - "library_set.$": "$.Item.library_set.SS" + "library_set.$": "$.Items[*].id" }, "ResultPath": "$.get_subject_library_set_step", "Next": "Collect All Libraries in Subject" @@ -78,7 +87,7 @@ "Type": "Map", "ItemsPath": "$.get_subject_library_set_step.library_set", "ItemSelector": { - "library_id.$": "$$.Map.Item.Value" + "library_orcabus_id.$": "$$.Map.Item.Value" }, "ItemProcessor": { "ProcessorConfig": { @@ -92,7 +101,7 @@ "Parameters": { "TableName": "${__table_name__}", "Key": { - "id.$": "$.library_id", + "id.$": "$.library_orcabus_id", "id_type": "${__library_partition_name__}" } }, @@ -116,13 +125,14 @@ "Parameters": { "output": { "library": { - "id.$": "$.library_id", + "orcabus_id.$": "$.get_library_map_step.Item.id.S", + "library_id.$": "$.get_library_map_step.Item.library_id.S", "phenotype.$": "$.get_library_map_step.Item.phenotype.S", "workflow.$": "$.get_library_map_step.Item.workflow.S", "type.$": "$.get_library_map_step.Item.type.S", "assay.$": "$.get_library_map_step.Item.assay.S", "subject_id.$": "$.get_library_map_step.Item.subject_id.S", - "orcabus_id.$": "$.get_library_map_step.Item.orcabus_id.S", + "subject_orcabus_id.$": "$.get_library_map_step.Item.subject_orcabus_id.S", "fastq_list_row_id_set.$": "$.get_library_map_step.Item.fastq_list_row_id_set.SS" } } @@ -153,13 +163,13 @@ "FunctionName": "${__get_complement_library_pair_lambda_function_arn__}", "Payload": { "library_obj": { - "id.$": "$.payload_data.libraryId", + "orcabus_id.$": "$.payload_data.library.orcabusId", + "library_id.$": "$.payload_data.library.libraryId", "phenotype.$": "$.get_library_item_step.Item.phenotype.S", "workflow.$": "$.get_library_item_step.Item.workflow.S", "type.$": "$.get_library_item_step.Item.type.S", "assay.$": "$.get_library_item_step.Item.assay.S", - "subject_id.$": "$.get_library_item_step.Item.subject_id.S", - "orcabus_id.$": "$.get_library_item_step.Item.orcabus_id.S", + "subject_orcabus_id.$": "$.get_library_item_step.Item.subject_orcabus_id.S", "fastq_list_row_id_set.$": "$.get_library_item_step.Item.fastq_list_row_id_set.SS" }, "complementary_library_obj_list.$": "$.get_complementary_libraries_step.complementary_library_obj_list" @@ -325,7 +335,7 @@ } } ], - "Next": "Generate Draft Event Payload", + "Next": "Get Subject Object From Orcabus Id", "ResultSelector": { "tumor_fastq_list_rows.$": "$.[0].get_fastq_list_rows.tumor_fastq_list_rows", "tumor_fastq_list_row_ids.$": "$.[0].get_fastq_list_rows.tumor_fastq_list_row_ids", @@ -336,6 +346,35 @@ }, "ResultPath": "$.get_parameters_step" }, + "Get Subject Object From Orcabus Id": { + "Type": "Task", + "Resource": "arn:aws:states:::lambda:invoke", + "Parameters": { + "FunctionName": "${__get_orcabus_obj_from_subject_id_lambda_function_arn__}", + "Payload": { + "value.$": "$.get_library_item_step.Item.subject_orcabus_id.S" + } + }, + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 1, + "MaxAttempts": 3, + "BackoffRate": 2 + } + ], + "ResultSelector": { + "subject_obj.$": "$.Payload", + "individual_id.$": "States.ArrayGetItem($.Payload.individualSet[?(@.individualId =~ /SBJ.*?/i)].individualId, 0)" + }, + "ResultPath": "$.get_subject_obj_step", + "Next": "Generate Draft Event Payload" + }, "Generate Draft Event Payload": { "Type": "Task", "Resource": "arn:aws:states:::lambda:invoke", @@ -343,8 +382,9 @@ "FunctionName": "${__generate_draft_event_payload_lambda_function_arn__}", "Payload": { "subject_id.$": "$.get_library_item_step.Item.subject_id.S", - "tumor_library_id.$": "$.get_tn_pair_step.tumor_library.id", - "normal_library_id.$": "$.get_tn_pair_step.normal_library.id", + "individual_id.$": "$.get_subject_obj_step.individual_id", + "tumor_library_id.$": "$.get_tn_pair_step.tumor_library.library_id", + "normal_library_id.$": "$.get_tn_pair_step.normal_library.library_id", "tumor_fastq_list_rows.$": "$.get_parameters_step.tumor_fastq_list_rows", "tumor_fastq_list_row_ids.$": "$.get_parameters_step.tumor_fastq_list_row_ids", "fastq_list_rows.$": "$.get_parameters_step.fastq_list_rows", @@ -365,48 +405,35 @@ } ], "ResultPath": "$.generate_draft_event_payload_data_step", - "Next": "Push TN Draft Event", + "Next": "Push TN Ready Event", "ResultSelector": { "input_event_data.$": "$.Payload.input_event_data", "event_tags.$": "$.Payload.event_tags" } }, - "Push TN Draft Event": { + "Push TN Ready Event": { "Type": "Task", - "Resource": "arn:aws:states:::events:putEvents", + "Resource": "arn:aws:states:::states:startExecution.sync:2", "Parameters": { - "Entries": [ - { - "Detail": { - "portalRunId.$": "$.get_parameters_step.portal_run_id", - "timestamp.$": "$$.State.EnteredTime", - "status": "${__output_status__}", - "workflowName": "${__workflow_name__}", - "workflowVersion": "${__workflow_version__}", - "workflowRunName.$": "$.get_parameters_step.workflow_run_name", - "linkedLibraries": [ - { - "libraryId.$": "$.get_tn_pair_step.tumor_library.id", - "orcabusId.$": "$.get_tn_pair_step.tumor_library.orcabus_id" - }, - { - "libraryId.$": "$.get_tn_pair_step.normal_library.id", - "orcabusId.$": "$.get_tn_pair_step.normal_library.orcabus_id" - } - ], - "payload": { - "version": "${__payload_version__}", - "data": { - "inputs.$": "$.generate_draft_event_payload_data_step.input_event_data", - "tags.$": "$.generate_draft_event_payload_data_step.event_tags" - } + "StateMachineArn": "${__launch_ready_event_sfn_arn__}", + "Input": { + "StatePayload": { + "portal_run_id.$": "$.get_parameters_step.portal_run_id", + "workflow_run_name.$": "$.get_parameters_step.workflow_run_name", + "linked_libraries": [ + { + "libraryId.$": "$.get_tn_pair_step.tumor_library.library_id", + "orcabusId.$": "$.get_tn_pair_step.tumor_library.orcabus_id" + }, + { + "libraryId.$": "$.get_tn_pair_step.normal_library.library_id", + "orcabusId.$": "$.get_tn_pair_step.normal_library.orcabus_id" } - }, - "DetailType": "${__detail_type__}", - "EventBusName": "${__event_bus_name__}", - "Source": "${__event_source__}" + ], + "data_inputs.$": "$.generate_draft_event_payload_data_step.input_event_data", + "data_tags.$": "$.generate_draft_event_payload_data_step.event_tags" } - ] + } }, "ResultPath": null, "End": true diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_4/update-fastq-list-row-qc-complete-dbs/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_4/update-fastq-list-row-qc-complete-dbs/index.ts deleted file mode 100644 index 27c716088..000000000 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_4/update-fastq-list-row-qc-complete-dbs/index.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { Construct } from 'constructs'; -import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; -import path from 'path'; -import * as sfn from 'aws-cdk-lib/aws-stepfunctions'; -import * as events from 'aws-cdk-lib/aws-events'; -import * as eventsTargets from 'aws-cdk-lib/aws-events-targets'; - -/* -Part 4 - -Input Event Source: `orcabus.wgtsqcinputeventglue` -Input Event DetailType: `FastqListRowStateChange` -Input Event status: `QcComplete` - -* Populate the fastq list row attributes for the rgid for this workflow -*/ - -export interface TnFastqListRowQcCompleteDbRowConstructProps { - tableObj: dynamodb.ITableV2; - eventBusObj: events.IEventBus; -} - -export class TnFastqListRowQcCompleteConstruct extends Construct { - public readonly TnFastqListRowQcCompleteDbRowMap = { - prefix: 'loctite-fqlr-qc-complete-to-db', - tablePartition: 'fastq_list_row', - triggerSource: 'orcabus.wgtsqcinputeventglue', - triggerStatus: 'QC_COMPLETE', - triggerDetailType: 'FastqListRowStateChange', - }; - - constructor(scope: Construct, id: string, props: TnFastqListRowQcCompleteDbRowConstructProps) { - super(scope, id); - - /* - Part 1: Build the internal sfn - */ - const inputMakerSfn = new sfn.StateMachine(this, 'fastq_list_row_qc_complete', { - stateMachineName: `${this.TnFastqListRowQcCompleteDbRowMap.prefix}-sfn`, - definitionBody: sfn.DefinitionBody.fromFile( - path.join( - __dirname, - 'step_functions_templates', - 'add_fastq_list_row_qc_complete_to_db_sfn_template.asl.json' - ) - ), - definitionSubstitutions: { - __table_name__: props.tableObj.tableName, - __fastq_list_row_partition_name__: this.TnFastqListRowQcCompleteDbRowMap.tablePartition, - }, - }); - - /* - Part 2: Grant the internal sfn permissions - */ - // access the dynamodb table - props.tableObj.grantReadWriteData(inputMakerSfn.role); - - /* - Part 3: Subscribe to the event bus for this event type - */ - const rule = new events.Rule(this, 'tn_populate_fastq_list_row', { - ruleName: `stacky-${this.TnFastqListRowQcCompleteDbRowMap.prefix}-event-rule`, - eventBus: props.eventBusObj, - eventPattern: { - source: [this.TnFastqListRowQcCompleteDbRowMap.triggerSource], - detailType: [this.TnFastqListRowQcCompleteDbRowMap.triggerDetailType], - detail: { - status: [{ 'equals-ignore-case': this.TnFastqListRowQcCompleteDbRowMap.triggerStatus }], - }, - }, - }); - - // Add target of event to be the state machine - rule.addTarget( - new eventsTargets.SfnStateMachine(inputMakerSfn, { - input: events.RuleTargetInput.fromEventPath('$.detail'), - }) - ); - } -} diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_4/update-fastq-list-row-qc-complete-dbs/step_functions_templates/add_fastq_list_row_qc_complete_to_db_sfn_template.asl.json b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_4/update-fastq-list-row-qc-complete-dbs/step_functions_templates/add_fastq_list_row_qc_complete_to_db_sfn_template.asl.json deleted file mode 100644 index 6558f00ad..000000000 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_4/update-fastq-list-row-qc-complete-dbs/step_functions_templates/add_fastq_list_row_qc_complete_to_db_sfn_template.asl.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "Comment": "A description of my state machine", - "StartAt": "Move Inputs", - "States": { - "Move Inputs": { - "Type": "Pass", - "Parameters": { - "payload_data.$": "$.payload.data" - }, - "Next": "Get Fastq List Row Item" - }, - "Get Fastq List Row Item": { - "Type": "Task", - "Resource": "arn:aws:states:::dynamodb:getItem", - "Parameters": { - "TableName": "${__table_name__}", - "Key": { - "id.$": "$.payload_data.fastqListRowId", - "id_type": "${__fastq_list_row_partition_name__}" - } - }, - "ResultPath": "$.get_fastq_list_row_item_step", - "Next": "Fastq List Row In DataBase" - }, - "Fastq List Row In DataBase": { - "Type": "Choice", - "Choices": [ - { - "Variable": "$.get_fastq_list_row_item_step.Item", - "IsPresent": true, - "Comment": "Fastq List Row In Database", - "Next": "Add QC Metrics To Fastq List Row" - } - ], - "Default": "Pass" - }, - "Add QC Metrics To Fastq List Row": { - "Type": "Task", - "Resource": "arn:aws:states:::dynamodb:updateItem", - "Parameters": { - "TableName": "${__table_name__}", - "Key": { - "id.$": "$.payload_data.fastqListRowId", - "id_type": "${__fastq_list_row_partition_name__}" - }, - "UpdateExpression": "SET qc_metrics_json = :qc_metrics_json", - "ExpressionAttributeValues": { - ":qc_metrics_json": { - "S.$": "States.JsonToString($.payload_data.qcMetrics)" - } - } - }, - "ResultPath": null, - "End": true - }, - "Pass": { - "Type": "Pass", - "End": true - } - } -} diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_5/library-qc-complete-db-to-tn-draft/lambdas/find_complement_library_pair_py/find_complement_library_pair.py b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_5/library-qc-complete-db-to-tn-draft/lambdas/find_complement_library_pair_py/find_complement_library_pair.py deleted file mode 100644 index 33e7831c5..000000000 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_5/library-qc-complete-db-to-tn-draft/lambdas/find_complement_library_pair_py/find_complement_library_pair.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env python3 - -""" -Given a library object and a list of complementary library objects, find a matching pair for the library object. - -The library objects must match on 'workflow' and 'type' attributes, but must be the opposite phenotype. -""" - -from typing import Dict, List - - -def find_complement_library_pair(library: Dict, complement_libraries: List[Dict]): - """ - Given a library object and a list of complementary library objects, find a matching pair for the library object - within the complement library list. - :param library: - :param complement_libraries: - :return: - """ - for complement_library in complement_libraries: - if library['workflow'] == complement_library['workflow'] and library['type'] == complement_library['type']: - if library['phenotype'] != complement_library['phenotype']: - return library, complement_library - return None, None - - -def handler(event, context): - """ - Lambda handler function - :param event: - :param context: - :return: - """ - - library_obj: Dict = event['library_obj'] - complement_libraries: List[Dict] = event['complementary_library_obj_list'] - - library, complement_library = find_complement_library_pair(library_obj, complement_libraries) - - if library is None: - return { - 'successful_pairing': False, - 'tumor_library': None, - 'normal_library': None - } - - if library['phenotype'] == 'tumor': - tumor_library = library - normal_library = complement_library - else: - tumor_library = complement_library - normal_library = library - - return { - 'successful_pairing': True, - 'tumor_library': tumor_library, - 'normal_library': normal_library - } diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_6/tn-draft-to-ready/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_6/tn-draft-to-ready/index.ts deleted file mode 100644 index 7e533d704..000000000 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/loctite/part_6/tn-draft-to-ready/index.ts +++ /dev/null @@ -1,105 +0,0 @@ -import { Construct } from 'constructs'; -import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; -import * as ssm from 'aws-cdk-lib/aws-ssm'; -import * as events from 'aws-cdk-lib/aws-events'; -import * as secretsManager from 'aws-cdk-lib/aws-secretsmanager'; -import { WorkflowDraftRunStateChangeToWorkflowRunStateChangeReadyConstruct } from '../../../../../../../components/event-workflowdraftrunstatechange-to-workflowrunstatechange-ready'; - -/* -Part 6 - -Input Event source: `orcabus.tninputeventglue` -Input Event DetailType: `WorkflowDraftRunStateChange` -Input Event status: `draft` - -Output Event source: `orcabus.tninputeventglue` -Output Event DetailType: `WorkflowRunStateChange` -Output Event status: `ready` - -* The tnInputMaker, subscribes to the tn input event glue (itself) and generates a ready event for the tnReadySfn - * However, in order to be 'READY' we need to use a few more variables such as - * icaLogsUri, - * analysisOutputUri - * cacheUri - * projectId - * userReference -*/ - -export interface TnInputMakerConstructProps { - /* Event bus object */ - eventBusObj: events.IEventBus; - /* Tables */ - inputMakerTableObj: dynamodb.ITableV2; - /* SSM Parameter Objects */ - icav2ProjectIdSsmParameterObj: ssm.IStringParameter; - outputUriSsmParameterObj: ssm.IStringParameter; - logsUriSsmParameterObj: ssm.IStringParameter; - cacheUriSsmParameterObj: ssm.IStringParameter; - /* Secrets */ - icav2AccessTokenSecretObj: secretsManager.ISecret; -} - -export class TnInputMakerConstruct extends Construct { - public readonly tnInputMakerEventMap = { - prefix: 'loctite-tn', - tablePartition: 'tn', - triggerSource: 'orcabus.tninputeventglue', - triggerStatus: 'DRAFT', - triggerDetailType: 'WorkflowDraftRunStateChange', - outputSource: 'orcabus.tninputeventglue', - outputStatus: 'READY', - payloadVersion: '2024.07.16', - workflowName: 'tumor_normal', - workflowVersion: '4.2.4', - }; - - constructor(scope: Construct, id: string, props: TnInputMakerConstructProps) { - super(scope, id); - - /* - Part 3: Build the external sfn - */ - new WorkflowDraftRunStateChangeToWorkflowRunStateChangeReadyConstruct( - this, - 'tn_internal_input_maker', - { - /* - Set Input StateMachine Object - */ - lambdaPrefix: this.tnInputMakerEventMap.prefix, - payloadVersion: this.tnInputMakerEventMap.payloadVersion, - stateMachinePrefix: this.tnInputMakerEventMap.prefix, - rulePrefix: `stacky-${this.tnInputMakerEventMap.prefix}`, - - /* - Table objects - */ - tableObj: props.inputMakerTableObj, - tablePartitionName: this.tnInputMakerEventMap.tablePartition, - - /* - Event Triggers - */ - eventBusObj: props.eventBusObj, - triggerDetailType: this.tnInputMakerEventMap.triggerDetailType, - triggerSource: this.tnInputMakerEventMap.triggerSource, - triggerStatus: this.tnInputMakerEventMap.triggerStatus, - outputSource: this.tnInputMakerEventMap.outputSource, - workflowName: this.tnInputMakerEventMap.workflowName, - workflowVersion: this.tnInputMakerEventMap.workflowVersion, - - /* - SSM Parameter Objects - */ - icav2ProjectIdSsmParameterObj: props.icav2ProjectIdSsmParameterObj, - outputUriSsmParameterObj: props.outputUriSsmParameterObj, - logsUriSsmParameterObj: props.logsUriSsmParameterObj, - - /* - Secrets - */ - icav2AccessTokenSecretObj: props.icav2AccessTokenSecretObj, - } - ); - } -} diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/mod-podge/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/mod-podge/index.ts index 87338250e..4bfa25d66 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/mod-podge/index.ts +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/mod-podge/index.ts @@ -5,9 +5,7 @@ import * as ssm from 'aws-cdk-lib/aws-ssm'; import * as secretsManager from 'aws-cdk-lib/aws-secretsmanager'; import { WtsInitialiseLibraryAndFastqListRowConstruct } from './part_1/initialise-wts-library-dbs'; import { WtsPopulateFastqListRowConstruct } from './part_2/update-fastq-list-rows-dbs'; -import { WtsFastqListRowQcCompleteConstruct } from './part_3/update-fastq-list-row-qc-complete-dbs'; -import { LibraryQcCompleteToWtsDraftConstruct } from './part_4/library-qc-complete-to-wts-draft'; -import { WtsInputMakerConstruct } from './part_5/wts-draft-to-ready'; +import { LibraryQcCompleteToWtsReadyConstruct } from './part_3/library-qc-complete-to-wts'; /* Provide the glue to get from the bssh fastq copy manager to submitting wgts qc analyses @@ -18,7 +16,6 @@ export interface wtsGlueHandlerConstructProps { eventBusObj: events.IEventBus; /* Tables */ wtsGlueTableObj: dynamodb.ITableV2; - inputMakerTableObj: dynamodb.ITableV2; /* SSM Parameters */ analysisOutputUriSsmParameterObj: ssm.IStringParameter; analysisLogsUriSsmParameterObj: ssm.IStringParameter; @@ -39,31 +36,11 @@ export class WtsGlueHandlerConstruct extends Construct { Input Event DetailType: `SamplesheetMetadataUnion` Input Event status: `LibraryInSamplesheet` - * Initialise wts instrument db construct + * Initialise wts library qc complete */ - const wts_initialise_library_and_fastq_list_row = - new WtsInitialiseLibraryAndFastqListRowConstruct( - this, - 'wts_initialise_library_and_fastq_list_row', - { - eventBusObj: props.eventBusObj, - tableObj: props.wtsGlueTableObj, - } - ); - - /* - Part 2 - - Input Event Source: `orcabus.instrumentrunmanager` - Input Event DetailType: `FastqListRowStateChange` - Input Event status: `newFastqListRow` - - * Populate the fastq list row attributes for the rgid for this workflow - */ - - const wts_populate_fastq_list_row = new WtsPopulateFastqListRowConstruct( + const wtsInitialiseLibraryAndFastqListRow = new WtsInitialiseLibraryAndFastqListRowConstruct( this, - 'wts_populate_fastq_list_row', + 'wts_initialise_library_and_fastq_list_row', { eventBusObj: props.eventBusObj, tableObj: props.wtsGlueTableObj, @@ -71,18 +48,18 @@ export class WtsGlueHandlerConstruct extends Construct { ); /* - Part 3 + Part 2 - Input Event Source: `orcabus.wgtsqcinputeventglue` + Input Event Source: `orcabus.instrumentrunmanager` Input Event DetailType: `FastqListRowStateChange` - Input Event status: `QcComplete` + Input Event status: `newFastqListRow` * Populate the fastq list row attributes for the rgid for this workflow */ - const wts_fastq_list_row_qc_complete = new WtsFastqListRowQcCompleteConstruct( + const wtsPopulateFastqListRow = new WtsPopulateFastqListRowConstruct( this, - 'wts_fastq_list_row_qc_complete', + 'wts_populate_fastq_list_row', { eventBusObj: props.eventBusObj, tableObj: props.wtsGlueTableObj, @@ -90,7 +67,7 @@ export class WtsGlueHandlerConstruct extends Construct { ); /* - Part 4 + Part 3 Input Event Source: `orcabus.wgtsqcinputeventglue` Input Event DetailType: `LibraryStateChange` @@ -103,49 +80,22 @@ export class WtsGlueHandlerConstruct extends Construct { * Subscribe to the wgts input event glue, library complete event. * Launch a draft event for the wts pipeline if the libraries' subject has a complement library that is also complete */ - const libraryQcCompleteToWtsDraft = new LibraryQcCompleteToWtsDraftConstruct( + const libraryQcCompleteToWtsDraft = new LibraryQcCompleteToWtsReadyConstruct( this, 'library_qc_complete_to_wts_draft', { - // Event bus + /* Event bus */ eventBusObj: props.eventBusObj, - // SSM Param objects + /* Tables */ tableObj: props.wtsGlueTableObj, - workflowsTableObj: props.inputMakerTableObj, + /* SSM Param objects */ + icav2ProjectIdSsmParameterObj: props.icav2ProjectIdSsmParameterObj, + outputUriSsmParameterObj: props.analysisOutputUriSsmParameterObj, + cacheUriSsmParameterObj: props.analysisCacheUriSsmParameterObj, + logsUriSsmParameterObj: props.analysisLogsUriSsmParameterObj, + /* Secrets Manager */ + icav2AccessTokenSecretObj: props.icav2AccessTokenSecretObj, } ); - - /* - Part 5 - - Input Event source: `orcabus.wtsinputeventglue` - Input Event DetailType: `WorkflowDraftRunStateChange` - Input Event status: `draft` - - Output Event source: `orcabus.wtsinputeventglue` - Output Event DetailType: `WorkflowRunStateChange` - Output Event status: `ready` - - * The wtsInputMaker, subscribes to the wts input event glue (itself) and generates a ready event for the wtsReadySfn - * However, in order to be 'READY' we need to use a few more variables such as - * icaLogsUri, - * analysisOutputUri - * cacheUri - * projectId - * userReference - */ - const wtsInputMaker = new WtsInputMakerConstruct(this, 'fastq_list_row_qc_complete', { - /* Event bus */ - eventBusObj: props.eventBusObj, - /* Tables */ - inputMakerTableObj: props.inputMakerTableObj, - /* SSM Param objects */ - icav2ProjectIdSsmParameterObj: props.icav2ProjectIdSsmParameterObj, - outputUriSsmParameterObj: props.analysisOutputUriSsmParameterObj, - cacheUriSsmParameterObj: props.analysisCacheUriSsmParameterObj, - logsUriSsmParameterObj: props.analysisLogsUriSsmParameterObj, - /* Secrets Manager */ - icav2AccessTokenSecretObj: props.icav2AccessTokenSecretObj, - }); } } diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/mod-podge/part_1/initialise-wts-library-dbs/step_functions_templates/initialise_wts_library_db_sfn_template.asl.json b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/mod-podge/part_1/initialise-wts-library-dbs/step_functions_templates/initialise_wts_library_db_sfn_template.asl.json index e5d7eaf3c..714ae9436 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/mod-podge/part_1/initialise-wts-library-dbs/step_functions_templates/initialise_wts_library_db_sfn_template.asl.json +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/mod-podge/part_1/initialise-wts-library-dbs/step_functions_templates/initialise_wts_library_db_sfn_template.asl.json @@ -15,7 +15,7 @@ "Parameters": { "TableName": "${__table_name__}", "Key": { - "id.$": "$.payload_data.library.libraryId", + "id.$": "$.payload_data.library.orcabusId", "id_type": "${__library_partition_name__}" } }, @@ -40,10 +40,10 @@ "Parameters": { "TableName": "${__table_name__}", "Item": { - "id.$": "$.payload_data.library.libraryId", + "id.$": "$.payload_data.library.orcabusId", "id_type": "${__library_partition_name__}", - "orcabus_id": { - "S.$": "$.payload_data.library.orcabusId" + "library_id": { + "S.$": "$.payload_data.library.libraryId" }, "phenotype": { "S.$": "$.payload_data.library.phenotype" @@ -58,7 +58,10 @@ "S.$": "$.payload_data.library.assay" }, "subject_id": { - "S.$": "$.payload_data.library.subject.subjectId" + "S.$": "$.payload_data.subject.subjectId" + }, + "subject_orcabus_id": { + "S.$": "$.payload_data.subject.orcabusId" } } }, @@ -73,7 +76,8 @@ "index.$": "$$.Map.Item.Index", "fastq_list_row_objs.$": "$.payload_data.fastqListRows", "instrument_run_id.$": "$.payload_data.instrumentRunId", - "library_id.$": "$.payload_data.library.libraryId" + "library_id.$": "$.payload_data.library.libraryId", + "library_orcabus_id.$": "$.payload_data.library.orcabusId" }, "ItemProcessor": { "ProcessorConfig": { @@ -105,6 +109,9 @@ "id_type": "${__fastq_list_row_partition_name__}", "library_id": { "S.$": "$.library_id" + }, + "library_orcabus_id": { + "S.$": "$.library_orcabus_id" } } }, @@ -122,7 +129,7 @@ "Parameters": { "TableName": "${__table_name__}", "Key": { - "id.$": "$.library_id", + "id.$": "$.library_orcabus_id", "id_type": "${__library_partition_name__}" }, "UpdateExpression": "ADD fastq_list_row_id_set :fastq_list_row_id_set", diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/mod-podge/part_4/library-qc-complete-to-wts-draft/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/mod-podge/part_3/library-qc-complete-to-wts/index.ts similarity index 61% rename from lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/mod-podge/part_4/library-qc-complete-to-wts-draft/index.ts rename to lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/mod-podge/part_3/library-qc-complete-to-wts/index.ts index c7e6c4418..ec20034b0 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/mod-podge/part_4/library-qc-complete-to-wts-draft/index.ts +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/mod-podge/part_3/library-qc-complete-to-wts/index.ts @@ -8,7 +8,11 @@ import * as events from 'aws-cdk-lib/aws-events'; import * as eventsTargets from 'aws-cdk-lib/aws-events-targets'; import { PythonFunction } from '@aws-cdk/aws-lambda-python-alpha'; import * as lambda from 'aws-cdk-lib/aws-lambda'; +import * as ssm from 'aws-cdk-lib/aws-ssm'; +import * as secretsManager from 'aws-cdk-lib/aws-secretsmanager'; import { WorkflowDraftRunStateChangeCommonPreambleConstruct } from '../../../../../../../components/sfn-workflowdraftrunstatechange-common-preamble'; +import { GenerateWorkflowRunStateChangeReadyConstruct } from '../../../../../../../components/sfn-generate-workflowrunstatechange-ready-event'; +import { GetMetadataLambdaConstruct } from '../../../../../../../components/python-lambda-metadata-mapper'; /* Part 4 @@ -26,25 +30,30 @@ Output Event status: `draft` */ export interface LibraryQcCompleteToWtsDraftConstructProps { + /* Events */ eventBusObj: events.IEventBus; + /* Tables */ tableObj: dynamodb.ITableV2; - workflowsTableObj: dynamodb.ITableV2; + /* SSM */ + outputUriSsmParameterObj: ssm.IStringParameter; + icav2ProjectIdSsmParameterObj: ssm.IStringParameter; + logsUriSsmParameterObj: ssm.IStringParameter; + cacheUriSsmParameterObj: ssm.IStringParameter; + /* Secrets */ + icav2AccessTokenSecretObj: secretsManager.ISecret; } -export class LibraryQcCompleteToWtsDraftConstruct extends Construct { - public readonly WtsDraftMap = { +export class LibraryQcCompleteToWtsReadyConstruct extends Construct { + public readonly WtsReadyMap = { prefix: 'modpodge-library-qc-to-wts', tablePartition: { library: 'library', fastq_list_row: 'fastq_list_row', }, - portalRunPartitionName: 'portal_run', triggerSource: 'orcabus.wgtsqcinputeventglue', triggerStatus: 'QC_COMPLETE', triggerDetailType: 'LibraryStateChange', outputSource: 'orcabus.wtsinputeventglue', - outputDetailType: 'WorkflowDraftRunStateChange', - outputStatus: 'DRAFT', payloadVersion: '2024.07.23', workflowName: 'wts', workflowVersion: '4.2.4', @@ -69,15 +78,42 @@ export class LibraryQcCompleteToWtsDraftConstruct extends Construct { /* Part 1: Generate the preamble (sfn to generate the portal run id and the workflow run name) */ - const sfn_preamble = new WorkflowDraftRunStateChangeCommonPreambleConstruct( + const sfnPreamble = new WorkflowDraftRunStateChangeCommonPreambleConstruct( this, - `${this.WtsDraftMap.prefix}_sfn_preamble`, + `${this.WtsReadyMap.prefix}_sfn_preamble`, { - portalRunTablePartitionName: this.WtsDraftMap.portalRunPartitionName, - stateMachinePrefix: this.WtsDraftMap.prefix, - tableObj: props.workflowsTableObj, - workflowName: this.WtsDraftMap.workflowName, - workflowVersion: this.WtsDraftMap.workflowVersion, + stateMachinePrefix: this.WtsReadyMap.prefix, + workflowName: this.WtsReadyMap.workflowName, + workflowVersion: this.WtsReadyMap.workflowVersion, + } + ).stepFunctionObj; + + /* + Part 2: Build the engine parameters sfn + */ + const engineParameterAndReadyEventMakerSfn = new GenerateWorkflowRunStateChangeReadyConstruct( + this, + 'fastqlistrow_complete_to_wgtsqc_ready_submitter', + { + /* Event Placeholders */ + eventBusObj: props.eventBusObj, + outputSource: this.WtsReadyMap.outputSource, + payloadVersion: this.WtsReadyMap.payloadVersion, + workflowName: this.WtsReadyMap.workflowName, + workflowVersion: this.WtsReadyMap.workflowVersion, + + /* SSM Parameters */ + outputUriSsmParameterObj: props.outputUriSsmParameterObj, + icav2ProjectIdSsmParameterObj: props.icav2ProjectIdSsmParameterObj, + logsUriSsmParameterObj: props.logsUriSsmParameterObj, + cacheUriSsmParameterObj: props.cacheUriSsmParameterObj, + + /* Secrets */ + icav2AccessTokenSecretObj: props.icav2AccessTokenSecretObj, + + /* Prefixes */ + lambdaPrefix: this.WtsReadyMap.prefix, + stateMachinePrefix: this.WtsReadyMap.prefix, } ).stepFunctionObj; @@ -88,7 +124,7 @@ export class LibraryQcCompleteToWtsDraftConstruct extends Construct { this, 'library_qc_complete_sfn_to_wts_draft', { - stateMachineName: `${this.WtsDraftMap.prefix}-sfn`, + stateMachineName: `${this.WtsReadyMap.prefix}-sfn`, definitionBody: sfn.DefinitionBody.fromFile( path.join( __dirname, @@ -97,26 +133,18 @@ export class LibraryQcCompleteToWtsDraftConstruct extends Construct { ) ), definitionSubstitutions: { - /* Events */ - __event_bus_name__: props.eventBusObj.eventBusName, - __event_source__: this.WtsDraftMap.outputSource, - __detail_type__: this.WtsDraftMap.outputDetailType, - __output_status__: this.WtsDraftMap.outputStatus, - __payload_version__: this.WtsDraftMap.payloadVersion, - __workflow_name__: this.WtsDraftMap.workflowName, - __workflow_version__: this.WtsDraftMap.workflowVersion, - /* Lambdas */ __generate_draft_event_payload_lambda_function_arn__: generateEventDataLambdaObj.currentVersion.functionArn, /* Tables */ __table_name__: props.tableObj.tableName, - __library_partition_name__: this.WtsDraftMap.tablePartition.library, - __fastq_list_row_partition_name__: this.WtsDraftMap.tablePartition.fastq_list_row, + __library_partition_name__: this.WtsReadyMap.tablePartition.library, + __fastq_list_row_partition_name__: this.WtsReadyMap.tablePartition.fastq_list_row, - // State Machines - __sfn_preamble_state_machine_arn__: sfn_preamble.stateMachineArn, + /* State Machines */ + __sfn_preamble_state_machine_arn__: sfnPreamble.stateMachineArn, + __launch_ready_event_sfn_arn__: engineParameterAndReadyEventMakerSfn.stateMachineArn, }, } ); @@ -127,9 +155,6 @@ export class LibraryQcCompleteToWtsDraftConstruct extends Construct { // access the dynamodb table props.tableObj.grantReadWriteData(qcCompleteToDraftSfn); - // allow the step function to submit events - props.eventBusObj.grantPutEventsTo(qcCompleteToDraftSfn); - // allow the step function to invoke the lambdas generateEventDataLambdaObj.currentVersion.grantInvoke(qcCompleteToDraftSfn); @@ -145,19 +170,20 @@ export class LibraryQcCompleteToWtsDraftConstruct extends Construct { }) ); // Allow the state machine to be able to invoke the preamble sfn - sfn_preamble.grantStartExecution(qcCompleteToDraftSfn); + sfnPreamble.grantStartExecution(qcCompleteToDraftSfn); + engineParameterAndReadyEventMakerSfn.grantStartExecution(qcCompleteToDraftSfn); /* Part 3: Subscribe to the event bus and trigger the internal sfn */ const rule = new events.Rule(this, 'library_qc_complete_to_tn_draft', { - ruleName: `stacky-${this.WtsDraftMap.prefix}-rule`, + ruleName: `stacky-${this.WtsReadyMap.prefix}-rule`, eventBus: props.eventBusObj, eventPattern: { - source: [this.WtsDraftMap.triggerSource], - detailType: [this.WtsDraftMap.triggerDetailType], + source: [this.WtsReadyMap.triggerSource], + detailType: [this.WtsReadyMap.triggerDetailType], detail: { - status: [{ 'equals-ignore-case': this.WtsDraftMap.triggerStatus }], + status: [{ 'equals-ignore-case': this.WtsReadyMap.triggerStatus }], }, }, }); diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/mod-podge/part_4/library-qc-complete-to-wts-draft/lambdas/generate_draft_event_payload_py/generate_draft_event_payload.py b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/mod-podge/part_3/library-qc-complete-to-wts/lambdas/generate_draft_event_payload_py/generate_draft_event_payload.py similarity index 100% rename from lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/mod-podge/part_4/library-qc-complete-to-wts-draft/lambdas/generate_draft_event_payload_py/generate_draft_event_payload.py rename to lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/mod-podge/part_3/library-qc-complete-to-wts/lambdas/generate_draft_event_payload_py/generate_draft_event_payload.py diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/mod-podge/part_4/library-qc-complete-to-wts-draft/step_functions_templates/add_library_qc_complete_to_wts_draft_sfn_template.asl.json b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/mod-podge/part_3/library-qc-complete-to-wts/step_functions_templates/add_library_qc_complete_to_wts_draft_sfn_template.asl.json similarity index 79% rename from lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/mod-podge/part_4/library-qc-complete-to-wts-draft/step_functions_templates/add_library_qc_complete_to_wts_draft_sfn_template.asl.json rename to lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/mod-podge/part_3/library-qc-complete-to-wts/step_functions_templates/add_library_qc_complete_to_wts_draft_sfn_template.asl.json index c755fba5c..ba189ec14 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/mod-podge/part_4/library-qc-complete-to-wts-draft/step_functions_templates/add_library_qc_complete_to_wts_draft_sfn_template.asl.json +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/mod-podge/part_3/library-qc-complete-to-wts/step_functions_templates/add_library_qc_complete_to_wts_draft_sfn_template.asl.json @@ -15,7 +15,7 @@ "Parameters": { "TableName": "${__table_name__}", "Key": { - "id.$": "$.payload_data.libraryId", + "id.$": "$.payload_data.library.orcabusId", "id_type": "${__library_partition_name__}" } }, @@ -115,7 +115,7 @@ "Parameters": { "FunctionName": "${__generate_draft_event_payload_lambda_function_arn__}", "Payload": { - "tumor_library_id.$": "$.payload_data.libraryId", + "tumor_library_id.$": "$.payload_data.library.libraryId", "tumor_fastq_list_rows.$": "$.get_parameters_step.tumor_fastq_list_rows", "tumor_fastq_list_row_ids.$": "$.get_parameters_step.tumor_fastq_list_row_ids", "subject_id.$": "$.get_library_item_step.Item.subject_id.S" @@ -135,44 +135,31 @@ } ], "ResultPath": "$.generate_draft_event_payload_data_step", - "Next": "Push WTS Draft Event", + "Next": "Push WTS Ready Event", "ResultSelector": { "input_event_data.$": "$.Payload.input_event_data", "event_tags.$": "$.Payload.event_tags" } }, - "Push WTS Draft Event": { + "Push WTS Ready Event": { "Type": "Task", - "Resource": "arn:aws:states:::events:putEvents", + "Resource": "arn:aws:states:::states:startExecution.sync:2", "Parameters": { - "Entries": [ - { - "EventBusName": "${__event_bus_name__}", - "Source": "${__event_source__}", - "DetailType": "${__detail_type__}", - "Detail": { - "portalRunId.$": "$.get_parameters_step.portal_run_id", - "timestamp.$": "$$.State.EnteredTime", - "status": "${__output_status__}", - "workflowName": "${__workflow_name__}", - "workflowVersion": "${__workflow_version__}", - "workflowRunName.$": "$.get_parameters_step.workflow_run_name", - "linkedLibraries": [ - { - "libraryId.$": "$.get_library_item_step.Item.id.S", - "orcabusId.$": "$.get_library_item_step.Item.orcabus_id.S" - } - ], - "payload": { - "version": "${__payload_version__}", - "data": { - "inputs.$": "$.generate_draft_event_payload_data_step.input_event_data", - "tags.$": "$.generate_draft_event_payload_data_step.event_tags" - } + "StateMachineArn": "${__launch_ready_event_sfn_arn__}", + "Input": { + "StatePayload": { + "portal_run_id.$": "$.get_parameters_step.portal_run_id", + "workflow_run_name.$": "$.get_parameters_step.workflow_run_name", + "linked_libraries": [ + { + "libraryId.$": "$.get_library_item_step.Item.library_id.S", + "orcabusId.$": "$.get_library_item_step.Item.id.S" } - } + ], + "data_inputs.$": "$.generate_draft_event_payload_data_step.input_event_data", + "data_tags.$": "$.generate_draft_event_payload_data_step.event_tags" } - ] + } }, "ResultPath": null, "End": true diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/mod-podge/part_3/update-fastq-list-row-qc-complete-dbs/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/mod-podge/part_3/update-fastq-list-row-qc-complete-dbs/index.ts deleted file mode 100644 index d1e9d8e52..000000000 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/mod-podge/part_3/update-fastq-list-row-qc-complete-dbs/index.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { Construct } from 'constructs'; -import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; -import path from 'path'; -import * as sfn from 'aws-cdk-lib/aws-stepfunctions'; -import * as events from 'aws-cdk-lib/aws-events'; -import * as eventsTargets from 'aws-cdk-lib/aws-events-targets'; - -/* -Part 3 - -Input Event Source: `orcabus.wgtsqcinputeventglue` -Input Event DetailType: `FastqListRowStateChange` -Input Event status: `QcComplete` - -* Populate the fastq list row attributes for the rgid for this workflow -*/ - -export interface WtsFastqListRowQcCompleteDbRowConstructProps { - tableObj: dynamodb.ITableV2; - eventBusObj: events.IEventBus; -} - -export class WtsFastqListRowQcCompleteConstruct extends Construct { - public readonly WtsFastqListRowQcCompleteDbRowMap = { - prefix: 'modpodge-fqlr-qc-complete', - tablePartition: 'fastq_list_row', - triggerSource: 'orcabus.wgtsqcinputeventglue', - triggerStatus: 'QC_COMPLETE', - triggerDetailType: 'FastqListRowStateChange', - }; - - constructor(scope: Construct, id: string, props: WtsFastqListRowQcCompleteDbRowConstructProps) { - super(scope, id); - - /* - Part 1: Build the internal sfn - */ - const inputMakerSfn = new sfn.StateMachine(this, 'fastq_list_row_qc_complete', { - stateMachineName: `${this.WtsFastqListRowQcCompleteDbRowMap.prefix}-sfn`, - definitionBody: sfn.DefinitionBody.fromFile( - path.join( - __dirname, - 'step_functions_templates', - 'add_fastq_list_row_qc_complete_to_db_sfn_template.asl.json' - ) - ), - definitionSubstitutions: { - __table_name__: props.tableObj.tableName, - __fastq_list_row_partition_name__: this.WtsFastqListRowQcCompleteDbRowMap.tablePartition, - }, - }); - - /* - Part 2: Grant the internal sfn permissions - */ - // access the dynamodb table - props.tableObj.grantReadWriteData(inputMakerSfn.role); - - /* - Part 3: Subscribe to the event bus for this event type - */ - const rule = new events.Rule(this, 'wts_populate_fastq_list_row', { - ruleName: `stacky-${this.WtsFastqListRowQcCompleteDbRowMap.prefix}-event-rule`, - eventBus: props.eventBusObj, - eventPattern: { - source: [this.WtsFastqListRowQcCompleteDbRowMap.triggerSource], - detailType: [this.WtsFastqListRowQcCompleteDbRowMap.triggerDetailType], - detail: { - status: [{ 'equals-ignore-case': this.WtsFastqListRowQcCompleteDbRowMap.triggerStatus }], - }, - }, - }); - - // Add target of event to be the state machine - rule.addTarget( - new eventsTargets.SfnStateMachine(inputMakerSfn, { - input: events.RuleTargetInput.fromEventPath('$.detail'), - }) - ); - } -} diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/mod-podge/part_3/update-fastq-list-row-qc-complete-dbs/step_functions_templates/add_fastq_list_row_qc_complete_to_db_sfn_template.asl.json b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/mod-podge/part_3/update-fastq-list-row-qc-complete-dbs/step_functions_templates/add_fastq_list_row_qc_complete_to_db_sfn_template.asl.json deleted file mode 100644 index 6558f00ad..000000000 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/mod-podge/part_3/update-fastq-list-row-qc-complete-dbs/step_functions_templates/add_fastq_list_row_qc_complete_to_db_sfn_template.asl.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "Comment": "A description of my state machine", - "StartAt": "Move Inputs", - "States": { - "Move Inputs": { - "Type": "Pass", - "Parameters": { - "payload_data.$": "$.payload.data" - }, - "Next": "Get Fastq List Row Item" - }, - "Get Fastq List Row Item": { - "Type": "Task", - "Resource": "arn:aws:states:::dynamodb:getItem", - "Parameters": { - "TableName": "${__table_name__}", - "Key": { - "id.$": "$.payload_data.fastqListRowId", - "id_type": "${__fastq_list_row_partition_name__}" - } - }, - "ResultPath": "$.get_fastq_list_row_item_step", - "Next": "Fastq List Row In DataBase" - }, - "Fastq List Row In DataBase": { - "Type": "Choice", - "Choices": [ - { - "Variable": "$.get_fastq_list_row_item_step.Item", - "IsPresent": true, - "Comment": "Fastq List Row In Database", - "Next": "Add QC Metrics To Fastq List Row" - } - ], - "Default": "Pass" - }, - "Add QC Metrics To Fastq List Row": { - "Type": "Task", - "Resource": "arn:aws:states:::dynamodb:updateItem", - "Parameters": { - "TableName": "${__table_name__}", - "Key": { - "id.$": "$.payload_data.fastqListRowId", - "id_type": "${__fastq_list_row_partition_name__}" - }, - "UpdateExpression": "SET qc_metrics_json = :qc_metrics_json", - "ExpressionAttributeValues": { - ":qc_metrics_json": { - "S.$": "States.JsonToString($.payload_data.qcMetrics)" - } - } - }, - "ResultPath": null, - "End": true - }, - "Pass": { - "Type": "Pass", - "End": true - } - } -} diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/mod-podge/part_5/wts-draft-to-ready/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/mod-podge/part_5/wts-draft-to-ready/index.ts deleted file mode 100644 index 570dfb022..000000000 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/mod-podge/part_5/wts-draft-to-ready/index.ts +++ /dev/null @@ -1,105 +0,0 @@ -import { Construct } from 'constructs'; -import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; -import * as ssm from 'aws-cdk-lib/aws-ssm'; -import * as secretsManager from 'aws-cdk-lib/aws-secretsmanager'; -import * as events from 'aws-cdk-lib/aws-events'; -import { WorkflowDraftRunStateChangeToWorkflowRunStateChangeReadyConstruct } from '../../../../../../../components/event-workflowdraftrunstatechange-to-workflowrunstatechange-ready'; - -/* -Part 5 - -Input Event source: `orcabus.wtsinputeventglue` -Input Event DetailType: `WorkflowDraftRunStateChange` -Input Event status: `draft` - -Output Event source: `orcabus.wtsinputeventglue` -Output Event DetailType: `WorkflowRunStateChange` -Output Event status: `ready` - -* The wtsInputMaker, subscribes to the wts input event glue (itself) and generates a ready event for the wtsReadySfn - * However, in order to be 'READY' we need to use a few more variables such as - * icaLogsUri, - * analysisOutputUri - * cacheUri - * projectId - * userReference -*/ - -export interface WtsInputMakerConstructProps { - /* Event bus object */ - eventBusObj: events.IEventBus; - /* Tables */ - inputMakerTableObj: dynamodb.ITableV2; - /* SSM Parameter Objects */ - icav2ProjectIdSsmParameterObj: ssm.IStringParameter; - outputUriSsmParameterObj: ssm.IStringParameter; - logsUriSsmParameterObj: ssm.IStringParameter; - cacheUriSsmParameterObj: ssm.IStringParameter; - /* Secrets */ - icav2AccessTokenSecretObj: secretsManager.ISecret; -} - -export class WtsInputMakerConstruct extends Construct { - public readonly wtsInputMakerEventMap = { - prefix: 'modpodge-wts', - tablePartition: 'wts', - triggerSource: 'orcabus.wtsinputeventglue', - triggerStatus: 'DRAFT', - triggerDetailType: 'WorkflowDraftRunStateChange', - outputSource: 'orcabus.wtsinputeventglue', - outputStatus: 'READY', - payloadVersion: '2024.07.16', - workflowName: 'wts', - workflowVersion: '4.2.4', - }; - - constructor(scope: Construct, id: string, props: WtsInputMakerConstructProps) { - super(scope, id); - - /* - Part 3: Build the external sfn - */ - new WorkflowDraftRunStateChangeToWorkflowRunStateChangeReadyConstruct( - this, - 'wts_internal_input_maker', - { - /* - Set Input StateMachine Object - */ - lambdaPrefix: this.wtsInputMakerEventMap.prefix, - payloadVersion: this.wtsInputMakerEventMap.payloadVersion, - stateMachinePrefix: this.wtsInputMakerEventMap.prefix, - rulePrefix: `stacky-${this.wtsInputMakerEventMap.prefix}`, - - /* - Table objects - */ - tableObj: props.inputMakerTableObj, - tablePartitionName: this.wtsInputMakerEventMap.tablePartition, - - /* - Event Triggers - */ - eventBusObj: props.eventBusObj, - triggerDetailType: this.wtsInputMakerEventMap.triggerDetailType, - triggerSource: this.wtsInputMakerEventMap.triggerSource, - triggerStatus: this.wtsInputMakerEventMap.triggerStatus, - outputSource: this.wtsInputMakerEventMap.outputSource, - workflowName: this.wtsInputMakerEventMap.workflowName, - workflowVersion: this.wtsInputMakerEventMap.workflowVersion, - - /* - SSM Parameter Objects - */ - icav2ProjectIdSsmParameterObj: props.icav2ProjectIdSsmParameterObj, - outputUriSsmParameterObj: props.outputUriSsmParameterObj, - logsUriSsmParameterObj: props.logsUriSsmParameterObj, - - /* - Secrets - */ - icav2AccessTokenSecretObj: props.icav2AccessTokenSecretObj, - } - ); - } -} diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/index.ts new file mode 100644 index 000000000..1c5b0075d --- /dev/null +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/index.ts @@ -0,0 +1,88 @@ +/* + +Construct the stacky glue for generating a cttso v2 glue stack + +Connect the cttso v2 outputs to pieriandx + +*/ + +import { Construct } from 'constructs'; +import * as events from 'aws-cdk-lib/aws-events'; +import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; +import * as secretsManager from 'aws-cdk-lib/aws-secretsmanager'; +import * as ssm from 'aws-cdk-lib/aws-ssm'; +import * as lambda from 'aws-cdk-lib/aws-lambda'; +import { PieriandxInitialiseLibraryConstruct } from './part_1/initialise-library-db'; +import { Cttsov2CompleteToPieriandxConstruct } from './part_2/cttso-v2-output-to-pieriandx-ready-event'; + +/* +Provide the glue to get from the bssh fastq copy manager to submitting wgts qc analyses +*/ + +export interface pieriandxGlueHandlerConstructProps { + /* General */ + eventBusObj: events.IEventBus; + /* Tables */ + pieriandxGlueTableObj: dynamodb.ITableV2; + /* Secrets */ + icav2AccessTokenSecretObj: secretsManager.ISecret; + /* Extras */ + pieriandxProjectInfoSsmParameterObj: ssm.IStringParameter; + redcapLambdaObj: lambda.IFunction; +} + +export class PieriandxGlueHandlerConstruct extends Construct { + constructor(scope: Construct, id: string, props: pieriandxGlueHandlerConstructProps) { + super(scope, id); + /* + Part 1 + + Input Event Source: `orcabus.instrumentrunmanager` + Input Event DetailType: `SamplesheetMetadataUnion` + Input Event status: `LibraryInSamplesheet` + + * Initialise pieriandx instrument db construct + */ + const PieriandxInitialiseLibrary = new PieriandxInitialiseLibraryConstruct( + this, + 'pieriandx_initialise_library', + { + eventBusObj: props.eventBusObj, + tableObj: props.pieriandxGlueTableObj, + } + ); + + /* + Part 2 + + Input Event Source: `orcabus.workflowmanager` + Input Event DetailType: `WorkflowRunStateChange` + Input Event status: `succeeded` + + Output Event source: `orcabus.pieriandxinputeventglue` + Output Event DetailType: `WorkflowDraftRunStateChange` + Output Event status: `draft` + + * Populate the fastq list row attributes for the rgid for this workflow + */ + + const cttsov2CompleteToPieriandxReady = new Cttsov2CompleteToPieriandxConstruct( + this, + 'cttsov2_to_pieriandx', + { + /* Events*/ + eventBusObj: props.eventBusObj, + + /* Tables */ + tableObj: props.pieriandxGlueTableObj, + + /* Secrets Manager */ + icav2AccessTokenSecretObj: props.icav2AccessTokenSecretObj, + + /* Extras */ + projectInfoSsmParameterObj: props.pieriandxProjectInfoSsmParameterObj, + redcapLambdaObj: props.redcapLambdaObj, + } + ); + } +} diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_1/initialise-library-db/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_1/initialise-library-db/index.ts new file mode 100644 index 000000000..aad69e93e --- /dev/null +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_1/initialise-library-db/index.ts @@ -0,0 +1,112 @@ +/* + +Populate the library database for an instrument run + +We need to collect the following attributes in the library database for each sample in pieriandx: + +subject_id +project_name +project_owner +instrument_run_id +external_subject_id +external_sample_id + +We store this under the library table + +*/ + +import { Construct } from 'constructs'; +import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; +import path from 'path'; +import * as sfn from 'aws-cdk-lib/aws-stepfunctions'; +import * as events from 'aws-cdk-lib/aws-events'; +import * as eventsTargets from 'aws-cdk-lib/aws-events-targets'; + +export interface PieriandxInitialiseLibraryConstructProps { + tableObj: dynamodb.ITableV2; + eventBusObj: events.IEventBus; +} + +export class PieriandxInitialiseLibraryConstruct extends Construct { + public readonly PieriandxInitialiseLibrary = { + prefix: 'nails-make-library', + tablePartition: { + library: 'library', + }, + triggerSource: 'orcabus.instrumentrunmanager', + triggerStatus: 'LibraryInSamplesheet', + triggerDetailType: 'SamplesheetMetadataUnion', + triggerAssayType: { + v1: 'cttso', + v2: 'cttsov2', + }, + }; + + constructor(scope: Construct, id: string, props: PieriandxInitialiseLibraryConstructProps) { + super(scope, id); + + /* + Part 1: Build the internal sfn + */ + const inputMakerSfn = new sfn.StateMachine(this, 'initialise_pieriandx_library_db_row', { + stateMachineName: `${this.PieriandxInitialiseLibrary.prefix}-initialise-pieriandx-library-db`, + definitionBody: sfn.DefinitionBody.fromFile( + path.join( + __dirname, + 'step_functions_templates', + 'store_cttsov2_metadata_sfn_template.asl.json' + ) + ), + definitionSubstitutions: { + /* General */ + __table_name__: props.tableObj.tableName, + + /* Table Partitions */ + __library_partition_name__: this.PieriandxInitialiseLibrary.tablePartition.library, + }, + }); + + /* + Part 2: Grant the sfn permissions + */ + // access the dynamodb table + props.tableObj.grantReadWriteData(inputMakerSfn.role); + + /* + Part 3: Subscribe to the library events from the event bus where the library assay type + is WGS and the workflow is RESEARCH or CLINICAL + and where the phenotype is NORMAL or TUMOR + */ + const rule = new events.Rule(this, 'initialise_library_assay', { + ruleName: `stacky-${this.PieriandxInitialiseLibrary.prefix}-rule`, + eventBus: props.eventBusObj, + eventPattern: { + source: [this.PieriandxInitialiseLibrary.triggerSource], + detailType: [this.PieriandxInitialiseLibrary.triggerDetailType], + detail: { + payload: { + data: { + library: { + assay: [ + { + 'equals-ignore-case': this.PieriandxInitialiseLibrary.triggerAssayType.v1, + }, + { + 'equals-ignore-case': this.PieriandxInitialiseLibrary.triggerAssayType.v2, + }, + ], + }, + }, + }, + }, + }, + }); + + // Add target of event to be the state machine + rule.addTarget( + new eventsTargets.SfnStateMachine(inputMakerSfn, { + input: events.RuleTargetInput.fromEventPath('$.detail'), + }) + ); + } +} diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_1/initialise-library-db/step_functions_templates/store_cttsov2_metadata_sfn_template.asl.json b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_1/initialise-library-db/step_functions_templates/store_cttsov2_metadata_sfn_template.asl.json new file mode 100644 index 000000000..de06d01db --- /dev/null +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_1/initialise-library-db/step_functions_templates/store_cttsov2_metadata_sfn_template.asl.json @@ -0,0 +1,48 @@ +{ + "Comment": "A description of my state machine", + "StartAt": "Move Inputs", + "States": { + "Move Inputs": { + "Type": "Pass", + "Next": "Initialise Library ID", + "Parameters": { + "inputs.$": "$", + "input_payload_data.$": "$.payload.data" + } + }, + "Initialise Library ID": { + "Type": "Task", + "Resource": "arn:aws:states:::dynamodb:putItem", + "Parameters": { + "TableName": "${__table_name__}", + "Item": { + "id": { + "S.$": "$.input_payload_data.library.orcabusId" + }, + "id_type": { + "S": "${__library_partition_name__}" + }, + "library_id": { + "S.$": "$.input_payload_data.library.libraryId" + }, + "library_obj": { + "S.$": "States.JsonToString($.input_payload_data.library)" + }, + "project_id": { + "S.$": "$.input_payload_data.projectSet[0].projectId" + }, + "external_sample_id": { + "S.$": "$.input_payload_data.sample.externalSampleId" + }, + "external_subject_id": { + "S.$": "$.input_payload_data.subject.subjectId" + }, + "instrument_run_id": { + "S.$": "$.input_payload_data.instrumentRunId" + } + } + }, + "End": true + } + } +} diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_2/cttso-v2-output-to-pieriandx-ready-event/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_2/cttso-v2-output-to-pieriandx-ready-event/index.ts new file mode 100644 index 000000000..78c82234d --- /dev/null +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_2/cttso-v2-output-to-pieriandx-ready-event/index.ts @@ -0,0 +1,267 @@ +/* + +Given a cttsov2 success event we need to + +1. Generate a portal run id +2. Collect any data available from redcap for this given subject / library combination +3. Collect the project owner and project name configuration for pieriandx +4. Collect the cttsov2 outputs +5. Send the data to pieriandx for processing + +*/ + +import { Construct } from 'constructs'; +import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; +import path from 'path'; +import * as sfn from 'aws-cdk-lib/aws-stepfunctions'; +import * as ssm from 'aws-cdk-lib/aws-ssm'; +import * as secretsManager from 'aws-cdk-lib/aws-secretsmanager'; +import * as events from 'aws-cdk-lib/aws-events'; +import * as eventsTargets from 'aws-cdk-lib/aws-events-targets'; +import { PythonFunction } from '@aws-cdk/aws-lambda-python-alpha'; +import * as lambda from 'aws-cdk-lib/aws-lambda'; +import { Duration } from 'aws-cdk-lib'; + +/* +Part 2 + +Input Event Source: `orcabus.workflowmanager` +Input Event DetailType: `WorkflowRunStateChange` +Input Event status: `COMPLETE` +Input Event Workflow Name: `cttsov2` + +Output Event Source: `orcabus.tninputeventglue` +Output Event DetailType: `WorkflowRunStateChange` +Output Event status: `READY` +Output Event Workflow Name: `pieriandx` + +* Subscribe to the wgts input event glue, library complete event. +* Launch a draft event for the tumor normal pipeline if the libraries' subject has a complement library that is also complete +*/ + +export interface Cttsov2CompleteToPieriandxConstructProps { + /* Events */ + eventBusObj: events.IEventBus; + + /* Tables */ + tableObj: dynamodb.ITableV2; + + /* Secrets */ + icav2AccessTokenSecretObj: secretsManager.ISecret; + + /* Extras */ + projectInfoSsmParameterObj: ssm.IStringParameter; + redcapLambdaObj: lambda.IFunction; +} + +export class Cttsov2CompleteToPieriandxConstruct extends Construct { + public readonly PierianDxMap = { + prefix: 'nails-cttsov2-complete-to-pdx', + tablePartition: { + subject: 'subject', + library: 'library', + fastq_list_row: 'fastq_list_row', + }, + + /* Input Rules */ + triggerSource: 'orcabus.workflowmanager', + triggerStatus: 'succeeded', + triggerWorkflowName: 'cttsov2', + triggerDetailType: 'WorkflowRunStateChange', + + /* Output Events */ + eventDetailType: 'WorkflowRunStateChange', + eventStatus: 'READY', + outputSource: 'orcabus.pieriandxinputeventglue', + payloadVersion: '2024.07.23', + workflowName: 'pieriandx', + workflowVersion: '2.1', + + /* Default values */ + defaultSpecimenCode: '122561005', + defaultSpecimenLabel: 'primarySpecimen', + }; + + constructor(scope: Construct, id: string, props: Cttsov2CompleteToPieriandxConstructProps) { + super(scope, id); + + /* + Part 1: Build the lambdas + */ + const generatePortalRunIdPyLambdaObj = new PythonFunction( + this, + 'generatePortalRunIdPyLambdaObj', + { + entry: path.join(__dirname, '/lambdas/generate_portal_run_id_py'), + runtime: lambda.Runtime.PYTHON_3_12, + architecture: lambda.Architecture.ARM_64, + index: 'generate_portal_run_id.py', + handler: 'handler', + } + ); + + const getDataFromRedCapPyLambdaObj = new PythonFunction(this, 'getDataFromRedCapPyLambdaObj', { + entry: path.join(__dirname, '/lambdas/get_data_from_redcap_py'), + runtime: lambda.Runtime.PYTHON_3_12, + architecture: lambda.Architecture.ARM_64, + index: 'get_data_from_redcap.py', + handler: 'handler', + timeout: Duration.seconds(60), + }); + + const getDeidentifiedCaseMetadataPyLambdaObj = new PythonFunction( + this, + 'getDeidentifiedCaseMetadataPyLambdaObj', + { + entry: path.join(__dirname, '/lambdas/get_deidentified_case_metadata_py'), + runtime: lambda.Runtime.PYTHON_3_12, + architecture: lambda.Architecture.ARM_64, + index: 'get_deidentified_case_metadata.py', + handler: 'handler', + } + ); + const getIdentifiedCaseMetadataPyLambdaObj = new PythonFunction( + this, + 'getIdentifiedCaseMetadataPyLambdaObj', + { + entry: path.join(__dirname, '/lambdas/get_identified_case_metadata_py'), + runtime: lambda.Runtime.PYTHON_3_12, + architecture: lambda.Architecture.ARM_64, + index: 'get_identified_case_metadata.py', + handler: 'handler', + } + ); + const getPieriandxDataFilesPyLambdaObj = new PythonFunction( + this, + 'getPieriandxDataFilesPyLambdaObj', + { + entry: path.join(__dirname, '/lambdas/get_pieriandx_data_files_py'), + runtime: lambda.Runtime.PYTHON_3_12, + architecture: lambda.Architecture.ARM_64, + index: 'get_pieriandx_data_files.py', + handler: 'handler', + timeout: Duration.seconds(300), + environment: { + ICAV2_ACCESS_TOKEN_SECRET_ID: props.icav2AccessTokenSecretObj.secretName, + }, + } + ); + const getProjectInfoPyLambdaObj = new PythonFunction(this, 'getProjectInfoPyLambdaObj', { + entry: path.join(__dirname, '/lambdas/get_project_info_py'), + runtime: lambda.Runtime.PYTHON_3_12, + architecture: lambda.Architecture.ARM_64, + index: 'get_project_info.py', + handler: 'handler', + }); + + /* + Handle lambda permissions + */ + props.redcapLambdaObj.latestVersion.grantInvoke(getDataFromRedCapPyLambdaObj.currentVersion); + getDataFromRedCapPyLambdaObj.addEnvironment( + 'REDCAP_LAMBDA_FUNCTION_NAME', + props.redcapLambdaObj.functionName + ); + + // Allow the getPieriandxDataFilesPyLambdaObj to read the secret + props.icav2AccessTokenSecretObj.grantRead(getPieriandxDataFilesPyLambdaObj.currentVersion); + + // Allow the getProjectInfoPyLambdaObj to read the ssm parameters + props.projectInfoSsmParameterObj.grantRead(getProjectInfoPyLambdaObj.currentVersion); + getProjectInfoPyLambdaObj.addEnvironment( + 'PIERIANDX_SAMPLE_CONFIGURATION_SSM_PARAMETER_NAME', + props.projectInfoSsmParameterObj.parameterName + ); + + /* + Part 2: Build the sfn + */ + const inputMakerSfn = new sfn.StateMachine(this, 'cttsov2_outputs_to_pieriandx', { + stateMachineName: `${this.PierianDxMap.prefix}-sfn`, + definitionBody: sfn.DefinitionBody.fromFile( + path.join( + __dirname, + 'step_functions_templates', + 'cttso_v2_outputs_to_pieriandx_ready_event_sfn_template.asl.json' + ) + ), + definitionSubstitutions: { + /* Event handlers */ + __event_bus_name__: props.eventBusObj.eventBusName, + __event_detail_type__: this.PierianDxMap.eventDetailType, + __event_source__: this.PierianDxMap.outputSource, + __event_status__: this.PierianDxMap.eventStatus, + __event_version__: this.PierianDxMap.payloadVersion, + __workflow_name__: this.PierianDxMap.workflowName, + __workflow_version__: this.PierianDxMap.workflowVersion, + __workflow_version_sub__: this.PierianDxMap.workflowVersion.replace(/\./g, '-'), + + /* Lambdas */ + __generate_portal_run_id_lambda_function_arn__: + generatePortalRunIdPyLambdaObj.currentVersion.functionArn, + __get_deidentified_case_metadata_lambda_function_arn__: + getDeidentifiedCaseMetadataPyLambdaObj.currentVersion.functionArn, + __get_identified_case_metadata_lambda_function_arn__: + getIdentifiedCaseMetadataPyLambdaObj.currentVersion.functionArn, + __get_pieriandx_project_pathway_mapping_lambda_function_arn__: + getProjectInfoPyLambdaObj.currentVersion.functionArn, + __get_project_data_files_lambda_function_arn__: + getPieriandxDataFilesPyLambdaObj.currentVersion.functionArn, + __get_sample_redcap_info_lambda_function_arn__: + getDataFromRedCapPyLambdaObj.currentVersion.functionArn, + + /* Tables */ + __table_name__: props.tableObj.tableName, + __library_table_partition_name__: this.PierianDxMap.tablePartition.library, + + /* Extras */ + __specimen_code__: this.PierianDxMap.defaultSpecimenCode, + __specimen_label__: this.PierianDxMap.defaultSpecimenLabel, + }, + }); + + /* + Part 2: Grant the sfn permissions + */ + // access the dynamodb table + props.tableObj.grantReadWriteData(inputMakerSfn); + + // allow the step function to invoke the lambdas + [ + generatePortalRunIdPyLambdaObj, + getDataFromRedCapPyLambdaObj, + getDeidentifiedCaseMetadataPyLambdaObj, + getIdentifiedCaseMetadataPyLambdaObj, + getPieriandxDataFilesPyLambdaObj, + getProjectInfoPyLambdaObj, + ].forEach((lambdaObj) => { + lambdaObj.currentVersion.grantInvoke(inputMakerSfn); + }); + + // Allow step function to submit events to the event bus + props.eventBusObj.grantPutEventsTo(inputMakerSfn); + + /* + Part 3: Subscribe to the event bus and trigger the internal sfn + */ + const rule = new events.Rule(this, 'library_qc_complete_to_tn_draft', { + ruleName: `stacky-${this.PierianDxMap.prefix}-rule`, + eventBus: props.eventBusObj, + eventPattern: { + source: [this.PierianDxMap.triggerSource], + detailType: [this.PierianDxMap.triggerDetailType], + detail: { + status: [{ 'equals-ignore-case': this.PierianDxMap.triggerStatus }], + workflowName: [{ 'equals-ignore-case': this.PierianDxMap.triggerWorkflowName }], + }, + }, + }); + + // Add target of event to be the state machine + rule.addTarget( + new eventsTargets.SfnStateMachine(inputMakerSfn, { + input: events.RuleTargetInput.fromEventPath('$.detail'), + }) + ); + } +} diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_2/cttso-v2-output-to-pieriandx-ready-event/lambdas/generate_portal_run_id_py/generate_portal_run_id.py b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_2/cttso-v2-output-to-pieriandx-ready-event/lambdas/generate_portal_run_id_py/generate_portal_run_id.py new file mode 100644 index 000000000..b4eb606b4 --- /dev/null +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_2/cttso-v2-output-to-pieriandx-ready-event/lambdas/generate_portal_run_id_py/generate_portal_run_id.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 + +""" +Generate a portal run id +""" + +# Imports +from datetime import timezone, datetime +import os + + +def handler(event, context): + """ + Generate a portal run id + """ + return { + "portal_run_id": datetime.now(timezone.utc).strftime('%Y%m%d') + os.urandom(4).hex() + } + + +# if __name__ == '__main__': +# import json +# print(json.dumps(handler(None, None), indent=4)) +# +# # { +# # "portal_run_id": "20240923e75c96f6" +# # } \ No newline at end of file diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_2/cttso-v2-output-to-pieriandx-ready-event/lambdas/get_data_from_redcap_py/get_data_from_redcap.py b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_2/cttso-v2-output-to-pieriandx-ready-event/lambdas/get_data_from_redcap_py/get_data_from_redcap.py new file mode 100644 index 000000000..e441efd3a --- /dev/null +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_2/cttso-v2-output-to-pieriandx-ready-event/lambdas/get_data_from_redcap_py/get_data_from_redcap.py @@ -0,0 +1,339 @@ +#!/usr/bin/env python + +""" +Given a library id, retrieve the necessary information from the REDCap database for this library id + +We really only need the disease name if it exists +""" + +# Standard imports +import typing +from typing import List +from time import sleep +from typing import Dict +from os import environ +import pandas as pd +import boto3 +from botocore.exceptions import ClientError +import json +import pytz +from datetime import datetime +import logging + +if typing.TYPE_CHECKING: + from mypy_boto3_lambda import LambdaClient + +# Set logger +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +# Globals +AUS_TIMEZONE = pytz.timezone("Australia/Melbourne") +AUS_TIME = datetime.now(AUS_TIMEZONE) +AUS_TIME_CURRENT_DEFAULT_DICT = { + "date_accessioned": AUS_TIME.date().isoformat(), + "date_collected": AUS_TIME.date().isoformat(), + "time_collected": AUS_TIME.strftime("%H:%M"), + "date_received": AUS_TIME.date().isoformat() +} +AUS_TIMEZONE_SUFFIX = AUS_TIME.strftime("%z") + +REDCAP_RAW_FIELDS_CLINICAL: List = [ + "record_id", + "clinician_firstname", + "clinician_lastname", + "patient_urn", + "disease", + "date_collection", + "time_collected", + "date_receipt", + "id_sbj", + "libraryid" +] + +REDCAP_LABEL_FIELDS_CLINICAL: List = [ + "record_id", + "report_type", + "disease", + "patient_gender", + "id_sbj", + "libraryid", + "pierian_metadata_complete" +] + + +def get_lambda_client() -> 'LambdaClient': + return boto3.client('lambda') + + +def get_redcap_lambda_from_env(): + """ + Get the redcap lambda from the environment + :return: + """ + return environ['REDCAP_LAMBDA_FUNCTION_NAME'] + + +def warm_up_lambda(): + """ + Warm up the lambda function + :return: + """ + try: + get_lambda_client().invoke( + FunctionName=get_redcap_lambda_from_env(), + InvocationType='RequestResponse' + ) + return True + except ClientError as e: + logger.info(f"Error warming up lambda: {e}") + return False + + +def launch_redcap_raw_lambda(library_id: str) -> pd.DataFrame: + """ + Launch the redcap lambda + :param library_id: + :return: + """ + redcap_raw_df: pd.DataFrame = pd.DataFrame(columns=REDCAP_RAW_FIELDS_CLINICAL) + + raw_list: List = json.loads( + json.loads( + get_lambda_client().invoke( + FunctionName=get_redcap_lambda_from_env(), + InvocationType='RequestResponse', + Payload=json.dumps( + { + "redcapProjectName": "TinyCT", + "queryStringParameters": { + "filter_logic": f"[libraryid] = \"{library_id}\"", + "fields": REDCAP_RAW_FIELDS_CLINICAL, + "raw_or_label": "raw", + } + } + ) + )['Payload'].read() + )['body'] + ) + + # Concat the raw data to the redcap raw df + redcap_raw_df = pd.concat( + [ + redcap_raw_df, + pd.DataFrame(raw_list, columns=REDCAP_RAW_FIELDS_CLINICAL) + ] + ) + + # Rename columns + redcap_raw_df.rename( + columns={ + "clinician_firstname": "requesting_physicians_first_name", + "clinician_lastname": "requesting_physicians_last_name", + "libraryid": "library_id", + "mrn": "patient_urn", + "disease": "disease_id", + "date_collection": "date_collected", + "date_receipt": "date_received" + }, + inplace=True + ) + + # Replace null values with NAs + redcap_raw_df = redcap_raw_df.replace({None: pd.NA, "": pd.NA}) + + return redcap_raw_df + + + +def launch_redcap_label_lambda(library_id: str) -> pd.DataFrame: + """ + Launch the redcap lambda + :param library_id: + :return: + """ + redcap_label_df: pd.DataFrame = pd.DataFrame(columns=REDCAP_LABEL_FIELDS_CLINICAL) + + label_list: List = json.loads( + json.loads( + get_lambda_client().invoke( + FunctionName=get_redcap_lambda_from_env(), + InvocationType='RequestResponse', + Payload=json.dumps( + { + "redcapProjectName": "TinyCT", + "queryStringParameters": { + "filter_logic": f"[libraryid] = \"{library_id}\"", + "fields": REDCAP_LABEL_FIELDS_CLINICAL, + "raw_or_label": "label", + } + } + ) + )['Payload'].read() + )['body'] + ) + + # Concatenate dict with empty columns + redcap_label_df = pd.concat( + [ + redcap_label_df, + pd.DataFrame(label_list, columns=REDCAP_LABEL_FIELDS_CLINICAL) + ] + ) + + # Rename columns + redcap_label_df.rename( + columns={ + "report_type": "sample_type", + "patient_gender": "gender", + "disease": "disease_name", + "libraryid": "library_id" + }, + inplace=True + ) + + # Filter to select columns + redcap_label_df = redcap_label_df[ + [ + "sample_type", + "disease_name", + "gender", + "library_id", + "pierian_metadata_complete" + ] + ] + + return redcap_label_df + +def get_and_merge_raw_and_label_data(library_id: str) -> Dict: + """ + Get the raw and label data from redcap and merge it + :param library_id: + :return: + """ + redcap_raw_df = launch_redcap_raw_lambda(library_id) + redcap_label_df = launch_redcap_label_lambda(library_id) + + # Check we have at least one entry + if redcap_raw_df.shape[0] == 0: + logger.info(f"No entries found for library '{library_id}'") + raise ValueError + + # Update the date field with na values if not set (for validation samples only) + validation_samples_index = redcap_label_df.query( + "sample_type.str.lower()=='validation'" + ).index + # For clinical samples, we only need to update the time_collected field + clinical_samples_index = redcap_label_df.query( + "not sample_type=='validation'" + ).index + + # Replace na values for date_collection or date_received, or date_receipt if None or null + # Update time_collected field in both since it might not exist + # Update for validation samples + for date_column in ["date_collected", "date_received", "time_collected"]: + redcap_raw_df.loc[validation_samples_index, date_column] = \ + redcap_raw_df.loc[validation_samples_index, date_column].fillna(AUS_TIME_CURRENT_DEFAULT_DICT[date_column]) + # Update for clinical samples + for date_column in ["time_collected"]: + redcap_raw_df.loc[clinical_samples_index, date_column] = \ + redcap_raw_df.loc[clinical_samples_index, date_column].fillna(AUS_TIME_CURRENT_DEFAULT_DICT[date_column]) + + # Update date fields + redcap_raw_df["date_collected"] = redcap_raw_df.apply( + lambda date_str: date_str.date_collected + "T" + date_str.time_collected + f":00{AUS_TIMEZONE_SUFFIX}", + axis="columns" + ) + + # Add time to 'date_receipt' string + redcap_raw_df["date_received"] = redcap_raw_df.apply( + lambda x: x.date_received + f"T00:00:00{AUS_TIMEZONE_SUFFIX}", + axis="columns" + ) + + # Subset columns for redcap raw df + redcap_raw_df = redcap_raw_df[ + [ + "disease_id", + "requesting_physicians_first_name", + "requesting_physicians_last_name", + "library_id", + "date_collected", + "date_received", + "patient_urn" + ] + ] + + # Merge redcap data + redcap_df: pd.DataFrame = pd.merge( + redcap_raw_df, redcap_label_df, + on=["library_id"] + ) + + # Merge redcap information and then return + num_entries: int + if not (num_entries := redcap_df.shape[0]) == 1: + logger.info(f"Expected dataframe to be of length 1, not {num_entries}") + raise ValueError(f"Expected dataframe to be of length 1, not {num_entries}") + + return redcap_df.to_dict(orient='records')[0] + + +def handler(event, context) -> Dict: + """ + Handler for the lambda function + :param event: + :param context: + :return: + """ + # Wait for lambda to warm up + while not warm_up_lambda(): + sleep(10) + + # Return + try: + return { + "redcap_data": get_and_merge_raw_and_label_data(event['library_id']), + "in_redcap": True + } + except ValueError: + return { + "redcap_data": None, + "in_redcap": False + } + + +# if __name__ == '__main__': +# # Or 'umccr-staging' / 'umccr-production' +# environ['AWS_PROFILE'] = 'umccr-development' +# environ['AWS_REGION'] = 'ap-southeast-2' +# # Or 'redcap-apis-stg-lambda-function' / 'redcap-apis-prod-lambda-function' +# environ['REDCAP_LAMBDA_FUNCTION_NAME'] = 'redcap-apis-dev-lambda-function' +# print( +# json.dumps( +# handler( +# event={ +# "library_id": 'L2401380' +# }, +# context=None +# ), +# indent=4 +# ) +# ) +# +# # { +# # "redcap_data": { +# # "disease_id": 254637007, +# # "requesting_physicians_first_name": "XXX", +# # "requesting_physicians_last_name": "XXX", +# # "library_id": "L2401380", +# # "date_collected": "2024-09-06T23:00:00+1000", +# # "date_received": "2024-09-06T00:00:00+1000", +# # "patient_urn": "0038-61302", +# # "sample_type": "Patient Care Sample", +# # "disease_name": "Non-small cell lung cancer", +# # "gender": "Unknown", +# # "pierian_metadata_complete": "Complete" +# # }, +# # "in_redcap": true +# # } diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_2/cttso-v2-output-to-pieriandx-ready-event/lambdas/get_data_from_redcap_py/requirements.txt b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_2/cttso-v2-output-to-pieriandx-ready-event/lambdas/get_data_from_redcap_py/requirements.txt new file mode 100644 index 000000000..ff6c06e3e --- /dev/null +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_2/cttso-v2-output-to-pieriandx-ready-event/lambdas/get_data_from_redcap_py/requirements.txt @@ -0,0 +1,2 @@ +pytz==2024.2 +pandas==2.2.3 diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_2/cttso-v2-output-to-pieriandx-ready-event/lambdas/get_deidentified_case_metadata_py/get_deidentified_case_metadata.py b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_2/cttso-v2-output-to-pieriandx-ready-event/lambdas/get_deidentified_case_metadata_py/get_deidentified_case_metadata.py new file mode 100644 index 000000000..214a20a59 --- /dev/null +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_2/cttso-v2-output-to-pieriandx-ready-event/lambdas/get_deidentified_case_metadata_py/get_deidentified_case_metadata.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 + +""" +Return payload of de-identified case metadata + +Payload will look a bit like this: + +{ + "isIdentified": false, + "caseAccessionNumber": "SBJ04407__L2400161__V2__abcd1238", + "externalSpecimenId": "externalspecimenid", + "sampleType": "PatientCare", + "specimenLabel": "primarySpecimen", + "indication": "Test", + "diseaseCode": 64572001, + "specimenCode": 122561005, + "sampleReception": { + "dateAccessioned": "2021-01-01T00:00:00Z", + "dateCollected": "2024-02-20T20:17:00Z", + "dateReceived": "2021-01-01T00:00:00Z" + }, + "study": { + "id": "studyid", + "subjectIdentifier": "subject" + } +} + +""" + +# Imports +import typing +from typing import Dict + +import pytz +from datetime import datetime +import logging + + +# Set logger +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +# Globals +AUS_TIMEZONE = pytz.timezone("Australia/Melbourne") +AUS_TIME = datetime.now(AUS_TIMEZONE) +AUS_TIME_AS_STR = f"{AUS_TIME.date().isoformat()}T{AUS_TIME.time().isoformat(timespec='seconds')}{AUS_TIME.strftime("%z")}" + +DEFAULT_INDICATION = "NA" + + +def handler(event, context) -> Dict: + # Return payload of de-identified case metadata + + # Get the case accession number from the event + case_accession_number = event.get("case_accession_number") + + # Get the external specimen id from the event + external_sample_id = event.get("external_sample_id") + external_subject_id = event.get("external_subject_id") + project_id = event.get("project_id") + + # Get the sample type from the event + sample_type = event.get("sample_type") + + # Get the specimen label from the event + specimen_label = event.get("specimen_label") + + # Get the indication from the event + indication = event.get("indication", DEFAULT_INDICATION) + + # Get the specimen code from the event + specimen_code = event.get("specimen_code") + + # Get redcap information from the event + # For deidentified samples, we only need to + redcap_dict = event.get("redcap_dict", None) + if redcap_dict is None: + redcap_dict = {} + + # Get the sample reception from the redcap data if it exists + date_accessioned = redcap_dict.get("date_accessioned", AUS_TIME_AS_STR) + date_collected = redcap_dict.get("date_collected", AUS_TIME_AS_STR) + date_received = redcap_dict.get("date_received", AUS_TIME_AS_STR) + + # Set the sample reception dictionary + # Set as camel case for event type + sample_reception = { + "dateAccessioned": date_accessioned, + "dateCollected": date_collected, + "dateReceived": date_received + } + + # Get the disease code from the event if it exists or + disease_code = redcap_dict.get("disease_id", event.get("default_disease_code")) + + # Return the payload + return { + "case_metadata": { + "isIdentified": False, + "caseAccessionNumber": case_accession_number, + "externalSpecimenId": external_sample_id, + "sampleType": sample_type, + "specimenLabel": specimen_label, + "indication": indication, + "diseaseCode": disease_code, + "specimenCode": specimen_code, + "sampleReception": sample_reception, + "study": { + "id": project_id, + "subjectIdentifier": external_subject_id + } + } + } + + +# if __name__ == "__main__": +# import json +# print( +# json.dumps( +# handler( +# { +# "specimen_label": "primarySpecimen", +# "indication": None, +# "specimen_code": "122561005", +# "redcap_dict": None, +# "external_subject_id": "CMM1pc-10646259ilm", +# "default_disease_code": 55342001, +# "external_sample_id": "SSq-CompMM-1pc-10646259ilm", +# "case_accession_number": "L2400160__V2__20241003fc695a2c", +# "sample_type": "patient_care_sample" +# }, +# None +# ), +# indent=4 +# ) +# ) +# +# # { +# # "case_metadata": { +# # "case_metadata": { +# # "isIdentified": false, +# # "caseAccessionNumber": "L2400160__V2__20241003fc695a2c", +# # "externalSpecimenId": "SSq-CompMM-1pc-10646259ilm", +# # "sampleType": "patient_care_sample", +# # "specimenLabel": "primarySpecimen", +# # "indication": null, +# # "diseaseCode": 55342001, +# # "specimenCode": "122561005", +# # "sampleReception": { +# # "dateAccessioned": "2024-10-04T09:17:27+1000", +# # "dateCollected": "2024-10-04T09:17:27+1000", +# # "dateReceived": "2024-10-04T09:17:27+1000" +# # }, +# # "study": { +# # "id": null, +# # "subjectIdentifier": "CMM1pc-10646259ilm" +# # } +# # } +# # } \ No newline at end of file diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_2/cttso-v2-output-to-pieriandx-ready-event/lambdas/get_deidentified_case_metadata_py/requirements.txt b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_2/cttso-v2-output-to-pieriandx-ready-event/lambdas/get_deidentified_case_metadata_py/requirements.txt new file mode 100644 index 000000000..9d3eb6965 --- /dev/null +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_2/cttso-v2-output-to-pieriandx-ready-event/lambdas/get_deidentified_case_metadata_py/requirements.txt @@ -0,0 +1 @@ +pytz==2024.2 diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_2/cttso-v2-output-to-pieriandx-ready-event/lambdas/get_identified_case_metadata_py/get_identified_case_metadata.py b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_2/cttso-v2-output-to-pieriandx-ready-event/lambdas/get_identified_case_metadata_py/get_identified_case_metadata.py new file mode 100644 index 000000000..40b07d516 --- /dev/null +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_2/cttso-v2-output-to-pieriandx-ready-event/lambdas/get_identified_case_metadata_py/get_identified_case_metadata.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 + +""" +Return payload of de-identified case metadata + +Payload will look a bit like this: + +{ + "case_metadata": { + "isIdentified": True, + "caseAccessionNumber": "SBJ04407__L2301368__V2__abcd1234", + "externalSpecimenId": "externalspecimenid", + "sampleType": "PatientCare", + "specimenLabel": "primarySpecimen", + "indication": "Test", + "diseaseCode": 64572001, + "specimenCode": 122561005, + "sampleReception": { + "dateAccessioned": "2021-01-01T00:00:00Z", + "dateCollected": "2024-02-20T20:17:00Z", + "dateReceived": "2021-01-01T00:00:00Z" + }, + "patientInformation": { + "dateOfBirth": "1970-01-01", + "firstName": "John", + "lastName": "Doe" + }, + "medicalRecordNumbers": { + "mrn": "3069999", + "medicalFacility": { + "facility": "Not Available", + "hospitalNumber": "99" + } + }, + "requestingPhysician": { + "firstName": "Meredith", + "lastName": "Gray" + } + } +} + +""" + +# Imports +from typing import Dict + +import pytz +from datetime import datetime +import logging + +# Set logger +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +# Globals +AUS_TIMEZONE = pytz.timezone("Australia/Melbourne") +AUS_TIME = datetime.now(AUS_TIMEZONE) +AUS_TIME_CURRENT_DEFAULT_DICT = { + "date_accessioned": AUS_TIME.isoformat(timespec='seconds'), + "date_collected": AUS_TIME.isoformat(timespec='seconds'), + "date_received": AUS_TIME.isoformat(timespec='seconds'), +} + +DEFAULT_REQUESTING_PHYSICIAN = { + "first_name": "Sean", + "last_name": "Grimmond" +} + +DEFAULT_INDICATION = "NA" +DEFAULT_HOSPITAL_NUMBER = "99" + + +def handler(event, context) -> Dict: + # Return payload of de-identified case metadata + + # Get the case accession number from the event + case_accession_number = event.get("case_accession_number") + + # Get the external specimen id from the event + external_sample_id = event.get("external_sample_id") + external_subject_id = event.get("external_subject_id") + + # Get the sample type from the event + sample_type = event.get("sample_type") + + # Get the specimen label from the event + specimen_label = event.get("specimen_label") + + # Get the indication from the event + indication = event.get("indication", DEFAULT_INDICATION) + + # Get the specimen code from the event + specimen_code = event.get("specimen_code") + + # Get redcap information from the event + # For deidentified samples, we only need to + redcap_dict = event.get("redcap_dict", None) + if redcap_dict is None: + redcap_dict = {} + + # Get the sample reception from the redcap data if it exists + # Get the sample reception from the redcap data if it exists + date_accessioned = redcap_dict.get("date_accessioned", AUS_TIME_CURRENT_DEFAULT_DICT["date_accessioned"]) + date_collected = redcap_dict.get("date_collected", AUS_TIME_CURRENT_DEFAULT_DICT["date_collected"]) + date_received = redcap_dict.get("date_received", AUS_TIME_CURRENT_DEFAULT_DICT["date_received"]) + + # Set the sample reception dictionary + # Set as camel case for event type + sample_reception = { + "dateAccessioned": date_accessioned, + "dateCollected": date_collected, + "dateReceived": date_received + } + + # Get the disease code from the event if it exists or + disease_code = redcap_dict.get("disease_id", event.get("default_disease_code")) + + # Get patient information from redcap + if redcap_dict is not None: + patient_information = { + "dateOfBirth": "1970-01-01", + "firstName": "Jane" if redcap_dict.get("gender", "female") else "John", + "lastName": "Doe" + } + else: + patient_information = { + "dateOfBirth": "1970-01-01", + "firstName": "John", + "lastName": "Doe" + } + + # Get medical record numbers from redcap + medical_record_numbers = { + "mrn": external_subject_id, + "medicalFacility": { + "facility": "Not Available", + "hospitalNumber": DEFAULT_HOSPITAL_NUMBER + } + } + + # Get requesting physician from redcap + requesting_physician = { + "firstName": redcap_dict.get("requesting_physician_first_name", DEFAULT_REQUESTING_PHYSICIAN["first_name"]), + "lastName": redcap_dict.get("requesting_physician_last_name", DEFAULT_REQUESTING_PHYSICIAN["last_name"]) + } + + # Return the payload + return { + "case_metadata": { + "isIdentified": True, + "caseAccessionNumber": case_accession_number, + "externalSpecimenId": external_sample_id, + "sampleType": sample_type, + "specimenLabel": specimen_label, + "indication": indication, + "diseaseCode": disease_code, + "specimenCode": specimen_code, + "sampleReception": sample_reception, + "patientInformation": patient_information, + "medicalRecordNumbers": medical_record_numbers, + "requestingPhysician": requesting_physician + } + } diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_2/cttso-v2-output-to-pieriandx-ready-event/lambdas/get_identified_case_metadata_py/requirements.txt b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_2/cttso-v2-output-to-pieriandx-ready-event/lambdas/get_identified_case_metadata_py/requirements.txt new file mode 100644 index 000000000..9d3eb6965 --- /dev/null +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_2/cttso-v2-output-to-pieriandx-ready-event/lambdas/get_identified_case_metadata_py/requirements.txt @@ -0,0 +1 @@ +pytz==2024.2 diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_2/cttso-v2-output-to-pieriandx-ready-event/lambdas/get_pieriandx_data_files_py/get_pieriandx_data_files.py b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_2/cttso-v2-output-to-pieriandx-ready-event/lambdas/get_pieriandx_data_files_py/get_pieriandx_data_files.py new file mode 100644 index 000000000..f671a684c --- /dev/null +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_2/cttso-v2-output-to-pieriandx-ready-event/lambdas/get_pieriandx_data_files_py/get_pieriandx_data_files.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 + +""" +Given the output uri, get the data files ready to upload into the pieriandx s3 bucket + +So given the output uri +'s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/' +And a sample id +'L2400161' + +We would expect the following files to be returned: +{ + "microsatOutputUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Logs_Intermediates/DragenCaller/L2400161/L2400161.microsat_output.json", + "tmbMetricsUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Logs_Intermediates/Tmb/L2400161/L2400161.tmb.metrics.csv", + "cnvVcfUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Results/L2400161/L2400161.cnv.vcf.gz", + "hardFilteredVcfUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Results/L2400161/L2400161.hard-filtered.vcf.gz", + "fusionsUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Results/L2400161/L2400161_Fusions.csv", + "metricsOutputUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Results/L2400161/L2400161_MetricsOutput.tsv", + "samplesheetUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Logs_Intermediates/SampleSheetValidation/SampleSheet_Intermediate.csv" +} +""" + +# Standard imports +from pathlib import Path +import typing +import logging +import boto3 +from os import environ +from urllib.parse import urlparse, urlunparse + +# Custom imports +from wrapica.project_data import convert_uri_to_project_data_obj +from wrapica.libica_exceptions import ApiException + +# Typing imports +from typing import Dict +if typing.TYPE_CHECKING: + from mypy_boto3_secretsmanager import SecretsManagerClient + + +# Globals +ICAV2_BASE_URL = "https://ica.illumina.com/ica/rest" + +URL_EXTENSION_MAP = { + "microsat_output_uri": "Logs_Intermediates/DragenCaller/{sample_id}/{sample_id}.microsat_output.json", + "tmb_metrics_uri": "Logs_Intermediates/Tmb/{sample_id}/{sample_id}.tmb.metrics.csv", + "cnv_vcf_uri": "Results/{sample_id}/{sample_id}.cnv.vcf.gz", + "hard_filtered_vcf_uri": "Results/{sample_id}/{sample_id}.hard-filtered.vcf.gz", + "fusions_uri": "Results/{sample_id}/{sample_id}_Fusions.csv", + "metrics_output_uri": "Results/{sample_id}/{sample_id}_MetricsOutput.tsv", + "samplesheet_uri": "Logs_Intermediates/SampleSheetValidation/SampleSheet_Intermediate.csv", +} + +# Set loggers +logger = logging.getLogger() +logger.setLevel(logging.INFO) + + +def get_secrets_manager_client() -> 'SecretsManagerClient': + """ + Return Secrets Manager client + """ + return boto3.client("secretsmanager") + + +def get_secret(secret_id: str) -> str: + """ + Return secret value + """ + return get_secrets_manager_client().get_secret_value(SecretId=secret_id)["SecretString"] + + +# Functions +def set_icav2_env_vars(): + """ + Set the icav2 environment variables + :return: + """ + environ["ICAV2_BASE_URL"] = ICAV2_BASE_URL + environ["ICAV2_ACCESS_TOKEN"] = get_secret( + environ["ICAV2_ACCESS_TOKEN_SECRET_ID"] + ) + + +def extend_url_path(base_url: str, path_ext: Path): + """ + Given a base url, convert to a url object, extend the base url path with the path extension and return the new url + :param base_url: + :param path_ext: + :return: + """ + + base_url_obj = urlparse(base_url) + + # Extend the path + new_path = Path(base_url_obj.path).joinpath(path_ext) + + # Return the new url + return str(urlunparse( + ( + base_url_obj.scheme, + base_url_obj.netloc, + str(new_path), + None, None, None + ) + )) + + +def handler(event, context) -> Dict[str, Dict]: + """ + Firse we need to set the icav2 env vars + :param event: + :param context: + :return: + """ + # Set ICAv2 Env Vars + set_icav2_env_vars() + + # Get the output uri and sample id from the event dict + output_uri = event["output_uri"] + sample_id = event["sample_id"] + + # Get the project name and owner from the output uri + + # Camel case data_files values for the event dict + data_files = { + "microsatOutputUri": extend_url_path(output_uri, Path(URL_EXTENSION_MAP["microsat_output_uri"].format(sample_id=sample_id))), + "tmbMetricsUri": extend_url_path(output_uri, Path(URL_EXTENSION_MAP["tmb_metrics_uri"].format(sample_id=sample_id))), + "cnvVcfUri": extend_url_path(output_uri, Path(URL_EXTENSION_MAP["cnv_vcf_uri"].format(sample_id=sample_id))), + "hardFilteredVcfUri": extend_url_path(output_uri, Path(URL_EXTENSION_MAP["hard_filtered_vcf_uri"].format(sample_id=sample_id))), + "fusionsUri": extend_url_path(output_uri, Path(URL_EXTENSION_MAP["fusions_uri"].format(sample_id=sample_id))), + "metricsOutputUri": extend_url_path(output_uri, Path(URL_EXTENSION_MAP["metrics_output_uri"].format(sample_id=sample_id))), + "samplesheetUri": extend_url_path(output_uri, Path(URL_EXTENSION_MAP["samplesheet_uri"].format(sample_id=sample_id))), + } + + # Check all data files exist + for key, value in data_files.items(): + logger.info(f"Checking {key} exists: {value}") + # Check the url exists + try: + convert_uri_to_project_data_obj(value) + except ApiException as e: + logger.error(f"Error checking {key} exists: {e}") + raise e + + return { + "data_files": data_files + } + + +if __name__ == "__main__": + import json + from os import environ + environ['AWS_PROFILE'] = 'umccr-development' + environ['AWS_DEFAULT_REGION'] = 'ap-southeast-2' + environ['ICAV2_ACCESS_TOKEN_SECRET_ID'] = "ICAv2JWTKey-umccr-prod-service-dev" + + print( + json.dumps( + handler( + { + "output_uri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20241003ead8ad9f/", + "sample_id": "L2400160" + }, + None + ), + indent=4 + ) + ) + + # { + # "data_files": { + # "microsatOutputUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Logs_Intermediates/DragenCaller/L2400161/L2400161.microsat_output.json", + # "tmbMetricsUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Logs_Intermediates/Tmb/L2400161/L2400161.tmb.metrics.csv", + # "cnvVcfUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Results/L2400161/L2400161.cnv.vcf.gz", + # "hardFilteredVcfUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Results/L2400161/L2400161.hard-filtered.vcf.gz", + # "fusionsUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Results/L2400161/L2400161_Fusions.csv", + # "metricsOutputUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Results/L2400161/L2400161_MetricsOutput.tsv", + # "samplesheetUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/20240910d260200d/Logs_Intermediates/SampleSheetValidation/SampleSheet_Intermediate.csv" + # } + # } diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_2/cttso-v2-output-to-pieriandx-ready-event/lambdas/get_pieriandx_data_files_py/requirements.txt b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_2/cttso-v2-output-to-pieriandx-ready-event/lambdas/get_pieriandx_data_files_py/requirements.txt new file mode 100644 index 000000000..1fa14ec17 --- /dev/null +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_2/cttso-v2-output-to-pieriandx-ready-event/lambdas/get_pieriandx_data_files_py/requirements.txt @@ -0,0 +1 @@ +wrapica==2.27.1.post20240830140737 diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_2/cttso-v2-output-to-pieriandx-ready-event/lambdas/get_project_info_py/get_project_info.py b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_2/cttso-v2-output-to-pieriandx-ready-event/lambdas/get_project_info_py/get_project_info.py new file mode 100644 index 000000000..6450d591b --- /dev/null +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_2/cttso-v2-output-to-pieriandx-ready-event/lambdas/get_project_info_py/get_project_info.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 + +""" +Given the results directory of the cttso output directory + +Generate the inputs json for the PierianDx manager + +cttso-lims-project-name-to-pieriandx-mapping + +[ + { + "project_id": "PO", + "panel": "subpanel", + "sample_type": "patient_care_sample", + "is_identified": "identified", + "default_snomed_disease_code": null + }, + { + "project_id": "COUMN", + "panel": "subpanel", + "sample_type": "patient_care_sample", + "is_identified": "identified", + "default_snomed_disease_code": null + }, + { + "project_id": "CUP", + "panel": "main", + "sample_type": "patient_care_sample", + "is_identified": "identified", + "default_snomed_disease_code": 285645000 + }, + { + "project_id": "PPGL", + "panel": "main", + "sample_type": "patient_care_sample", + "is_identified": "identified", + "default_snomed_disease_code": null + }, + { + "project_id": "MESO", + "panel": "subpanel", + "sample_type": "patient_care_sample", + "is_identified": "identified", + "default_snomed_disease_code": null + }, + { + "project_id": "OCEANiC", + "panel": "subpanel", + "sample_type": "patient_care_sample", + "is_identified": "deidentified", + "default_snomed_disease_code": null + }, + { + "project_id": "SOLACE2", + "panel": "main", + "sample_type": "patient_care_sample", + "is_identified": "deidentified", + "default_snomed_disease_code": 55342001 + }, + { + "project_id": "IMPARP", + "panel": "main", + "sample_type": "patient_care_sample", + "is_identified": "deidentified", + "default_snomed_disease_code": 55342001 + }, + { + "project_id": "Control", + "panel": "main", + "sample_type": "validation", + "is_identified": "deidentified", + "default_snomed_disease_code": 55342001 + }, + { + "project_id": "QAP", + "panel": "subpanel", + "sample_type": "patient_care_sample", + "is_identified": "identified", + "default_snomed_disease_code": null + }, + { + "project_id": "iPredict2", + "panel": "subpanel", + "sample_type": "patient_care_sample", + "is_identified": "identified", + "default_snomed_disease_code": null + }, + { + "project_id": "*", + "panel": "main", + "sample_type": "patient_care_sample", + "is_identified": "deidentified", + "default_snomed_disease_code": 55342001 + } +] + + +""" + +# Standard imports +from os import environ +import boto3 +import typing +import json + + +if typing.TYPE_CHECKING: + from mypy_boto3_ssm import SSMClient + + +def get_ssm_client() -> 'SSMClient': + return boto3.client('ssm') + + +def get_ssm_parameter_value(name: str) -> str: + client = get_ssm_client() + response = client.get_parameter(Name=name, WithDecryption=True) + return response['Parameter']['Value'] + + +def handler(event, context) -> typing.Dict[str, str]: + pieriandx_configuration_dict = json.loads(get_ssm_parameter_value(environ['PIERIANDX_SAMPLE_CONFIGURATION_SSM_PARAMETER_NAME'])) + + # Get the project name / owner + project_id = event.get('project_id', None) + + return { + "project_info": next( + filter( + lambda project_iter: ( + ( + project_iter.get("project_id") == project_id or + project_iter.get("project_id") == "*" + ) + ), + pieriandx_configuration_dict + ) + ) + } + +# # PO +# if __name__ == "__main__": +# import json +# environ['AWS_PROFILE'] = 'umccr-development' +# environ['AWS_REGION'] = 'ap-southeast-2' +# environ['PIERIANDX_SAMPLE_CONFIGURATION_SSM_PARAMETER_NAME'] = '/umccr/orcabus/pieriandx/project_info' +# +# print( +# json.dumps( +# handler( +# { +# "project_id": "PO" +# }, +# None +# ), +# indent=4 +# ) +# ) +# +# # { +# # "project_info": { +# # "project_id": "PO", +# # "panel": "subpanel", +# # "sample_type": "patient_care_sample", +# # "is_identified": "identified", +# # "default_snomed_term": null +# # } +# # } + + + +# # Default value for projects that do not match in list +# if __name__ == "__main__": +# import json +# environ['AWS_PROFILE'] = 'umccr-development' +# environ['AWS_REGION'] = 'ap-southeast-2' +# environ['PIERIANDX_SAMPLE_CONFIGURATION_SSM_PARAMETER_NAME'] = '/umccr/orcabus/pieriandx/project_info' +# +# print( +# json.dumps( +# handler( +# { +# "project_id": "Testing" +# }, +# None +# ), +# indent=4 +# ) +# ) +# +# # { +# # "project_info": { +# # "project_id": "*", +# # "panel": "main", +# # "sample_type": "patient_care_sample", +# # "is_identified": "deidentified", +# # "default_snomed_term": "Neoplastic disease" +# # } +# # } diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_2/cttso-v2-output-to-pieriandx-ready-event/step_functions_templates/cttso_v2_outputs_to_pieriandx_ready_event_sfn_template.asl.json b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_2/cttso-v2-output-to-pieriandx-ready-event/step_functions_templates/cttso_v2_outputs_to_pieriandx_ready_event_sfn_template.asl.json new file mode 100644 index 000000000..b69033d3b --- /dev/null +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_2/cttso-v2-output-to-pieriandx-ready-event/step_functions_templates/cttso_v2_outputs_to_pieriandx_ready_event_sfn_template.asl.json @@ -0,0 +1,390 @@ +{ + "Comment": "A description of my state machine", + "StartAt": "Move Inputs", + "States": { + "Move Inputs": { + "Type": "Pass", + "Parameters": { + "inputs.$": "$" + }, + "Next": "Get Library Info from DB" + }, + "Get Library Info from DB": { + "Type": "Task", + "Resource": "arn:aws:states:::dynamodb:getItem", + "Parameters": { + "TableName": "${__table_name__}", + "Key": { + "id.$": "$.inputs.linkedLibraries[0].orcabusId", + "id_type": "${__library_table_partition_name__}" + } + }, + "ResultPath": "$.get_library_info_from_db_step", + "Next": "Is Library In DataBase" + }, + "Is Library In DataBase": { + "Type": "Choice", + "Choices": [ + { + "Variable": "$.get_library_info_from_db_step.Item", + "IsPresent": true, + "Next": "Generate Portal Run ID" + } + ], + "Default": "Pass" + }, + "Generate Portal Run ID": { + "Type": "Task", + "Resource": "arn:aws:states:::lambda:invoke", + "Parameters": { + "FunctionName": "${__generate_portal_run_id_lambda_function_arn__}", + "Payload": {} + }, + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 1, + "MaxAttempts": 3, + "BackoffRate": 2 + } + ], + "ResultSelector": { + "portal_run_id.$": "$.Payload.portal_run_id" + }, + "ResultPath": "$.generate_portal_run_id_step", + "Next": "Generate Case Accession Number" + }, + "Generate Case Accession Number": { + "Type": "Pass", + "Parameters": { + "case_accession_number.$": "States.Format('{}__V2__{}', $.get_library_info_from_db_step.Item.library_id.S, $.generate_portal_run_id_step.portal_run_id)" + }, + "ResultPath": "$.generate_case_accession_number_step", + "Next": "Get Project Info and Project Data Files" + }, + "Get Project Info and Project Data Files": { + "Type": "Parallel", + "Branches": [ + { + "StartAt": "Get PierianDx Project Pathway mapping", + "States": { + "Get PierianDx Project Pathway mapping": { + "Type": "Task", + "Resource": "arn:aws:states:::lambda:invoke", + "Parameters": { + "Payload": { + "project_id.$": "$.get_library_info_from_db_step.Item.project_id.S" + }, + "FunctionName": "${__get_pieriandx_project_pathway_mapping_lambda_function_arn__}" + }, + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 1, + "MaxAttempts": 3, + "BackoffRate": 2 + } + ], + "ResultSelector": { + "project_info.$": "$.Payload.project_info" + }, + "ResultPath": "$.get_pieriandx_project_pathway_mapping_step", + "Next": "Get RedCap Information" + }, + "Get RedCap Information": { + "Type": "Parallel", + "Branches": [ + { + "StartAt": "Get sample redcap info", + "States": { + "Get sample redcap info": { + "Type": "Task", + "Resource": "arn:aws:states:::lambda:invoke", + "Parameters": { + "Payload": { + "library_id.$": "$.get_library_info_from_db_step.Item.library_id.S" + }, + "FunctionName": "${__get_sample_redcap_info_lambda_function_arn__}" + }, + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 1, + "MaxAttempts": 3, + "BackoffRate": 2 + } + ], + "ResultSelector": { + "redcap_data.$": "$.Payload.redcap_data", + "in_redcap.$": "$.Payload.in_redcap" + }, + "ResultPath": "$.get_sample_redcap_info_step", + "Next": "Has Default Disease Code OR In Redcap" + }, + "Has Default Disease Code OR In Redcap": { + "Type": "Choice", + "Choices": [ + { + "And": [ + { + "Variable": "$.get_sample_redcap_info_step.in_redcap", + "BooleanEquals": false + }, + { + "Variable": "$.get_pieriandx_project_pathway_mapping_step.project_info.default_snomed_disease_code", + "IsNull": true + } + ], + "Next": "Fail" + } + ], + "Default": "Option Placeholder 1" + }, + "Fail": { + "Type": "Fail" + }, + "Option Placeholder 1": { + "Type": "Pass", + "End": true + } + } + }, + { + "StartAt": "Parallel Placeholder 1", + "States": { + "Parallel Placeholder 1": { + "Type": "Pass", + "End": true + } + } + } + ], + "ResultSelector": { + "redcap_data.$": "$[0].get_sample_redcap_info_step.redcap_data", + "in_redcap.$": "$[0].get_sample_redcap_info_step.in_redcap" + }, + "ResultPath": "$.get_redcap_info_step", + "Next": "Generate Case Metadata" + }, + "Generate Case Metadata": { + "Type": "Parallel", + "Branches": [ + { + "StartAt": "Is Identified Sample", + "States": { + "Is Identified Sample": { + "Type": "Choice", + "Choices": [ + { + "Variable": "$.get_pieriandx_project_pathway_mapping_step.project_info.is_identified", + "StringEquals": "identified", + "Next": "Generate Identified Case Metadata" + } + ], + "Default": "Generate DeIdentified case metadata" + }, + "Generate Identified Case Metadata": { + "Type": "Task", + "Resource": "arn:aws:states:::lambda:invoke", + "Parameters": { + "FunctionName": "${__get_identified_case_metadata_lambda_function_arn__}", + "Payload": { + "case_accession_number.$": "$.generate_case_accession_number_step.case_accession_number", + "external_sample_id.$": "$.get_library_info_from_db_step.Item.external_sample_id.S", + "external_subject_id.$": "$.get_library_info_from_db_step.Item.external_subject_id.S", + "sample_type.$": "$.get_pieriandx_project_pathway_mapping_step.project_info.sample_type", + "specimen_label": "${__specimen_label__}", + "specimen_code": "${__specimen_code__}", + "redcap_dict.$": "$.get_redcap_info_step.redcap_data", + "default_disease_code.$": "$.get_pieriandx_project_pathway_mapping_step.project_info.default_snomed_disease_code" + } + }, + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 1, + "MaxAttempts": 3, + "BackoffRate": 2 + } + ], + "ResultSelector": { + "case_metadata.$": "$.Payload.case_metadata" + }, + "ResultPath": "$.get_case_metadata_step", + "End": true + }, + "Generate DeIdentified case metadata": { + "Type": "Task", + "Resource": "arn:aws:states:::lambda:invoke", + "Parameters": { + "FunctionName": "${__get_deidentified_case_metadata_lambda_function_arn__}", + "Payload": { + "case_accession_number.$": "$.generate_case_accession_number_step.case_accession_number", + "external_sample_id.$": "$.get_library_info_from_db_step.Item.external_sample_id.S", + "external_subject_id.$": "$.get_library_info_from_db_step.Item.external_subject_id.S", + "sample_type.$": "$.get_pieriandx_project_pathway_mapping_step.project_info.sample_type", + "project_id.$": "$.get_library_info_from_db_step.Item.project_id.S", + "specimen_label": "${__specimen_label__}", + "specimen_code": "${__specimen_code__}", + "redcap_dict.$": "$.get_redcap_info_step.redcap_data", + "default_disease_code.$": "$.get_pieriandx_project_pathway_mapping_step.project_info.default_snomed_disease_code" + } + }, + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 1, + "MaxAttempts": 3, + "BackoffRate": 2 + } + ], + "ResultSelector": { + "case_metadata.$": "$.Payload.case_metadata" + }, + "ResultPath": "$.get_case_metadata_step", + "End": true + } + } + }, + { + "StartAt": "Parallel Placeholder 2", + "States": { + "Parallel Placeholder 2": { + "Type": "Pass", + "End": true + } + } + } + ], + "ResultSelector": { + "case_metadata.$": "$[0].get_case_metadata_step.case_metadata" + }, + "ResultPath": "$.get_case_metadata_step", + "End": true + } + } + }, + { + "StartAt": "Get Project Data Files", + "States": { + "Get Project Data Files": { + "Type": "Task", + "Resource": "arn:aws:states:::lambda:invoke", + "Parameters": { + "Payload": { + "output_uri.$": "$.inputs.payload.data.engineParameters.outputUri", + "sample_id.$": "$.inputs.payload.data.inputs.sampleId" + }, + "FunctionName": "${__get_project_data_files_lambda_function_arn__}" + }, + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 1, + "MaxAttempts": 3, + "BackoffRate": 2 + } + ], + "ResultSelector": { + "project_data_files.$": "$.Payload.data_files" + }, + "ResultPath": "$.get_project_data_files_step", + "End": true + } + } + } + ], + "ResultSelector": { + "case_metadata.$": "$[0].get_case_metadata_step.case_metadata", + "project_info.$": "$[0].get_pieriandx_project_pathway_mapping_step.project_info", + "project_data_files.$": "$[1].get_project_data_files_step.project_data_files", + "in_redcap.$": "$[0].get_redcap_info_step.in_redcap" + }, + "ResultPath": "$.get_project_info_and_project_data_files_step", + "Next": "Push PierianDx Ready Event" + }, + "Push PierianDx Ready Event": { + "Type": "Task", + "Resource": "arn:aws:states:::events:putEvents", + "Parameters": { + "Entries": [ + { + "Detail": { + "portalRunId.$": "$.generate_portal_run_id_step.portal_run_id", + "status": "${__event_status__}", + "workflowName": "${__workflow_name__}", + "workflowVersion": "${__workflow_version__}", + "workflowRunName.$": "States.Format('${__workflow_name__}--${__workflow_version_sub__}--{}', $.generate_portal_run_id_step.portal_run_id)", + "timestamp.$": "$$.State.EnteredTime", + "linkedLibraries": [ + { + "libraryId.$": "$.get_library_info_from_db_step.Item.library_id.S", + "orcabusId.$": "$.get_library_info_from_db_step.Item.id.S" + } + ], + "payload": { + "version": "${__event_version__}", + "data": { + "inputs": { + "instrumentRunId.$": "$.get_library_info_from_db_step.Item.instrument_run_id.S", + "panelVersion.$": "$.get_project_info_and_project_data_files_step.project_info.panel", + "caseMetadata.$": "$.get_project_info_and_project_data_files_step.case_metadata", + "dataFiles.$": "$.get_project_info_and_project_data_files_step.project_data_files" + }, + "engineParameters": {}, + "tags": { + "projectId.$": "$.get_library_info_from_db_step.Item.project_id.S", + "libraryId.$": "$.get_library_info_from_db_step.Item.library_id.S", + "instrumentRunId.$": "$.get_library_info_from_db_step.Item.instrument_run_id.S", + "isIdentified.$": "$.get_project_info_and_project_data_files_step.case_metadata.isIdentified", + "metadataFromRedCap.$": "$.get_project_info_and_project_data_files_step.in_redcap", + "sampleType.$": "$.get_project_info_and_project_data_files_step.case_metadata.sampleType", + "panelVersion.$": "$.get_project_info_and_project_data_files_step.project_info.panel" + } + } + } + }, + "DetailType": "${__event_detail_type__}", + "EventBusName": "${__event_bus_name__}", + "Source": "${__event_source__}" + } + ] + }, + "End": true + }, + "Pass": { + "Type": "Pass", + "End": true + } + } +} diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/index.ts index b196e1dfa..3900faa0f 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/index.ts +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/index.ts @@ -3,11 +3,8 @@ import * as events from 'aws-cdk-lib/aws-events'; import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; import * as ssm from 'aws-cdk-lib/aws-ssm'; import * as secretsManager from 'aws-cdk-lib/aws-secretsmanager'; -import { UmccriseInitialiseSubjectDbRowConstruct } from './part_1/initialise-umccrise-subject-dbs'; -import { UmccriseInitialiseLibraryAndFastqListRowConstruct } from './part_2/initialise-umccrise-library-dbs'; -import { UmccrisePopulateFastqListRowConstruct } from './part_3/update-fastq-list-rows-dbs'; -import { TnCompleteToUmccriseDraftConstruct } from './part_4/tn-complete-to-umccrise-draft'; -import { UmccriseInputMakerConstruct } from './part_5/umccrise-draft-to-ready'; +import { UmccriseInitialiseLibraryConstruct } from './part_1/initialise-umccrise-library-dbs'; +import { TnCompleteToUmccriseReadyConstruct } from './part_2/tn-complete-to-umccrise-draft'; /* Provide the glue to get from the bssh fastq copy manager to submitting wgts qc analyses @@ -18,7 +15,6 @@ export interface umccriseGlueHandlerConstructProps { eventBusObj: events.IEventBus; /* Tables */ umccriseGlueTableObj: dynamodb.ITableV2; - inputMakerTableObj: dynamodb.ITableV2; /* SSM Parameters */ analysisOutputUriSsmParameterObj: ssm.IStringParameter; analysisLogsUriSsmParameterObj: ssm.IStringParameter; @@ -31,57 +27,18 @@ export interface umccriseGlueHandlerConstructProps { export class UmccriseGlueHandlerConstruct extends Construct { constructor(scope: Construct, id: string, props: umccriseGlueHandlerConstructProps) { super(scope, id); - /* Part 1 - Input Event Source: `orcabus.instrumentrunmanager` - Input Event DetailType: `SamplesheetMetadataUnion` - Input Event status: `SubjectInSamplesheet` - - * Initialise umccrise instrument db construct - */ - const umccrise_initialise_subject = new UmccriseInitialiseSubjectDbRowConstruct( - this, - 'umccrise_initialise_subject', - { - eventBusObj: props.eventBusObj, - tableObj: props.umccriseGlueTableObj, - } - ); - - /* - Part 2 - Input Event Source: `orcabus.instrumentrunmanager` Input Event DetailType: `SamplesheetMetadataUnion` Input Event status: `LibraryInSamplesheet` * Initialise umccrise instrument db construct */ - const umccrise_initialise_library_and_fastq_list_row = - new UmccriseInitialiseLibraryAndFastqListRowConstruct( - this, - 'umccrise_initialise_library_and_fastq_list_row', - { - eventBusObj: props.eventBusObj, - tableObj: props.umccriseGlueTableObj, - } - ); - - /* - Part 3 - - Input Event Source: `orcabus.instrumentrunmanager` - Input Event DetailType: `FastqListRowStateChange` - Input Event status: `newFastqListRow` - - * Populate the fastq list row attributes for the rgid for this workflow - */ - - const umccrise_populate_fastq_list_row = new UmccrisePopulateFastqListRowConstruct( + const UmccriseInitialiseLibrary = new UmccriseInitialiseLibraryConstruct( this, - 'umccrise_populate_fastq_list_row', + 'umccrise_initialise_library', { eventBusObj: props.eventBusObj, tableObj: props.umccriseGlueTableObj, @@ -89,7 +46,7 @@ export class UmccriseGlueHandlerConstruct extends Construct { ); /* - Part 4 + Part 2 Input Event Source: `orcabus.workflowmanager` Input Event DetailType: `WorkflowRunStateChange` @@ -102,47 +59,22 @@ export class UmccriseGlueHandlerConstruct extends Construct { * Populate the fastq list row attributes for the rgid for this workflow */ - const tn_to_umccrise_draft = new TnCompleteToUmccriseDraftConstruct( + const tnCompleteToUmccriseReady = new TnCompleteToUmccriseReadyConstruct( this, - 'tn_to_umccrise_draft', + 'tn_to_umccrise', { + /* Events*/ eventBusObj: props.eventBusObj, + /* Tables */ tableObj: props.umccriseGlueTableObj, - workflowsTableObj: props.inputMakerTableObj, + /* SSM Param objects */ + icav2ProjectIdSsmParameterObj: props.icav2ProjectIdSsmParameterObj, + outputUriSsmParameterObj: props.analysisOutputUriSsmParameterObj, + cacheUriSsmParameterObj: props.analysisCacheUriSsmParameterObj, + logsUriSsmParameterObj: props.analysisLogsUriSsmParameterObj, + /* Secrets Manager */ + icav2AccessTokenSecretObj: props.icav2AccessTokenSecretObj, } ); - - /* - Part 5 - - Input Event source: `orcabus.umccriseinputeventglue` - Input Event DetailType: `WorkflowDraftRunStateChange` - Input Event status: `draft` - - Output Event source: `orcabus.umccriseinputeventglue` - Output Event DetailType: `WorkflowRunStateChange` - Output Event status: `ready` - - * The umccriseInputMaker, subscribes to the umccrise input event glue (itself) and generates a ready event for the umccriseReadySfn - * However, in order to be 'READY' we need to use a few more variables such as - * icaLogsUri, - * analysisOutputUri - * cacheUri - * projectId - * userReference - */ - const umccriseInputMaker = new UmccriseInputMakerConstruct(this, 'fastq_list_row_qc_complete', { - /* Event bus */ - eventBusObj: props.eventBusObj, - /* Tables */ - inputMakerTableObj: props.inputMakerTableObj, - /* SSM Param objects */ - icav2ProjectIdSsmParameterObj: props.icav2ProjectIdSsmParameterObj, - outputUriSsmParameterObj: props.analysisOutputUriSsmParameterObj, - cacheUriSsmParameterObj: props.analysisCacheUriSsmParameterObj, - logsUriSsmParameterObj: props.analysisLogsUriSsmParameterObj, - /* Secrets Manager */ - icav2AccessTokenSecretObj: props.icav2AccessTokenSecretObj, - }); } } diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/part_2/initialise-umccrise-library-dbs/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/part_1/initialise-umccrise-library-dbs/index.ts similarity index 82% rename from lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/part_2/initialise-umccrise-library-dbs/index.ts rename to lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/part_1/initialise-umccrise-library-dbs/index.ts index c70a0cc2d..756d1ad59 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/part_2/initialise-umccrise-library-dbs/index.ts +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/part_1/initialise-umccrise-library-dbs/index.ts @@ -15,18 +15,16 @@ Input Event status: `LibraryInSamplesheet` * Initialise umccrise instrument db construct */ -export interface UmccriseInitialiseLibraryAndFastqListRowConstructProps { +export interface UmccriseInitialiseLibraryConstructProps { tableObj: dynamodb.ITableV2; eventBusObj: events.IEventBus; } -export class UmccriseInitialiseLibraryAndFastqListRowConstruct extends Construct { +export class UmccriseInitialiseLibraryConstruct extends Construct { public readonly UmccriseInitialiseLibraryAndFastqListRowMap = { prefix: 'pva-make-library-and-fqlr-row', tablePartition: { - subject: 'subject', library: 'library', - fastqListRow: 'fastq_list_row', }, triggerSource: 'orcabus.instrumentrunmanager', triggerStatus: 'LibraryInSamplesheet', @@ -39,16 +37,11 @@ export class UmccriseInitialiseLibraryAndFastqListRowConstruct extends Construct CLINICAL: 'clinical', }, triggerPhenotypeType: { - NORMAL: 'normal', TUMOR: 'tumor', }, }; - constructor( - scope: Construct, - id: string, - props: UmccriseInitialiseLibraryAndFastqListRowConstructProps - ) { + constructor(scope: Construct, id: string, props: UmccriseInitialiseLibraryConstructProps) { super(scope, id); /* @@ -68,12 +61,8 @@ export class UmccriseInitialiseLibraryAndFastqListRowConstruct extends Construct __table_name__: props.tableObj.tableName, /* Table Partitions */ - __subject_partition_name__: - this.UmccriseInitialiseLibraryAndFastqListRowMap.tablePartition.subject, __library_partition_name__: this.UmccriseInitialiseLibraryAndFastqListRowMap.tablePartition.library, - __fastq_list_row_partition_name__: - this.UmccriseInitialiseLibraryAndFastqListRowMap.tablePartition.fastqListRow, }, }); @@ -115,10 +104,6 @@ export class UmccriseInitialiseLibraryAndFastqListRowConstruct extends Construct }, ], phenotype: [ - { - 'equals-ignore-case': - this.UmccriseInitialiseLibraryAndFastqListRowMap.triggerPhenotypeType.NORMAL, - }, { 'equals-ignore-case': this.UmccriseInitialiseLibraryAndFastqListRowMap.triggerPhenotypeType.TUMOR, diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/part_1/initialise-umccrise-library-dbs/step_functions_templates/initialise_umccrise_library_db_sfn_template.asl.json b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/part_1/initialise-umccrise-library-dbs/step_functions_templates/initialise_umccrise_library_db_sfn_template.asl.json new file mode 100644 index 000000000..fbbae715e --- /dev/null +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/part_1/initialise-umccrise-library-dbs/step_functions_templates/initialise_umccrise_library_db_sfn_template.asl.json @@ -0,0 +1,98 @@ +{ + "Comment": "A description of my state machine", + "StartAt": "Move Inputs", + "States": { + "Move Inputs": { + "Type": "Pass", + "Parameters": { + "payload_data.$": "$.payload.data" + }, + "Next": "Get Library Item" + }, + "Get Library Item": { + "Type": "Task", + "Resource": "arn:aws:states:::dynamodb:getItem", + "Parameters": { + "TableName": "${__table_name__}", + "Key": { + "id.$": "$.payload_data.library.orcabusId", + "id_type": "${__library_partition_name__}" + } + }, + "ResultPath": "$.get_library_item_step", + "Next": "Update Databases" + }, + "Update Databases": { + "Type": "Parallel", + "Branches": [ + { + "StartAt": "Pass", + "States": { + "Pass": { + "Type": "Pass", + "End": true + } + } + }, + { + "StartAt": "Library in Database", + "States": { + "Library in Database": { + "Type": "Choice", + "Choices": [ + { + "Variable": "$.get_library_item_step.Item", + "IsPresent": false, + "Comment": "Library Not In Database", + "Next": "Initialise Library" + } + ], + "Default": "No Need to Initialise Library" + }, + "Initialise Library": { + "Type": "Task", + "Resource": "arn:aws:states:::dynamodb:putItem", + "Parameters": { + "TableName": "${__table_name__}", + "Item": { + "id.$": "$.payload_data.library.orcabusId", + "id_type": "${__library_partition_name__}", + "library_id": { + "S.$": "$.payload_data.library.libraryId" + }, + "phenotype": { + "S.$": "$.payload_data.library.phenotype" + }, + "workflow": { + "S.$": "$.payload_data.library.workflow" + }, + "type": { + "S.$": "$.payload_data.library.type" + }, + "assay": { + "S.$": "$.payload_data.library.assay" + }, + "subject_id": { + "S.$": "$.payload_data.subject.subjectId" + }, + "subject_orcabus_id": { + "S.$": "$.payload_data.subject.orcabusId" + } + } + }, + "ResultPath": null, + "End": true + }, + "No Need to Initialise Library": { + "Type": "Pass", + "End": true, + "ResultPath": null + } + } + } + ], + "ResultPath": null, + "End": true + } + } +} diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/part_1/initialise-umccrise-subject-dbs/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/part_1/initialise-umccrise-subject-dbs/index.ts deleted file mode 100644 index dfcc7e994..000000000 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/part_1/initialise-umccrise-subject-dbs/index.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { Construct } from 'constructs'; -import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; -import path from 'path'; -import * as sfn from 'aws-cdk-lib/aws-stepfunctions'; -import * as events from 'aws-cdk-lib/aws-events'; -import * as eventsTargets from 'aws-cdk-lib/aws-events-targets'; - -/* -Part 1 - -Input Event Source: `orcabus.instrumentrunmanager` -Input Event DetailType: `SamplesheetMetadataUnion` -Input Event status: `SubjectInSamplesheet` - -* Initialise umccrise subject db construct -*/ - -export interface UmccriseInitialiseSubjectDbRowConstructProps { - tableObj: dynamodb.ITableV2; - eventBusObj: events.IEventBus; -} - -export class UmccriseInitialiseSubjectDbRowConstruct extends Construct { - public readonly UmccriseInitialiseSubjectDbRowMap = { - prefix: 'pva-make-subject-row', - tablePartition: 'subject', - triggerSource: 'orcabus.instrumentrunmanager', - triggerStatus: 'SubjectInSamplesheet', - triggerDetailType: 'SamplesheetMetadataUnion', - }; - - constructor(scope: Construct, id: string, props: UmccriseInitialiseSubjectDbRowConstructProps) { - super(scope, id); - - /* - Part 1: Build the internal sfn - */ - const inputMakerSfn = new sfn.StateMachine(this, 'initialise_subject_db_row', { - stateMachineName: `${this.UmccriseInitialiseSubjectDbRowMap.prefix}-initialise-subject`, - definitionBody: sfn.DefinitionBody.fromFile( - path.join( - __dirname, - 'step_functions_templates', - 'initialise_umccrise_subject_db_sfn_template.asl.json' - ) - ), - definitionSubstitutions: { - __table_name__: props.tableObj.tableName, - __subject_partition_name__: this.UmccriseInitialiseSubjectDbRowMap.tablePartition, - }, - }); - - /* - Part 2: Grant the internal sfn permissions - */ - // access the dynamodb table - props.tableObj.grantReadWriteData(inputMakerSfn.role); - - /* - Part 3: Subscribe to the event bus and trigger the internal sfn - */ - const rule = new events.Rule(this, 'umccrise_subscribe_to_samplesheet_shower_subject', { - ruleName: `stacky-${this.UmccriseInitialiseSubjectDbRowMap.prefix}-rule`, - eventBus: props.eventBusObj, - eventPattern: { - source: [this.UmccriseInitialiseSubjectDbRowMap.triggerSource], - detailType: [this.UmccriseInitialiseSubjectDbRowMap.triggerDetailType], - detail: { - status: [{ 'equals-ignore-case': this.UmccriseInitialiseSubjectDbRowMap.triggerStatus }], - }, - }, - }); - - // Add target of event to be the state machine - rule.addTarget( - new eventsTargets.SfnStateMachine(inputMakerSfn, { - input: events.RuleTargetInput.fromEventPath('$.detail'), - }) - ); - } -} diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/part_1/initialise-umccrise-subject-dbs/step_functions_templates/initialise_umccrise_subject_db_sfn_template.asl.json b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/part_1/initialise-umccrise-subject-dbs/step_functions_templates/initialise_umccrise_subject_db_sfn_template.asl.json deleted file mode 100644 index e0c23b290..000000000 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/part_1/initialise-umccrise-subject-dbs/step_functions_templates/initialise_umccrise_subject_db_sfn_template.asl.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "Comment": "A description of my state machine", - "StartAt": "Move Inputs", - "States": { - "Move Inputs": { - "Type": "Pass", - "Parameters": { - "payload_data.$": "$.payload.data" - }, - "Next": "Get Subject Item" - }, - "Get Subject Item": { - "Type": "Task", - "Resource": "arn:aws:states:::dynamodb:getItem", - "Parameters": { - "TableName": "${__table_name__}", - "Key": { - "id.$": "$.payload_data.subject.subjectId", - "id_type": "${__subject_partition_name__}" - } - }, - "ResultPath": "$.get_subject_item_step", - "Next": "Subject in Database" - }, - "Subject in Database": { - "Type": "Choice", - "Choices": [ - { - "Variable": "$.get_subject_item_step.Item", - "IsPresent": false, - "Comment": "Subject Not In Database", - "Next": "Initialise Subject" - } - ], - "Default": "Pass" - }, - "Pass": { - "Type": "Pass", - "End": true - }, - "Initialise Subject": { - "Type": "Task", - "Resource": "arn:aws:states:::dynamodb:putItem", - "Parameters": { - "TableName": "${__table_name__}", - "Item": { - "id.$": "$.payload_data.subject.subjectId", - "id_type": "${__subject_partition_name__}", - "orcabus_id": { - "S.$": "$.payload_data.subject.orcabusId" - } - } - }, - "End": true - } - } -} diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/part_2/initialise-umccrise-library-dbs/step_functions_templates/initialise_umccrise_library_db_sfn_template.asl.json b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/part_2/initialise-umccrise-library-dbs/step_functions_templates/initialise_umccrise_library_db_sfn_template.asl.json deleted file mode 100644 index 924a9d807..000000000 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/part_2/initialise-umccrise-library-dbs/step_functions_templates/initialise_umccrise_library_db_sfn_template.asl.json +++ /dev/null @@ -1,190 +0,0 @@ -{ - "Comment": "A description of my state machine", - "StartAt": "Move Inputs", - "States": { - "Move Inputs": { - "Type": "Pass", - "Parameters": { - "payload_data.$": "$.payload.data" - }, - "Next": "Get Library Item" - }, - "Get Library Item": { - "Type": "Task", - "Resource": "arn:aws:states:::dynamodb:getItem", - "Parameters": { - "TableName": "${__table_name__}", - "Key": { - "id.$": "$.payload_data.library.libraryId", - "id_type": "${__library_partition_name__}" - } - }, - "ResultPath": "$.get_library_item_step", - "Next": "Update Databases" - }, - "Update Databases": { - "Type": "Parallel", - "Branches": [ - { - "StartAt": "Add Library to Subject", - "States": { - "Add Library to Subject": { - "Type": "Task", - "Resource": "arn:aws:states:::dynamodb:updateItem", - "Parameters": { - "TableName": "${__table_name__}", - "Key": { - "id.$": "$.payload_data.library.subject.subjectId", - "id_type": "${__subject_partition_name__}" - }, - "UpdateExpression": "ADD library_set :library_set", - "ExpressionAttributeValues": { - ":library_set": { - "SS.$": "States.Array($.payload_data.library.libraryId)" - } - } - }, - "ResultPath": null, - "End": true - } - } - }, - { - "StartAt": "Library in Database", - "States": { - "Library in Database": { - "Type": "Choice", - "Choices": [ - { - "Variable": "$.get_library_item_step.Item", - "IsPresent": false, - "Comment": "Library Not In Database", - "Next": "Initialise Library" - } - ], - "Default": "No Need to Initialise Library" - }, - "Initialise Library": { - "Type": "Task", - "Resource": "arn:aws:states:::dynamodb:putItem", - "Parameters": { - "TableName": "${__table_name__}", - "Item": { - "id.$": "$.payload_data.library.libraryId", - "id_type": "${__library_partition_name__}", - "orcabus_id": { - "S.$": "$.payload_data.library.orcabusId" - }, - "phenotype": { - "S.$": "$.payload_data.library.phenotype" - }, - "workflow": { - "S.$": "$.payload_data.library.workflow" - }, - "type": { - "S.$": "$.payload_data.library.type" - }, - "assay": { - "S.$": "$.payload_data.library.assay" - }, - "subject_id": { - "S.$": "$.payload_data.library.subject.subjectId" - } - } - }, - "ResultPath": null, - "End": true - }, - "No Need to Initialise Library": { - "Type": "Pass", - "End": true, - "ResultPath": null - } - } - } - ], - "Next": "Initialise Fastq List Rows", - "ResultPath": null - }, - "Initialise Fastq List Rows": { - "Type": "Map", - "ItemsPath": "$.payload_data.bclconvertDataRows", - "ItemSelector": { - "bclconvert_data_row.$": "$$.Map.Item.Value", - "index.$": "$$.Map.Item.Index", - "fastq_list_row_objs.$": "$.payload_data.fastqListRows", - "instrument_run_id.$": "$.payload_data.instrumentRunId", - "library_id.$": "$.payload_data.library.libraryId" - }, - "ItemProcessor": { - "ProcessorConfig": { - "Mode": "INLINE" - }, - "StartAt": "Get Fastq List Row Id", - "States": { - "Get Fastq List Row Id": { - "Type": "Pass", - "Next": "Initialise Fastq List Row and Update Library", - "Parameters": { - "fastq_list_row_id.$": "States.ArrayGetItem($.fastq_list_row_objs[*].fastqListRowRgid, $.index)" - }, - "ResultPath": "$.get_fastq_list_row_id_step" - }, - "Initialise Fastq List Row and Update Library": { - "Type": "Parallel", - "Branches": [ - { - "StartAt": "Initialise Fastq List Row", - "States": { - "Initialise Fastq List Row": { - "Type": "Task", - "Resource": "arn:aws:states:::dynamodb:putItem", - "Parameters": { - "TableName": "${__table_name__}", - "Item": { - "id.$": "$.get_fastq_list_row_id_step.fastq_list_row_id", - "id_type": "${__fastq_list_row_partition_name__}", - "library_id": { - "S.$": "$.library_id" - } - } - }, - "ResultPath": null, - "End": true - } - } - }, - { - "StartAt": "Append Fastq List Row to Library", - "States": { - "Append Fastq List Row to Library": { - "Type": "Task", - "Resource": "arn:aws:states:::dynamodb:updateItem", - "Parameters": { - "TableName": "${__table_name__}", - "Key": { - "id.$": "$.library_id", - "id_type": "${__library_partition_name__}" - }, - "UpdateExpression": "ADD fastq_list_row_id_set :fastq_list_row_id_set", - "ExpressionAttributeValues": { - ":fastq_list_row_id_set": { - "SS.$": "States.Array($.get_fastq_list_row_id_step.fastq_list_row_id)" - } - } - }, - "End": true, - "ResultPath": null - } - } - } - ], - "End": true - } - } - }, - "End": true, - "ResultPath": null - } - } -} diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/part_4/tn-complete-to-umccrise-draft/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/part_2/tn-complete-to-umccrise-draft/index.ts similarity index 58% rename from lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/part_4/tn-complete-to-umccrise-draft/index.ts rename to lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/part_2/tn-complete-to-umccrise-draft/index.ts index 3916e8010..abe258157 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/part_4/tn-complete-to-umccrise-draft/index.ts +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/part_2/tn-complete-to-umccrise-draft/index.ts @@ -8,45 +8,53 @@ import * as events from 'aws-cdk-lib/aws-events'; import * as eventsTargets from 'aws-cdk-lib/aws-events-targets'; import { PythonFunction } from '@aws-cdk/aws-lambda-python-alpha'; import * as lambda from 'aws-cdk-lib/aws-lambda'; +import * as ssm from 'aws-cdk-lib/aws-ssm'; +import * as secretsManager from 'aws-cdk-lib/aws-secretsmanager'; import { WorkflowDraftRunStateChangeCommonPreambleConstruct } from '../../../../../../../components/sfn-workflowdraftrunstatechange-common-preamble'; +import { GenerateWorkflowRunStateChangeReadyConstruct } from '../../../../../../../components/sfn-generate-workflowrunstatechange-ready-event'; /* -Part 4 +Part 2 Input Event Source: `orcabus.workflowmanager` Input Event DetailType: `WorkflowRunStateChange` -Input Event WorkflowName: tumor_normal -Input Event status: `succeeded` +Input Event WorkflowName: tumor-normal +Input Event status: `SUCCEEDED` Output Event Source: `orcabus.umccriseinputeventglue` Output Event DetailType: `WorkflowDraftRunStateChange` -Output Event status: `draft` +Output Event status: `READY` * Subscribe to the workflow manager succeeded event for tumor normal libraries. * Launch a draft event for the umccrise pipeline */ export interface TnCompleteToUmccriseDraftConstructProps { + /* Events */ eventBusObj: events.IEventBus; + /* Tables */ tableObj: dynamodb.ITableV2; - workflowsTableObj: dynamodb.ITableV2; + /* SSM Parameters */ + outputUriSsmParameterObj: ssm.IStringParameter; + icav2ProjectIdSsmParameterObj: ssm.IStringParameter; + logsUriSsmParameterObj: ssm.IStringParameter; + cacheUriSsmParameterObj: ssm.IStringParameter; + /* Secrets */ + icav2AccessTokenSecretObj: secretsManager.ISecret; } -export class TnCompleteToUmccriseDraftConstruct extends Construct { - public readonly UmccriseDraftMap = { - prefix: 'pva-tn-complete-to-umccrise-draft', - portalRunPartitionName: 'portal_run', +export class TnCompleteToUmccriseReadyConstruct extends Construct { + public readonly UmccriseReadyMap = { + prefix: 'pva-tn-complete-to-umccrise', triggerSource: 'orcabus.workflowmanager', triggerStatus: 'succeeded', - triggerWorkflowName: 'tumor_normal', + triggerWorkflowName: 'tumor-normal', triggerDetailType: 'WorkflowRunStateChange', outputSource: 'orcabus.umccriseinputeventglue', - outputDetailType: 'WorkflowDraftRunStateChange', - outputStatus: 'DRAFT', payloadVersion: '2024.07.23', workflowName: 'umccrise', workflowVersion: '2.3.1', - tablePartitionName: 'subject', + tablePartitionName: 'library', }; constructor(scope: Construct, id: string, props: TnCompleteToUmccriseDraftConstructProps) { @@ -68,15 +76,42 @@ export class TnCompleteToUmccriseDraftConstruct extends Construct { /* Part 1: Generate the preamble (sfn to generate the portal run id and the workflow run name) */ - const sfn_preamble = new WorkflowDraftRunStateChangeCommonPreambleConstruct( + const sfnPreamble = new WorkflowDraftRunStateChangeCommonPreambleConstruct( this, - `${this.UmccriseDraftMap.prefix}_sfn_preamble`, + `${this.UmccriseReadyMap.prefix}_sfn_preamble`, { - portalRunTablePartitionName: this.UmccriseDraftMap.portalRunPartitionName, - stateMachinePrefix: this.UmccriseDraftMap.prefix, - tableObj: props.workflowsTableObj, - workflowName: this.UmccriseDraftMap.workflowName, - workflowVersion: this.UmccriseDraftMap.workflowVersion, + stateMachinePrefix: this.UmccriseReadyMap.prefix, + workflowName: this.UmccriseReadyMap.workflowName, + workflowVersion: this.UmccriseReadyMap.workflowVersion, + } + ).stepFunctionObj; + + /* + Part 2: Build the engine parameters sfn + */ + const engineParameterAndReadyEventMakerSfn = new GenerateWorkflowRunStateChangeReadyConstruct( + this, + 'fastqlistrow_complete_to_wgtsqc_ready_submitter', + { + /* Event Placeholders */ + eventBusObj: props.eventBusObj, + outputSource: this.UmccriseReadyMap.outputSource, + payloadVersion: this.UmccriseReadyMap.payloadVersion, + workflowName: this.UmccriseReadyMap.workflowName, + workflowVersion: this.UmccriseReadyMap.workflowVersion, + + /* SSM Parameters */ + outputUriSsmParameterObj: props.outputUriSsmParameterObj, + icav2ProjectIdSsmParameterObj: props.icav2ProjectIdSsmParameterObj, + logsUriSsmParameterObj: props.logsUriSsmParameterObj, + cacheUriSsmParameterObj: props.cacheUriSsmParameterObj, + + /* Secrets */ + icav2AccessTokenSecretObj: props.icav2AccessTokenSecretObj, + + /* Prefixes */ + lambdaPrefix: this.UmccriseReadyMap.prefix, + stateMachinePrefix: this.UmccriseReadyMap.prefix, } ).stepFunctionObj; @@ -84,7 +119,7 @@ export class TnCompleteToUmccriseDraftConstruct extends Construct { Part 2: Build the sfn */ const qcCompleteToDraftSfn = new sfn.StateMachine(this, 'tn_complete_to_umccrise_draft_sfn', { - stateMachineName: `${this.UmccriseDraftMap.prefix}-sfn`, + stateMachineName: `${this.UmccriseReadyMap.prefix}-sfn`, definitionBody: sfn.DefinitionBody.fromFile( path.join( __dirname, @@ -93,15 +128,6 @@ export class TnCompleteToUmccriseDraftConstruct extends Construct { ) ), definitionSubstitutions: { - /* Events */ - __event_bus_name__: props.eventBusObj.eventBusName, - __event_source__: this.UmccriseDraftMap.outputSource, - __detail_type__: this.UmccriseDraftMap.outputDetailType, - __output_status__: this.UmccriseDraftMap.outputStatus, - __payload_version__: this.UmccriseDraftMap.payloadVersion, - __workflow_name__: this.UmccriseDraftMap.workflowName, - __workflow_version__: this.UmccriseDraftMap.workflowVersion, - /* Lambdas */ __generate_draft_event_payload_lambda_function_arn__: generateEventDataLambdaObj.currentVersion.functionArn, @@ -110,10 +136,11 @@ export class TnCompleteToUmccriseDraftConstruct extends Construct { __table_name__: props.tableObj.tableName, /* Table Partitions */ - __subject_table_partition_name__: this.UmccriseDraftMap.tablePartitionName, + __library_partition_name__: this.UmccriseReadyMap.tablePartitionName, // State Machines - __sfn_preamble_state_machine_arn__: sfn_preamble.stateMachineArn, + __sfn_preamble_state_machine_arn__: sfnPreamble.stateMachineArn, + __launch_ready_event_sfn_arn__: engineParameterAndReadyEventMakerSfn.stateMachineArn, }, }); @@ -123,9 +150,6 @@ export class TnCompleteToUmccriseDraftConstruct extends Construct { // access the dynamodb table props.tableObj.grantReadWriteData(qcCompleteToDraftSfn); - // allow the step function to submit events - props.eventBusObj.grantPutEventsTo(qcCompleteToDraftSfn); - // allow the step function to invoke the lambdas generateEventDataLambdaObj.currentVersion.grantInvoke(qcCompleteToDraftSfn); @@ -141,22 +165,23 @@ export class TnCompleteToUmccriseDraftConstruct extends Construct { }) ); // Allow the state machine to be able to invoke the preamble sfn - sfn_preamble.grantStartExecution(qcCompleteToDraftSfn); + sfnPreamble.grantStartExecution(qcCompleteToDraftSfn); + engineParameterAndReadyEventMakerSfn.grantStartExecution(qcCompleteToDraftSfn); /* Part 3: Subscribe to the event bus and trigger the internal sfn */ const rule = new events.Rule(this, 'tn_complete_to_umccrise_draft_rule', { - ruleName: `stacky-${this.UmccriseDraftMap.prefix}-rule`, + ruleName: `stacky-${this.UmccriseReadyMap.prefix}-rule`, eventBus: props.eventBusObj, eventPattern: { - source: [this.UmccriseDraftMap.triggerSource], - detailType: [this.UmccriseDraftMap.triggerDetailType], + source: [this.UmccriseReadyMap.triggerSource], + detailType: [this.UmccriseReadyMap.triggerDetailType], detail: { - status: [{ 'equals-ignore-case': this.UmccriseDraftMap.triggerStatus }], + status: [{ 'equals-ignore-case': this.UmccriseReadyMap.triggerStatus }], workflowName: [ { - 'equals-ignore-case': this.UmccriseDraftMap.triggerWorkflowName, + 'equals-ignore-case': this.UmccriseReadyMap.triggerWorkflowName, }, ], }, diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/part_4/tn-complete-to-umccrise-draft/lambdas/generate_draft_event_payload_py/generate_draft_event_payload.py b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/part_2/tn-complete-to-umccrise-draft/lambdas/generate_draft_event_payload_py/generate_draft_event_payload.py similarity index 93% rename from lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/part_4/tn-complete-to-umccrise-draft/lambdas/generate_draft_event_payload_py/generate_draft_event_payload.py rename to lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/part_2/tn-complete-to-umccrise-draft/lambdas/generate_draft_event_payload_py/generate_draft_event_payload.py index 043bd1660..3c764dd53 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/part_4/tn-complete-to-umccrise-draft/lambdas/generate_draft_event_payload_py/generate_draft_event_payload.py +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/part_2/tn-complete-to-umccrise-draft/lambdas/generate_draft_event_payload_py/generate_draft_event_payload.py @@ -44,6 +44,7 @@ def handler(event, context) -> Dict: """ subject_id = event['subject_id'] + individual_id = event['individual_id'] tumor_library_id = event['tumor_library_id'] normal_library_id = event['normal_library_id'] dragen_somatic_output_s3_uri = event['dragen_somatic_output_s3_uri'] @@ -53,7 +54,7 @@ def handler(event, context) -> Dict: return { "input_event_data": { - "subjectId": subject_id, + "subjectId": individual_id, "dragenSomaticLibraryId": tumor_library_id, "dragenGermlineLibraryId": normal_library_id, "dragenSomaticOutputUri": dragen_somatic_output_s3_uri, @@ -61,6 +62,7 @@ def handler(event, context) -> Dict: }, "event_tags": { "subjectId": subject_id, + "individualId": individual_id, "tumorLibraryId": tumor_library_id, "normalLibraryId": normal_library_id, "tumorFastqListRowIds": tumor_fastq_list_row_ids, diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/part_4/tn-complete-to-umccrise-draft/step_functions_templates/tn_complete_to_umccrise_draft_sfn_template.asl.json b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/part_2/tn-complete-to-umccrise-draft/step_functions_templates/tn_complete_to_umccrise_draft_sfn_template.asl.json similarity index 66% rename from lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/part_4/tn-complete-to-umccrise-draft/step_functions_templates/tn_complete_to_umccrise_draft_sfn_template.asl.json rename to lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/part_2/tn-complete-to-umccrise-draft/step_functions_templates/tn_complete_to_umccrise_draft_sfn_template.asl.json index 756332981..19b4d8556 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/part_4/tn-complete-to-umccrise-draft/step_functions_templates/tn_complete_to_umccrise_draft_sfn_template.asl.json +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/part_2/tn-complete-to-umccrise-draft/step_functions_templates/tn_complete_to_umccrise_draft_sfn_template.asl.json @@ -8,26 +8,34 @@ "linked_libraries.$": "$.linkedLibraries", "payload_data.$": "$.payload.data" }, - "Next": "Get Subject Item" + "Next": "Get Tumor Library Orcabus Id" }, - "Get Subject Item": { + "Get Tumor Library Orcabus Id": { + "Type": "Pass", + "Next": "Get Library Item", + "Parameters": { + "tumor_orcabus_id.$": "States.ArrayGetItem($.linked_libraries[?(@.libraryId==$.payload_data.tags.tumorLibraryId)].orcabusId, 0)" + }, + "ResultPath": "$.get_tumor_orcabus_id_step" + }, + "Get Library Item": { "Type": "Task", "Resource": "arn:aws:states:::dynamodb:getItem", "Parameters": { "TableName": "${__table_name__}", "Key": { - "id.$": "$.payload_data.tags.subjectId", - "id_type": "${__subject_table_partition_name__}" + "id.$": "$.get_tumor_orcabus_id_step.tumor_orcabus_id", + "id_type": "${__library_partition_name__}" } }, - "ResultPath": "$.get_subject_item_step", - "Next": "Subject Item In DataBase" + "ResultPath": "$.get_library_item_step", + "Next": "Library Item In DataBase" }, - "Subject Item In DataBase": { + "Library Item In DataBase": { "Type": "Choice", "Choices": [ { - "Variable": "$.get_subject_item_step.Item", + "Variable": "$.get_library_item_step.Item", "IsPresent": true, "Comment": "Subject Item In DataBase", "Next": "Get Portal Run Id and Workflow Run Name" @@ -56,6 +64,7 @@ "FunctionName": "${__generate_draft_event_payload_lambda_function_arn__}", "Payload": { "subject_id.$": "$.payload_data.tags.subjectId", + "individual_id.$": "$.payload_data.tags.individualId", "tumor_library_id.$": "$.payload_data.tags.tumorLibraryId", "normal_library_id.$": "$.payload_data.tags.normalLibraryId", "tumor_fastq_list_row_ids.$": "$.payload_data.tags.tumorFastqListRowIds", @@ -82,35 +91,22 @@ "event_tags.$": "$.Payload.event_tags" }, "ResultPath": "$.generate_draft_event_payload_data_step", - "Next": "Push UMCCRise Draft Event" + "Next": "Push UMCCRise Ready Event" }, - "Push UMCCRise Draft Event": { + "Push UMCCRise Ready Event": { "Type": "Task", - "Resource": "arn:aws:states:::events:putEvents", + "Resource": "arn:aws:states:::states:startExecution.sync:2", "Parameters": { - "Entries": [ - { - "Detail": { - "portalRunId.$": "$.get_portal_and_run_name_step.portal_run_id", - "timestamp.$": "$$.State.EnteredTime", - "status": "${__output_status__}", - "workflowName": "${__workflow_name__}", - "workflowVersion": "${__workflow_version__}", - "workflowRunName.$": "$.get_portal_and_run_name_step.workflow_run_name", - "linkedLibraries.$": "$.linked_libraries", - "payload": { - "version": "${__payload_version__}", - "data": { - "inputs.$": "$.generate_draft_event_payload_data_step.input_event_data", - "tags.$": "$.generate_draft_event_payload_data_step.event_tags" - } - } - }, - "DetailType": "${__detail_type__}", - "EventBusName": "${__event_bus_name__}", - "Source": "${__event_source__}" + "StateMachineArn": "${__launch_ready_event_sfn_arn__}", + "Input": { + "StatePayload": { + "portal_run_id.$": "$.get_portal_and_run_name_step.portal_run_id", + "workflow_run_name.$": "$.get_portal_and_run_name_step.workflow_run_name", + "linked_libraries.$": "$.linked_libraries", + "data_inputs.$": "$.generate_draft_event_payload_data_step.input_event_data", + "data_tags.$": "$.generate_draft_event_payload_data_step.event_tags" } - ] + } }, "ResultPath": null, "End": true diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/part_3/update-fastq-list-rows-dbs/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/part_3/update-fastq-list-rows-dbs/index.ts deleted file mode 100644 index 890bbbf92..000000000 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/part_3/update-fastq-list-rows-dbs/index.ts +++ /dev/null @@ -1,87 +0,0 @@ -import { Construct } from 'constructs'; -import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; -import path from 'path'; -import * as sfn from 'aws-cdk-lib/aws-stepfunctions'; -import * as events from 'aws-cdk-lib/aws-events'; -import * as eventsTargets from 'aws-cdk-lib/aws-events-targets'; - -/* -Part 3 - -Input Event Source: `orcabus.instrumentrunmanager` -Input Event DetailType: `FastqListRowStateChange` -Input Event status: `newFastqListRow` - -* Populate the fastq list row attributes for the rgid for this workflow -*/ - -export interface UmccrisePopulateFastqListRowDbRowConstructProps { - tableObj: dynamodb.ITableV2; - eventBusObj: events.IEventBus; -} - -export class UmccrisePopulateFastqListRowConstruct extends Construct { - public readonly UmccrisePopulateFastqListRowDbRowMap = { - prefix: 'pva-populate-fqlr-row', - tablePartition: 'fastq_list_row', - triggerSource: 'orcabus.instrumentrunmanager', - triggerStatus: 'newFastqListRow', - triggerDetailType: 'FastqListRowStateChange', - }; - - constructor( - scope: Construct, - id: string, - props: UmccrisePopulateFastqListRowDbRowConstructProps - ) { - super(scope, id); - - /* - Part 1: Build the internal sfn - */ - const inputMakerSfn = new sfn.StateMachine(this, 'update_fastq-list-row_db_row', { - stateMachineName: `${this.UmccrisePopulateFastqListRowDbRowMap.prefix}-sfn`, - definitionBody: sfn.DefinitionBody.fromFile( - path.join( - __dirname, - 'step_functions_templates', - 'add_fastq_list_rows_db_sfn_template.asl.json' - ) - ), - definitionSubstitutions: { - __table_name__: props.tableObj.tableName, - __fastq_list_row_partition_name__: this.UmccrisePopulateFastqListRowDbRowMap.tablePartition, - }, - }); - - /* - Part 2: Grant the internal sfn permissions - */ - // access the dynamodb table - props.tableObj.grantReadWriteData(inputMakerSfn.role); - - /* - Part 3: Subscribe to the event bus for this event type - */ - const rule = new events.Rule(this, 'umccrise_populate_fastq_list_row', { - ruleName: `stacky-${this.UmccrisePopulateFastqListRowDbRowMap.prefix}-event-rule`, - eventBus: props.eventBusObj, - eventPattern: { - source: [this.UmccrisePopulateFastqListRowDbRowMap.triggerSource], - detailType: [this.UmccrisePopulateFastqListRowDbRowMap.triggerDetailType], - detail: { - status: [ - { 'equals-ignore-case': this.UmccrisePopulateFastqListRowDbRowMap.triggerStatus }, - ], - }, - }, - }); - - // Add target of event to be the state machine - rule.addTarget( - new eventsTargets.SfnStateMachine(inputMakerSfn, { - input: events.RuleTargetInput.fromEventPath('$.detail'), - }) - ); - } -} diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/part_3/update-fastq-list-rows-dbs/step_functions_templates/add_fastq_list_rows_db_sfn_template.asl.json b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/part_3/update-fastq-list-rows-dbs/step_functions_templates/add_fastq_list_rows_db_sfn_template.asl.json deleted file mode 100644 index e5cdfae90..000000000 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/part_3/update-fastq-list-rows-dbs/step_functions_templates/add_fastq_list_rows_db_sfn_template.asl.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "Comment": "A description of my state machine", - "StartAt": "Move Inputs", - "States": { - "Move Inputs": { - "Type": "Pass", - "Parameters": { - "payload_data.$": "$.payload.data" - }, - "Next": "Get Fastq List Row Item" - }, - "Get Fastq List Row Item": { - "Type": "Task", - "Resource": "arn:aws:states:::dynamodb:getItem", - "Parameters": { - "TableName": "${__table_name__}", - "Key": { - "id.$": "$.payload_data.fastqListRow.rgid", - "id_type": "${__fastq_list_row_partition_name__}" - } - }, - "ResultPath": "$.get_fastq_list_row_item_step", - "Next": "Is Fastq List Row In Db" - }, - "Is Fastq List Row In Db": { - "Type": "Choice", - "Choices": [ - { - "Variable": "$.get_fastq_list_row_item_step.Item", - "IsPresent": true, - "Comment": "Fastq List Row In Database", - "Next": "Populate Fastq List Row" - } - ], - "Default": "Pass" - }, - "Populate Fastq List Row": { - "Type": "Task", - "Resource": "arn:aws:states:::dynamodb:updateItem", - "Parameters": { - "TableName": "${__table_name__}", - "Key": { - "id.$": "$.payload_data.id", - "id_type": "${__fastq_list_row_partition_name__}" - }, - "UpdateExpression": "SET fastq_list_row_json = :fastq_list_row_json", - "ExpressionAttributeValues": { - ":fastq_list_row_json": { - "S.$": "States.JsonToString($.payload_data.fastqListRow)" - } - } - }, - "ResultPath": null, - "End": true - }, - "Pass": { - "Type": "Pass", - "End": true - } - } -} diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/part_5/umccrise-draft-to-ready/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/part_5/umccrise-draft-to-ready/index.ts deleted file mode 100644 index 41aa3cc1d..000000000 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/pva/part_5/umccrise-draft-to-ready/index.ts +++ /dev/null @@ -1,105 +0,0 @@ -import { Construct } from 'constructs'; -import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; -import * as ssm from 'aws-cdk-lib/aws-ssm'; -import * as secretsManager from 'aws-cdk-lib/aws-secretsmanager'; -import * as events from 'aws-cdk-lib/aws-events'; -import { WorkflowDraftRunStateChangeToWorkflowRunStateChangeReadyConstruct } from '../../../../../../../components/event-workflowdraftrunstatechange-to-workflowrunstatechange-ready'; - -/* -Part 5 - -Input Event source: `orcabus.umccriseinputeventglue` -Input Event DetailType: `WorkflowDraftRunStateChange` -Input Event status: `draft` - -Output Event source: `orcabus.umccriseinputeventglue` -Output Event DetailType: `WorkflowRunStateChange` -Output Event status: `ready` - -* The umccriseInputMaker, subscribes to the umccrise input event glue (itself) and generates a ready event for the umccriseReadySfn - * However, in order to be 'READY' we need to use a few more variables such as - * icaLogsUri, - * analysisOutputUri - * cacheUri - * projectId - * userReference -*/ - -export interface UmccriseInputMakerConstructProps { - /* Event bus object */ - eventBusObj: events.IEventBus; - /* Tables */ - inputMakerTableObj: dynamodb.ITableV2; - /* SSM Parameter Objects */ - icav2ProjectIdSsmParameterObj: ssm.IStringParameter; - outputUriSsmParameterObj: ssm.IStringParameter; - logsUriSsmParameterObj: ssm.IStringParameter; - cacheUriSsmParameterObj: ssm.IStringParameter; - /* Secrets */ - icav2AccessTokenSecretObj: secretsManager.ISecret; -} - -export class UmccriseInputMakerConstruct extends Construct { - public readonly umccriseInputMakerEventMap = { - prefix: 'pva-umccrise', - tablePartition: 'umccrise', - triggerSource: 'orcabus.umccriseinputeventglue', - triggerStatus: 'DRAFT', - triggerDetailType: 'WorkflowDraftRunStateChange', - outputSource: 'orcabus.umccriseinputeventglue', - outputStatus: 'READY', - payloadVersion: '2024.07.16', - workflowName: 'umccrise', - workflowVersion: '4.2.4', - }; - - constructor(scope: Construct, id: string, props: UmccriseInputMakerConstructProps) { - super(scope, id); - - /* - Part 3: Build the external sfn - */ - new WorkflowDraftRunStateChangeToWorkflowRunStateChangeReadyConstruct( - this, - 'umccrise_internal_input_maker', - { - /* - Set Input StateMachine Object - */ - lambdaPrefix: this.umccriseInputMakerEventMap.prefix, - payloadVersion: this.umccriseInputMakerEventMap.payloadVersion, - stateMachinePrefix: this.umccriseInputMakerEventMap.prefix, - rulePrefix: this.umccriseInputMakerEventMap.prefix, - - /* - Table objects - */ - tableObj: props.inputMakerTableObj, - tablePartitionName: this.umccriseInputMakerEventMap.tablePartition, - - /* - Event Triggers - */ - eventBusObj: props.eventBusObj, - triggerDetailType: this.umccriseInputMakerEventMap.triggerDetailType, - triggerSource: this.umccriseInputMakerEventMap.triggerSource, - triggerStatus: this.umccriseInputMakerEventMap.triggerStatus, - outputSource: this.umccriseInputMakerEventMap.outputSource, - workflowName: this.umccriseInputMakerEventMap.workflowName, - workflowVersion: this.umccriseInputMakerEventMap.workflowVersion, - - /* - SSM Parameter Objects - */ - icav2ProjectIdSsmParameterObj: props.icav2ProjectIdSsmParameterObj, - outputUriSsmParameterObj: props.outputUriSsmParameterObj, - logsUriSsmParameterObj: props.logsUriSsmParameterObj, - - /* - Secrets - */ - icav2AccessTokenSecretObj: props.icav2AccessTokenSecretObj, - } - ); - } -} diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/roket/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/roket/index.ts index 228a07e94..fef1872b8 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/roket/index.ts +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/roket/index.ts @@ -3,9 +3,9 @@ import * as events from 'aws-cdk-lib/aws-events'; import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; import * as ssm from 'aws-cdk-lib/aws-ssm'; import * as secretsManager from 'aws-cdk-lib/aws-secretsmanager'; -import { RnasumInitialiseSubjectDbRowConstruct } from './part_1/initialise-rnasum-subject-dbs'; -import { UmccriseAndWtsCompleteToRnasumDraftDraftConstruct } from './part_2/umccrise-and-wts-complete-to-rnasum-draft'; -import { RnasumInputMakerConstruct } from './part_3/rnasum-draft-to-ready'; +import { UmccriseAndWtsCompleteToRnasumReadyConstruct } from './part_2/umccrise-and-wts-complete-to-rnasum-draft'; +import { TnInitialiseLibraryAndFastqListRowConstruct } from '../loctite/part_1/initialise-tn-library-dbs'; +import { RnasumInitialiseLibraryConstruct } from './part_1/initialise-rnasum-library-dbs'; /* Provide the glue to get from the bssh fastq copy manager to submitting wgts qc analyses @@ -16,7 +16,6 @@ export interface umccriseGlueHandlerConstructProps { eventBusObj: events.IEventBus; /* Tables */ rnasumGlueTableObj: dynamodb.ITableV2; - inputMakerTableObj: dynamodb.ITableV2; /* SSM Parameters */ analysisOutputUriSsmParameterObj: ssm.IStringParameter; analysisLogsUriSsmParameterObj: ssm.IStringParameter; @@ -32,16 +31,16 @@ export class RnasumGlueHandlerConstruct extends Construct { /* Part 1 - Input Event Source: `orcabus.instrumentrunmanager` Input Event DetailType: `SamplesheetMetadataUnion` - Input Event status: `SubjectInSamplesheet` + Input Event status: `LibraryInSamplesheet` - * Initialise rnasum instrument db construct + * Initialise rnasum library and fastq list row constructs */ - const rnasum_initialise_subject = new RnasumInitialiseSubjectDbRowConstruct( + + const rnasumInitialiseLibraryAndFastqListRow = new RnasumInitialiseLibraryConstruct( this, - 'rnasum_initialise_subject', + 'rnasum_initialise_library_and_fastq_list_row', { eventBusObj: props.eventBusObj, tableObj: props.rnasumGlueTableObj, @@ -57,52 +56,29 @@ export class RnasumGlueHandlerConstruct extends Construct { Output Event source: `orcabus.umccriseinputeventglue` Output Event DetailType: `WorkflowDraftRunStateChange` - Output Event status: `draft` + Output Event status: `ready` * Populate the fastq list row attributes for the rgid for this workflow */ - - const umccrise_and_wts_to_rnasum_draft = new UmccriseAndWtsCompleteToRnasumDraftDraftConstruct( + const umccriseAndWtsToRnasumDraft = new UmccriseAndWtsCompleteToRnasumReadyConstruct( this, 'umccrise_and_wts_to_rnasum_draft', { + /* Events */ eventBusObj: props.eventBusObj, - tableObj: props.rnasumGlueTableObj, - workflowsTableObj: props.inputMakerTableObj, - } - ); - /* - Part 3 - - Input Event source: `orcabus.rnasuminputeventglue` - Input Event DetailType: `WorkflowDraftRunStateChange` - Input Event status: `draft` + /* Tables */ + tableObj: props.rnasumGlueTableObj, - Output Event source: `orcabus.rnasuminputeventglue` - Output Event DetailType: `WorkflowRunStateChange` - Output Event status: `ready` + /* SSM Parameters */ + outputUriSsmParameterObj: props.analysisOutputUriSsmParameterObj, + cacheUriSsmParameterObj: props.analysisCacheUriSsmParameterObj, + logsUriSsmParameterObj: props.analysisLogsUriSsmParameterObj, + icav2ProjectIdSsmParameterObj: props.icav2ProjectIdSsmParameterObj, - * The rnasumInputMaker, subscribes to the rnasum input event glue (itself) and generates a ready event for the rnasumReadySfn - * However, in order to be 'READY' we need to use a few more variables such as - * icaLogsUri, - * analysisOutputUri - * cacheUri - * projectId - * userReference - */ - const rnasumInputMaker = new RnasumInputMakerConstruct(this, 'rnasum_input_maker_construct', { - /* Event bus */ - eventBusObj: props.eventBusObj, - /* Tables */ - inputMakerTableObj: props.inputMakerTableObj, - /* SSM Param objects */ - icav2ProjectIdSsmParameterObj: props.icav2ProjectIdSsmParameterObj, - outputUriSsmParameterObj: props.analysisOutputUriSsmParameterObj, - cacheUriSsmParameterObj: props.analysisCacheUriSsmParameterObj, - logsUriSsmParameterObj: props.analysisLogsUriSsmParameterObj, - /* Secrets Manager */ - icav2AccessTokenSecretObj: props.icav2AccessTokenSecretObj, - }); + /* Secrets */ + icav2AccessTokenSecretObj: props.icav2AccessTokenSecretObj, + } + ); } } diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/roket/part_1/initialise-rnasum-library-dbs/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/roket/part_1/initialise-rnasum-library-dbs/index.ts new file mode 100644 index 000000000..368841c3a --- /dev/null +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/roket/part_1/initialise-rnasum-library-dbs/index.ts @@ -0,0 +1,127 @@ +import { Construct } from 'constructs'; +import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; +import path from 'path'; +import * as sfn from 'aws-cdk-lib/aws-stepfunctions'; +import * as events from 'aws-cdk-lib/aws-events'; +import * as eventsTargets from 'aws-cdk-lib/aws-events-targets'; + +/* +Part 2 + +Input Event Source: `orcabus.instrumentrunmanager` +Input Event DetailType: `SamplesheetMetadataUnion` +Input Event status: `LibraryInSamplesheet` + +* Initialise rnasum instrument db construct +*/ + +export interface RnasumInitialiseLibraryConstructProps { + tableObj: dynamodb.ITableV2; + eventBusObj: events.IEventBus; +} + +export class RnasumInitialiseLibraryConstruct extends Construct { + public readonly RnasumInitialiseLibraryAndFastqListRowMap = { + prefix: 'roket-make-library', + tablePartition: { + library: 'library', + }, + triggerSource: 'orcabus.instrumentrunmanager', + triggerStatus: 'LibraryInSamplesheet', + triggerDetailType: 'SamplesheetMetadataUnion', + triggerSampleType: { + WGS: 'WGS', + WTS: 'WTS', + }, + triggerWorkflowType: { + RESEARCH: 'research', + CLINICAL: 'clinical', + }, + triggerPhenotypeType: { + TUMOR: 'tumor', + }, + }; + + constructor(scope: Construct, id: string, props: RnasumInitialiseLibraryConstructProps) { + super(scope, id); + + /* + Part 1: Build the internal sfn + */ + const inputMakerSfn = new sfn.StateMachine(this, 'initialise_rnasum_library_db_row', { + stateMachineName: `${this.RnasumInitialiseLibraryAndFastqListRowMap.prefix}-initialise-rnasum-library-db`, + definitionBody: sfn.DefinitionBody.fromFile( + path.join(__dirname, 'step_functions_templates', 'initialise_library_db_template.asl.json') + ), + definitionSubstitutions: { + /* General */ + __table_name__: props.tableObj.tableName, + + /* Table Partitions */ + __library_partition_name__: + this.RnasumInitialiseLibraryAndFastqListRowMap.tablePartition.library, + }, + }); + + /* + Part 2: Grant the sfn permissions + */ + // access the dynamodb table + props.tableObj.grantReadWriteData(inputMakerSfn.role); + + /* + Part 3: Subscribe to the library events from the event bus where the library assay type + is WGS and the workflow is RESEARCH or CLINICAL + and where the phenotype is NORMAL or TUMOR + */ + const rule = new events.Rule(this, 'initialise_library_assay', { + ruleName: `stacky-${this.RnasumInitialiseLibraryAndFastqListRowMap.prefix}-rule`, + eventBus: props.eventBusObj, + eventPattern: { + source: [this.RnasumInitialiseLibraryAndFastqListRowMap.triggerSource], + detailType: [this.RnasumInitialiseLibraryAndFastqListRowMap.triggerDetailType], + detail: { + payload: { + data: { + library: { + type: [ + { + 'equals-ignore-case': + this.RnasumInitialiseLibraryAndFastqListRowMap.triggerSampleType.WGS, + }, + { + 'equals-ignore-case': + this.RnasumInitialiseLibraryAndFastqListRowMap.triggerSampleType.WTS, + }, + ], + workflow: [ + { + 'equals-ignore-case': + this.RnasumInitialiseLibraryAndFastqListRowMap.triggerWorkflowType.RESEARCH, + }, + { + 'equals-ignore-case': + this.RnasumInitialiseLibraryAndFastqListRowMap.triggerWorkflowType.CLINICAL, + }, + ], + phenotype: [ + { + 'equals-ignore-case': + this.RnasumInitialiseLibraryAndFastqListRowMap.triggerPhenotypeType.TUMOR, + }, + ], + }, + }, + }, + }, + }, + }); + + // Add target of event to be the state machine + rule.addTarget( + new eventsTargets.SfnStateMachine(inputMakerSfn, { + input: events.RuleTargetInput.fromEventPath('$.detail'), + }) + ); + } +} diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/roket/part_1/initialise-rnasum-library-dbs/step_functions_templates/initialise_library_db_template.asl.json b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/roket/part_1/initialise-rnasum-library-dbs/step_functions_templates/initialise_library_db_template.asl.json new file mode 100644 index 000000000..fbbae715e --- /dev/null +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/roket/part_1/initialise-rnasum-library-dbs/step_functions_templates/initialise_library_db_template.asl.json @@ -0,0 +1,98 @@ +{ + "Comment": "A description of my state machine", + "StartAt": "Move Inputs", + "States": { + "Move Inputs": { + "Type": "Pass", + "Parameters": { + "payload_data.$": "$.payload.data" + }, + "Next": "Get Library Item" + }, + "Get Library Item": { + "Type": "Task", + "Resource": "arn:aws:states:::dynamodb:getItem", + "Parameters": { + "TableName": "${__table_name__}", + "Key": { + "id.$": "$.payload_data.library.orcabusId", + "id_type": "${__library_partition_name__}" + } + }, + "ResultPath": "$.get_library_item_step", + "Next": "Update Databases" + }, + "Update Databases": { + "Type": "Parallel", + "Branches": [ + { + "StartAt": "Pass", + "States": { + "Pass": { + "Type": "Pass", + "End": true + } + } + }, + { + "StartAt": "Library in Database", + "States": { + "Library in Database": { + "Type": "Choice", + "Choices": [ + { + "Variable": "$.get_library_item_step.Item", + "IsPresent": false, + "Comment": "Library Not In Database", + "Next": "Initialise Library" + } + ], + "Default": "No Need to Initialise Library" + }, + "Initialise Library": { + "Type": "Task", + "Resource": "arn:aws:states:::dynamodb:putItem", + "Parameters": { + "TableName": "${__table_name__}", + "Item": { + "id.$": "$.payload_data.library.orcabusId", + "id_type": "${__library_partition_name__}", + "library_id": { + "S.$": "$.payload_data.library.libraryId" + }, + "phenotype": { + "S.$": "$.payload_data.library.phenotype" + }, + "workflow": { + "S.$": "$.payload_data.library.workflow" + }, + "type": { + "S.$": "$.payload_data.library.type" + }, + "assay": { + "S.$": "$.payload_data.library.assay" + }, + "subject_id": { + "S.$": "$.payload_data.subject.subjectId" + }, + "subject_orcabus_id": { + "S.$": "$.payload_data.subject.orcabusId" + } + } + }, + "ResultPath": null, + "End": true + }, + "No Need to Initialise Library": { + "Type": "Pass", + "End": true, + "ResultPath": null + } + } + } + ], + "ResultPath": null, + "End": true + } + } +} diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/roket/part_1/initialise-rnasum-subject-dbs/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/roket/part_1/initialise-rnasum-subject-dbs/index.ts deleted file mode 100644 index 1fc8d11f8..000000000 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/roket/part_1/initialise-rnasum-subject-dbs/index.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { Construct } from 'constructs'; -import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; -import path from 'path'; -import * as sfn from 'aws-cdk-lib/aws-stepfunctions'; -import * as events from 'aws-cdk-lib/aws-events'; -import * as eventsTargets from 'aws-cdk-lib/aws-events-targets'; - -/* -Part 1 - -Input Event Source: `orcabus.instrumentrunmanager` -Input Event DetailType: `SamplesheetMetadataUnion` -Input Event status: `SubjectInSamplesheet` - -* Initialise rnasum subject db construct -*/ - -export interface RnasumInitialiseSubjectDbRowConstructProps { - tableObj: dynamodb.ITableV2; - eventBusObj: events.IEventBus; -} - -export class RnasumInitialiseSubjectDbRowConstruct extends Construct { - public readonly RnasumInitialiseSubjectDbRowMap = { - prefix: 'roket-make-subject-row', - tablePartition: 'subject', - triggerSource: 'orcabus.instrumentrunmanager', - triggerStatus: 'SubjectInSamplesheet', - triggerDetailType: 'SamplesheetMetadataUnion', - }; - - constructor(scope: Construct, id: string, props: RnasumInitialiseSubjectDbRowConstructProps) { - super(scope, id); - - /* - Part 1: Build the internal sfn - */ - const inputMakerSfn = new sfn.StateMachine(this, 'initialise_subject_db_row', { - stateMachineName: `${this.RnasumInitialiseSubjectDbRowMap.prefix}-initialise-subject`, - definitionBody: sfn.DefinitionBody.fromFile( - path.join( - __dirname, - 'step_functions_templates', - 'initialise_rnasum_subject_db_sfn_template.asl.json' - ) - ), - definitionSubstitutions: { - __table_name__: props.tableObj.tableName, - __subject_partition_name__: this.RnasumInitialiseSubjectDbRowMap.tablePartition, - }, - }); - - /* - Part 2: Grant the internal sfn permissions - */ - // access the dynamodb table - props.tableObj.grantReadWriteData(inputMakerSfn.role); - - /* - Part 3: Subscribe to the event bus and trigger the internal sfn - */ - const rule = new events.Rule(this, 'rnasum_subscribe_to_samplesheet_shower_subject', { - ruleName: `stacky-${this.RnasumInitialiseSubjectDbRowMap.prefix}-rule`, - eventBus: props.eventBusObj, - eventPattern: { - source: [this.RnasumInitialiseSubjectDbRowMap.triggerSource], - detailType: [this.RnasumInitialiseSubjectDbRowMap.triggerDetailType], - detail: { - status: [{ 'equals-ignore-case': this.RnasumInitialiseSubjectDbRowMap.triggerStatus }], - }, - }, - }); - - // Add target of event to be the state machine - rule.addTarget( - new eventsTargets.SfnStateMachine(inputMakerSfn, { - input: events.RuleTargetInput.fromEventPath('$.detail'), - }) - ); - } -} diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/roket/part_1/initialise-rnasum-subject-dbs/step_functions_templates/initialise_rnasum_subject_db_sfn_template.asl.json b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/roket/part_1/initialise-rnasum-subject-dbs/step_functions_templates/initialise_rnasum_subject_db_sfn_template.asl.json deleted file mode 100644 index e0c23b290..000000000 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/roket/part_1/initialise-rnasum-subject-dbs/step_functions_templates/initialise_rnasum_subject_db_sfn_template.asl.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "Comment": "A description of my state machine", - "StartAt": "Move Inputs", - "States": { - "Move Inputs": { - "Type": "Pass", - "Parameters": { - "payload_data.$": "$.payload.data" - }, - "Next": "Get Subject Item" - }, - "Get Subject Item": { - "Type": "Task", - "Resource": "arn:aws:states:::dynamodb:getItem", - "Parameters": { - "TableName": "${__table_name__}", - "Key": { - "id.$": "$.payload_data.subject.subjectId", - "id_type": "${__subject_partition_name__}" - } - }, - "ResultPath": "$.get_subject_item_step", - "Next": "Subject in Database" - }, - "Subject in Database": { - "Type": "Choice", - "Choices": [ - { - "Variable": "$.get_subject_item_step.Item", - "IsPresent": false, - "Comment": "Subject Not In Database", - "Next": "Initialise Subject" - } - ], - "Default": "Pass" - }, - "Pass": { - "Type": "Pass", - "End": true - }, - "Initialise Subject": { - "Type": "Task", - "Resource": "arn:aws:states:::dynamodb:putItem", - "Parameters": { - "TableName": "${__table_name__}", - "Item": { - "id.$": "$.payload_data.subject.subjectId", - "id_type": "${__subject_partition_name__}", - "orcabus_id": { - "S.$": "$.payload_data.subject.orcabusId" - } - } - }, - "End": true - } - } -} diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/roket/part_2/umccrise-and-wts-complete-to-rnasum-draft/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/roket/part_2/umccrise-and-wts-complete-to-rnasum-draft/index.ts index 915903244..38b268284 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/roket/part_2/umccrise-and-wts-complete-to-rnasum-draft/index.ts +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/roket/part_2/umccrise-and-wts-complete-to-rnasum-draft/index.ts @@ -4,18 +4,22 @@ import * as iam from 'aws-cdk-lib/aws-iam'; import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; import path from 'path'; import * as sfn from 'aws-cdk-lib/aws-stepfunctions'; +import * as ssm from 'aws-cdk-lib/aws-ssm'; +import * as secretsManager from 'aws-cdk-lib/aws-secretsmanager'; import * as events from 'aws-cdk-lib/aws-events'; import * as eventsTargets from 'aws-cdk-lib/aws-events-targets'; import { PythonFunction } from '@aws-cdk/aws-lambda-python-alpha'; import * as lambda from 'aws-cdk-lib/aws-lambda'; import { WorkflowDraftRunStateChangeCommonPreambleConstruct } from '../../../../../../../components/sfn-workflowdraftrunstatechange-common-preamble'; +import { GetMetadataLambdaConstruct } from '../../../../../../../components/python-lambda-metadata-mapper'; +import { GenerateWorkflowRunStateChangeReadyConstruct } from '../../../../../../../components/sfn-generate-workflowrunstatechange-ready-event'; /* Part 4 Input Event Source: `orcabus.workflowmanager` Input Event DetailType: `WorkflowRunStateChange` -Input Event WorkflowName: tumor_normal +Input Event WorkflowName: tumor-normal Input Event status: `succeeded` Output Event Source: `orcabus.inputeventglue` @@ -27,16 +31,23 @@ Output Event status: `draft` pipeline */ -export interface UmccriseAndWtsCompleteToRnasumDraftDraftConstructProps { +export interface UmccriseAndWtsCompleteToRnasumReadyConstructProps { + /* Event bus */ eventBusObj: events.IEventBus; + /* Table */ tableObj: dynamodb.ITableV2; - workflowsTableObj: dynamodb.ITableV2; + /* SSM Parameters */ + outputUriSsmParameterObj: ssm.IStringParameter; + icav2ProjectIdSsmParameterObj: ssm.IStringParameter; + logsUriSsmParameterObj: ssm.IStringParameter; + cacheUriSsmParameterObj: ssm.IStringParameter; + /* Secrets */ + icav2AccessTokenSecretObj: secretsManager.ISecret; } -export class UmccriseAndWtsCompleteToRnasumDraftDraftConstruct extends Construct { +export class UmccriseAndWtsCompleteToRnasumReadyConstruct extends Construct { public readonly RnasumDraftMap = { - prefix: 'roket-umccrise-wts-complete-to-rnasum-draft', - portalRunPartitionName: 'portal_run', + prefix: 'roket-umccrise-or-wts-to-rnasum-draft', triggerSource: 'orcabus.workflowmanager', triggerStatus: 'succeeded', triggerWorkflowName: { @@ -47,11 +58,10 @@ export class UmccriseAndWtsCompleteToRnasumDraftDraftConstruct extends Construct wtsWorkflowName: 'wts', triggerDetailType: 'WorkflowRunStateChange', tablePartitions: { + library: 'library', subject: 'subject', }, outputSource: 'orcabus.rnasuminputeventglue', - outputDetailType: 'WorkflowDraftRunStateChange', - outputStatus: 'DRAFT', payloadVersion: '2024.07.23', workflowName: 'rnasum', workflowVersion: '4.2.4', @@ -60,7 +70,7 @@ export class UmccriseAndWtsCompleteToRnasumDraftDraftConstruct extends Construct constructor( scope: Construct, id: string, - props: UmccriseAndWtsCompleteToRnasumDraftDraftConstructProps + props: UmccriseAndWtsCompleteToRnasumReadyConstructProps ) { super(scope, id); @@ -77,21 +87,62 @@ export class UmccriseAndWtsCompleteToRnasumDraftDraftConstruct extends Construct memorySize: 1024, }); + // Generate the lambda to collect the orcabus id from the subject id + const collectOrcaBusIdLambdaObj = new GetMetadataLambdaConstruct( + this, + 'get_orcabus_id_from_subject_id', + { + functionNamePrefix: this.RnasumDraftMap.prefix, + } + ).lambdaObj; + + // Add CONTEXT, FROM_ID and RETURN_STR environment variables to the lambda + collectOrcaBusIdLambdaObj.addEnvironment('CONTEXT', 'subject'); + collectOrcaBusIdLambdaObj.addEnvironment('FROM_ID', ''); + collectOrcaBusIdLambdaObj.addEnvironment('RETURN_STR', ''); + /* Part 1: Generate the preamble (sfn to generate the portal run id and the workflow run name) */ - const sfn_preamble = new WorkflowDraftRunStateChangeCommonPreambleConstruct( + const sfnPreamble = new WorkflowDraftRunStateChangeCommonPreambleConstruct( this, `${this.RnasumDraftMap.prefix}_sfn_preamble`, { - portalRunTablePartitionName: this.RnasumDraftMap.portalRunPartitionName, stateMachinePrefix: this.RnasumDraftMap.prefix, - tableObj: props.workflowsTableObj, workflowName: this.RnasumDraftMap.workflowName, workflowVersion: this.RnasumDraftMap.workflowVersion, } ).stepFunctionObj; + /* + Part 2: Build the engine parameters sfn + */ + const engineParameterAndReadyEventMakerSfn = new GenerateWorkflowRunStateChangeReadyConstruct( + this, + 'fastqlistrow_complete_to_wgtsqc_ready_submitter', + { + /* Event Placeholders */ + eventBusObj: props.eventBusObj, + outputSource: this.RnasumDraftMap.outputSource, + payloadVersion: this.RnasumDraftMap.payloadVersion, + workflowName: this.RnasumDraftMap.workflowName, + workflowVersion: this.RnasumDraftMap.workflowVersion, + + /* SSM Parameters */ + outputUriSsmParameterObj: props.outputUriSsmParameterObj, + icav2ProjectIdSsmParameterObj: props.icav2ProjectIdSsmParameterObj, + logsUriSsmParameterObj: props.logsUriSsmParameterObj, + cacheUriSsmParameterObj: props.cacheUriSsmParameterObj, + + /* Secrets */ + icav2AccessTokenSecretObj: props.icav2AccessTokenSecretObj, + + /* Prefixes */ + lambdaPrefix: this.RnasumDraftMap.prefix, + stateMachinePrefix: this.RnasumDraftMap.prefix, + } + ).stepFunctionObj; + /* Part 2: Build the sfn */ @@ -108,23 +159,17 @@ export class UmccriseAndWtsCompleteToRnasumDraftDraftConstruct extends Construct ) ), definitionSubstitutions: { - /* Events */ - __event_bus_name__: props.eventBusObj.eventBusName, - __event_source__: this.RnasumDraftMap.outputSource, - __detail_type__: this.RnasumDraftMap.outputDetailType, - __output_status__: this.RnasumDraftMap.outputStatus, - __payload_version__: this.RnasumDraftMap.payloadVersion, - __workflow_name__: this.RnasumDraftMap.workflowName, - __workflow_version__: this.RnasumDraftMap.workflowVersion, - /* Lambdas */ __generate_workflow_inputs_lambda_function_arn__: generateEventDataLambdaObj.currentVersion.functionArn, + __get_orcabus_id_from_subject_id_lambda_function_arn__: + collectOrcaBusIdLambdaObj.currentVersion.functionArn, /* Tables */ __table_name__: props.tableObj.tableName, /* Table partitions */ + __library_table_partition_name__: this.RnasumDraftMap.tablePartitions.library, __subject_table_partition_name__: this.RnasumDraftMap.tablePartitions.subject, /* Statuses */ @@ -134,7 +179,8 @@ export class UmccriseAndWtsCompleteToRnasumDraftDraftConstruct extends Construct __wts_workflow_name__: this.RnasumDraftMap.wtsWorkflowName, // State Machines - __sfn_preamble_state_machine_arn__: sfn_preamble.stateMachineArn, + __sfn_preamble_state_machine_arn__: sfnPreamble.stateMachineArn, + __launch_ready_event_sfn_arn__: engineParameterAndReadyEventMakerSfn.stateMachineArn, }, } ); @@ -145,11 +191,10 @@ export class UmccriseAndWtsCompleteToRnasumDraftDraftConstruct extends Construct // access the dynamodb table props.tableObj.grantReadWriteData(umccriseAndWtsCompleteToDraftSfn); - // allow the step function to submit events - props.eventBusObj.grantPutEventsTo(umccriseAndWtsCompleteToDraftSfn); - // allow the step function to invoke the lambdas - generateEventDataLambdaObj.currentVersion.grantInvoke(umccriseAndWtsCompleteToDraftSfn); + [generateEventDataLambdaObj, collectOrcaBusIdLambdaObj].forEach((lambda) => { + lambda.currentVersion.grantInvoke(umccriseAndWtsCompleteToDraftSfn); + }); /* Allow step function to call nested state machine */ // Because we run a nested state machine, we need to add the permissions to the state machine role @@ -163,7 +208,8 @@ export class UmccriseAndWtsCompleteToRnasumDraftDraftConstruct extends Construct }) ); // Allow the state machine to be able to invoke the preamble sfn - sfn_preamble.grantStartExecution(umccriseAndWtsCompleteToDraftSfn); + sfnPreamble.grantStartExecution(umccriseAndWtsCompleteToDraftSfn); + engineParameterAndReadyEventMakerSfn.grantStartExecution(umccriseAndWtsCompleteToDraftSfn); /* Part 3: Subscribe to the event bus and trigger the internal sfn diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/roket/part_2/umccrise-and-wts-complete-to-rnasum-draft/lambdas/generate_workflow_inputs_py/generate_workflow_inputs.py b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/roket/part_2/umccrise-and-wts-complete-to-rnasum-draft/lambdas/generate_workflow_inputs_py/generate_workflow_inputs.py index cab540cca..14f5953d4 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/roket/part_2/umccrise-and-wts-complete-to-rnasum-draft/lambdas/generate_workflow_inputs_py/generate_workflow_inputs.py +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/roket/part_2/umccrise-and-wts-complete-to-rnasum-draft/lambdas/generate_workflow_inputs_py/generate_workflow_inputs.py @@ -35,6 +35,7 @@ def handler(event, context): wts_tumor_fastq_list_row_ids = event['wts_tumor_fastq_list_row_ids'] wgs_tumor_fastq_list_row_ids = event['wgs_tumor_fastq_list_row_ids'] wgs_normal_fastq_list_row_ids = event['wgs_normal_fastq_list_row_ids'] + individual_id = event['individual_id'] subject_id = event['subject_id'] # Outputs @@ -43,11 +44,12 @@ def handler(event, context): "dragenTranscriptomeUri": dragen_wts_output_uri, "umccriseUri": umccrise_output_uri, "wtsTumorLibraryId": wts_tumor_library_id, - "subjectId": subject_id + "subjectId": individual_id } event_tags = { "subjectId": subject_id, + "individualId": individual_id, "wtsTumorLibraryId": wts_tumor_library_id, "wgsTumorLibraryId": wgs_tumor_library_id, "wgsNormalLibraryId": wgs_normal_library_id, diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/roket/part_2/umccrise-and-wts-complete-to-rnasum-draft/step_functions_templates/umccrise_and_wts_complete_to_rnasum_draft.asl.json b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/roket/part_2/umccrise-and-wts-complete-to-rnasum-draft/step_functions_templates/umccrise_and_wts_complete_to_rnasum_draft.asl.json index 7dc3c1e47..62743c14b 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/roket/part_2/umccrise-and-wts-complete-to-rnasum-draft/step_functions_templates/umccrise_and_wts_complete_to_rnasum_draft.asl.json +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/roket/part_2/umccrise-and-wts-complete-to-rnasum-draft/step_functions_templates/umccrise_and_wts_complete_to_rnasum_draft.asl.json @@ -7,19 +7,121 @@ "Parameters": { "workflow_inputs.$": "$" }, - "Next": "Get Subject ID from Tags" + "Next": "Scan Libraries for subject" }, - "Get Subject ID from Tags": { + "Scan Libraries for subject": { + "Type": "Task", + "Resource": "arn:aws:states:::aws-sdk:dynamodb:scan", + "Parameters": { + "TableName": "${__table_name__}", + "ExpressionAttributeValues": { + ":subject_id": { + "S.$": "$.workflow_inputs.payload.data.tags.subjectId" + }, + ":id_type": { + "S": "${__library_table_partition_name__}" + } + }, + "ExpressionAttributeNames": { + "#subject_id": "subject_id", + "#id_type": "id_type" + }, + "FilterExpression": "#subject_id = :subject_id AND #id_type = :id_type" + }, + + "ResultPath": "$.scan_libraries_step", + "Next": "Subject In Library DB" + }, + "Subject In Library DB": { + "Type": "Choice", + "Choices": [ + { + "Variable": "$.scan_libraries_step.Items", + "IsPresent": false, + "Next": "Success (1)" + } + ], + "Default": "Get Subject Orcabus Id from Subject Id in tags", + "Comment": "Subject Does not exist in db" + }, + "Success (1)": { + "Type": "Succeed" + }, + "Get Subject Orcabus Id from Subject Id in tags": { + "Type": "Task", + "Resource": "arn:aws:states:::lambda:invoke", + "Parameters": { + "FunctionName": "${__get_orcabus_id_from_subject_id_lambda_function_arn__}", + "Payload": { + "value.$": "$.workflow_inputs.payload.data.tags.subjectId" + } + }, + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 1, + "MaxAttempts": 3, + "BackoffRate": 2 + } + ], + "ResultSelector": { + "subject_orcabus_id.$": "$.Payload.orcabus_id" + }, + "ResultPath": "$.get_subject_orcabus_id_step", + "Next": "Get Subject Orcabus ID" + }, + "Get Subject Orcabus ID": { "Type": "Task", "Resource": "arn:aws:states:::dynamodb:getItem", "Parameters": { "TableName": "${__table_name__}", "Key": { - "id.$": "$.workflow_inputs.payload.data.tags.subjectId", + "id.$": "$.get_subject_orcabus_id_step.subject_orcabus_id", "id_type": "${__subject_table_partition_name__}" } }, "ResultPath": "$.get_subject_id_step", + "Next": "Is Subject Item Initialised" + }, + "Is Subject Item Initialised": { + "Type": "Choice", + "Choices": [ + { + "Variable": "$.get_subject_id_step.Item", + "IsPresent": false, + "Next": "Initialise Subject", + "Comment": "Subject Needs Initialising" + } + ], + "Default": "Update DB" + }, + "Initialise Subject": { + "Type": "Task", + "Resource": "arn:aws:states:::dynamodb:putItem", + "Parameters": { + "TableName": "${__table_name__}", + "Item": { + "id.$": "$.get_subject_orcabus_id_step.subject_orcabus_id", + "id_type": "${__subject_table_partition_name__}", + "individual_id": { + "S.$": "$.workflow_inputs.payload.data.tags.individualId" + }, + "subject_id": { + "S.$": "$.workflow_inputs.payload.data.tags.subjectId" + } + } + }, + "ResultPath": null, + "Next": "Wait For DB Sync" + }, + "Wait For DB Sync": { + "Type": "Wait", + "Seconds": 1, "Next": "Update DB" }, "Update DB": { @@ -47,7 +149,7 @@ "Parameters": { "TableName": "${__table_name__}", "Key": { - "id.$": "$.workflow_inputs.payload.data.tags.subjectId", + "id.$": "$.get_subject_orcabus_id_step.subject_orcabus_id", "id_type": "${__subject_table_partition_name__}" }, "UpdateExpression": "SET wts_workflow_status = :wts_workflow_status, wts_tumor_library_id = :wts_tumor_library_id, wts_tumor_fastq_list_row_ids = :wts_tumor_fastq_list_row_ids, arriba_output_uri = :arriba_output_uri, dragen_wts_output_uri = :dragen_wts_output_uri", @@ -78,7 +180,7 @@ "Parameters": { "TableName": "${__table_name__}", "Key": { - "id.$": "$.workflow_inputs.payload.data.tags.subjectId", + "id.$": "$.get_subject_orcabus_id_step.subject_orcabus_id", "id_type": "${__subject_table_partition_name__}" }, "UpdateExpression": "SET umccrise_workflow_status = :umccrise_workflow_status, umccrise_tumor_library_id = :umccrise_tumor_library_id, umccrise_normal_library_id = :umccrise_normal_library_id, umccrise_output_uri = :umccrise_output_uri, umccrise_tumor_fastq_list_row_ids = :umccrise_tumor_fastq_list_row_ids, umccrise_normal_fastq_list_row_ids = :umccrise_normal_fastq_list_row_ids", @@ -150,7 +252,7 @@ "Parameters": { "TableName": "${__table_name__}", "Key": { - "id.$": "$.workflow_inputs.payload.data.tags.subjectId", + "id.$": "$.get_subject_orcabus_id_step.subject_orcabus_id", "id_type": "${__subject_table_partition_name__}" }, "UpdateExpression": "ADD linked_libraries_set :linked_libraries_set", @@ -161,6 +263,11 @@ } }, "ResultPath": null, + "Next": "Wait For Linked Library DB Sync" + }, + "Wait For Linked Library DB Sync": { + "Type": "Wait", + "Seconds": 1, "End": true } } @@ -174,7 +281,7 @@ "Parameters": { "TableName": "${__table_name__}", "Key": { - "id.$": "$.workflow_inputs.payload.data.tags.subjectId", + "id.$": "$.get_subject_orcabus_id_step.subject_orcabus_id", "id_type": "${__subject_table_partition_name__}" } }, @@ -210,7 +317,7 @@ }, "Get workflow inputs": { "Type": "Parallel", - "Next": "Generate RNASum Draft Event", + "Next": "Generate RNASum Ready Event", "Branches": [ { "StartAt": "Generate workflow inputs", @@ -230,7 +337,8 @@ "wts_tumor_fastq_list_row_ids.$": "States.StringToJson($.get_subject_item_step.Item.wts_tumor_fastq_list_row_ids.S)", "wgs_tumor_fastq_list_row_ids.$": "States.StringToJson($.get_subject_item_step.Item.umccrise_tumor_fastq_list_row_ids.S)", "wgs_normal_fastq_list_row_ids.$": "States.StringToJson($.get_subject_item_step.Item.umccrise_normal_fastq_list_row_ids.S)", - "subject_id.$": "$.get_subject_item_step.Item.id.S" + "subject_id.$": "$.get_subject_item_step.Item.subject_id.S", + "individual_id.$": "$.get_subject_item_step.Item.individual_id.S" } }, "Retry": [ @@ -313,34 +421,22 @@ }, "ResultPath": "$.get_parameters_step" }, - "Generate RNASum Draft Event": { + "Generate RNASum Ready Event": { "Type": "Task", - "Resource": "arn:aws:states:::events:putEvents", + "Resource": "arn:aws:states:::states:startExecution.sync:2", "Parameters": { - "Entries": [ - { - "EventBusName": "${__event_bus_name__}", - "Source": "${__event_source__}", - "DetailType": "${__detail_type__}", - "Detail": { - "portalRunId.$": "$.get_parameters_step.portal_run_id", - "timestamp.$": "$$.State.EnteredTime", - "status": "${__output_status__}", - "workflowName": "${__workflow_name__}", - "workflowVersion": "${__workflow_version__}", - "workflowRunName.$": "$.get_parameters_step.workflow_run_name", - "linkedLibraries.$": "$.get_parameters_step.linked_libraries", - "payload": { - "version": "${__payload_version__}", - "data": { - "inputs.$": "$.get_parameters_step.input_event_data", - "tags.$": "$.get_parameters_step.event_tags" - } - } - } + "StateMachineArn": "${__launch_ready_event_sfn_arn__}", + "Input": { + "StatePayload": { + "portal_run_id.$": "$.get_parameters_step.portal_run_id", + "workflow_run_name.$": "$.get_parameters_step.workflow_run_name", + "linked_libraries.$": "$.get_parameters_step.linked_libraries", + "data_inputs.$": "$.get_parameters_step.input_event_data", + "data_tags.$": "$.get_parameters_step.event_tags" } - ] + } }, + "ResultPath": null, "End": true }, "Success": { diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/roket/part_3/rnasum-draft-to-ready/index.ts b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/roket/part_3/rnasum-draft-to-ready/index.ts deleted file mode 100644 index da6073b94..000000000 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/roket/part_3/rnasum-draft-to-ready/index.ts +++ /dev/null @@ -1,105 +0,0 @@ -import { Construct } from 'constructs'; -import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; -import * as ssm from 'aws-cdk-lib/aws-ssm'; -import * as secretsManager from 'aws-cdk-lib/aws-secretsmanager'; -import * as events from 'aws-cdk-lib/aws-events'; -import { WorkflowDraftRunStateChangeToWorkflowRunStateChangeReadyConstruct } from '../../../../../../../components/event-workflowdraftrunstatechange-to-workflowrunstatechange-ready'; - -/* -Part 5 - -Input Event source: `orcabus.rnasuminputeventglue` -Input Event DetailType: `WorkflowDraftRunStateChange` -Input Event status: `draft` - -Output Event source: `orcabus.rnasuminputeventglue` -Output Event DetailType: `WorkflowRunStateChange` -Output Event status: `ready` - -* The rnasumInputMaker, subscribes to the rnasum input event glue (itself) and generates a ready event for the rnasumReadySfn - * However, in order to be 'READY' we need to use a few more variables such as - * icaLogsUri, - * analysisOutputUri - * cacheUri - * projectId - * userReference -*/ - -export interface RnasumInputMakerConstructProps { - /* Event bus object */ - eventBusObj: events.IEventBus; - /* Tables */ - inputMakerTableObj: dynamodb.ITableV2; - /* SSM Parameter Objects */ - icav2ProjectIdSsmParameterObj: ssm.IStringParameter; - outputUriSsmParameterObj: ssm.IStringParameter; - logsUriSsmParameterObj: ssm.IStringParameter; - cacheUriSsmParameterObj: ssm.IStringParameter; - /* Secrets */ - icav2AccessTokenSecretObj: secretsManager.ISecret; -} - -export class RnasumInputMakerConstruct extends Construct { - public readonly rnasumInputMakerEventMap = { - prefix: 'roket-rnasum', - tablePartition: 'rnasum', - triggerSource: 'orcabus.rnasuminputeventglue', - triggerStatus: 'DRAFT', - triggerDetailType: 'WorkflowDraftRunStateChange', - outputSource: 'orcabus.rnasuminputeventglue', - outputStatus: 'READY', - payloadVersion: '2024.07.16', - workflowName: 'rnasum', - workflowVersion: '4.2.4', - }; - - constructor(scope: Construct, id: string, props: RnasumInputMakerConstructProps) { - super(scope, id); - - /* - Part 3: Build the external sfn - */ - new WorkflowDraftRunStateChangeToWorkflowRunStateChangeReadyConstruct( - this, - 'rnasum_internal_input_maker', - { - /* - Set Input StateMachine Object - */ - lambdaPrefix: this.rnasumInputMakerEventMap.prefix, - payloadVersion: this.rnasumInputMakerEventMap.payloadVersion, - stateMachinePrefix: this.rnasumInputMakerEventMap.prefix, - rulePrefix: `stacky-${this.rnasumInputMakerEventMap.prefix}`, - - /* - Table objects - */ - tableObj: props.inputMakerTableObj, - tablePartitionName: this.rnasumInputMakerEventMap.tablePartition, - - /* - Event Triggers - */ - eventBusObj: props.eventBusObj, - triggerDetailType: this.rnasumInputMakerEventMap.triggerDetailType, - triggerSource: this.rnasumInputMakerEventMap.triggerSource, - triggerStatus: this.rnasumInputMakerEventMap.triggerStatus, - outputSource: this.rnasumInputMakerEventMap.outputSource, - workflowName: this.rnasumInputMakerEventMap.workflowName, - workflowVersion: this.rnasumInputMakerEventMap.workflowVersion, - - /* - SSM Parameter Objects - */ - icav2ProjectIdSsmParameterObj: props.icav2ProjectIdSsmParameterObj, - outputUriSsmParameterObj: props.outputUriSsmParameterObj, - logsUriSsmParameterObj: props.logsUriSsmParameterObj, - - /* - Secrets - */ - icav2AccessTokenSecretObj: props.icav2AccessTokenSecretObj, - } - ); - } -} diff --git a/lib/workload/stateless/statelessStackCollectionClass.ts b/lib/workload/stateless/statelessStackCollectionClass.ts index e60a40749..fe58667a3 100644 --- a/lib/workload/stateless/statelessStackCollectionClass.ts +++ b/lib/workload/stateless/statelessStackCollectionClass.ts @@ -58,6 +58,10 @@ import { RnasumIcav2PipelineManagerStackProps, } from './stacks/rnasum-pipeline-manager/deploy'; import { FMAnnotator, FMAnnotatorConfigurableProps } from './stacks/fmannotator/deploy/stack'; +import { + PieriandxPipelineManagerStack, + PierianDxPipelineManagerStackProps, +} from './stacks/pieriandx-pipeline-manager/deploy'; export interface StatelessStackCollectionProps { metadataManagerStackProps: MetadataManagerStackProps; @@ -72,6 +76,7 @@ export interface StatelessStackCollectionProps { wtsIcav2PipelineManagerStackProps: WtsIcav2PipelineManagerStackProps; umccriseIcav2PipelineManagerStackProps: UmccriseIcav2PipelineManagerStackProps; rnasumIcav2PipelineManagerStackProps: RnasumIcav2PipelineManagerStackProps; + pieriandxPipelineManagerStackProps: PierianDxPipelineManagerStackProps; eventSchemaStackProps: SchemaStackProps; dataSchemaStackProps: SchemaStackProps; bclConvertManagerStackProps: BclConvertManagerStackProps; @@ -94,6 +99,7 @@ export class StatelessStackCollection { readonly wtsIcav2PipelineManagerStack: Stack; readonly umccriseIcav2PipelineManagerStack: Stack; readonly rnasumIcav2PipelineManagerStack: Stack; + readonly pieriandxPipelineManagerStack: Stack; readonly eventSchemaStack: Stack; readonly dataSchemaStack: Stack; readonly bclConvertManagerStack: Stack; @@ -214,6 +220,15 @@ export class StatelessStackCollection { } ); + this.pieriandxPipelineManagerStack = new PieriandxPipelineManagerStack( + scope, + 'PieriandxPipelineManagerStack', + { + ...this.createTemplateProps(env, 'PieriandxPipelineManagerStack'), + ...statelessConfiguration.pieriandxPipelineManagerStackProps, + } + ); + this.bclConvertManagerStack = new BclConvertManagerStack(scope, 'BclConvertManagerStack', { ...this.createTemplateProps(env, 'BclConvertManagerStack'), ...statelessConfiguration.bclConvertManagerStackProps,