diff --git a/config/constants.ts b/config/constants.ts index 82d91e723..46a47363e 100644 --- a/config/constants.ts +++ b/config/constants.ts @@ -560,7 +560,7 @@ export const pieriandxDagSsmParameterPath = '/umccr/orcabus/stateful/pieriandx/d /* "s3://pdx-cgwxfer-test/melbournetest" // development "s3://pdx-cgwxfer-test/melbournetest" // staging -"s3://pdx-cgwxfer/melbourne" // production +"s3://pdx-xfer/melbourne" // production */ export const pieriandxS3SequencerRunRootSsmParameterPath = '/umccr/orcabus/pieriandx/s3_sequencer_run_root'; diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/deploy/index.ts b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/deploy/index.ts index 44d53bc61..3cbd73e8a 100644 --- a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/deploy/index.ts +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/deploy/index.ts @@ -13,6 +13,7 @@ import { PythonFunction } from '@aws-cdk/aws-lambda-python-alpha'; import { Duration } from 'aws-cdk-lib'; import { PieriandxMonitorRunsStepFunctionStateMachineConstruct } from './constructs/pieriandx_monitor_runs_step_function'; import { PythonLambdaLayerConstruct } from '../../../../components/python-lambda-layer'; +import { NagSuppressions } from 'cdk-nag'; export interface PierianDxPipelineManagerConfig { /* DynamoDB Table */ @@ -157,7 +158,7 @@ export class PieriandxPipelineManagerStack extends cdk.Stack { handler: 'handler', memorySize: 1024, layers: [lambdaLayerObj.lambdaLayerVersionObj], - timeout: Duration.seconds(20), + timeout: Duration.seconds(60), environment: icav2Envs, } ); @@ -208,6 +209,7 @@ export class PieriandxPipelineManagerStack extends cdk.Stack { memorySize: 1024, layers: [lambdaLayerObj.lambdaLayerVersionObj], environment: pieriandxEnvs, + timeout: Duration.seconds(60), } ); @@ -286,30 +288,40 @@ export class PieriandxPipelineManagerStack extends cdk.Stack { getInformaticsjobAndReportStatusLambdaObj, ].forEach((lambdaFunction) => { // Give the lambda permission to access the pieriandx apis - pieriandxTokenCollectionLambdaObj.latestVersion.grantInvoke(lambdaFunction); + // Fixme, no way to give access to only the current version + pieriandxTokenCollectionLambdaObj.grantInvoke(lambdaFunction.currentVersion); + NagSuppressions.addResourceSuppressions( + lambdaFunction, + [ + { + id: 'AwsSolutions-IAM5', + reason: + 'Cannot get latest version of pieriandx collect access token function ($LATEST) will not work', + }, + ], + true + ); }); /* - Give the upload lambda access to the pieriandx s3 bucket - */ - // @ts-ignore + Give the upload lambda access to the pieriandx s3 bucket + */ pieriandxS3AccessTokenSecretObj.grantRead(uploadDataToS3LambdaObj); /* - Give the lambdas permission to access the icav2 apis - */ + Give the lambdas permission to access the icav2 apis + */ [ generatePieriandxObjectsLambdaObj, generateSamplesheetLambdaObj, uploadDataToS3LambdaObj, ].forEach((lambdaFunction) => { - // @ts-ignore icav2AccessTokenSecretObj.grantRead(lambdaFunction); }); /* - Generate State Machines - */ + Generate State Machines + */ /* Generate case creation statemachine object */ const pieriandxLaunchCaseCreationStateMachine = diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/lambdas/generate_output_data_payload_py/generate_output_data_payload.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/lambdas/generate_output_data_payload_py/generate_output_data_payload.py index 5b67785cf..381a25fcd 100644 --- a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/lambdas/generate_output_data_payload_py/generate_output_data_payload.py +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/lambdas/generate_output_data_payload_py/generate_output_data_payload.py @@ -16,6 +16,53 @@ """ +from urllib.parse import ( + urlparse, urlunparse +) +from pathlib import Path + + +def strip_path_from_url(url: str) -> str: + """ + Strip the path from the url + :param url: + :return: + """ + url_obj = urlparse(url) + + return str( + urlunparse( + ( + url_obj.scheme, + url_obj.netloc, + "/", + None, None, None + ) + ) + ) + + +def join_url_paths(url: str, path_ext: str) -> str: + """ + Join the url paths + :param url: str + :param path_ext: str + :return: url + """ + url_obj = urlparse(url) + url_path = Path(url_obj.path) / path_ext + + return str( + urlunparse( + ( + url_obj.scheme, + url_obj.netloc, + str(url_path), + None, None, None + ) + ) + ) + def handler(event, context): """ @@ -53,15 +100,28 @@ def handler(event, context): # Set the outputs return_dict["data_payload"]["outputs"] = { - "caseUrl": f"{pieriandx_base_url.replace("cgw-api/v2.0.0/", "")}/cgw/order/viewOrderDetails/{case_id}", - "vcfOutputUrl": f"{pieriandx_base_url.replace("cgw-api/v2.0.0/", "")}/cgw/informatics/downloadJobOutputAnalysisFile?caseId={case_id}&jobId={job_id}&accessionNumber={case_accession_number}&fileName=main.vcf", - "reportPdfUrl": f"{pieriandx_base_url.replace("cgw-api/v2.0.0/", "")}/cgw/report/openPdfReport/{report_id}", - "biomarkerReportUrl": f"{pieriandx_base_url.replace("cgw-api/v2.0.0/", "")}/cgw/informatics/downloadJobOutputAnalysisFile?caseId={case_id}&jobId={job_id}&accessionNumber={case_accession_number}&fileName={sample_name}_BiomarkerReport.txt" + "caseUrl": join_url_paths( + strip_path_from_url(pieriandx_base_url), + f"/cgw/order/viewOrderDetails/{case_id}" + ), + "vcfOutputUrl": join_url_paths( + strip_path_from_url(pieriandx_base_url), + f"/cgw/informatics/downloadJobOutputAnalysisFile?caseId={case_id}&jobId={job_id}&accessionNumber={case_accession_number}&fileName=main.vcf" + ), + "reportPdfUrl": join_url_paths( + strip_path_from_url(pieriandx_base_url), + f"/cgw/report/openPdfReport/{report_id}" + ), + "biomarkerReportUrl": join_url_paths( + strip_path_from_url(pieriandx_base_url), + f"/cgw/informatics/downloadJobOutputAnalysisFile?caseId={case_id}&jobId={job_id}&accessionNumber={case_accession_number}&fileName={sample_name}_BiomarkerReport.txt" + ), } return return_dict +# DEV # if __name__ == "__main__": # import json # @@ -181,3 +241,127 @@ def handler(event, context): # # } # # } # # } + + +# # PROD +# if __name__ == "__main__": +# import json +# +# print( +# json.dumps( +# handler( +# { +# "pieriandx_base_url": "https://app.pieriandx.com/cgw-api/v2.0.0", +# "inputs": { +# "instrumentRunId": "241101_A01052_0236_BHVJNMDMXY", +# "panelVersion": "main", +# "caseMetadata": { +# "isIdentified": False, +# "caseAccessionNumber": "L2401562__V2__20241106c25602c7", +# "externalSpecimenId": "CMM0.5pc-10646979", +# "sampleType": "validation", +# "specimenLabel": "primarySpecimen", +# "indication": "NA", +# "diseaseCode": 55342001, +# "specimenCode": "122561005", +# "sampleReception": { +# "dateAccessioned": "2024-11-07T08:39:55+1100", +# "dateCollected": "2024-11-07T08:39:55+1100", +# "dateReceived": "2024-11-07T08:39:55+1100" +# }, +# "study": { +# "id": "Control", +# "subjectIdentifier": "Sera-ctDNA-Comp05pc" +# } +# }, +# "dataFiles": { +# "microsatOutputUri": "s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/analysis/cttsov2/202411052080dbdd/Logs_Intermediates/DragenCaller/L2401562/L2401562.microsat_output.json", +# "tmbMetricsUri": "s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/analysis/cttsov2/202411052080dbdd/Logs_Intermediates/Tmb/L2401562/L2401562.tmb.metrics.csv", +# "cnvVcfUri": "s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/analysis/cttsov2/202411052080dbdd/Results/L2401562/L2401562.cnv.vcf.gz", +# "hardFilteredVcfUri": "s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/analysis/cttsov2/202411052080dbdd/Results/L2401562/L2401562.hard-filtered.vcf.gz", +# "fusionsUri": "s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/analysis/cttsov2/202411052080dbdd/Results/L2401562/L2401562_Fusions.csv", +# "metricsOutputUri": "s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/analysis/cttsov2/202411052080dbdd/Results/L2401562/L2401562_MetricsOutput.tsv", +# "samplesheetUri": "s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/analysis/cttsov2/202411052080dbdd/Logs_Intermediates/SampleSheetValidation/SampleSheet_Intermediate.csv" +# } +# }, +# "job_id": "302091", +# "report_id": "312010", +# "case_id": "336486", +# "report_status": "complete", +# "case_accession_number": "L2401562__V2__20241106c25602c7", +# "engine_parameters": { +# "caseId": "336486", +# "informaticsJobId": "302091" +# }, +# "tags": { +# "metadataFromRedCap": False, +# "isIdentified": False, +# "libraryId": "L2401562", +# "sampleType": "validation", +# "projectId": "Control", +# "instrumentRunId": "241101_A01052_0236_BHVJNMDMXY", +# "panelVersion": "main" +# }, +# "sample_name": "L2401562" +# }, +# None +# ), +# indent=4 +# ) +# ) +# +# # { +# # "data_payload": { +# # "inputs": { +# # "instrumentRunId": "241101_A01052_0236_BHVJNMDMXY", +# # "panelVersion": "main", +# # "caseMetadata": { +# # "isIdentified": false, +# # "caseAccessionNumber": "L2401562__V2__20241106c25602c7", +# # "externalSpecimenId": "CMM0.5pc-10646979", +# # "sampleType": "validation", +# # "specimenLabel": "primarySpecimen", +# # "indication": "NA", +# # "diseaseCode": 55342001, +# # "specimenCode": "122561005", +# # "sampleReception": { +# # "dateAccessioned": "2024-11-07T08:39:55+1100", +# # "dateCollected": "2024-11-07T08:39:55+1100", +# # "dateReceived": "2024-11-07T08:39:55+1100" +# # }, +# # "study": { +# # "id": "Control", +# # "subjectIdentifier": "Sera-ctDNA-Comp05pc" +# # } +# # }, +# # "dataFiles": { +# # "microsatOutputUri": "s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/analysis/cttsov2/202411052080dbdd/Logs_Intermediates/DragenCaller/L2401562/L2401562.microsat_output.json", +# # "tmbMetricsUri": "s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/analysis/cttsov2/202411052080dbdd/Logs_Intermediates/Tmb/L2401562/L2401562.tmb.metrics.csv", +# # "cnvVcfUri": "s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/analysis/cttsov2/202411052080dbdd/Results/L2401562/L2401562.cnv.vcf.gz", +# # "hardFilteredVcfUri": "s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/analysis/cttsov2/202411052080dbdd/Results/L2401562/L2401562.hard-filtered.vcf.gz", +# # "fusionsUri": "s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/analysis/cttsov2/202411052080dbdd/Results/L2401562/L2401562_Fusions.csv", +# # "metricsOutputUri": "s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/analysis/cttsov2/202411052080dbdd/Results/L2401562/L2401562_MetricsOutput.tsv", +# # "samplesheetUri": "s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/analysis/cttsov2/202411052080dbdd/Logs_Intermediates/SampleSheetValidation/SampleSheet_Intermediate.csv" +# # } +# # }, +# # "engineParameters": { +# # "caseId": "336486", +# # "informaticsJobId": "302091" +# # }, +# # "tags": { +# # "metadataFromRedCap": false, +# # "isIdentified": false, +# # "libraryId": "L2401562", +# # "sampleType": "validation", +# # "projectId": "Control", +# # "instrumentRunId": "241101_A01052_0236_BHVJNMDMXY", +# # "panelVersion": "main" +# # }, +# # "outputs": { +# # "caseUrl": "https://app.pieriandx.com/cgw/order/viewOrderDetails/336486", +# # "vcfOutputUrl": "https://app.pieriandx.com/cgw/informatics/downloadJobOutputAnalysisFile?caseId=336486&jobId=302091&accessionNumber=L2401562__V2__20241106c25602c7&fileName=main.vcf", +# # "reportPdfUrl": "https://app.pieriandx.com/cgw/report/openPdfReport/312010", +# # "biomarkerReportUrl": "https://app.pieriandx.com/cgw/informatics/downloadJobOutputAnalysisFile?caseId=336486&jobId=302091&accessionNumber=L2401562__V2__20241106c25602c7&fileName=L2401562_BiomarkerReport.txt" +# # } +# # } +# # } \ No newline at end of file diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/lambdas/get_informaticsjob_and_report_status_py/get_informaticsjob_and_report_status.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/lambdas/get_informaticsjob_and_report_status_py/get_informaticsjob_and_report_status.py index 640507ec4..72a51521e 100644 --- a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/lambdas/get_informaticsjob_and_report_status_py/get_informaticsjob_and_report_status.py +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/lambdas/get_informaticsjob_and_report_status_py/get_informaticsjob_and_report_status.py @@ -200,7 +200,7 @@ def handler(event, context): "S": job_status }, ":report_id": { - "S": reportjob_obj.get("id") + "N": reportjob_obj.get("id") }, ":report_status": { "S": report_status diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/lambdas/upload_pieriandx_sample_data_to_s3_py/upload_pieriandx_sample_data_to_s3.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/lambdas/upload_pieriandx_sample_data_to_s3_py/upload_pieriandx_sample_data_to_s3.py index 23d6f945e..f45060122 100644 --- a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/lambdas/upload_pieriandx_sample_data_to_s3_py/upload_pieriandx_sample_data_to_s3.py +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/lambdas/upload_pieriandx_sample_data_to_s3_py/upload_pieriandx_sample_data_to_s3.py @@ -23,15 +23,22 @@ """ +# Standard imports from tempfile import TemporaryDirectory from pathlib import Path from urllib.parse import urlparse from wrapica.project_data import read_icav2_file_contents, convert_uri_to_project_data_obj +import logging +# Layer imports from pieriandx_pipeline_tools.utils.s3_helpers import set_s3_access_cred_env_vars, upload_file from pieriandx_pipeline_tools.utils.secretsmanager_helpers import set_icav2_env_vars from pieriandx_pipeline_tools.utils.compression_helpers import decompress_file +# Logger +logger = logging.getLogger() +logger.setLevel(logging.INFO) + def handler(event, context): """ @@ -107,3 +114,21 @@ def handler(event, context): # }, # None # ) + +# if __name__ == "__main__": +# from os import environ +# environ["AWS_PROFILE"] = 'umccr-production' +# environ['AWS_REGION'] = 'ap-southeast-2' +# environ["ICAV2_ACCESS_TOKEN_SECRET_ID"] = "ICAv2JWTKey-umccr-prod-service-production" +# environ['PIERIANDX_S3_ACCESS_CREDENTIALS_SECRET_ID'] = "PierianDx/S3Credentials" +# +# handler( +# { +# "needs_decompression": False, +# "src_uri": "s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/analysis/cttsov2/202411053da6481e/Results/L2401560/L2401560_MetricsOutput.tsv", +# "contents": None, +# "dest_uri": "s3://pdx-xfer/melbourne/241101_A01052_0236_BHVJNMDMXY__L2401560__V2__20241105f6bc3fb9__20241105f6bc3fb9/Data/Intensities/BaseCalls/L2401560_MetricsOutput.tsv" +# }, +# None +# ) +# diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/utils/s3_helpers.py b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/utils/s3_helpers.py index dd17f71e0..de51d5157 100644 --- a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/utils/s3_helpers.py +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/utils/s3_helpers.py @@ -24,7 +24,14 @@ def get_s3_client() -> 'S3Client': def upload_file(bucket: str, key: str, input_file_path: Path) -> None: s3 = get_s3_client() - s3.upload_file(str(input_file_path), bucket, key.lstrip("/")) + s3.upload_file( + str(input_file_path), + bucket, + key.lstrip("/"), + ExtraArgs={ + 'ServerSideEncryption': 'AES256' + } + ) def set_s3_access_cred_env_vars(): diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/step_function_templates/launch_pieriandx.asl.json b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/step_function_templates/launch_pieriandx.asl.json index 67dd8f620..b6d424958 100644 --- a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/step_function_templates/launch_pieriandx.asl.json +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/step_function_templates/launch_pieriandx.asl.json @@ -85,7 +85,7 @@ "Type": "Choice", "Choices": [ { - "Variable": "$.workflowInputs.panelName", + "Variable": "$.workflow_inputs.payload.data.inputs.panelVersion", "IsPresent": true, "Next": "Get Panel Name from inputs", "Comment": "Panel Name is set" @@ -105,7 +105,7 @@ "Type": "Pass", "Next": "Get Panel Name Value from SSM Parameter", "Parameters": { - "panel_name.$": "$.workflow_inputs.panelName" + "panel_name.$": "$.workflow_inputs.payload.data.inputs.panelVersion" }, "ResultPath": "$.get_panel_name_step" }, @@ -402,10 +402,13 @@ "id.$": "$.workflow_inputs.portalRunId", "id_type": "portal_run_id" }, - "UpdateExpression": "SET engine_parameters = :engine_parameters", + "UpdateExpression": "SET engine_parameters = :engine_parameters, workflow_status = :workflow_status", "ExpressionAttributeValues": { ":engine_parameters": { "S.$": "States.JsonToString($.set_engine_parameters_step.engine_parameters)" + }, + ":workflow_status": { + "S": "RUNNING" } } }, diff --git a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/step_function_templates/monitor_runs.asl.json b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/step_function_templates/monitor_runs.asl.json index f6e4f19fe..70975273d 100644 --- a/lib/workload/stateless/stacks/pieriandx-pipeline-manager/step_function_templates/monitor_runs.asl.json +++ b/lib/workload/stateless/stacks/pieriandx-pipeline-manager/step_function_templates/monitor_runs.asl.json @@ -212,7 +212,7 @@ "job_id.$": "$.get_portal_run_db_step.Item.informaticsjob_id.N", "case_accession_number.$": "$.get_portal_run_db_step.Item.case_accession_number.S", "report_id.$": "$.get_portal_run_db_step.Item.report_id.N", - "pieriandx_base_url": "https://app.uat.pieriandx.com/cgw-api/v2.0.0", + "pieriandx_base_url": "${__pieriandx_base_url__}", "sample_name.$": "$.get_portal_run_db_step.Item.sample_name.S" } }, @@ -252,7 +252,7 @@ "workflowName": "${__workflow_name__}", "workflowVersion": "${__workflow_version__}", "workflowRunName.$": "$.get_portal_run_db_step.Item.workflow_run_name.S", - "linkedLibraries.$": "States.StringToJson($.get_portal_run_db_step.Item.linkedLibraries.S)", + "linkedLibraries.$": "States.StringToJson($.get_portal_run_db_step.Item.linked_libraries.S)", "payload": { "version": "${__payload_version__}", "data.$": "$.get_data_payload_step.data_payload" diff --git a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_2/cttso-v2-output-to-pieriandx-ready-event/lambdas/get_data_from_redcap_py/get_data_from_redcap.py b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_2/cttso-v2-output-to-pieriandx-ready-event/lambdas/get_data_from_redcap_py/get_data_from_redcap.py index 93ba97ac6..3f9256297 100644 --- a/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_2/cttso-v2-output-to-pieriandx-ready-event/lambdas/get_data_from_redcap_py/get_data_from_redcap.py +++ b/lib/workload/stateless/stacks/stacky-mcstackface/glue-constructs/nails/part_2/cttso-v2-output-to-pieriandx-ready-event/lambdas/get_data_from_redcap_py/get_data_from_redcap.py @@ -130,8 +130,8 @@ def launch_redcap_raw_lambda(library_id: str) -> pd.DataFrame: # Rename columns redcap_raw_df.rename( columns={ - "clinician_firstname": "requesting_physicians_first_name", - "clinician_lastname": "requesting_physicians_last_name", + "clinician_firstname": "requesting_physician_first_name", + "clinician_lastname": "requesting_physician_last_name", "libraryid": "library_id", "mrn": "patient_urn", "disease": "disease_id", @@ -257,8 +257,8 @@ def get_and_merge_raw_and_label_data(library_id: str) -> Dict: redcap_raw_df = redcap_raw_df[ [ "disease_id", - "requesting_physicians_first_name", - "requesting_physicians_last_name", + "requesting_physician_first_name", + "requesting_physician_last_name", "library_id", "date_collected", "date_received",