Skip to content

Commit

Permalink
Merge pull request #659 from umccr/bump-pieriandx-generate-outputs-timer
Browse files Browse the repository at this point in the history
  • Loading branch information
@alexiswl
alexiswl authored Nov 6, 2024
2 parents 29aee0a + deb8862 commit 626de1a
Show file tree
Hide file tree
Showing 9 changed files with 257 additions and 26 deletions.
2 changes: 1 addition & 1 deletion config/constants.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -560,7 +560,7 @@ export const pieriandxDagSsmParameterPath = '/umccr/orcabus/stateful/pieriandx/d
/*
"s3://pdx-cgwxfer-test/melbournetest" // development
"s3://pdx-cgwxfer-test/melbournetest" // staging
"s3://pdx-cgwxfer/melbourne" // production
"s3://pdx-xfer/melbourne" // production
*/
export const pieriandxS3SequencerRunRootSsmParameterPath =
'/umccr/orcabus/pieriandx/s3_sequencer_run_root';
Expand Down
32 changes: 22 additions & 10 deletions lib/workload/stateless/stacks/pieriandx-pipeline-manager/deploy/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import { PythonFunction } from '@aws-cdk/aws-lambda-python-alpha';
import { Duration } from 'aws-cdk-lib';
import { PieriandxMonitorRunsStepFunctionStateMachineConstruct } from './constructs/pieriandx_monitor_runs_step_function';
import { PythonLambdaLayerConstruct } from '../../../../components/python-lambda-layer';
import { NagSuppressions } from 'cdk-nag';

export interface PierianDxPipelineManagerConfig {
/* DynamoDB Table */
Expand Down Expand Up @@ -157,7 +158,7 @@ export class PieriandxPipelineManagerStack extends cdk.Stack {
handler: 'handler',
memorySize: 1024,
layers: [lambdaLayerObj.lambdaLayerVersionObj],
timeout: Duration.seconds(20),
timeout: Duration.seconds(60),
environment: icav2Envs,
}
);
Expand Down Expand Up @@ -208,6 +209,7 @@ export class PieriandxPipelineManagerStack extends cdk.Stack {
memorySize: 1024,
layers: [lambdaLayerObj.lambdaLayerVersionObj],
environment: pieriandxEnvs,
timeout: Duration.seconds(60),
}
);

Expand Down Expand Up @@ -286,30 +288,40 @@ export class PieriandxPipelineManagerStack extends cdk.Stack {
getInformaticsjobAndReportStatusLambdaObj,
].forEach((lambdaFunction) => {
// Give the lambda permission to access the pieriandx apis
pieriandxTokenCollectionLambdaObj.latestVersion.grantInvoke(lambdaFunction);
// Fixme, no way to give access to only the current version
pieriandxTokenCollectionLambdaObj.grantInvoke(lambdaFunction.currentVersion);
NagSuppressions.addResourceSuppressions(
lambdaFunction,
[
{
id: 'AwsSolutions-IAM5',
reason:
'Cannot get latest version of pieriandx collect access token function ($LATEST) will not work',
},
],
true
);
});

/*
Give the upload lambda access to the pieriandx s3 bucket
*/
// @ts-ignore
Give the upload lambda access to the pieriandx s3 bucket
*/
pieriandxS3AccessTokenSecretObj.grantRead(uploadDataToS3LambdaObj);

/*
Give the lambdas permission to access the icav2 apis
*/
Give the lambdas permission to access the icav2 apis
*/
[
generatePieriandxObjectsLambdaObj,
generateSamplesheetLambdaObj,
uploadDataToS3LambdaObj,
].forEach((lambdaFunction) => {
// @ts-ignore
icav2AccessTokenSecretObj.grantRead(lambdaFunction);
});

/*
Generate State Machines
*/
Generate State Machines
*/

/* Generate case creation statemachine object */
const pieriandxLaunchCaseCreationStateMachine =
Expand Down
192 changes: 188 additions & 4 deletions ...-pipeline-manager/lambdas/generate_output_data_payload_py/generate_output_data_payload.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,53 @@
"""

from urllib.parse import (
urlparse, urlunparse
)
from pathlib import Path


def strip_path_from_url(url: str) -> str:
"""
Strip the path from the url
:param url:
:return:
"""
url_obj = urlparse(url)

return str(
urlunparse(
(
url_obj.scheme,
url_obj.netloc,
"/",
None, None, None
)
)
)


def join_url_paths(url: str, path_ext: str) -> str:
"""
Join the url paths
:param url: str
:param path_ext: str
:return: url
"""
url_obj = urlparse(url)
url_path = Path(url_obj.path) / path_ext

return str(
urlunparse(
(
url_obj.scheme,
url_obj.netloc,
str(url_path),
None, None, None
)
)
)


def handler(event, context):
"""
Expand Down Expand Up @@ -53,15 +100,28 @@ def handler(event, context):

# Set the outputs
return_dict["data_payload"]["outputs"] = {
"caseUrl": f"{pieriandx_base_url.replace("cgw-api/v2.0.0/", "")}/cgw/order/viewOrderDetails/{case_id}",
"vcfOutputUrl": f"{pieriandx_base_url.replace("cgw-api/v2.0.0/", "")}/cgw/informatics/downloadJobOutputAnalysisFile?caseId={case_id}&jobId={job_id}&accessionNumber={case_accession_number}&fileName=main.vcf",
"reportPdfUrl": f"{pieriandx_base_url.replace("cgw-api/v2.0.0/", "")}/cgw/report/openPdfReport/{report_id}",
"biomarkerReportUrl": f"{pieriandx_base_url.replace("cgw-api/v2.0.0/", "")}/cgw/informatics/downloadJobOutputAnalysisFile?caseId={case_id}&jobId={job_id}&accessionNumber={case_accession_number}&fileName={sample_name}_BiomarkerReport.txt"
"caseUrl": join_url_paths(
strip_path_from_url(pieriandx_base_url),
f"/cgw/order/viewOrderDetails/{case_id}"
),
"vcfOutputUrl": join_url_paths(
strip_path_from_url(pieriandx_base_url),
f"/cgw/informatics/downloadJobOutputAnalysisFile?caseId={case_id}&jobId={job_id}&accessionNumber={case_accession_number}&fileName=main.vcf"
),
"reportPdfUrl": join_url_paths(
strip_path_from_url(pieriandx_base_url),
f"/cgw/report/openPdfReport/{report_id}"
),
"biomarkerReportUrl": join_url_paths(
strip_path_from_url(pieriandx_base_url),
f"/cgw/informatics/downloadJobOutputAnalysisFile?caseId={case_id}&jobId={job_id}&accessionNumber={case_accession_number}&fileName={sample_name}_BiomarkerReport.txt"
),
}

return return_dict


# DEV
# if __name__ == "__main__":
# import json
#
Expand Down Expand Up @@ -181,3 +241,127 @@ def handler(event, context):
# # }
# # }
# # }


# # PROD
# if __name__ == "__main__":
# import json
#
# print(
# json.dumps(
# handler(
# {
# "pieriandx_base_url": "https://app.pieriandx.com/cgw-api/v2.0.0",
# "inputs": {
# "instrumentRunId": "241101_A01052_0236_BHVJNMDMXY",
# "panelVersion": "main",
# "caseMetadata": {
# "isIdentified": False,
# "caseAccessionNumber": "L2401562__V2__20241106c25602c7",
# "externalSpecimenId": "CMM0.5pc-10646979",
# "sampleType": "validation",
# "specimenLabel": "primarySpecimen",
# "indication": "NA",
# "diseaseCode": 55342001,
# "specimenCode": "122561005",
# "sampleReception": {
# "dateAccessioned": "2024-11-07T08:39:55+1100",
# "dateCollected": "2024-11-07T08:39:55+1100",
# "dateReceived": "2024-11-07T08:39:55+1100"
# },
# "study": {
# "id": "Control",
# "subjectIdentifier": "Sera-ctDNA-Comp05pc"
# }
# },
# "dataFiles": {
# "microsatOutputUri": "s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/analysis/cttsov2/202411052080dbdd/Logs_Intermediates/DragenCaller/L2401562/L2401562.microsat_output.json",
# "tmbMetricsUri": "s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/analysis/cttsov2/202411052080dbdd/Logs_Intermediates/Tmb/L2401562/L2401562.tmb.metrics.csv",
# "cnvVcfUri": "s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/analysis/cttsov2/202411052080dbdd/Results/L2401562/L2401562.cnv.vcf.gz",
# "hardFilteredVcfUri": "s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/analysis/cttsov2/202411052080dbdd/Results/L2401562/L2401562.hard-filtered.vcf.gz",
# "fusionsUri": "s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/analysis/cttsov2/202411052080dbdd/Results/L2401562/L2401562_Fusions.csv",
# "metricsOutputUri": "s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/analysis/cttsov2/202411052080dbdd/Results/L2401562/L2401562_MetricsOutput.tsv",
# "samplesheetUri": "s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/analysis/cttsov2/202411052080dbdd/Logs_Intermediates/SampleSheetValidation/SampleSheet_Intermediate.csv"
# }
# },
# "job_id": "302091",
# "report_id": "312010",
# "case_id": "336486",
# "report_status": "complete",
# "case_accession_number": "L2401562__V2__20241106c25602c7",
# "engine_parameters": {
# "caseId": "336486",
# "informaticsJobId": "302091"
# },
# "tags": {
# "metadataFromRedCap": False,
# "isIdentified": False,
# "libraryId": "L2401562",
# "sampleType": "validation",
# "projectId": "Control",
# "instrumentRunId": "241101_A01052_0236_BHVJNMDMXY",
# "panelVersion": "main"
# },
# "sample_name": "L2401562"
# },
# None
# ),
# indent=4
# )
# )
#
# # {
# # "data_payload": {
# # "inputs": {
# # "instrumentRunId": "241101_A01052_0236_BHVJNMDMXY",
# # "panelVersion": "main",
# # "caseMetadata": {
# # "isIdentified": false,
# # "caseAccessionNumber": "L2401562__V2__20241106c25602c7",
# # "externalSpecimenId": "CMM0.5pc-10646979",
# # "sampleType": "validation",
# # "specimenLabel": "primarySpecimen",
# # "indication": "NA",
# # "diseaseCode": 55342001,
# # "specimenCode": "122561005",
# # "sampleReception": {
# # "dateAccessioned": "2024-11-07T08:39:55+1100",
# # "dateCollected": "2024-11-07T08:39:55+1100",
# # "dateReceived": "2024-11-07T08:39:55+1100"
# # },
# # "study": {
# # "id": "Control",
# # "subjectIdentifier": "Sera-ctDNA-Comp05pc"
# # }
# # },
# # "dataFiles": {
# # "microsatOutputUri": "s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/analysis/cttsov2/202411052080dbdd/Logs_Intermediates/DragenCaller/L2401562/L2401562.microsat_output.json",
# # "tmbMetricsUri": "s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/analysis/cttsov2/202411052080dbdd/Logs_Intermediates/Tmb/L2401562/L2401562.tmb.metrics.csv",
# # "cnvVcfUri": "s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/analysis/cttsov2/202411052080dbdd/Results/L2401562/L2401562.cnv.vcf.gz",
# # "hardFilteredVcfUri": "s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/analysis/cttsov2/202411052080dbdd/Results/L2401562/L2401562.hard-filtered.vcf.gz",
# # "fusionsUri": "s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/analysis/cttsov2/202411052080dbdd/Results/L2401562/L2401562_Fusions.csv",
# # "metricsOutputUri": "s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/analysis/cttsov2/202411052080dbdd/Results/L2401562/L2401562_MetricsOutput.tsv",
# # "samplesheetUri": "s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/analysis/cttsov2/202411052080dbdd/Logs_Intermediates/SampleSheetValidation/SampleSheet_Intermediate.csv"
# # }
# # },
# # "engineParameters": {
# # "caseId": "336486",
# # "informaticsJobId": "302091"
# # },
# # "tags": {
# # "metadataFromRedCap": false,
# # "isIdentified": false,
# # "libraryId": "L2401562",
# # "sampleType": "validation",
# # "projectId": "Control",
# # "instrumentRunId": "241101_A01052_0236_BHVJNMDMXY",
# # "panelVersion": "main"
# # },
# # "outputs": {
# # "caseUrl": "https://app.pieriandx.com/cgw/order/viewOrderDetails/336486",
# # "vcfOutputUrl": "https://app.pieriandx.com/cgw/informatics/downloadJobOutputAnalysisFile?caseId=336486&jobId=302091&accessionNumber=L2401562__V2__20241106c25602c7&fileName=main.vcf",
# # "reportPdfUrl": "https://app.pieriandx.com/cgw/report/openPdfReport/312010",
# # "biomarkerReportUrl": "https://app.pieriandx.com/cgw/informatics/downloadJobOutputAnalysisFile?caseId=336486&jobId=302091&accessionNumber=L2401562__V2__20241106c25602c7&fileName=L2401562_BiomarkerReport.txt"
# # }
# # }
# # }
2 changes: 1 addition & 1 deletion ...r/lambdas/get_informaticsjob_and_report_status_py/get_informaticsjob_and_report_status.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,7 @@ def handler(event, context):
"S": job_status
},
":report_id": {
"S": reportjob_obj.get("id")
"N": reportjob_obj.get("id")
},
":report_status": {
"S": report_status
Expand Down
25 changes: 25 additions & 0 deletions ...nager/lambdas/upload_pieriandx_sample_data_to_s3_py/upload_pieriandx_sample_data_to_s3.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,15 +23,22 @@
"""

# Standard imports
from tempfile import TemporaryDirectory
from pathlib import Path
from urllib.parse import urlparse
from wrapica.project_data import read_icav2_file_contents, convert_uri_to_project_data_obj
import logging

# Layer imports
from pieriandx_pipeline_tools.utils.s3_helpers import set_s3_access_cred_env_vars, upload_file
from pieriandx_pipeline_tools.utils.secretsmanager_helpers import set_icav2_env_vars
from pieriandx_pipeline_tools.utils.compression_helpers import decompress_file

# Logger
logger = logging.getLogger()
logger.setLevel(logging.INFO)


def handler(event, context):
"""
Expand Down Expand Up @@ -107,3 +114,21 @@ def handler(event, context):
# },
# None
# )

# if __name__ == "__main__":
# from os import environ
# environ["AWS_PROFILE"] = 'umccr-production'
# environ['AWS_REGION'] = 'ap-southeast-2'
# environ["ICAV2_ACCESS_TOKEN_SECRET_ID"] = "ICAv2JWTKey-umccr-prod-service-production"
# environ['PIERIANDX_S3_ACCESS_CREDENTIALS_SECRET_ID'] = "PierianDx/S3Credentials"
#
# handler(
# {
# "needs_decompression": False,
# "src_uri": "s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/analysis/cttsov2/202411053da6481e/Results/L2401560/L2401560_MetricsOutput.tsv",
# "contents": None,
# "dest_uri": "s3://pdx-xfer/melbourne/241101_A01052_0236_BHVJNMDMXY__L2401560__V2__20241105f6bc3fb9__20241105f6bc3fb9/Data/Intensities/BaseCalls/L2401560_MetricsOutput.tsv"
# },
# None
# )
#
9 changes: 8 additions & 1 deletion ...stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/utils/s3_helpers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,14 @@ def get_s3_client() -> 'S3Client':

def upload_file(bucket: str, key: str, input_file_path: Path) -> None:
s3 = get_s3_client()
s3.upload_file(str(input_file_path), bucket, key.lstrip("/"))
s3.upload_file(
str(input_file_path),
bucket,
key.lstrip("/"),
ExtraArgs={
'ServerSideEncryption': 'AES256'
}
)


def set_s3_access_cred_env_vars():
Expand Down
Loading

0 comments on commit 626de1a

Please sign in to comment.