Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

PierianDx Production Fixes #659

Merged
merged 11 commits into from
Nov 6, 2024
Merged

PierianDx Production Fixes #659

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
192187f
Timeout error since pulling files from github now is taking time
alexiswl Nov 6, 2024
334e527
Fix ssm parameter for s3 bucket path
alexiswl Nov 6, 2024
b2b904d
Don't use 'latest' for lambda version, use * and nag suppression
alexiswl Nov 6, 2024
f4e073a
Use KMS encryption when upload data to s3
alexiswl Nov 6, 2024
56c245d
Wrong json path used to collect the panel version
alexiswl Nov 6, 2024
2373e94
Typo on the requesting physician first name / last name meant default…
alexiswl Nov 6, 2024
ed0e7f3
Initialise workflow status after launch for referencing when monitori…
alexiswl Nov 6, 2024
c097cd6
Report ID should be stored in db as an integer not as a string
alexiswl Nov 6, 2024
136d863
Missed requesting physician 's' on collection of data from redcap raw…
alexiswl Nov 6, 2024
83a6cb9
Don't hardcode url, fixed linked libraries get from dynamodb step
alexiswl Nov 6, 2024
deb8862
Fix analysis path generation for outputs
alexiswl Nov 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion config/constants.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -560,7 +560,7 @@ export const pieriandxDagSsmParameterPath = '/umccr/orcabus/stateful/pieriandx/d
/*
"s3://pdx-cgwxfer-test/melbournetest" // development
"s3://pdx-cgwxfer-test/melbournetest" // staging
"s3://pdx-cgwxfer/melbourne" // production
"s3://pdx-xfer/melbourne" // production
*/
export const pieriandxS3SequencerRunRootSsmParameterPath =
'/umccr/orcabus/pieriandx/s3_sequencer_run_root';
Expand Down
32 changes: 22 additions & 10 deletions lib/workload/stateless/stacks/pieriandx-pipeline-manager/deploy/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import { PythonFunction } from '@aws-cdk/aws-lambda-python-alpha';
import { Duration } from 'aws-cdk-lib';
import { PieriandxMonitorRunsStepFunctionStateMachineConstruct } from './constructs/pieriandx_monitor_runs_step_function';
import { PythonLambdaLayerConstruct } from '../../../../components/python-lambda-layer';
import { NagSuppressions } from 'cdk-nag';

export interface PierianDxPipelineManagerConfig {
/* DynamoDB Table */
Expand Down Expand Up @@ -157,7 +158,7 @@ export class PieriandxPipelineManagerStack extends cdk.Stack {
handler: 'handler',
memorySize: 1024,
layers: [lambdaLayerObj.lambdaLayerVersionObj],
timeout: Duration.seconds(20),
timeout: Duration.seconds(60),
environment: icav2Envs,
}
);
Expand Down Expand Up @@ -208,6 +209,7 @@ export class PieriandxPipelineManagerStack extends cdk.Stack {
memorySize: 1024,
layers: [lambdaLayerObj.lambdaLayerVersionObj],
environment: pieriandxEnvs,
timeout: Duration.seconds(60),
}
);

Expand Down Expand Up @@ -286,30 +288,40 @@ export class PieriandxPipelineManagerStack extends cdk.Stack {
getInformaticsjobAndReportStatusLambdaObj,
].forEach((lambdaFunction) => {
// Give the lambda permission to access the pieriandx apis
pieriandxTokenCollectionLambdaObj.latestVersion.grantInvoke(lambdaFunction);
// Fixme, no way to give access to only the current version
pieriandxTokenCollectionLambdaObj.grantInvoke(lambdaFunction.currentVersion);
NagSuppressions.addResourceSuppressions(
lambdaFunction,
[
{
id: 'AwsSolutions-IAM5',
reason:
'Cannot get latest version of pieriandx collect access token function ($LATEST) will not work',
},
],
true
);
});

/*
Give the upload lambda access to the pieriandx s3 bucket
*/
// @ts-ignore
Give the upload lambda access to the pieriandx s3 bucket
*/
pieriandxS3AccessTokenSecretObj.grantRead(uploadDataToS3LambdaObj);

/*
Give the lambdas permission to access the icav2 apis
*/
Give the lambdas permission to access the icav2 apis
*/
[
generatePieriandxObjectsLambdaObj,
generateSamplesheetLambdaObj,
uploadDataToS3LambdaObj,
].forEach((lambdaFunction) => {
// @ts-ignore
icav2AccessTokenSecretObj.grantRead(lambdaFunction);
});

/*
Generate State Machines
*/
Generate State Machines
*/

/* Generate case creation statemachine object */
const pieriandxLaunchCaseCreationStateMachine =
Expand Down
192 changes: 188 additions & 4 deletions ...-pipeline-manager/lambdas/generate_output_data_payload_py/generate_output_data_payload.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,53 @@

"""

from urllib.parse import (
urlparse, urlunparse
)
from pathlib import Path


def strip_path_from_url(url: str) -> str:
"""
Strip the path from the url
:param url:
:return:
"""
url_obj = urlparse(url)

return str(
urlunparse(
(
url_obj.scheme,
url_obj.netloc,
"/",
None, None, None
)
)
)


def join_url_paths(url: str, path_ext: str) -> str:
"""
Join the url paths
:param url: str
:param path_ext: str
:return: url
"""
url_obj = urlparse(url)
url_path = Path(url_obj.path) / path_ext

return str(
urlunparse(
(
url_obj.scheme,
url_obj.netloc,
str(url_path),
None, None, None
)
)
)


def handler(event, context):
"""
Expand Down Expand Up @@ -53,15 +100,28 @@ def handler(event, context):

# Set the outputs
return_dict["data_payload"]["outputs"] = {
"caseUrl": f"{pieriandx_base_url.replace("cgw-api/v2.0.0/", "")}/cgw/order/viewOrderDetails/{case_id}",
"vcfOutputUrl": f"{pieriandx_base_url.replace("cgw-api/v2.0.0/", "")}/cgw/informatics/downloadJobOutputAnalysisFile?caseId={case_id}&jobId={job_id}&accessionNumber={case_accession_number}&fileName=main.vcf",
"reportPdfUrl": f"{pieriandx_base_url.replace("cgw-api/v2.0.0/", "")}/cgw/report/openPdfReport/{report_id}",
"biomarkerReportUrl": f"{pieriandx_base_url.replace("cgw-api/v2.0.0/", "")}/cgw/informatics/downloadJobOutputAnalysisFile?caseId={case_id}&jobId={job_id}&accessionNumber={case_accession_number}&fileName={sample_name}_BiomarkerReport.txt"
"caseUrl": join_url_paths(
strip_path_from_url(pieriandx_base_url),
f"/cgw/order/viewOrderDetails/{case_id}"
),
"vcfOutputUrl": join_url_paths(
strip_path_from_url(pieriandx_base_url),
f"/cgw/informatics/downloadJobOutputAnalysisFile?caseId={case_id}&jobId={job_id}&accessionNumber={case_accession_number}&fileName=main.vcf"
),
"reportPdfUrl": join_url_paths(
strip_path_from_url(pieriandx_base_url),
f"/cgw/report/openPdfReport/{report_id}"
),
"biomarkerReportUrl": join_url_paths(
strip_path_from_url(pieriandx_base_url),
f"/cgw/informatics/downloadJobOutputAnalysisFile?caseId={case_id}&jobId={job_id}&accessionNumber={case_accession_number}&fileName={sample_name}_BiomarkerReport.txt"
),
}

return return_dict


# DEV
# if __name__ == "__main__":
# import json
#
Expand Down Expand Up @@ -181,3 +241,127 @@ def handler(event, context):
# # }
# # }
# # }


# # PROD
# if __name__ == "__main__":
# import json
#
# print(
# json.dumps(
# handler(
# {
# "pieriandx_base_url": "https://app.pieriandx.com/cgw-api/v2.0.0",
# "inputs": {
# "instrumentRunId": "241101_A01052_0236_BHVJNMDMXY",
# "panelVersion": "main",
# "caseMetadata": {
# "isIdentified": False,
# "caseAccessionNumber": "L2401562__V2__20241106c25602c7",
# "externalSpecimenId": "CMM0.5pc-10646979",
# "sampleType": "validation",
# "specimenLabel": "primarySpecimen",
# "indication": "NA",
# "diseaseCode": 55342001,
# "specimenCode": "122561005",
# "sampleReception": {
# "dateAccessioned": "2024-11-07T08:39:55+1100",
# "dateCollected": "2024-11-07T08:39:55+1100",
# "dateReceived": "2024-11-07T08:39:55+1100"
# },
# "study": {
# "id": "Control",
# "subjectIdentifier": "Sera-ctDNA-Comp05pc"
# }
# },
# "dataFiles": {
# "microsatOutputUri": "s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/analysis/cttsov2/202411052080dbdd/Logs_Intermediates/DragenCaller/L2401562/L2401562.microsat_output.json",
# "tmbMetricsUri": "s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/analysis/cttsov2/202411052080dbdd/Logs_Intermediates/Tmb/L2401562/L2401562.tmb.metrics.csv",
# "cnvVcfUri": "s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/analysis/cttsov2/202411052080dbdd/Results/L2401562/L2401562.cnv.vcf.gz",
# "hardFilteredVcfUri": "s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/analysis/cttsov2/202411052080dbdd/Results/L2401562/L2401562.hard-filtered.vcf.gz",
# "fusionsUri": "s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/analysis/cttsov2/202411052080dbdd/Results/L2401562/L2401562_Fusions.csv",
# "metricsOutputUri": "s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/analysis/cttsov2/202411052080dbdd/Results/L2401562/L2401562_MetricsOutput.tsv",
# "samplesheetUri": "s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/analysis/cttsov2/202411052080dbdd/Logs_Intermediates/SampleSheetValidation/SampleSheet_Intermediate.csv"
# }
# },
# "job_id": "302091",
# "report_id": "312010",
# "case_id": "336486",
# "report_status": "complete",
# "case_accession_number": "L2401562__V2__20241106c25602c7",
# "engine_parameters": {
# "caseId": "336486",
# "informaticsJobId": "302091"
# },
# "tags": {
# "metadataFromRedCap": False,
# "isIdentified": False,
# "libraryId": "L2401562",
# "sampleType": "validation",
# "projectId": "Control",
# "instrumentRunId": "241101_A01052_0236_BHVJNMDMXY",
# "panelVersion": "main"
# },
# "sample_name": "L2401562"
# },
# None
# ),
# indent=4
# )
# )
#
# # {
# # "data_payload": {
# # "inputs": {
# # "instrumentRunId": "241101_A01052_0236_BHVJNMDMXY",
# # "panelVersion": "main",
# # "caseMetadata": {
# # "isIdentified": false,
# # "caseAccessionNumber": "L2401562__V2__20241106c25602c7",
# # "externalSpecimenId": "CMM0.5pc-10646979",
# # "sampleType": "validation",
# # "specimenLabel": "primarySpecimen",
# # "indication": "NA",
# # "diseaseCode": 55342001,
# # "specimenCode": "122561005",
# # "sampleReception": {
# # "dateAccessioned": "2024-11-07T08:39:55+1100",
# # "dateCollected": "2024-11-07T08:39:55+1100",
# # "dateReceived": "2024-11-07T08:39:55+1100"
# # },
# # "study": {
# # "id": "Control",
# # "subjectIdentifier": "Sera-ctDNA-Comp05pc"
# # }
# # },
# # "dataFiles": {
# # "microsatOutputUri": "s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/analysis/cttsov2/202411052080dbdd/Logs_Intermediates/DragenCaller/L2401562/L2401562.microsat_output.json",
# # "tmbMetricsUri": "s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/analysis/cttsov2/202411052080dbdd/Logs_Intermediates/Tmb/L2401562/L2401562.tmb.metrics.csv",
# # "cnvVcfUri": "s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/analysis/cttsov2/202411052080dbdd/Results/L2401562/L2401562.cnv.vcf.gz",
# # "hardFilteredVcfUri": "s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/analysis/cttsov2/202411052080dbdd/Results/L2401562/L2401562.hard-filtered.vcf.gz",
# # "fusionsUri": "s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/analysis/cttsov2/202411052080dbdd/Results/L2401562/L2401562_Fusions.csv",
# # "metricsOutputUri": "s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/analysis/cttsov2/202411052080dbdd/Results/L2401562/L2401562_MetricsOutput.tsv",
# # "samplesheetUri": "s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/analysis/cttsov2/202411052080dbdd/Logs_Intermediates/SampleSheetValidation/SampleSheet_Intermediate.csv"
# # }
# # },
# # "engineParameters": {
# # "caseId": "336486",
# # "informaticsJobId": "302091"
# # },
# # "tags": {
# # "metadataFromRedCap": false,
# # "isIdentified": false,
# # "libraryId": "L2401562",
# # "sampleType": "validation",
# # "projectId": "Control",
# # "instrumentRunId": "241101_A01052_0236_BHVJNMDMXY",
# # "panelVersion": "main"
# # },
# # "outputs": {
# # "caseUrl": "https://app.pieriandx.com/cgw/order/viewOrderDetails/336486",
# # "vcfOutputUrl": "https://app.pieriandx.com/cgw/informatics/downloadJobOutputAnalysisFile?caseId=336486&jobId=302091&accessionNumber=L2401562__V2__20241106c25602c7&fileName=main.vcf",
# # "reportPdfUrl": "https://app.pieriandx.com/cgw/report/openPdfReport/312010",
# # "biomarkerReportUrl": "https://app.pieriandx.com/cgw/informatics/downloadJobOutputAnalysisFile?caseId=336486&jobId=302091&accessionNumber=L2401562__V2__20241106c25602c7&fileName=L2401562_BiomarkerReport.txt"
# # }
# # }
# # }
2 changes: 1 addition & 1 deletion ...r/lambdas/get_informaticsjob_and_report_status_py/get_informaticsjob_and_report_status.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,7 @@ def handler(event, context):
"S": job_status
},
":report_id": {
"S": reportjob_obj.get("id")
"N": reportjob_obj.get("id")
},
":report_status": {
"S": report_status
Expand Down
25 changes: 25 additions & 0 deletions ...nager/lambdas/upload_pieriandx_sample_data_to_s3_py/upload_pieriandx_sample_data_to_s3.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,15 +23,22 @@

"""

# Standard imports
from tempfile import TemporaryDirectory
from pathlib import Path
from urllib.parse import urlparse
from wrapica.project_data import read_icav2_file_contents, convert_uri_to_project_data_obj
import logging

# Layer imports
from pieriandx_pipeline_tools.utils.s3_helpers import set_s3_access_cred_env_vars, upload_file
from pieriandx_pipeline_tools.utils.secretsmanager_helpers import set_icav2_env_vars
from pieriandx_pipeline_tools.utils.compression_helpers import decompress_file

# Logger
logger = logging.getLogger()
logger.setLevel(logging.INFO)


def handler(event, context):
"""
Expand Down Expand Up @@ -107,3 +114,21 @@ def handler(event, context):
# },
# None
# )

# if __name__ == "__main__":
# from os import environ
# environ["AWS_PROFILE"] = 'umccr-production'
# environ['AWS_REGION'] = 'ap-southeast-2'
# environ["ICAV2_ACCESS_TOKEN_SECRET_ID"] = "ICAv2JWTKey-umccr-prod-service-production"
# environ['PIERIANDX_S3_ACCESS_CREDENTIALS_SECRET_ID'] = "PierianDx/S3Credentials"
#
# handler(
# {
# "needs_decompression": False,
# "src_uri": "s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/analysis/cttsov2/202411053da6481e/Results/L2401560/L2401560_MetricsOutput.tsv",
# "contents": None,
# "dest_uri": "s3://pdx-xfer/melbourne/241101_A01052_0236_BHVJNMDMXY__L2401560__V2__20241105f6bc3fb9__20241105f6bc3fb9/Data/Intensities/BaseCalls/L2401560_MetricsOutput.tsv"
# },
# None
# )
#
9 changes: 8 additions & 1 deletion ...stacks/pieriandx-pipeline-manager/layers/src/pieriandx_pipeline_tools/utils/s3_helpers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,14 @@ def get_s3_client() -> 'S3Client':

def upload_file(bucket: str, key: str, input_file_path: Path) -> None:
s3 = get_s3_client()
s3.upload_file(str(input_file_path), bucket, key.lstrip("/"))
s3.upload_file(
str(input_file_path),
bucket,
key.lstrip("/"),
ExtraArgs={
'ServerSideEncryption': 'AES256'
}
)


def set_s3_access_cred_env_vars():
Expand Down
Loading