-
Notifications
You must be signed in to change notification settings - Fork 0
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Friday changes: #589
Merged
Merged
Friday changes: #589
Changes from all commits
Commits
Show all changes
7 commits
Select commit
Hold shift + click to select a range
2f8477b
Friday changes:
alexiswl 9facfdf
Validate make-check
alexiswl 00ae37d
Only give latest versions permissions
alexiswl f20da82
Only give latest permissions to sfn on lambda objects
alexiswl 2104ee6
Fix cdk-nag permisisons
alexiswl be48b09
CDK Nag fix
alexiswl 6355897
Fixes cdk-nag error on grant read to statemachine object
alexiswl File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,2 +1,3 @@ | ||
| /.yarn/releases/** binary | ||
| /.yarn/plugins/** binary | ||
| *.json.gz filter=lfs diff=lfs merge=lfs -text |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -291,7 +291,7 @@ External resources required by the wgtsqc Stack | |
| // Deployed under dev/stg/prod | ||
| export const wgtsQcIcav2PipelineIdSSMParameterPath = '/icav2/umccr-prod/wgts_qc_4.2.4_pipeline_id'; // 03689516-b7f8-4dca-bba9-8405b85fae45 | ||
|
|
||
| export const wgtsQcIcav2PipelineWorkflowType = 'wgtsQc'; | ||
| export const wgtsQcIcav2PipelineWorkflowType = 'wgts-qc'; | ||
| export const wgtsQcIcav2PipelineWorkflowTypeVersion = '4.2.4'; | ||
| export const wgtsQcIcav2ServiceVersion = '2024.07.01'; | ||
|
|
||
|
|
@@ -324,7 +324,7 @@ TN Stateless stack | |
|
|
||
| // Deployed under dev/stg/prod | ||
| export const tnIcav2PipelineIdSSMParameterPath = '/icav2/umccr-prod/tumor_normal_4.2.4_pipeline_id'; // 0f5575bc-6cf8-4a90-a80e-05088aae8ed7 | ||
| export const tnIcav2PipelineWorkflowType = 'tumor_normal'; | ||
| export const tnIcav2PipelineWorkflowType = 'tumor-normal'; | ||
| export const tnIcav2PipelineWorkflowTypeVersion = '4.2.4'; | ||
| export const tnIcav2ServiceVersion = '2024.07.01'; | ||
| export const tnIcav2ReadyEventSource = 'orcabus.workflowmanager'; | ||
|
|
@@ -500,6 +500,183 @@ export const rnasumIcav2EventDetailType = 'WorkflowRunStateChange'; | |
| export const rnasumStateMachinePrefix = 'rnasumSfn'; | ||
| export const rnasumDefaultDatasetVersion = 'PANCAN'; | ||
|
|
||
| /* | ||
| Ora Compression Stateless Stack | ||
| */ | ||
|
|
||
| // Deployed in dev | ||
| // export const oraCompressionTarSSMParameterPath = '/icav2/umccr-prod/ora_compression_tar_uri'; // icav2://reference-data/dragen-ora/v2/ora_reference_v2.tar.gz | ||
|
|
||
| /* | ||
| PierianDx Stateful and Stateless stacks | ||
| */ | ||
| export const pieriandxPrefix = 'pieriandx'; | ||
| export const pieriandxTriggerLaunchSource = 'orcabus.workflowmanager'; | ||
| export const pieriandxWorkflowName = 'pieriandx'; | ||
| export const pieriandxWorkflowVersion = '2.1'; | ||
| export const pieriandxDetailType = 'WorkflowRunStateChange'; | ||
| export const pieriandxEventSource = 'orcabus.pieriandx'; | ||
| export const pieriandxPayloadVersion = '2024.10.01'; | ||
| export const pieriandxDynamodbTable = 'PierianDxPipelineDynamoDbTable'; | ||
|
|
||
| /* | ||
| [ | ||
| { | ||
| "panelName": "main", | ||
| "panelId": "tso500_DRAGEN_ctDNA_v2_1_Universityofmelbourne" // pragma: allowlist secret | ||
| }, | ||
| { | ||
| "panelName": "subpanel", | ||
| "panelId": "tso500_DRAGEN_ctDNA_v2_1_subpanel_Universityofmelbourne" // pragma: allowlist secret | ||
| } | ||
| ] | ||
|
|
||
| */ | ||
| export const pieriandxDefaultPanelName = 'main'; | ||
| export const pieriandxPanelMapSsmParameterPath = '/umccr/orcabus/stateful/pieriandx/panel_map'; | ||
|
|
||
| /* | ||
| [ | ||
| { | ||
| "dagName": "cromwell_tso500_ctdna_workflow_1.0.4", | ||
| "dagDescription": "tso500_ctdna_workflow" | ||
| } | ||
| ] | ||
| */ | ||
| export const pieriandxDefaultDagName = 'cromwell_tso500_ctdna_workflow_1.0.4'; | ||
| export const pieriandxDagSsmParameterPath = '/umccr/orcabus/stateful/pieriandx/dag_map'; | ||
|
|
||
| /* | ||
| "s3://pdx-cgwxfer-test/melbournetest" // development | ||
| "s3://pdx-cgwxfer-test/melbournetest" // staging | ||
| "s3://pdx-cgwxfer/melbourne" // production | ||
| */ | ||
| export const pieriandxS3SequencerRunRootSsmParameterPath = | ||
| '/umccr/orcabus/pieriandx/s3_sequencer_run_root'; | ||
|
|
||
| /* | ||
| "[email protected]" // development | ||
| "[email protected]" // staging | ||
| "[email protected]" // production | ||
| */ | ||
| export const pieriandxUserEmailSsmParameterPath = '/umccr/orcabus/pieriandx/user_email'; | ||
|
|
||
| /* | ||
| "melbournetest" // development | ||
| "melbournetest" // staging | ||
| "melbourne" // production | ||
| */ | ||
| export const pieriandxInstitutionSsmParameterPath = '/umccr/orcabus/pieriandx/institution'; | ||
|
|
||
| /* | ||
| "https://app.uat.pieriandx.com/cgw-api/v2.0.0" // development | ||
| "https://app.uat.pieriandx.com/cgw-api/v2.0.0" // staging | ||
| "https://app.pieriandx.com/cgw-api/v2.0.0" // production | ||
| */ | ||
| export const pieriandxBaseUrlSsmParameterPath = '/umccr/orcabus/pieriandx/base_url'; | ||
|
|
||
| // Constant for all environments | ||
| export const pieriandxAuthTokeSsmParameterPath = 'collectPierianDxAccessToken'; | ||
|
|
||
| // Secret name for PierianDx S3 credentials (test bucket for dev and staging, prod bucket for prod) | ||
| export const pieriandxS3CredentialsSecretsManagerId = 'PierianDx/S3Credentials'; // pragma: allowlist secret | ||
|
|
||
| /* | ||
| [ | ||
| { | ||
| "project_id": "PO", | ||
| "panel": "subpanel", | ||
| "sample_type": "patientcare", | ||
| "is_identified": "identified", | ||
| "default_snomed_disease_code": null | ||
| }, | ||
| { | ||
| "project_id": "COUMN", | ||
| "panel": "subpanel", | ||
| "sample_type": "patientcare", | ||
| "is_identified": "identified", | ||
| "default_snomed_disease_code": null | ||
| }, | ||
| { | ||
| "project_id": "CUP", | ||
| "panel": "main", | ||
| "sample_type": "patientcare", | ||
| "is_identified": "identified", | ||
| "default_snomed_disease_code": 285645000 | ||
| }, | ||
| { | ||
| "project_id": "PPGL", | ||
| "panel": "main", | ||
| "sample_type": "patientcare", | ||
| "is_identified": "identified", | ||
| "default_snomed_disease_code": null | ||
| }, | ||
| { | ||
| "project_id": "MESO", | ||
| "panel": "subpanel", | ||
| "sample_type": "patientcare", | ||
| "is_identified": "identified", | ||
| "default_snomed_disease_code": null | ||
| }, | ||
| { | ||
| "project_id": "OCEANiC", | ||
| "panel": "subpanel", | ||
| "sample_type": "patientcare", | ||
| "is_identified": "deidentified", | ||
| "default_snomed_disease_code": null | ||
| }, | ||
| { | ||
| "project_id": "SOLACE2", | ||
| "panel": "main", | ||
| "sample_type": "patientcare", | ||
| "is_identified": "deidentified", | ||
| "default_snomed_disease_code": 55342001 | ||
| }, | ||
| { | ||
| "project_id": "IMPARP", | ||
| "panel": "main", | ||
| "sample_type": "patientcare", | ||
| "is_identified": "deidentified", | ||
| "default_snomed_disease_code": 55342001 | ||
| }, | ||
| { | ||
| "project_id": "Control", | ||
| "panel": "main", | ||
| "sample_type": "validation", | ||
| "is_identified": "deidentified", | ||
| "default_snomed_disease_code": 55342001 | ||
| }, | ||
| { | ||
| "project_id": "QAP", | ||
| "panel": "subpanel", | ||
| "sample_type": "patientcare", | ||
| "is_identified": "identified", | ||
| "default_snomed_disease_code": null | ||
| }, | ||
| { | ||
| "project_id": "iPredict2", | ||
| "panel": "subpanel", | ||
| "sample_type": "patientcare", | ||
| "is_identified": "identified", | ||
| "default_snomed_disease_code":null | ||
| }, | ||
| { | ||
| "project_id": "*", | ||
| "panel": "main", | ||
| "sample_type": "patientcare", | ||
| "is_identified": "deidentified", | ||
| "default_snomed_disease_code": 55342001 | ||
| } | ||
| ] | ||
| */ | ||
| export const pieriandxProjectInfoSsmParameterPath = '/umccr/orcabus/pieriandx/project_info'; | ||
|
|
||
| export const redcapLambdaFunctionName: Record<AppStage, string> = { | ||
| [AppStage.BETA]: 'redcap-apis-dev-lambda-function', | ||
| [AppStage.GAMMA]: 'redcap-apis-stg-lambda-function', | ||
| [AppStage.PROD]: 'redcap-apis-prod-lambda-function', | ||
| }; | ||
|
|
||
| // Mock Stack | ||
| export const mockEventBusName = eventBusName; | ||
| export const mockInstrumentRunTableName = 'stacky-instrument-run-table'; | ||
|
|
@@ -510,6 +687,7 @@ export const mockTnGlueTableName = 'stacky-tn-glue-table'; | |
| export const mockWtsGlueTableName = 'stacky-wts-glue-table'; | ||
| export const mockUmccriseGlueTableName = 'stacky-umccrise-glue-table'; | ||
| export const mockRnasumGlueTableName = 'stacky-rnasum-glue-table'; | ||
| export const mockPierianDxGlueTableName = 'stacky-pieriandx-glue-table'; | ||
| export const mockWorkflowManagerTableName = 'stacky-workflow-manager-table'; | ||
|
|
||
| // { | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,71 @@ | ||
| import { | ||
| AppStage, | ||
| eventBusName, | ||
| icav2AccessTokenSecretName, | ||
| pieriandxAuthTokeSsmParameterPath, | ||
| pieriandxBaseUrlSsmParameterPath, | ||
| pieriandxDagSsmParameterPath, | ||
| pieriandxDefaultDagName, | ||
| pieriandxDefaultPanelName, | ||
| pieriandxDetailType, | ||
| pieriandxDynamodbTable, | ||
| pieriandxEventSource, | ||
| pieriandxInstitutionSsmParameterPath, | ||
| pieriandxPanelMapSsmParameterPath, | ||
| pieriandxPayloadVersion, | ||
| pieriandxPrefix, | ||
| pieriandxS3CredentialsSecretsManagerId, | ||
| pieriandxS3SequencerRunRootSsmParameterPath, | ||
| pieriandxTriggerLaunchSource, | ||
| pieriandxUserEmailSsmParameterPath, | ||
| pieriandxWorkflowName, | ||
| pieriandxWorkflowVersion, | ||
| } from '../constants'; | ||
| import { PierianDxPipelineTableConfig } from '../../lib/workload/stateful/stacks/pieriandx-pipeline-dynamo-db/deploy'; | ||
| import { PierianDxPipelineManagerConfig } from '../../lib/workload/stateless/stacks/pieriandx-pipeline-manager/deploy'; | ||
|
|
||
| // Stateful | ||
| export const getPierianDxPipelineTableStackProps = (): PierianDxPipelineTableConfig => { | ||
| return { | ||
| dynamodbTableName: pieriandxDynamodbTable, | ||
| }; | ||
| }; | ||
|
|
||
| // Stateless | ||
| export const getPierianDxPipelineManagerStackProps = ( | ||
| stage: AppStage | ||
| ): PierianDxPipelineManagerConfig => { | ||
| return { | ||
| /* DynamoDB Table */ | ||
| dynamodbTableName: pieriandxDynamodbTable, | ||
| /* Workflow knowledge */ | ||
| workflowName: pieriandxWorkflowName, | ||
| workflowVersion: pieriandxWorkflowVersion, | ||
| /* Default values */ | ||
| defaultDagVersion: pieriandxDefaultDagName, | ||
| defaultPanelName: pieriandxDefaultPanelName, | ||
| /* Secrets */ | ||
| /* ICAv2 Pipeline analysis essentials */ | ||
| icav2AccessTokenSecretId: icav2AccessTokenSecretName[stage], // "/icav2/umccr-prod/service-production-jwt-token-secret-arn" | ||
| pieriandxS3AccessTokenSecretId: pieriandxS3CredentialsSecretsManagerId, // "/pieriandx/s3AccessCredentials" | ||
| /* SSM Parameters */ | ||
| dagSsmParameterPath: pieriandxDagSsmParameterPath, | ||
| panelNameSsmParameterPath: pieriandxPanelMapSsmParameterPath, | ||
| s3SequencerRunRootSsmParameterPath: pieriandxS3SequencerRunRootSsmParameterPath, | ||
| /* | ||
| Pieriandx specific parameters | ||
| */ | ||
| pieriandxUserEmailSsmParameterPath: pieriandxUserEmailSsmParameterPath, | ||
| pieriandxInstitutionSsmParameterPath: pieriandxInstitutionSsmParameterPath, | ||
| pieriandxBaseUrlSsmParameterPath: pieriandxBaseUrlSsmParameterPath, | ||
| pieriandxAuthTokenCollectionLambdaFunctionName: pieriandxAuthTokeSsmParameterPath, | ||
| /* Event info */ | ||
| eventDetailType: pieriandxDetailType, | ||
| eventBusName: eventBusName, | ||
| eventSource: pieriandxEventSource, | ||
| payloadVersion: pieriandxPayloadVersion, | ||
| triggerLaunchSource: pieriandxTriggerLaunchSource, | ||
| /* Custom */ | ||
| prefix: pieriandxPrefix, | ||
| }; | ||
| }; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Most of those are ...
WorkflowTypeand ...WorkflowTypeVersioncombinations,but there are some that are ...
WorkflowNameand ...WorkflowTypeVersionor...
WorkflowName, ...WorkflowVersionThe rename/update does not affect all of those?
E.g.
bsshFastqCopyThere was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Ah let me fix bsshFastqCopy too, bsshFastqCopy is not used in any output paths so got less attention but I think it would be good to fix for workflow run name consistency
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
btw: are we OK with
cttsov2?First, we are adding a version where the workflow definition has already a
versionfield. That feels redundant and confusing and may limit our future uses of that workflow (e.g. what if there's a new version 3.0.1?)Second, we've seen now multiple other places where the analysis and the sample type are separated, e.g. tso-ctdna. I wonder if we are limiting the possible use cases combining those into one term, e.g. could there be a tso-ctrna, a tso600-ctdna or others?
I just want to take this opportunity to rethink our workflow naming based on the experience we have now, not simply following the convenience of what we used so far.
I am not the best person to judge this though, so I'll follow the consensus
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Maybe a discussion on Slack where we can reach a consensus.