Merge pull request #708 from umccr/feature/add-holmes-glue-service

Add holmes glue

config/constants.ts

@@ -207,8 +207,8 @@ export const bclconvertInteropQcStateMachinePrefix = 'bclconvertInteropQcSfn';
 Resources used by the bclConvert InteropQc Pipeline
 */
 
-// Release can be found here: https://github.com/umccr/cwl-ica/releases/tag/bclconvert-interop-qc%2F1.3.1--1.21__20240627051309
-// Pipeline ID is: 35cae57c-8895-4814-ae89-db4b5e9668b2
+// Release can be found here: https://github.com/umccr/cwl-ica/releases/tag/bclconvert-interop-qc%2F1.3.1--1.21__20241119001529
+// Pipeline ID is: a147ad9f-af8f-409d-95b7-49018782ab4d
 export const bclconvertInteropQcIcav2PipelineIdSSMParameterPath =
   '/icav2/umccr-prod/bclconvert_interop_qc_pipeline_id';
 

lib/workload/components/python-lambda-get-workflow-payload/get_workflow_payload_py/get_workflow_payload.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+
+"""
+Given either a portal run id or a orcabus workflow id, and a workflow status, return the payload of the workflow at that state
+"""
+from workflow_tools.utils.payload_helpers import get_payload
+from workflow_tools.utils.workflow_run_helpers import get_workflow_run_from_portal_run_id, get_workflow_run, \
+    get_workflow_run_state
+
+
+def handler(event, context):
+    """
+    Takes in either (portal_run_id or orcabus_workflow_id) and workflow_status
+
+    Gets the workflow payload at that state
+
+    Returns the payload object
+    :param event:
+    :param context:
+    :return:
+    """
+
+    if event.get("portal_run_id", None) is not None:
+        # Get the workflow object from the portal run id
+        workflow_obj = get_workflow_run_from_portal_run_id(event.get("portal_run_id"))
+    elif event.get("orcabus_workflow_id", None) is not None:
+        # Get the workflow object from the orcabus workflow id
+        workflow_obj = get_workflow_run(event.get("orcabus_workflow_id"))
+    else:
+        raise ValueError("Must provide either portal_run_id or orcabus_workflow_id")
+
+    status = event.get("workflow_status", None)
+
+    if status is None:
+        raise ValueError("Must provide a workflow_status, i.e 'READY' or 'SUCCEEDED'")
+
+    # Get the READY run state
+    workflow_ready_run_state_obj = get_workflow_run_state(workflow_obj.get("orcabusId"), status)
+    # Get the payload from the READY run state
+    return {
+      "payload": get_payload(workflow_ready_run_state_obj.get("payload"))
+    }
+
+
+# if __name__ == "__main__":
+#     import json
+#     from os import environ
+#     environ['ORCABUS_TOKEN_SECRET_ID'] = 'orcabus/token-service-jwt'
+#     environ['HOSTNAME_SSM_PARAMETER'] = '/hosted_zone/umccr/name'
+#     environ['AWS_PROFILE'] = 'umccr-development'
+#     print(
+#         json.dumps(
+#             handler(
+#                 {
+#                     "portal_run_id": "202411071a2c31a3",
+#                     "workflow_status": "SUCCEEDED"
+#                 },
+#                 None
+#             ),
+#             indent=4
+#         ),
+#     )
+#
+#     # {
+#     #     "payload": {
+#     #         "orcabusId": "pld.01JC28FD1GSVPDKH322NQKTYHR",
+#     #         "payloadRefId": "2b5a3f90-4da5-4183-be2d-a88df6c21f1b",
+#     #         "version": "2024.07.01",
+#     #         "data": {
+#     #             "tags": {
+#     #                 "libraryId": "L2401547",
+#     #                 "sampleType": "WGS",
+#     #                 "fastqListRowId": "GACCTGAA.CTCACCAA.3.241024_A00130_0336_BHW7MVDSXC.L2401547",
+#     #                 "instrumentRunId": "241024_A00130_0336_BHW7MVDSXC"
+#     #             },
+#     #             "inputs": {
+#     #                 "sampleType": "WGS",
+#     #                 "fastqListRow": {
+#     #                     "lane": 3,
+#     #                     "rgid": "GACCTGAA.CTCACCAA.3.241024_A00130_0336_BHW7MVDSXC.L2401547",
+#     #                     "rglb": "L2401547",
+#     #                     "rgsm": "L2401547",
+#     #                     "read1FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/241024_A00130_0336_BHW7MVDSXC/20241030c613872c/Samples/Lane_3/L2401547/L2401547_S16_L003_R1_001.fastq.gz",
+#     #                     "read2FileUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary/241024_A00130_0336_BHW7MVDSXC/20241030c613872c/Samples/Lane_3/L2401547/L2401547_S16_L003_R2_001.fastq.gz"
+#     #                 },
+#     #                 "outputPrefix": "L2401547",
+#     #                 "fastqListRowId": "GACCTGAA.CTCACCAA.3.241024_A00130_0336_BHW7MVDSXC.L2401547",
+#     #                 "dragenReferenceVersion": "v9-r3"
+#     #             },
+#     #             "outputs": {
+#     #                 "multiqcOutputUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/wgts-qc/202411071a2c31a3/L2401547_dragen_alignment_multiqc/",
+#     #                 "multiqcHtmlReportUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/wgts-qc/202411071a2c31a3/L2401547_dragen_alignment_multiqc/L2401547_dragen_alignment_multiqc.html",
+#     #                 "dragenAlignmentBamUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/wgts-qc/202411071a2c31a3/L2401547_dragen_alignment/L2401547.bam",
+#     #                 "dragenAlignmentOutputUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/wgts-qc/202411071a2c31a3/L2401547_dragen_alignment/"
+#     #             },
+#     #             "engineParameters": {
+#     #                 "logsUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/logs/wgts-qc/202411071a2c31a3/",
+#     #                 "cacheUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/cache/wgts-qc/202411071a2c31a3/",
+#     #                 "outputUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/wgts-qc/202411071a2c31a3/",
+#     #                 "projectId": "ea19a3f5-ec7c-4940-a474-c31cd91dbad4",
+#     #                 "analysisId": "0f2959fa-880a-4fa2-9908-8d12e3c998d5",
+#     #                 "pipelineId": "03689516-b7f8-4dca-bba9-8405b85fae45"
+#     #             }
+#     #         }
+#     #     }
+#     # }

lib/workload/components/python-lambda-get-workflow-payload/index.ts

@@ -0,0 +1,94 @@
+/*
+Quick and dirty way to map orcabus ids to complementary ids for each
+of the databases from the metadata manager
+
+Comes with the bells and whistles of metadata tools layer and
+permissions to use the orcabus token.
+
+User will need to use the 'addEnvironment' method on the returned lambda object in order
+to specify what command will be run
+
+User will need
+
+ENV:
+CONTEXT: One of the following:
+  - library
+  - subject
+  - individual
+  - sample
+  - project
+  - contact
+
+FROM_ORCABUS or FROM_ID
+RETURN_STR or RETURN_OBJ
+
+Look at ./map_metadata_py/map_metadata.py for examples of outputs
+
+*/
+
+import { Construct } from 'constructs';
+import * as lambda from 'aws-cdk-lib/aws-lambda';
+import { PythonFunction } from '@aws-cdk/aws-lambda-python-alpha';
+import path from 'path';
+import * as ssm from 'aws-cdk-lib/aws-ssm';
+import * as secretsmanager from 'aws-cdk-lib/aws-secretsmanager';
+import { Duration } from 'aws-cdk-lib';
+import { WorkflowToolsPythonLambdaLayer } from '../python-workflow-tools-layer';
+
+interface GetWorkflowPayloadLambdaObj {
+  functionNamePrefix: string;
+}
+
+export class GetWorkflowPayloadLambdaConstruct extends Construct {
+  public readonly lambdaObj: PythonFunction;
+
+  // Globals
+  private readonly hostnameSsmParameterPath = '/hosted_zone/umccr/name';
+  private readonly orcabusTokenSecretId = 'orcabus/token-service-jwt'; // pragma: allowlist secret
+
+  constructor(scope: Construct, id: string, props: GetWorkflowPayloadLambdaObj) {
+    super(scope, id);
+
+    // Get the metadata layer object
+    const workflowToolsLayer = new WorkflowToolsPythonLambdaLayer(this, 'workflow-tools-layer', {
+      layerPrefix: `${props.functionNamePrefix}-wtl`,
+    });
+
+    /*
+    Collect the required secret and ssm parameters for getting metadata
+    */
+    const hostnameSsmParameterObj = ssm.StringParameter.fromStringParameterName(
+      this,
+      'hostname_ssm_parameter',
+      this.hostnameSsmParameterPath
+    );
+    const orcabusTokenSecretObj = secretsmanager.Secret.fromSecretNameV2(
+      this,
+      'orcabus_token_secret',
+      this.orcabusTokenSecretId
+    );
+
+    // Get library objects
+    this.lambdaObj = new PythonFunction(this, 'get_workflow_payload_py', {
+      functionName: `${props.functionNamePrefix}-get-workflow-payload-py`,
+      entry: path.join(__dirname, 'get_workflow_payload_py'),
+      runtime: lambda.Runtime.PYTHON_3_12,
+      architecture: lambda.Architecture.ARM_64,
+      index: 'get_workflow_payload.py',
+      handler: 'handler',
+      memorySize: 1024,
+      layers: [workflowToolsLayer.lambdaLayerVersionObj],
+      environment: {
+        HOSTNAME_SSM_PARAMETER: hostnameSsmParameterObj.parameterName,
+        ORCABUS_TOKEN_SECRET_ID: orcabusTokenSecretObj.secretName,
+      },
+      timeout: Duration.seconds(60),
+    });
+
+    // Allow the lambda to read the secret
+    orcabusTokenSecretObj.grantRead(this.lambdaObj.currentVersion);
+
+    // Allow the lambda to read the ssm parameter
+    hostnameSsmParameterObj.grantRead(this.lambdaObj.currentVersion);
+  }
+}

lib/workload/components/python-lambda-list-service-instances/index.ts

@@ -0,0 +1,54 @@
+import { Construct } from 'constructs';
+import * as lambda from 'aws-cdk-lib/aws-lambda';
+import * as iam from 'aws-cdk-lib/aws-iam';
+import { PythonFunction } from '@aws-cdk/aws-lambda-python-alpha';
+import path from 'path';
+import { NagSuppressions } from 'cdk-nag';
+
+interface LambdaDiscoverInstancesProps {
+  functionNamePrefix: string;
+}
+
+export class LambdaDiscoverInstancesConstruct extends Construct {
+  public readonly lambdaObj: PythonFunction;
+
+  constructor(scope: Construct, id: string, props: LambdaDiscoverInstancesProps) {
+    super(scope, id);
+
+    // ServiceDiscovery lambda
+    this.lambdaObj = new PythonFunction(this, 'list_service_instances_py', {
+      functionName: `${props.functionNamePrefix}-list-service-instances`,
+      entry: path.join(__dirname, 'list_service_instances_py'),
+      runtime: lambda.Runtime.PYTHON_3_12,
+      architecture: lambda.Architecture.ARM_64,
+      index: 'list_service_instances.py',
+      handler: 'handler',
+      memorySize: 1024,
+    });
+
+    // Grant permissions to the lambda
+    this.lambdaObj.addToRolePolicy(
+      new iam.PolicyStatement({
+        actions: [
+          'servicediscovery:ListInstances',
+          'servicediscovery:DiscoverInstances',
+          'servicediscovery:GetService',
+        ],
+        resources: ['*'],
+      })
+    );
+
+    // Suppress CDK NAGs
+    NagSuppressions.addResourceSuppressions(
+      this.lambdaObj,
+      [
+        {
+          id: 'AwsSolutions-IAM5',
+          reason:
+            'Need to run the DiscoverInstances against all services since we dont know which one will be used',
+        },
+      ],
+      true
+    );
+  }
+}

lib/workload/components/python-lambda-list-service-instances/list_service_instances_py/list_service_instances.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+
+"""
+Use the boto3 api to list a service's instances
+"""
+
+import typing
+import boto3
+from os import environ
+
+if typing.TYPE_CHECKING:
+    from mypy_boto3_servicediscovery import ServiceDiscoveryClient
+
+
+def get_service_discovery_client() -> 'ServiceDiscoveryClient':
+    return boto3.client('servicediscovery')
+
+
+def handler(event, context):
+    """
+
+    :return:
+    """
+    service_id = event["service_id"]
+
+    service_discovery_client = get_service_discovery_client()
+
+    service_instances = list(
+        map(
+            lambda service_instance_iter_: {
+              "instance_id": service_instance_iter_['Id'],
+              "instance_attributes": list(
+                  map(
+                    lambda instance_kv_iter_: {
+                        "attr_key": instance_kv_iter_[0],
+                        "attr_value": instance_kv_iter_[1],
+                    },
+                    service_instance_iter_['Attributes'].items()
+                  )
+              ),
+            },
+            service_discovery_client.list_instances(ServiceId=service_id)['Instances']
+        )
+    )
+
+    return {
+        "service_instances": service_instances
+    }
+
+
+# if __name__ == "__main__":
+#     import json
+#     environ['AWS_PROFILE'] = 'umccr-development'
+#     print(
+#         json.dumps(
+#             handler(
+#                 {
+#                     "service_id": "srv-zuvfka3fyqxswwme"
+#                 },
+#                 None
+#             ),
+#             indent=4
+#         )
+#
+#     )
+#
+#     # {
+#     #     "service_instances": [
+#     #         {
+#     #             "instance_id": "HolmesLocalDevTestStackServiceNonIpD37CD7BB",
+#     #             "instance_attributes": [
+#     #                 {
+#     #                     "attr_key": "checkLambdaArn",
+#     #                     "attr_value": "arn:aws:lambda:ap-southeast-2:843407916570:function:HolmesLocalDevTestStack-CheckFunction82225D96-gtKl1C5USxIO"
+#     #                 },
+#     #                 {
+#     #                     "attr_key": "controlLambdaArn",
+#     #                     "attr_value": "arn:aws:lambda:ap-southeast-2:843407916570:function:HolmesLocalDevTestStack-ControlFunction8A0764F3-zunc6vUJbVGz"
+#     #                 },
+#     #                 {
+#     #                     "attr_key": "extractStepsArn",
+#     #                     "attr_value": "arn:aws:states:ap-southeast-2:843407916570:stateMachine:SomalierExtractStateMachine59E102CC-CEqwe36xUrru"
+#     #                 },
+#     #                 {
+#     #                     "attr_key": "listLambdaArn",
+#     #                     "attr_value": "arn:aws:lambda:ap-southeast-2:843407916570:function:HolmesLocalDevTestStack-ListFunction89E6AFAD-WXt699CK8CMD"
+#     #                 },
+#     #                 {
+#     #                     "attr_key": "relateLambdaArn",
+#     #                     "attr_value": "arn:aws:lambda:ap-southeast-2:843407916570:function:HolmesLocalDevTestStack-RelateFunction666B2CBC-2KKtCnoqHQ5F"
+#     #                 }
+#     #             ]
+#     #         }
+#     #     ]
+#     # }

lib/workload/components/python-lambda-metadata-mapper/index.ts

@@ -51,7 +51,7 @@ export class GetMetadataLambdaConstruct extends Construct {
 
     // Get the metadata layer object
     const metadataLayerObj = new MetadataToolsPythonLambdaLayer(this, 'metadata-tools-layer', {
-      layerPrefix: 'get-library-objects',
+      layerPrefix: `${props.functionNamePrefix}-mtl`,
     });
 
     /*