Merge pull request #336 from umccr/feature/update-to-bclconvert-manager

As this PR will not create any resource and change any resource, I will merge it first.
It will just fix a analysis id consistency issue in lambda and add non-succeeded event test in test case. Let's see....

lib/workload/stateless/stacks/bclconvert-manager/README.md

@@ -93,7 +93,7 @@ The AWS lambda functions put translated event following  ```WorkflowRunStateChan
     "portalRunId": '20xxxxxxxxxx',
     "executionId": "valid_payload_id",
     "timestamp": "2024-00-25T00:07:00Z",
-    "status": "INPROGRESS"
+    "status": "INPROGRESS",
     "workflowName": "BclConvert",
     "workflowVersion": "4.2.7",
     "workflowRunName": "123456_A1234_0000_TestingPattern",
@@ -121,7 +121,6 @@ The AWS lambda functions put translated event following  ```WorkflowRunStateChan
     "workflowVersion": "4.2.7",
     "workflowRunName": "540424_A01001_0193_BBBBMMDRX5_c754de_bd822f",
     "payload": {
-      "refId": null,
       "version": "0.1.0",
       "data": {
         "projectId": "bxxxxxxxx-dxxx-4xxxx-adcc-xxxxxxxxx",
@@ -172,7 +171,7 @@ Send the orignal input and orcabus event after translation to the dynamo db for
 
 Dynamodb format:
 | id | id_type | analysis_id | analysis_status | portal_run_id | db_uuid | original_external_event | translated_internal_ica_event | timestamp |
-| -------- | ---- | ------- | ------- | ---- | ------- | ------- | ------- |------- | 
+| -------- | ---- | ------- | ------- | ---- | ------- | ------- | ------- |------- |
 | 20xxxxxxxxxx | portal_run_id | dxxxxx-6xxxx-fxxx-1xxx-5xxxxx | | | 4xxxxx-4xxxx-4xxx-4xxx-4xxxxx | | | |
 | dxxxxx-6xxxx-fxxx-1xxx-5xxxxx | analysis_id |  |   | 20xxxxxxxxxx | 4xxxxx-4xxxx-4xxx-4xxx-4xxxxx | | | |
 | 1xxxxx-1xxxx-1xxx-1xxx-1xxxxx | db_uuid | dxxxxx-6xxxx-fxxx-1xxx-5xxxxx | INITIALIZING | 20xxxxxxxxxx | | {"correlationId": "",...} | {'portal_run_id': "",...} | 2024-01-01T00:11:35Z |

lib/workload/stateless/stacks/bclconvert-manager/translator_service/icav2_event_translator.py

@@ -153,12 +153,11 @@ def store_events_into_dynamodb(internal_ica_event, table_name, event_details) ->
             Item={
                 'id': {'S': db_uuid},
                 'id_type': {'S': 'db_uuid'},
-                'analysis_id': {'S': internal_ica_event.detail.payload.get('data', '').get("analysisId", '')},
-                'analysis_status': {'S': internal_ica_event.detail.status},
-                # 'SUCCEEDED', 'FAILED', 'ABORTED'
+                'analysis_id': {'S': event_details.get("payload", {}).get("id", '')},
+                'analysis_status': {'S': internal_ica_event.detail.status}, # 'SUCCEEDED', 'FAILED', 'ABORTED'
                 "portal_run_id": {'S': internal_ica_event.detail.portalRunId},
                 'original_external_event': {'S': json.dumps(event_details)},
-                'translated_internal_ica_event': {'S': json.dumps(internal_ica_event.detail.to_dict())},
+                'translated_internal_ica_event': {'S': json.dumps(WorkflowRunStateChangeMarshaller.marshall(internal_ica_event.detail))},
                 'timestamp': {'S': internal_ica_event.detail.timestamp}
             }
         )
@@ -168,7 +167,7 @@ def store_events_into_dynamodb(internal_ica_event, table_name, event_details) ->
         dynamodb.update_item(
             TableName=table_name,
             Key={
-                'id': {'S': internal_ica_event.detail.payload.get("data", '').get("analysisId", '')},
+                'id': {'S': event_details.get("payload", {}).get("id", '')},
                 'id_type': {'S': 'analysis_id'}
             },
             UpdateExpression='SET db_uuid = :db_uuid',

lib/workload/stateless/stacks/bclconvert-manager/translator_service/tests/test_icav2_event_translator.py

@@ -1,13 +1,25 @@
 import json
 import os
-import datetime 
 import unittest
 import botocore.session
 from botocore.stub import Stubber
 from unittest.mock import patch
 from freezegun import freeze_time
-from uuid import UUID
+
 '''
+assume output event is in the following format:
+(Non-SUCCEEDED Event without payload)
+{
+    "portalRunId": "2024010112345678",
+    "executionId": "valid_payload_id",
+    "timestamp": "2024-00-25T00:07:00Z",
+    "status": "FAILED", (Non-SUCCEEDED status)
+    "workflowName": "BclConvert",
+    "workflowVersion": "4.2.7",
+    workflowRunName: "123456_A1234_0000_TestingPattern",
+}
+
+(SUCCEEDED Event)
 {
     "portalRunId": "2024010112345678",
     "executionId": "valid_payload_id",
@@ -53,14 +65,14 @@ def mock_generate_db_uuid():
 
 class TestICAv2EventTranslator(unittest.TestCase):
     def setUp(self):
-        self.event = {
+        self.test_event = {
             "detail-type": "Test Event.",
             "detail": {
                 'ica-event': { 
                     "projectId": "valid_project_id",
                     "eventCode": "ICA_EXEC_028",
                     "eventParameters": {
-                        "analysisStatus": "SUCCEEDED"
+                        "analysisStatus": "UNSPECIFIED"
                     },
                     "payload": {
                         "id": "valid_payload_id",
@@ -111,8 +123,63 @@ def tearDown(self):
         patch.stopall()
         self.cleanup_environment_vars()
 
+    def test_succeeded_event(self):
+        self.setup_event("SUCCEEDED")
+        self.setup_expected_ica_event_details("SUCCEEDED")
+        self.run_event_test()
+
+    def test_progress_event(self):
+        self.setup_event("INPROGRESS")
+        self.setup_expected_ica_event_details("INPROGRESS")
+        self.run_event_test()
+
+    def setup_event(self, status):
+        """
+        Set up expected ICA event details based on the given status.
+    
+        :param status: A string representing the status of the event ('SUCCEEDED' or any non-success status).
+        """
+        self.test_event['detail']['ica-event']['eventParameters']['analysisStatus'] = status
+
+    def setup_expected_ica_event_details(self, status):
+        """
+        Set up expected ICA event details based on the given status.
+    
+        :param status: A string representing the status of the event ('SUCCEEDED' or any non-success status).
+        """
+        # Basic event details that are common to all statuses
+        self.expected_ica_event_details = {
+            "portalRunId": '2024010112345678',
+            "executionId": "valid_payload_id",
+            "timestamp": "2024-01-01T00:00:00Z",
+            "status": status,
+            "workflowName": "BclConvert",
+            "workflowVersion": "0.0.0",
+            "workflowRunName": "123456_A1234_0000_TestingPattern"
+        }
+
+    # Add payload details only if the status is "SUCCEEDED"
+        if status == "SUCCEEDED":
+            self.expected_ica_event_details["payload"] = {
+                "version": "0.1.0",
+                "data": {
+                    "projectId": "valid_project_id",
+                    "analysisId": "valid_payload_id",
+                    "userReference": "123456_A1234_0000_TestingPattern",
+                    "timeCreated": "2024-01-01T00:11:35Z",
+                    "timeModified": "2024-01-01T01:24:29Z",
+                    "pipelineId": "valid_pipeline_id",
+                    "pipelineCode": "BclConvert v0_0_0",
+                    "pipelineDescription": "This is an test autolaunch BclConvert pipeline.",
+                    "pipelineUrn": "urn:ilmn:ica:pipeline:123456-abcd-efghi-1234-acdefg1234a#BclConvert_v0_0_0",
+                    "instrumentRunId": "12345_A12345_1234_ABCDE12345",
+                    "basespaceRunId": "1234567",
+                    "samplesheetB64gz": "test_samplesheet_b64gz"
+                }
+            }
+
     @freeze_time("2024-01-1")
-    def test_valid_events_handler(self):
+    def run_event_test(self):
         """
         Arrange: stub responses in the order they are expected to be called
         1. dynamodb.query
@@ -153,32 +220,7 @@ def test_valid_events_handler(self):
 
 
         # expected internal event
-        expected_ica_event_details = {
-            "portalRunId": '2024010112345678',
-            "executionId": "valid_payload_id",
-            "timestamp": datetime.datetime.now(datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'),
-            "status": "SUCCEEDED",
-            "workflowName":"BclConvert",
-            "workflowVersion": "0.0.0",
-            "workflowRunName": "123456_A1234_0000_TestingPattern",
-            "payload": {
-                "version": "0.1.0",
-                "data": {
-                    "projectId": "valid_project_id",
-                    "analysisId": "valid_payload_id",
-                    "userReference": "123456_A1234_0000_TestingPattern",
-                    "timeCreated": "2024-01-01T00:11:35Z",
-                    "timeModified": "2024-01-01T01:24:29Z",
-                    "pipelineId": "valid_pipeline_id",
-                    "pipelineCode": "BclConvert v0_0_0",
-                    "pipelineDescription": "This is an test autolaunch BclConvert pipeline.",
-                    "pipelineUrn": "urn:ilmn:ica:pipeline:123456-abcd-efghi-1234-acdefg1234a#BclConvert_v0_0_0",
-                    "instrumentRunId": "12345_A12345_1234_ABCDE12345",
-                    "basespaceRunId": "1234567",
-                    "samplesheetB64gz": "test_samplesheet_b64gz"
-                }
-            }
-        }
+        expected_ica_event_details = self.expected_ica_event_details
         # Mock the response and setup stubbing
         response = {}
         expected_params = {
@@ -197,10 +239,10 @@ def test_valid_events_handler(self):
         expected_item = {
             'id': {'S': '12345678-1234-5678-1234-567812345678'},
             'id_type': {'S': 'db_uuid'},
-            'analysis_id': {'S': expected_ica_event_details['payload']['data']['analysisId']},
+            'analysis_id': {'S': "valid_payload_id"},
             'analysis_status': {'S': expected_ica_event_details['status']},
             "portal_run_id": {'S': expected_ica_event_details['portalRunId']},
-            'original_external_event': {'S': json.dumps(self.event['detail']['ica-event'])},
+            'original_external_event': {'S': json.dumps(self.test_event['detail']['ica-event'])},
             'translated_internal_ica_event': {'S': json.dumps(expected_ica_event_details)},
             'timestamp': {'S': expected_ica_event_details['timestamp']}
         }
@@ -237,7 +279,7 @@ def test_valid_events_handler(self):
         self.dynamodb_stubber.add_response('update_item', response, expected_params)
 
         # Execute the handler
-        response = handler(self.event, None)
+        response = handler(self.test_event, None)
 
         # Assertions to check handler response and if the stubs were called correctly
         self.assertEqual(response['statusCode'], 200)
@@ -248,28 +290,28 @@ def test_missing_environment_variables(self):
         # Remove environment variable to test error handling
         del os.environ['EVENT_BUS_NAME']
         with self.assertRaises(AssertionError):
-            handler(self.event, None)
+            handler(self.test_event, None)
         # Reset for next test
         os.environ['EVENT_BUS_NAME'] = 'test_event_bus'
 
         # Remove environment variable to test error handling
         del os.environ['TABLE_NAME']
         with self.assertRaises(AssertionError):
-            handler(self.event, None)
+            handler(self.test_event, None)
         # Reset for next test
         os.environ['TABLE_NAME'] = 'test_table'
 
         # Test missing ICAV2_BASE_URL
         del os.environ['ICAV2_BASE_URL']
         with self.assertRaises(AssertionError):
-            handler({}, None)
+            handler(self.test_event, None)
         # Reset for next test
         os.environ['ICAV2_BASE_URL'] = 'https://test.com'
 
         # Test missing ICAV2_ACCESS_TOKEN_SECRET_ID
         del os.environ['ICAV2_ACCESS_TOKEN_SECRET_ID']
         with self.assertRaises(AssertionError):
-            handler({}, None)
+            handler(self.test_event, None)
         # Reset for next test
         os.environ['ICAV2_ACCESS_TOKEN_SECRET_ID'] = 'test_secret_id'