From 3ee974ba4d739a30a963ae5fe6cd58b218ecc9a9 Mon Sep 17 00:00:00 2001 From: Ray Liu Date: Mon, 9 Sep 2024 12:02:39 +1000 Subject: [PATCH 1/8] add related name for each relationship and modify serialize and model to reverse query --- .../workflow_manager/models/state.py | 2 +- .../workflow_manager/models/workflow_run.py | 13 ++++++++++--- .../workflow_manager/serializers.py | 9 +++++++++ .../workflow_manager/viewsets/workflow_run.py | 2 +- 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/state.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/state.py index de8c607a5..5aeaa9f93 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/state.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/state.py @@ -92,7 +92,7 @@ class Meta: id = models.BigAutoField(primary_key=True) # --- mandatory fields - workflow_run = models.ForeignKey(WorkflowRun, on_delete=models.CASCADE) + workflow_run = models.ForeignKey(WorkflowRun, related_name='states', on_delete=models.CASCADE) status = models.CharField(max_length=255) # TODO: How and where to enforce conventions? timestamp = models.DateTimeField() diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow_run.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow_run.py index 41eb0b7fd..e0f31c340 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow_run.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow_run.py @@ -29,7 +29,7 @@ class WorkflowRun(OrcaBusBaseModel): workflow = models.ForeignKey(Workflow, null=True, blank=True, on_delete=models.SET_NULL) # Link to library table - libraries = models.ManyToManyField(Library, through="LibraryAssociation") + # libraries = models.ManyToManyField(Library, through="LibraryAssociation") objects = WorkflowRunManager() @@ -51,14 +51,21 @@ def get_all_states(self): # retrieve all states (DB records rather than a queryset) return list(self.state_set.all()) # TODO: ensure order by timestamp ? + def get_latest_state(self): + # retrieve all related states and get the latest one + return self.states.order_by('-timestamp').first() + + def get_libraries(self): + # retrieve all related libraries objects + return Library.objects.filter(workflow_run_association__workflow_run=self).distinct() class LibraryAssociationManager(OrcaBusBaseManager): pass class LibraryAssociation(OrcaBusBaseModel): - workflow_run = models.ForeignKey(WorkflowRun, on_delete=models.CASCADE) - library = models.ForeignKey(Library, on_delete=models.CASCADE) + workflow_run = models.ForeignKey(WorkflowRun, related_name="library_association", on_delete=models.CASCADE) + library = models.ForeignKey(Library, related_name="workflow_run_association", on_delete=models.CASCADE) association_date = models.DateTimeField() status = models.CharField(max_length=255) diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers.py index da6ba63f0..637410f8b 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers.py @@ -40,10 +40,19 @@ class Meta: class WorkflowRunModelSerializer(serializers.ModelSerializer): + current_state = serializers.SerializerMethodField() + libraries = serializers.SerializerMethodField() class Meta: model = WorkflowRun fields = '__all__' + def get_current_state(self, obj)->dict: + latest_state = obj.get_latest_state() + return StateModelSerializer(latest_state).data if latest_state else None + + def get_libraries(self, obj)->list: + libraries = obj.get_libraries() + return [library.library_id for library in libraries] class PayloadModelSerializer(serializers.ModelSerializer): class Meta: diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run.py index 8a3b1d1d4..516f4f27a 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run.py @@ -17,7 +17,7 @@ class WorkflowRunViewSet(ReadOnlyModelViewSet): search_fields = WorkflowRun.get_base_fields() def get_queryset(self): - return WorkflowRun.objects.get_by_keyword(**self.request.query_params) + return WorkflowRun.objects.get_by_keyword(**self.request.query_params).prefetch_related('states').prefetch_related('library_association__library') # add prefetch_related to reduce the number of queries @action(detail=False, methods=['GET']) def ongoing(self, request): From 04402c2c3f3151f8f5dc36291c3f140f560c643e Mon Sep 17 00:00:00 2001 From: Ray Liu Date: Mon, 9 Sep 2024 12:42:26 +1000 Subject: [PATCH 2/8] integrate workflow run type info in the model --- .../stacks/workflow-manager/workflow_manager/serializers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers.py index 637410f8b..5697424fd 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers.py @@ -42,6 +42,7 @@ class Meta: class WorkflowRunModelSerializer(serializers.ModelSerializer): current_state = serializers.SerializerMethodField() libraries = serializers.SerializerMethodField() + workflow = WorkflowModelSerializer(read_only=True) class Meta: model = WorkflowRun fields = '__all__' From b35b6f42df4be0a4b1e3bf803e7ab7ca34c220d3 Mon Sep 17 00:00:00 2001 From: Ray Liu Date: Mon, 9 Sep 2024 12:58:35 +1000 Subject: [PATCH 3/8] add select query for workflow run to reduce query --- .../workflow-manager/workflow_manager/viewsets/workflow_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run.py index 516f4f27a..444c791f9 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run.py @@ -17,7 +17,7 @@ class WorkflowRunViewSet(ReadOnlyModelViewSet): search_fields = WorkflowRun.get_base_fields() def get_queryset(self): - return WorkflowRun.objects.get_by_keyword(**self.request.query_params).prefetch_related('states').prefetch_related('library_association__library') # add prefetch_related to reduce the number of queries + return WorkflowRun.objects.get_by_keyword(**self.request.query_params).prefetch_related('states').prefetch_related('library_association__library').select_related('workflow') # add prefetch_related & select_related to reduce the number of queries @action(detail=False, methods=['GET']) def ongoing(self, request): From 5fce7a72dbc05c8e125909c23c98653c36671cf7 Mon Sep 17 00:00:00 2001 From: Ray Liu Date: Mon, 9 Sep 2024 14:42:14 +1000 Subject: [PATCH 4/8] modify test function for new workflowrun model --- .../workflow_manager/models/workflow_run.py | 2 +- .../tests/test_create_workflow_run_state.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow_run.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow_run.py index e0f31c340..943b424c1 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow_run.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow_run.py @@ -49,7 +49,7 @@ def to_dict(self): def get_all_states(self): # retrieve all states (DB records rather than a queryset) - return list(self.state_set.all()) # TODO: ensure order by timestamp ? + return list(self.states.all()) # TODO: ensure order by timestamp ? def get_latest_state(self): # retrieve all related states and get the latest one diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager_proc/tests/test_create_workflow_run_state.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager_proc/tests/test_create_workflow_run_state.py index 33cc9a9ca..52b278ed2 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager_proc/tests/test_create_workflow_run_state.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager_proc/tests/test_create_workflow_run_state.py @@ -56,7 +56,7 @@ def test_create_wrsc_no_library(self): db_wfr: WorkflowRun = wfr_qs.first() self.assertEqual("ctTSO500-L000002", db_wfr.workflow_run_name) # We don't expect any library associations here! - self.assertEqual(0, db_wfr.libraries.count()) + self.assertEqual(0, db_wfr.get_libraries().count()) def test_create_wrsc_library(self): """ @@ -116,8 +116,8 @@ def test_create_wrsc_library(self): db_wfr: WorkflowRun = wfr_qs.first() self.assertEqual("ctTSO500-L000002", db_wfr.workflow_run_name) # We do expect 2 library associations here! - self.assertEqual(2, db_wfr.libraries.count()) - for lib in db_wfr.libraries.all(): + self.assertEqual(2, db_wfr.get_libraries().count()) + for lib in db_wfr.get_libraries().all(): self.assertTrue(lib.library_id in library_ids) def test_create_wrsc_library_exists(self): @@ -184,8 +184,8 @@ def test_create_wrsc_library_exists(self): db_wfr: WorkflowRun = wfr_qs.first() self.assertEqual("ctTSO500-L000002", db_wfr.workflow_run_name) # We do expect 2 library associations here! - self.assertEqual(2, db_wfr.libraries.count()) - for lib in db_wfr.libraries.all(): + self.assertEqual(2, db_wfr.get_libraries().count()) + for lib in db_wfr.get_libraries().all(): self.assertTrue(lib.library_id in library_ids) def test_get_last_state(self): From d867c2f26dcadff710c8b59b238ddc35d95c0035 Mon Sep 17 00:00:00 2001 From: Ray Liu Date: Mon, 9 Sep 2024 15:19:50 +1000 Subject: [PATCH 5/8] add modification migration --- ...2_remove_workflowrun_libraries_and_more.py | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0002_remove_workflowrun_libraries_and_more.py diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0002_remove_workflowrun_libraries_and_more.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0002_remove_workflowrun_libraries_and_more.py new file mode 100644 index 000000000..096cc4be8 --- /dev/null +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0002_remove_workflowrun_libraries_and_more.py @@ -0,0 +1,45 @@ +# Generated by Django 5.1 on 2024-09-09 05:16 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("workflow_manager", "0001_initial"), + ] + + operations = [ + migrations.RemoveField( + model_name="workflowrun", + name="libraries", + ), + migrations.AlterField( + model_name="libraryassociation", + name="library", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="workflow_run_association", + to="workflow_manager.library", + ), + ), + migrations.AlterField( + model_name="libraryassociation", + name="workflow_run", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="library_association", + to="workflow_manager.workflowrun", + ), + ), + migrations.AlterField( + model_name="state", + name="workflow_run", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="states", + to="workflow_manager.workflowrun", + ), + ), + ] From 4c50acff01d5ced2e41c941b5167642c82106b2b Mon Sep 17 00:00:00 2001 From: Ray Liu Date: Mon, 9 Sep 2024 16:01:04 +1000 Subject: [PATCH 6/8] revert libraries attribute back for refernece of both orcabusid and libraryid --- .../0002_alter_state_workflow_run.py | 23 ++++++++++ ...2_remove_workflowrun_libraries_and_more.py | 45 ------------------- .../workflow_manager/models/workflow_run.py | 10 ++--- .../workflow_manager/serializers.py | 16 +++---- .../workflow_manager/viewsets/workflow_run.py | 2 +- .../tests/test_create_workflow_run_state.py | 10 ++--- 6 files changed, 38 insertions(+), 68 deletions(-) create mode 100644 lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0002_alter_state_workflow_run.py delete mode 100644 lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0002_remove_workflowrun_libraries_and_more.py diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0002_alter_state_workflow_run.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0002_alter_state_workflow_run.py new file mode 100644 index 000000000..442c0b3dd --- /dev/null +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0002_alter_state_workflow_run.py @@ -0,0 +1,23 @@ +# Generated by Django 5.1 on 2024-09-09 05:55 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("workflow_manager", "0001_initial"), + ] + + operations = [ + migrations.AlterField( + model_name="state", + name="workflow_run", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="states", + to="workflow_manager.workflowrun", + ), + ), + ] diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0002_remove_workflowrun_libraries_and_more.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0002_remove_workflowrun_libraries_and_more.py deleted file mode 100644 index 096cc4be8..000000000 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0002_remove_workflowrun_libraries_and_more.py +++ /dev/null @@ -1,45 +0,0 @@ -# Generated by Django 5.1 on 2024-09-09 05:16 - -import django.db.models.deletion -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ("workflow_manager", "0001_initial"), - ] - - operations = [ - migrations.RemoveField( - model_name="workflowrun", - name="libraries", - ), - migrations.AlterField( - model_name="libraryassociation", - name="library", - field=models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - related_name="workflow_run_association", - to="workflow_manager.library", - ), - ), - migrations.AlterField( - model_name="libraryassociation", - name="workflow_run", - field=models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - related_name="library_association", - to="workflow_manager.workflowrun", - ), - ), - migrations.AlterField( - model_name="state", - name="workflow_run", - field=models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - related_name="states", - to="workflow_manager.workflowrun", - ), - ), - ] diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow_run.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow_run.py index 943b424c1..3f1b6fd03 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow_run.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow_run.py @@ -29,7 +29,7 @@ class WorkflowRun(OrcaBusBaseModel): workflow = models.ForeignKey(Workflow, null=True, blank=True, on_delete=models.SET_NULL) # Link to library table - # libraries = models.ManyToManyField(Library, through="LibraryAssociation") + libraries = models.ManyToManyField(Library, through="LibraryAssociation") objects = WorkflowRunManager() @@ -54,18 +54,14 @@ def get_all_states(self): def get_latest_state(self): # retrieve all related states and get the latest one return self.states.order_by('-timestamp').first() - - def get_libraries(self): - # retrieve all related libraries objects - return Library.objects.filter(workflow_run_association__workflow_run=self).distinct() class LibraryAssociationManager(OrcaBusBaseManager): pass class LibraryAssociation(OrcaBusBaseModel): - workflow_run = models.ForeignKey(WorkflowRun, related_name="library_association", on_delete=models.CASCADE) - library = models.ForeignKey(Library, related_name="workflow_run_association", on_delete=models.CASCADE) + workflow_run = models.ForeignKey(WorkflowRun, on_delete=models.CASCADE) + library = models.ForeignKey(Library, on_delete=models.CASCADE) association_date = models.DateTimeField() status = models.CharField(max_length=255) diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers.py index 5697424fd..8de594ee3 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers.py @@ -39,9 +39,14 @@ class Meta: fields = '__all__' +class LibraryModelSerializer(serializers.ModelSerializer): + class Meta: + model = Library + fields = '__all__' + class WorkflowRunModelSerializer(serializers.ModelSerializer): current_state = serializers.SerializerMethodField() - libraries = serializers.SerializerMethodField() + libraries = LibraryModelSerializer(many=True, read_only=True) workflow = WorkflowModelSerializer(read_only=True) class Meta: model = WorkflowRun @@ -50,10 +55,6 @@ class Meta: def get_current_state(self, obj)->dict: latest_state = obj.get_latest_state() return StateModelSerializer(latest_state).data if latest_state else None - - def get_libraries(self, obj)->list: - libraries = obj.get_libraries() - return [library.library_id for library in libraries] class PayloadModelSerializer(serializers.ModelSerializer): class Meta: @@ -61,11 +62,6 @@ class Meta: fields = '__all__' -class LibraryModelSerializer(serializers.ModelSerializer): - class Meta: - model = Library - fields = '__all__' - class StateModelSerializer(serializers.ModelSerializer): class Meta: diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run.py index 444c791f9..83071da07 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run.py @@ -17,7 +17,7 @@ class WorkflowRunViewSet(ReadOnlyModelViewSet): search_fields = WorkflowRun.get_base_fields() def get_queryset(self): - return WorkflowRun.objects.get_by_keyword(**self.request.query_params).prefetch_related('states').prefetch_related('library_association__library').select_related('workflow') # add prefetch_related & select_related to reduce the number of queries + return WorkflowRun.objects.get_by_keyword(**self.request.query_params).prefetch_related('states').prefetch_related('libraries').select_related('workflow') # add prefetch_related & select_related to reduce the number of queries @action(detail=False, methods=['GET']) def ongoing(self, request): diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager_proc/tests/test_create_workflow_run_state.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager_proc/tests/test_create_workflow_run_state.py index 52b278ed2..33cc9a9ca 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager_proc/tests/test_create_workflow_run_state.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager_proc/tests/test_create_workflow_run_state.py @@ -56,7 +56,7 @@ def test_create_wrsc_no_library(self): db_wfr: WorkflowRun = wfr_qs.first() self.assertEqual("ctTSO500-L000002", db_wfr.workflow_run_name) # We don't expect any library associations here! - self.assertEqual(0, db_wfr.get_libraries().count()) + self.assertEqual(0, db_wfr.libraries.count()) def test_create_wrsc_library(self): """ @@ -116,8 +116,8 @@ def test_create_wrsc_library(self): db_wfr: WorkflowRun = wfr_qs.first() self.assertEqual("ctTSO500-L000002", db_wfr.workflow_run_name) # We do expect 2 library associations here! - self.assertEqual(2, db_wfr.get_libraries().count()) - for lib in db_wfr.get_libraries().all(): + self.assertEqual(2, db_wfr.libraries.count()) + for lib in db_wfr.libraries.all(): self.assertTrue(lib.library_id in library_ids) def test_create_wrsc_library_exists(self): @@ -184,8 +184,8 @@ def test_create_wrsc_library_exists(self): db_wfr: WorkflowRun = wfr_qs.first() self.assertEqual("ctTSO500-L000002", db_wfr.workflow_run_name) # We do expect 2 library associations here! - self.assertEqual(2, db_wfr.get_libraries().count()) - for lib in db_wfr.get_libraries().all(): + self.assertEqual(2, db_wfr.libraries.count()) + for lib in db_wfr.libraries.all(): self.assertTrue(lib.library_id in library_ids) def test_get_last_state(self): From 8b9dcb1d70d98edb2c4ac9b10982073683e100b0 Mon Sep 17 00:00:00 2001 From: Ray Liu Date: Tue, 10 Sep 2024 15:03:31 +1000 Subject: [PATCH 7/8] add custom query params for workflow run query --- .../workflow_manager/viewsets/workflow_run.py | 51 ++++++++++++++++--- 1 file changed, 45 insertions(+), 6 deletions(-) diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run.py index 83071da07..0d6871666 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run.py @@ -1,4 +1,4 @@ -from django.db.models import Q +from django.db.models import Q, Max from rest_framework import filters from rest_framework.decorators import action from rest_framework.viewsets import ReadOnlyModelViewSet @@ -17,7 +17,46 @@ class WorkflowRunViewSet(ReadOnlyModelViewSet): search_fields = WorkflowRun.get_base_fields() def get_queryset(self): - return WorkflowRun.objects.get_by_keyword(**self.request.query_params).prefetch_related('states').prefetch_related('libraries').select_related('workflow') # add prefetch_related & select_related to reduce the number of queries + + """ + custom queryset to filter by: + start_time, end_time : range of latest state timestamp + is_ongoing : filter by ongoing workflow runs + """ + # default start time is 0 + start_time = self.request.query_params.get('start_time', 0) + # default is current time + end_time = self.request.query_params.get('end_time', 0) + + # get is ongoing flag + is_ongoing = self.request.query_params.get('is_ongoing', 'false') + + def exclude_params(params): + for param in params: + self.request.query_params.pop(param) if param in self.request.query_params.keys() else None + + exclude_params([ + 'start_time', + 'end_time', + 'is_ongoing' + ]) + + # get all workflow runs with rest (build-in) of the query params + result_set = WorkflowRun.objects.get_by_keyword(**self.request.query_params).prefetch_related('states').prefetch_related('libraries').select_related('workflow') # add prefetch_related & select_related to reduce the number of queries + + if start_time and end_time: + result_set = result_set.annotate(latest_state_time=Max('states__timestamp')).filter( + latest_state_time__range=[start_time, end_time] + ) + + if is_ongoing.lower() == 'true': + result_set = result_set.filter( + ~Q(states__status="FAILED") & + ~Q(states__status="ABORTED") & + ~Q(states__status="SUCCEEDED") + ) + + return result_set @action(detail=False, methods=['GET']) def ongoing(self, request): @@ -29,14 +68,14 @@ def ongoing(self, request): if "status" in self.request.query_params.keys(): print("found status!") status = self.request.query_params.get('status') - result_set = WorkflowRun.objects.get_by_keyword(state__status=status).order_by(ordering) + result_set = WorkflowRun.objects.get_by_keyword(states__status=status).order_by(ordering) else: result_set = WorkflowRun.objects.get_by_keyword(**self.request.query_params).order_by(ordering) result_set = result_set.filter( - ~Q(state__status="FAILED") & - ~Q(state__status="ABORTED") & - ~Q(state__status="SUCCEEDED") + ~Q(states__status="FAILED") & + ~Q(states__status="ABORTED") & + ~Q(states__status="SUCCEEDED") ) pagw_qs = self.paginate_queryset(result_set) serializer = self.get_serializer(pagw_qs, many=True) From d30793ea6a6225bde6a378a593aeb03bc6903f76 Mon Sep 17 00:00:00 2001 From: Ray Liu Date: Tue, 10 Sep 2024 15:23:45 +1000 Subject: [PATCH 8/8] modify some comment [skip ci] --- .../workflow_manager/viewsets/workflow_run.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run.py index 0d6871666..cc150d0e5 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run.py @@ -23,14 +23,14 @@ def get_queryset(self): start_time, end_time : range of latest state timestamp is_ongoing : filter by ongoing workflow runs """ - # default start time is 0 + # default time is 0 start_time = self.request.query_params.get('start_time', 0) - # default is current time end_time = self.request.query_params.get('end_time', 0) # get is ongoing flag is_ongoing = self.request.query_params.get('is_ongoing', 'false') + # exclude the custom query params from the rest of the query params def exclude_params(params): for param in params: self.request.query_params.pop(param) if param in self.request.query_params.keys() else None @@ -41,7 +41,7 @@ def exclude_params(params): 'is_ongoing' ]) - # get all workflow runs with rest (build-in) of the query params + # get all workflow runs with rest of the query params result_set = WorkflowRun.objects.get_by_keyword(**self.request.query_params).prefetch_related('states').prefetch_related('libraries').select_related('workflow') # add prefetch_related & select_related to reduce the number of queries if start_time and end_time: