From d9ebd2923b5e0f12bfaa736f7fcfbdddfdcbaed0 Mon Sep 17 00:00:00 2001 From: Florian Reisinger Date: Tue, 3 Sep 2024 11:47:14 +1000 Subject: [PATCH 01/14] Add initial data Analysis model --- .../workflow_manager/models/__init__.py | 3 ++ .../workflow_manager/models/analysis.py | 36 +++++++++++++++++++ .../models/analysis_context.py | 31 ++++++++++++++++ .../workflow_manager/models/analysis_run.py | 34 ++++++++++++++++++ 4 files changed, 104 insertions(+) create mode 100644 lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis.py create mode 100644 lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_context.py create mode 100644 lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_run.py diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/__init__.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/__init__.py index 5e2e6da1d..3975261b9 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/__init__.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/__init__.py @@ -7,3 +7,6 @@ from .state import State from .state import Status from .utils import WorkflowRunUtil +from .analysis import Analysis +from .analysis_run import AnalysisRun +from .analysis_context import AnalysisContext diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis.py new file mode 100644 index 000000000..f7e729dff --- /dev/null +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis.py @@ -0,0 +1,36 @@ +from django.db import models + +from workflow_manager.models.base import OrcaBusBaseModel, OrcaBusBaseManager +from workflow_manager.models.analysis_context import AnalysisContext + + +class AnalysisManager(OrcaBusBaseManager): + pass + + +class Analysis(OrcaBusBaseModel): + class Meta: + unique_together = ["analysis_name", "analysis_version"] + + id = models.BigAutoField(primary_key=True) + + analysis_name = models.CharField(max_length=255) + analysis_version = models.CharField(max_length=255) + description = models.CharField(max_length=255) + status = models.CharField(max_length=255) + + allowed_contexts = models.ManyToManyField(AnalysisContext) + + objects = AnalysisManager() + + def __str__(self): + return f"ID: {self.id}, analysis_name: {self.analysis_name}, analysis_version: {self.analysis_version}" + + def to_dict(self): + return { + "id": self.id, + "analysis_name": self.analysis_name, + "analysis_version": self.analysis_version, + "description": self.description, + "status": self.status + } diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_context.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_context.py new file mode 100644 index 000000000..e32bb98b3 --- /dev/null +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_context.py @@ -0,0 +1,31 @@ +from django.db import models + +from workflow_manager.models.base import OrcaBusBaseModel, OrcaBusBaseManager + + +class AnalysisContextManager(OrcaBusBaseManager): + pass + + +class AnalysisContext(OrcaBusBaseModel): + + orcabus_id = models.CharField(primary_key=True, max_length=255) + context_id = models.CharField(max_length=255) + + name = models.CharField(max_length=255) + description = models.CharField(max_length=255) + status = models.CharField(max_length=255) + + objects = AnalysisContextManager() + + def __str__(self): + return f"ID: {self.orcabus_id}, name: {self.name}, status: {self.status}" + + def to_dict(self): + return { + "orcabus_id": self.orcabus_id, + "context_id": self.context_id, + "name": self.name, + "status": self.status, + "description": self.description + } diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_run.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_run.py new file mode 100644 index 000000000..52bf7de8d --- /dev/null +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_run.py @@ -0,0 +1,34 @@ +from django.db import models + +from workflow_manager.models.base import OrcaBusBaseModel, OrcaBusBaseManager +from workflow_manager.models.analysis_context import AnalysisContext + + +class AnalysisRunManager(OrcaBusBaseManager): + pass + + +class AnalysisRun(OrcaBusBaseModel): + id = models.BigAutoField(primary_key=True) + + analysis_run_id = models.CharField(max_length=255, unique=True) + + analysis_run_name = models.CharField(max_length=255) + comment = models.CharField(max_length=255) + + approval_context = models.ForeignKey(AnalysisContext, null=True, blank=True, on_delete=models.SET_NULL) + project_context = models.ForeignKey(AnalysisContext, null=True, blank=True, on_delete=models.SET_NULL) + + objects = AnalysisRunManager() + + def __str__(self): + return f"ID: {self.analysis_run_id}, analysis_run_name: {self.analysis_run_name}" + + def to_dict(self): + return { + "analysis_run_id": self.analysis_run_id, + "analysis_run_name": self.analysis_run_name, + "comment": self.comment, + "approval_context": self.approval_context, + "project_context": self.project_context + } From 3205e9268b29459f45135bc312dd0d5fb2001879 Mon Sep 17 00:00:00 2001 From: Florian Reisinger Date: Sat, 7 Sep 2024 14:35:19 +1000 Subject: [PATCH 02/14] generator for Analysis --- .../management/commands/clean_db.py | 22 ++- .../generate_analysis_for_metadata.py | 133 ++++++++++++++++++ ...02_analysiscontext_analysisrun_analysis.py | 56 ++++++++ ...name_allowed_contexts_analysis_contexts.py | 18 +++ .../workflow_manager/models/analysis.py | 5 +- .../workflow_manager/models/analysis_run.py | 8 +- 6 files changed, 236 insertions(+), 6 deletions(-) create mode 100644 lib/workload/stateless/stacks/workflow-manager/workflow_manager/management/commands/generate_analysis_for_metadata.py create mode 100644 lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0002_analysiscontext_analysisrun_analysis.py create mode 100644 lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0003_rename_allowed_contexts_analysis_contexts.py diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/management/commands/clean_db.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/management/commands/clean_db.py index e6504ebce..aba356356 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/management/commands/clean_db.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/management/commands/clean_db.py @@ -1,15 +1,31 @@ from django.core.management import BaseCommand -from django.db.models import QuerySet -from workflow_manager.models import WorkflowRun, Workflow, Payload +from workflow_manager.models import ( + WorkflowRun, + Workflow, + Payload, + State, + Library, + LibraryAssociation, + Analysis, + AnalysisRun, + AnalysisContext +) + # https://docs.djangoproject.com/en/5.0/howto/custom-management-commands/ class Command(BaseCommand): help = "Delete all DB data" def handle(self, *args, **options): + Workflow.objects.all().delete() WorkflowRun.objects.all().delete() + State.objects.all().delete() Payload.objects.all().delete() - Workflow.objects.all().delete() + Library.objects.all().delete() + LibraryAssociation.objects.all().delete() + AnalysisContext.objects.all().delete() + AnalysisRun.objects.all().delete() + Analysis.objects.all().delete() print("Done") diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/management/commands/generate_analysis_for_metadata.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/management/commands/generate_analysis_for_metadata.py new file mode 100644 index 000000000..dbb86ca92 --- /dev/null +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/management/commands/generate_analysis_for_metadata.py @@ -0,0 +1,133 @@ +from django.core.management import BaseCommand +from django.utils.timezone import make_aware + +from datetime import datetime, timedelta +from workflow_manager.models import Workflow, Analysis, AnalysisContext, AnalysisRun + + +# https://docs.djangoproject.com/en/5.0/howto/custom-management-commands/ +class Command(BaseCommand): + help = """ + Generate mock data and populate DB for local testing. + python manage.py generate_analysis_for_metadata + """ + + def handle(self, *args, **options): + + metadata = [ + { + "phenotype": "tumor", + "library_id": "L000001", + "assay": "TsqNano", + "type": "WGS", + "subject": "SBJ00001", + "workflow": "clinical" + }, + { + "phenotype": "normal", + "library_id": "L000002", + "assay": "TsqNano", + "type": "WGS", + "subject": "SBJ00001", + "workflow": "clinical" + }, + ] + + # Create needed workflows + # Create Analysis + _setup_requirements() + + # Use metadata to decide on Analysis + # Create AnalysisRun + # Create WorkflowRuns + + print("Done") + + +def _setup_requirements(): + + clinical_context = AnalysisContext( + orcabus_id="ctx.12345", + context_id="C12345", + name="NATA_Accredited", + description="Accredited by NATA", + status="ACTIVE", + ) + clinical_context.save() + + research_context = AnalysisContext( + orcabus_id="ctx.23456", + context_id="C23456", + name="Research", + description="For research use", + status="ACTIVE", + ) + research_context.save() + + wgs_workflow = Workflow( + workflow_name="tumor_normal", + workflow_version="1.0", + execution_engine="ICAv2", + execution_engine_pipeline_id="ica.pipeline.12345", + ) + wgs_workflow.save() + + cttsov2_workflow = Workflow( + workflow_name="cttsov2", + workflow_version="1.0", + execution_engine="ICAv2", + execution_engine_pipeline_id="ica.pipeline.23456", + ) + cttsov2_workflow.save() + + umccrise_workflow = Workflow( + workflow_name="umccrise", + workflow_version="1.0", + execution_engine="ICAv2", + execution_engine_pipeline_id="ica.pipeline.34567", + ) + umccrise_workflow.save() + + oa_wgs_workflow = Workflow( + workflow_name="oncoanalyser_wgs", + workflow_version="1.0", + execution_engine="Nextflow", + execution_engine_pipeline_id="nf.12345", + ) + oa_wgs_workflow.save() + + sash_workflow = Workflow( + workflow_name="sash", + workflow_version="1.0", + execution_engine="Nextflow", + execution_engine_pipeline_id="nf.23456", + ) + sash_workflow.save() + + wgs_clinical_analysis = Analysis( + analysis_name="Accredited_WGS", + analysis_version="1.0", + description="NATA accredited analysis for WGS", + status="ACTIVE", + ) + wgs_clinical_analysis.save() + wgs_clinical_analysis.contexts.add(clinical_context) + wgs_clinical_analysis.workflows.add(wgs_workflow) + wgs_clinical_analysis.workflows.add(umccrise_workflow) + + wgs_research_analysis = Analysis( + analysis_name="Research_WGS", + analysis_version="1.0", + description="Analysis for WGS research samples", + status="ACTIVE", + ) + wgs_research_analysis.save() + wgs_research_analysis.contexts.add(research_context) + wgs_research_analysis.workflows.add(wgs_workflow) + wgs_research_analysis.workflows.add(oa_wgs_workflow) + wgs_research_analysis.workflows.add(sash_workflow) + + +def _assign_workflows(metadata: dict): + + pass \ No newline at end of file diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0002_analysiscontext_analysisrun_analysis.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0002_analysiscontext_analysisrun_analysis.py new file mode 100644 index 000000000..98a051222 --- /dev/null +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0002_analysiscontext_analysisrun_analysis.py @@ -0,0 +1,56 @@ +# Generated by Django 5.1 on 2024-09-07 04:09 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('workflow_manager', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='AnalysisContext', + fields=[ + ('orcabus_id', models.CharField(max_length=255, primary_key=True, serialize=False)), + ('context_id', models.CharField(max_length=255)), + ('name', models.CharField(max_length=255)), + ('description', models.CharField(max_length=255)), + ('status', models.CharField(max_length=255)), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='AnalysisRun', + fields=[ + ('id', models.BigAutoField(primary_key=True, serialize=False)), + ('analysis_run_id', models.CharField(max_length=255, unique=True)), + ('analysis_run_name', models.CharField(max_length=255)), + ('comment', models.CharField(max_length=255)), + ('approval_context', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='approval_context', to='workflow_manager.analysiscontext')), + ('project_context', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='project_context', to='workflow_manager.analysiscontext')), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='Analysis', + fields=[ + ('id', models.BigAutoField(primary_key=True, serialize=False)), + ('analysis_name', models.CharField(max_length=255)), + ('analysis_version', models.CharField(max_length=255)), + ('description', models.CharField(max_length=255)), + ('status', models.CharField(max_length=255)), + ('workflows', models.ManyToManyField(to='workflow_manager.workflow')), + ('allowed_contexts', models.ManyToManyField(to='workflow_manager.analysiscontext')), + ], + options={ + 'unique_together': {('analysis_name', 'analysis_version')}, + }, + ), + ] diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0003_rename_allowed_contexts_analysis_contexts.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0003_rename_allowed_contexts_analysis_contexts.py new file mode 100644 index 000000000..936bd4d0c --- /dev/null +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0003_rename_allowed_contexts_analysis_contexts.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1 on 2024-09-07 04:17 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('workflow_manager', '0002_analysiscontext_analysisrun_analysis'), + ] + + operations = [ + migrations.RenameField( + model_name='analysis', + old_name='allowed_contexts', + new_name='contexts', + ), + ] diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis.py index f7e729dff..5380fce62 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis.py @@ -2,6 +2,7 @@ from workflow_manager.models.base import OrcaBusBaseModel, OrcaBusBaseManager from workflow_manager.models.analysis_context import AnalysisContext +from workflow_manager.models.workflow import Workflow class AnalysisManager(OrcaBusBaseManager): @@ -19,7 +20,9 @@ class Meta: description = models.CharField(max_length=255) status = models.CharField(max_length=255) - allowed_contexts = models.ManyToManyField(AnalysisContext) + # relationships + contexts = models.ManyToManyField(AnalysisContext) + workflows = models.ManyToManyField(Workflow) objects = AnalysisManager() diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_run.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_run.py index 52bf7de8d..9865bdc8c 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_run.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_run.py @@ -16,8 +16,12 @@ class AnalysisRun(OrcaBusBaseModel): analysis_run_name = models.CharField(max_length=255) comment = models.CharField(max_length=255) - approval_context = models.ForeignKey(AnalysisContext, null=True, blank=True, on_delete=models.SET_NULL) - project_context = models.ForeignKey(AnalysisContext, null=True, blank=True, on_delete=models.SET_NULL) + approval_context = models.ForeignKey(AnalysisContext, + null=True, blank=True, on_delete=models.SET_NULL, + related_name="approval_context") + project_context = models.ForeignKey(AnalysisContext, + null=True, blank=True, on_delete=models.SET_NULL, + related_name="project_context") objects = AnalysisRunManager() From 834536da2d61a09c48cff59b9eecf17f3e6d0466 Mon Sep 17 00:00:00 2001 From: Florian Reisinger Date: Sat, 7 Sep 2024 21:19:20 +1000 Subject: [PATCH 03/14] Add analysis generation from library list --- .../workflow-manager/deps/requirements.txt | 1 + .../generate_analysis_for_metadata.py | 241 ++++++++++++++++-- ...analysis_analysisrun_libraries_and_more.py | 34 +++ .../workflow_manager/models/analysis_run.py | 13 +- 4 files changed, 268 insertions(+), 21 deletions(-) create mode 100644 lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0004_analysisrun_analysis_analysisrun_libraries_and_more.py diff --git a/lib/workload/stateless/stacks/workflow-manager/deps/requirements.txt b/lib/workload/stateless/stacks/workflow-manager/deps/requirements.txt index 912901713..21e924e24 100644 --- a/lib/workload/stateless/stacks/workflow-manager/deps/requirements.txt +++ b/lib/workload/stateless/stacks/workflow-manager/deps/requirements.txt @@ -22,3 +22,4 @@ serverless-wsgi==3.0.4 # for sequencerunstatechange package six==1.16.0 regex==2024.7.24 +ulid-py==1.1.0 diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/management/commands/generate_analysis_for_metadata.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/management/commands/generate_analysis_for_metadata.py index dbb86ca92..81ddb766b 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/management/commands/generate_analysis_for_metadata.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/management/commands/generate_analysis_for_metadata.py @@ -1,11 +1,13 @@ +from typing import List +import ulid +from collections import defaultdict from django.core.management import BaseCommand -from django.utils.timezone import make_aware +from workflow_manager.models import Workflow, Analysis, AnalysisContext, AnalysisRun, Library -from datetime import datetime, timedelta -from workflow_manager.models import Workflow, Analysis, AnalysisContext, AnalysisRun +# https://docs.djangoproject.com/en/5.0/howto/custom-management-commands/ +from workflow_manager.tests.factories import PayloadFactory, LibraryFactory -# https://docs.djangoproject.com/en/5.0/howto/custom-management-commands/ class Command(BaseCommand): help = """ Generate mock data and populate DB for local testing. @@ -14,7 +16,7 @@ class Command(BaseCommand): def handle(self, *args, **options): - metadata = [ + libraries = [ { "phenotype": "tumor", "library_id": "L000001", @@ -31,6 +33,38 @@ def handle(self, *args, **options): "subject": "SBJ00001", "workflow": "clinical" }, + { + "phenotype": "tumor", + "library_id": "L000003", + "assay": "TsqNano", + "type": "WGS", + "subject": "SBJ00002", + "workflow": "research" + }, + { + "phenotype": "normal", + "library_id": "L000004", + "assay": "TsqNano", + "type": "WGS", + "subject": "SBJ00002", + "workflow": "research" + }, + { + "phenotype": "tumor", + "library_id": "L000005", + "assay": "ctTSOv2", + "type": "ctDNA", + "subject": "SBJ00003", + "workflow": "clinical" + }, + { + "phenotype": "tumor", + "library_id": "L000006", + "assay": "ctTSOv2", + "type": "ctDNA", + "subject": "SBJ00003", + "workflow": "research" + }, ] # Create needed workflows @@ -38,18 +72,24 @@ def handle(self, *args, **options): _setup_requirements() # Use metadata to decide on Analysis - # Create AnalysisRun - # Create WorkflowRuns + runs: List[AnalysisRun] = assign_analysis(libraries) + print(runs) print("Done") def _setup_requirements(): + """ + Create the resources assumed pre-existing. These are usually registered via external services or manual operation. + """ + + # ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- + # The contexts information available for analysis clinical_context = AnalysisContext( orcabus_id="ctx.12345", context_id="C12345", - name="NATA_Accredited", + name="accredited", description="Accredited by NATA", status="ACTIVE", ) @@ -58,12 +98,32 @@ def _setup_requirements(): research_context = AnalysisContext( orcabus_id="ctx.23456", context_id="C23456", - name="Research", + name="research", description="For research use", status="ACTIVE", ) research_context.save() + internal_context = AnalysisContext( + orcabus_id="ctx.00001", + context_id="C00001", + name="internal", + description="For internal use", + status="ACTIVE", + ) + internal_context.save() + + # ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- + # The workflows that are available to be run + + qc_workflow = Workflow( + workflow_name="wgts_alignment_qc", + workflow_version="1.0", + execution_engine="ICAv2", + execution_engine_pipeline_id="ica.pipeline.01234", + ) + qc_workflow.save() + wgs_workflow = Workflow( workflow_name="tumor_normal", workflow_version="1.0", @@ -104,21 +164,36 @@ def _setup_requirements(): ) sash_workflow.save() + # ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- + # The analysis options, based on the available workflows and conditions + + qc_assessment = Analysis( + analysis_name="QC_Assessment", + analysis_version="1.0", + description="Quality Control analysis for WGS and WTS", + status="ACTIVE", + ) + qc_assessment.save() + qc_assessment.contexts.add(clinical_context) + qc_assessment.contexts.add(research_context) + qc_assessment.workflows.add(qc_workflow) + wgs_clinical_analysis = Analysis( - analysis_name="Accredited_WGS", + analysis_name="WGS", analysis_version="1.0", - description="NATA accredited analysis for WGS", + description="Analysis for WGS samples", status="ACTIVE", ) wgs_clinical_analysis.save() wgs_clinical_analysis.contexts.add(clinical_context) + wgs_clinical_analysis.contexts.add(research_context) wgs_clinical_analysis.workflows.add(wgs_workflow) wgs_clinical_analysis.workflows.add(umccrise_workflow) wgs_research_analysis = Analysis( - analysis_name="Research_WGS", - analysis_version="1.0", - description="Analysis for WGS research samples", + analysis_name="WGS", + analysis_version="2.0", + description="Analysis for WGS samples", status="ACTIVE", ) wgs_research_analysis.save() @@ -127,7 +202,141 @@ def _setup_requirements(): wgs_research_analysis.workflows.add(oa_wgs_workflow) wgs_research_analysis.workflows.add(sash_workflow) + cttso_research_analysis = Analysis( + analysis_name="ctTSO500", + analysis_version="1.0", + description="Analysis for ctTSO samples", + status="ACTIVE", + ) + cttso_research_analysis.save() + cttso_research_analysis.contexts.add(research_context) + cttso_research_analysis.contexts.add(clinical_context) + cttso_research_analysis.workflows.add(cttsov2_workflow) + + # ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- + # Generate a Payload stub and some Libraries + + generic_payload = PayloadFactory() # Payload content is not important for now + LibraryFactory(orcabus_id="lib.01J5M2JFE1JPYV62RYQEG99CP1", library_id="L000001"), + LibraryFactory(orcabus_id="lib.02J5M2JFE1JPYV62RYQEG99CP2", library_id="L000002"), + LibraryFactory(orcabus_id="lib.03J5M2JFE1JPYV62RYQEG99CP3", library_id="L000003"), + LibraryFactory(orcabus_id="lib.03J5M2JFE1JPYV62RYQEG99CP4", library_id="L000004"), + LibraryFactory(orcabus_id="lib.03J5M2JFE1JPYV62RYQEG99CP5", library_id="L000005"), + LibraryFactory(orcabus_id="lib.04J5M2JFE1JPYV62RYQEG99CP6", library_id="L000006") + + +def assign_analysis(libraries: List[dict]) -> List[AnalysisRun]: + analysis_runs: List[AnalysisRun] = [] + + # Handle QC analysis + analysis_runs.extend(create_qc_analysis(libraries=libraries)) + + # Handle WGS analysis + analysis_runs.extend(create_wgs_analysis(libraries=libraries)) + + # Handle ctTSO analysis + analysis_runs.extend(create_cttso_analysis(libraries=libraries)) + + # handle WTS + + return analysis_runs + + +def create_qc_analysis(libraries: List[dict]) -> List[AnalysisRun]: + analysis_runs: List[AnalysisRun] = [] + context_internal = AnalysisContext.objects.get_by_keyword(name="internal").first() # FIXME + analysis_qc_qs = Analysis.objects.get_by_keyword(analysis_name='QC_Assessment') + analysis_qc = analysis_qc_qs.first() # FIXME: assume there are more than one and select by latest version, etc + + for lib in libraries: + lib_record: Library = Library.objects.get(library_id=lib['library_id']) + + # handle QC + if lib['type'] in ['WGS', 'WTS']: + # Create QC analysis + analysis_run = AnalysisRun( + analysis_run_id=f"ar.{ulid.new().str}", + analysis_run_name=f"automated__{analysis_qc.analysis_name}__{lib_record.library_id}", + status="DRAFT", + approval_context=context_internal, # FIXME: does this matter here? Internal? + analysis=analysis_qc + ) + analysis_run.save() + analysis_run.libraries.add(lib_record) + analysis_runs.append(analysis_run) + + return analysis_runs + + +def create_wgs_analysis(libraries: List[dict]) -> List[AnalysisRun]: + analysis_runs: List[AnalysisRun] = [] + context_clinical = AnalysisContext.objects.get_by_keyword(name="accredited").first() # FIXME + context_research = AnalysisContext.objects.get_by_keyword(name="research").first() # FIXME + analysis_wgs_clinical: Analysis = Analysis.objects.filter(analysis_name='WGS', contexts=context_clinical).first() # TODO: filter up front? + analysis_wgs_research: Analysis = Analysis.objects.filter(analysis_name='WGS', contexts=context_research).first() # TODO: filter up front? + + # FIXME: better pairing algorithm! + pairing = defaultdict(lambda: defaultdict(list)) + for lib in libraries: + pairing[lib['subject']][lib['phenotype']].append(lib) + + for sbj in pairing: + if pairing[sbj]['tumor'] and pairing[sbj]['normal']: + # noinspection PyTypeChecker + tumor_lib_record = normal_lib_record = None + if len(pairing[sbj]['tumor']) == 1: + tumor_lib_record: Library = Library.objects.get(library_id=pairing[sbj]['tumor'][0]['library_id']) + if len(pairing[sbj]['normal']) == 1: + normal_lib_record: Library = Library.objects.get(library_id=pairing[sbj]['normal'][0]['library_id']) + + if not tumor_lib_record or not normal_lib_record: + print("Not a valid pairing.") + break + + workflow = pairing[sbj]['tumor'][0]['workflow'] + context = context_clinical if workflow == 'clinical' else context_research + analysis = analysis_wgs_clinical if workflow == 'clinical' else analysis_wgs_research + analysis_run_name = f"automated__{analysis.analysis_name}__{context.name}__" + \ + f"{tumor_lib_record.library_id}__{normal_lib_record.library_id} " + ar_wgs = AnalysisRun( + analysis_run_id=f"ar.{ulid.new().str}", + analysis_run_name=analysis_run_name, + status="DRAFT", + approval_context=context, + analysis=analysis + ) + ar_wgs.save() + ar_wgs.libraries.add(tumor_lib_record) + ar_wgs.libraries.add(normal_lib_record) + analysis_runs.append(ar_wgs) + else: + print(f"No pairing for {sbj}.") + + return analysis_runs + + +def create_cttso_analysis(libraries: List[dict]) -> List[AnalysisRun]: + analysis_runs: List[AnalysisRun] = [] + context_clinical = AnalysisContext.objects.get_by_keyword(name="accredited").first() # FIXME + context_research = AnalysisContext.objects.get_by_keyword(name="research").first() # FIXME + analysis_cttso_qs = Analysis.objects.get_by_keyword(analysis_name='ctTSO500').first() # FIXME: allow for multiple + + for lib in libraries: + lib_record: Library = Library.objects.get(library_id=lib['library_id']) -def _assign_workflows(metadata: dict): + # handle QC + if lib['type'] in ['ctDNA'] and lib['assay'] in ['ctTSOv2']: + context: AnalysisContext = context_clinical if lib['workflow'] == 'clinical' else context_research + analysis_run_name = f"automated__{analysis_cttso_qs.analysis_name}__{context.name}__{lib_record.library_id}" + analysis_run = AnalysisRun( + analysis_run_id=f"ar.{ulid.new().str}", + analysis_run_name=analysis_run_name, + status="DRAFT", + approval_context=context, + analysis=analysis_cttso_qs + ) + analysis_run.save() + analysis_run.libraries.add(lib_record) + analysis_runs.append(analysis_run) - pass \ No newline at end of file + return analysis_runs diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0004_analysisrun_analysis_analysisrun_libraries_and_more.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0004_analysisrun_analysis_analysisrun_libraries_and_more.py new file mode 100644 index 000000000..a92c769a9 --- /dev/null +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0004_analysisrun_analysis_analysisrun_libraries_and_more.py @@ -0,0 +1,34 @@ +# Generated by Django 5.1 on 2024-09-07 06:53 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('workflow_manager', '0003_rename_allowed_contexts_analysis_contexts'), + ] + + operations = [ + migrations.AddField( + model_name='analysisrun', + name='analysis', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='workflow_manager.analysis'), + ), + migrations.AddField( + model_name='analysisrun', + name='libraries', + field=models.ManyToManyField(to='workflow_manager.library'), + ), + migrations.AddField( + model_name='analysisrun', + name='status', + field=models.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='analysisrun', + name='comment', + field=models.CharField(blank=True, max_length=255, null=True), + ), + ] diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_run.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_run.py index 9865bdc8c..f284306dd 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_run.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_run.py @@ -1,7 +1,9 @@ from django.db import models from workflow_manager.models.base import OrcaBusBaseModel, OrcaBusBaseManager +from workflow_manager.models.analysis import Analysis from workflow_manager.models.analysis_context import AnalysisContext +from workflow_manager.models.library import Library class AnalysisRunManager(OrcaBusBaseManager): @@ -14,14 +16,15 @@ class AnalysisRun(OrcaBusBaseModel): analysis_run_id = models.CharField(max_length=255, unique=True) analysis_run_name = models.CharField(max_length=255) - comment = models.CharField(max_length=255) + comment = models.CharField(max_length=255, null=True, blank=True) + status = models.CharField(max_length=255, null=True, blank=True) - approval_context = models.ForeignKey(AnalysisContext, - null=True, blank=True, on_delete=models.SET_NULL, + approval_context = models.ForeignKey(AnalysisContext, null=True, blank=True, on_delete=models.SET_NULL, related_name="approval_context") - project_context = models.ForeignKey(AnalysisContext, - null=True, blank=True, on_delete=models.SET_NULL, + project_context = models.ForeignKey(AnalysisContext, null=True, blank=True, on_delete=models.SET_NULL, related_name="project_context") + analysis = models.ForeignKey(Analysis, null=True, blank=True, on_delete=models.SET_NULL) + libraries = models.ManyToManyField(Library) objects = AnalysisRunManager() From f5ed78c14f67258aaeb366eea7fd7d4ba8af094c Mon Sep 17 00:00:00 2001 From: Florian Reisinger Date: Mon, 9 Sep 2024 11:15:50 +1000 Subject: [PATCH 04/14] Add WorkflowRun templates for Analysis to be run --- .../generate_analysis_for_metadata.py | 94 +++++++++++++++++-- .../0005_workflowrun_analysis_run.py | 19 ++++ .../workflow_manager/models/workflow_run.py | 6 +- 3 files changed, 110 insertions(+), 9 deletions(-) create mode 100644 lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0005_workflowrun_analysis_run.py diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/management/commands/generate_analysis_for_metadata.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/management/commands/generate_analysis_for_metadata.py index 81ddb766b..a9f06ee6f 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/management/commands/generate_analysis_for_metadata.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/management/commands/generate_analysis_for_metadata.py @@ -1,17 +1,40 @@ +from datetime import datetime, timezone from typing import List import ulid +import uuid from collections import defaultdict from django.core.management import BaseCommand -from workflow_manager.models import Workflow, Analysis, AnalysisContext, AnalysisRun, Library +from django.db.models import QuerySet -# https://docs.djangoproject.com/en/5.0/howto/custom-management-commands/ +from workflow_manager.models import Workflow, WorkflowRun, Analysis, AnalysisContext, AnalysisRun, \ + Library, LibraryAssociation from workflow_manager.tests.factories import PayloadFactory, LibraryFactory +# https://docs.djangoproject.com/en/5.0/howto/custom-management-commands/ + class Command(BaseCommand): help = """ Generate mock data and populate DB for local testing. python manage.py generate_analysis_for_metadata + + This is split into several parts: + 1. A set-up of core data that is assumed present in the DB, + e.g. Library, Workflow and Analysis definitions, etc. This is usually expected to be provided by + external processes: + - Workflows: from workflow deployment pipelines or manual/operator + - Analysis: manual/operator + - Library: from MetadataManager via publishing events + - Contexts: from MetadataManager via publishing events or manual/operator + 2. a process aiming to assign Analysis to provided libraries (and their metadata) + This could be an external rules engine acting on LibraryStateChange events from the MetadataManager + or triggered on start of sequencing. + It would query the available Analysis from the WorkflowManager and map the library (metadata) to the + most appropriate Analysis (if possible). Those mappings could be published via AnalysisStateChange + events and recorded by the WorkflowManager. + - a process that creates WorkflowRun drafts for an Analysis + This could be handled by the WorkflowManager e.g. on sequencing events + """ def handle(self, *args, **options): @@ -73,8 +96,9 @@ def handle(self, *args, **options): # Use metadata to decide on Analysis runs: List[AnalysisRun] = assign_analysis(libraries) - print(runs) + + prep_workflow_runs(libraries) print("Done") @@ -272,8 +296,8 @@ def create_wgs_analysis(libraries: List[dict]) -> List[AnalysisRun]: analysis_runs: List[AnalysisRun] = [] context_clinical = AnalysisContext.objects.get_by_keyword(name="accredited").first() # FIXME context_research = AnalysisContext.objects.get_by_keyword(name="research").first() # FIXME - analysis_wgs_clinical: Analysis = Analysis.objects.filter(analysis_name='WGS', contexts=context_clinical).first() # TODO: filter up front? - analysis_wgs_research: Analysis = Analysis.objects.filter(analysis_name='WGS', contexts=context_research).first() # TODO: filter up front? + analysis_wgs_clinical: Analysis = Analysis.objects.filter(analysis_name='WGS', contexts=context_clinical).first() # FIXME + analysis_wgs_research: Analysis = Analysis.objects.filter(analysis_name='WGS', contexts=context_research, analysis_version='2.0').first() # FIXME # FIXME: better pairing algorithm! pairing = defaultdict(lambda: defaultdict(list)) @@ -283,7 +307,9 @@ def create_wgs_analysis(libraries: List[dict]) -> List[AnalysisRun]: for sbj in pairing: if pairing[sbj]['tumor'] and pairing[sbj]['normal']: # noinspection PyTypeChecker - tumor_lib_record = normal_lib_record = None + tumor_lib_record = None + # noinspection PyTypeChecker + normal_lib_record = None if len(pairing[sbj]['tumor']) == 1: tumor_lib_record: Library = Library.objects.get(library_id=pairing[sbj]['tumor'][0]['library_id']) if len(pairing[sbj]['normal']) == 1: @@ -340,3 +366,59 @@ def create_cttso_analysis(libraries: List[dict]) -> List[AnalysisRun]: analysis_runs.append(analysis_run) return analysis_runs + + +def prep_workflow_runs(libraries: List[dict]): + # Find AnalysisRuns for the given libraries (all libs of AnalysisRun must match) + lids = set() + for lib in libraries: + lids.add(lib['library_id']) + + # Querying "valid" AnalysisRuns for libraries + # The AnalysisRun linked libraries need to be both in the input list + print(f"Finding valid AnalysisRuns for libraries {lids}") + valid_aruns = set() + qs: QuerySet = AnalysisRun.objects.filter(libraries__library_id__in=lids) + for arun in qs: + alids = set() + for l in arun.libraries.all(): + alids.add(l.library_id) + valid = all(x in lids for x in alids) + if valid: + valid_aruns.add(arun) + else: + print(f"Not a valid AnalysisRun: {arun} for libraries {lids}") + + # create Workflow drafts for each found AnalysisRuns + for valid_arun in valid_aruns: + print(valid_arun.analysis_run_name) + # create WorkflowRuns + create_workflowrun_for_analysis(valid_arun) + # update status of AnalysisRun from DRAFT to READY + valid_arun.status = "READY" # TODO: READY or PENDING or INITIALISED ... ?? + valid_arun.save() + + +def create_workflowrun_for_analysis(analysis_run: AnalysisRun): + analysis: Analysis = analysis_run.analysis + workflows = analysis.workflows + for workflow in workflows.all(): + wr: WorkflowRun = WorkflowRun( + portal_run_id=create_portal_run_id(), + workflow_run_name=f"{analysis_run.analysis_run_name}__{workflow.workflow_name}", + workflow=workflow, + analysis_run=analysis_run, + ) + wr.save() + for lib in analysis_run.libraries.all(): + LibraryAssociation.objects.create( + workflow_run=wr, + library=lib, + association_date=datetime.now(timezone.utc), + status="ACTIVE", + ) + + +def create_portal_run_id() -> str: + date = datetime.now(timezone.utc) + return f"{date.year}{date.month}{date.day}{str(uuid.uuid4())[:8]}" diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0005_workflowrun_analysis_run.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0005_workflowrun_analysis_run.py new file mode 100644 index 000000000..9b4a44800 --- /dev/null +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0005_workflowrun_analysis_run.py @@ -0,0 +1,19 @@ +# Generated by Django 5.1 on 2024-09-09 00:23 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('workflow_manager', '0004_analysisrun_analysis_analysisrun_libraries_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='workflowrun', + name='analysis_run', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='workflow_manager.analysisrun'), + ), + ] diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow_run.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow_run.py index 41eb0b7fd..31eef2733 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow_run.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow_run.py @@ -3,6 +3,7 @@ from workflow_manager.models.base import OrcaBusBaseModel, OrcaBusBaseManager from workflow_manager.models.library import Library from workflow_manager.models.workflow import Workflow +from workflow_manager.models.analysis_run import AnalysisRun class WorkflowRunManager(OrcaBusBaseManager): @@ -25,10 +26,9 @@ class WorkflowRun(OrcaBusBaseModel): # --- FK link to value objects - # Link to workflow table + # Relationships workflow = models.ForeignKey(Workflow, null=True, blank=True, on_delete=models.SET_NULL) - - # Link to library table + analysis_run = models.ForeignKey(AnalysisRun, null=True, blank=True, on_delete=models.SET_NULL) libraries = models.ManyToManyField(Library, through="LibraryAssociation") objects = WorkflowRunManager() From e9f156869b5944f6e6a12fdb0ef43cbe0b0efb9b Mon Sep 17 00:00:00 2001 From: Florian Reisinger Date: Mon, 30 Sep 2024 09:52:52 +1000 Subject: [PATCH 05/14] Add OrcaBus ID --- .../workflow_manager/fields.py | 26 ++++++++ .../generate_analysis_for_metadata.py | 5 +- .../migrations/0001_initial.py | 62 +++++++++++++++++-- ...02_analysiscontext_analysisrun_analysis.py | 56 ----------------- ...name_allowed_contexts_analysis_contexts.py | 18 ------ ...analysis_analysisrun_libraries_and_more.py | 34 ---------- .../0005_workflowrun_analysis_run.py | 19 ------ .../workflow_manager/models/analysis.py | 3 +- .../models/analysis_context.py | 3 +- .../workflow_manager/models/analysis_run.py | 3 +- .../workflow_manager/models/payload.py | 3 +- .../workflow_manager/models/state.py | 5 +- .../workflow_manager/models/workflow.py | 3 +- .../workflow_manager/models/workflow_run.py | 4 +- 14 files changed, 100 insertions(+), 144 deletions(-) delete mode 100644 lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0002_analysiscontext_analysisrun_analysis.py delete mode 100644 lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0003_rename_allowed_contexts_analysis_contexts.py delete mode 100644 lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0004_analysisrun_analysis_analysisrun_libraries_and_more.py delete mode 100644 lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0005_workflowrun_analysis_run.py diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/fields.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/fields.py index ba79fa4f7..26f591fb8 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/fields.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/fields.py @@ -1,8 +1,34 @@ import hashlib +import ulid from django.db import models +def orcabus_id(prefix: str) -> str: + oid = f"{prefix}.{ulid.new()}" + return oid + + +class OrcabusIdField(models.CharField): + description = "An OrcaBus internal ID composed of a 3 letter prefix followed by dot followed by a ULID" + + def __init__(self, prefix, *args, **kwargs): + self.prefix = prefix + kwargs["max_length"] = 30 # prefix + . + ULID = 3 + 1 + 26 = 30 + kwargs['unique'] = True + super().__init__(*args, **kwargs) + + def deconstruct(self): + name, path, args, kwargs = super().deconstruct() + if self.prefix is not None: + kwargs["prefix"] = self.prefix + return name, path, args, kwargs + + def pre_save(self, instance, add): + setattr(instance, self.attname, orcabus_id(self.prefix)) + return super().pre_save(instance, add) + + class HashField(models.CharField): description = ( "HashField is related to some base fields (other columns) in a model and" diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/management/commands/generate_analysis_for_metadata.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/management/commands/generate_analysis_for_metadata.py index a9f06ee6f..950affd76 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/management/commands/generate_analysis_for_metadata.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/management/commands/generate_analysis_for_metadata.py @@ -111,25 +111,24 @@ def _setup_requirements(): # The contexts information available for analysis clinical_context = AnalysisContext( - orcabus_id="ctx.12345", context_id="C12345", name="accredited", description="Accredited by NATA", status="ACTIVE", ) + print(clinical_context) clinical_context.save() research_context = AnalysisContext( - orcabus_id="ctx.23456", context_id="C23456", name="research", description="For research use", status="ACTIVE", ) + print(research_context) research_context.save() internal_context = AnalysisContext( - orcabus_id="ctx.00001", context_id="C00001", name="internal", description="For internal use", diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0001_initial.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0001_initial.py index 484ed0407..66eafbcef 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0001_initial.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0001_initial.py @@ -1,7 +1,8 @@ -# Generated by Django 5.1 on 2024-08-21 05:48 +# Generated by Django 5.1 on 2024-09-25 08:06 import django.core.serializers.json import django.db.models.deletion +import workflow_manager.fields from django.db import migrations, models @@ -13,6 +14,19 @@ class Migration(migrations.Migration): ] operations = [ + migrations.CreateModel( + name='AnalysisContext', + fields=[ + ('orcabus_id', workflow_manager.fields.OrcabusIdField(max_length=30, prefix='ctx', primary_key=True, serialize=False, unique=True)), + ('context_id', models.CharField(max_length=255)), + ('name', models.CharField(max_length=255)), + ('description', models.CharField(max_length=255)), + ('status', models.CharField(max_length=255)), + ], + options={ + 'abstract': False, + }, + ), migrations.CreateModel( name='Library', fields=[ @@ -26,7 +40,7 @@ class Migration(migrations.Migration): migrations.CreateModel( name='Payload', fields=[ - ('id', models.BigAutoField(primary_key=True, serialize=False)), + ('orcabus_id', workflow_manager.fields.OrcabusIdField(max_length=30, prefix='pld', primary_key=True, serialize=False, unique=True)), ('payload_ref_id', models.CharField(max_length=255, unique=True)), ('version', models.CharField(max_length=255)), ('data', models.JSONField(encoder=django.core.serializers.json.DjangoJSONEncoder)), @@ -35,6 +49,34 @@ class Migration(migrations.Migration): 'abstract': False, }, ), + migrations.CreateModel( + name='Analysis', + fields=[ + ('orcabus_id', workflow_manager.fields.OrcabusIdField(max_length=30, prefix='ana', primary_key=True, serialize=False, unique=True)), + ('analysis_name', models.CharField(max_length=255)), + ('analysis_version', models.CharField(max_length=255)), + ('description', models.CharField(max_length=255)), + ('status', models.CharField(max_length=255)), + ('contexts', models.ManyToManyField(to='workflow_manager.analysiscontext')), + ], + ), + migrations.CreateModel( + name='AnalysisRun', + fields=[ + ('orcabus_id', workflow_manager.fields.OrcabusIdField(max_length=30, prefix='anr', primary_key=True, serialize=False, unique=True)), + ('analysis_run_id', models.CharField(max_length=255, unique=True)), + ('analysis_run_name', models.CharField(max_length=255)), + ('comment', models.CharField(blank=True, max_length=255, null=True)), + ('status', models.CharField(blank=True, max_length=255, null=True)), + ('analysis', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='workflow_manager.analysis')), + ('approval_context', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='approval_context', to='workflow_manager.analysiscontext')), + ('project_context', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='project_context', to='workflow_manager.analysiscontext')), + ('libraries', models.ManyToManyField(to='workflow_manager.library')), + ], + options={ + 'abstract': False, + }, + ), migrations.CreateModel( name='LibraryAssociation', fields=[ @@ -50,7 +92,7 @@ class Migration(migrations.Migration): migrations.CreateModel( name='Workflow', fields=[ - ('id', models.BigAutoField(primary_key=True, serialize=False)), + ('orcabus_id', workflow_manager.fields.OrcabusIdField(max_length=30, prefix='wfl', primary_key=True, serialize=False, unique=True)), ('workflow_name', models.CharField(max_length=255)), ('workflow_version', models.CharField(max_length=255)), ('execution_engine', models.CharField(max_length=255)), @@ -61,14 +103,20 @@ class Migration(migrations.Migration): 'unique_together': {('workflow_name', 'workflow_version')}, }, ), + migrations.AddField( + model_name='analysis', + name='workflows', + field=models.ManyToManyField(to='workflow_manager.workflow'), + ), migrations.CreateModel( name='WorkflowRun', fields=[ - ('id', models.BigAutoField(primary_key=True, serialize=False)), + ('orcabus_id', workflow_manager.fields.OrcabusIdField(max_length=30, prefix='wfr', primary_key=True, serialize=False, unique=True)), ('portal_run_id', models.CharField(max_length=255, unique=True)), ('execution_id', models.CharField(blank=True, max_length=255, null=True)), ('workflow_run_name', models.CharField(blank=True, max_length=255, null=True)), ('comment', models.CharField(blank=True, max_length=255, null=True)), + ('analysis_run', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='workflow_manager.analysisrun')), ('libraries', models.ManyToManyField(through='workflow_manager.LibraryAssociation', to='workflow_manager.library')), ('workflow', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='workflow_manager.workflow')), ], @@ -81,10 +129,14 @@ class Migration(migrations.Migration): name='workflow_run', field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='workflow_manager.workflowrun'), ), + migrations.AlterUniqueTogether( + name='analysis', + unique_together={('analysis_name', 'analysis_version')}, + ), migrations.CreateModel( name='State', fields=[ - ('id', models.BigAutoField(primary_key=True, serialize=False)), + ('orcabus_id', workflow_manager.fields.OrcabusIdField(max_length=30, prefix='stt', primary_key=True, serialize=False, unique=True)), ('status', models.CharField(max_length=255)), ('timestamp', models.DateTimeField()), ('comment', models.CharField(blank=True, max_length=255, null=True)), diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0002_analysiscontext_analysisrun_analysis.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0002_analysiscontext_analysisrun_analysis.py deleted file mode 100644 index 98a051222..000000000 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0002_analysiscontext_analysisrun_analysis.py +++ /dev/null @@ -1,56 +0,0 @@ -# Generated by Django 5.1 on 2024-09-07 04:09 - -import django.db.models.deletion -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('workflow_manager', '0001_initial'), - ] - - operations = [ - migrations.CreateModel( - name='AnalysisContext', - fields=[ - ('orcabus_id', models.CharField(max_length=255, primary_key=True, serialize=False)), - ('context_id', models.CharField(max_length=255)), - ('name', models.CharField(max_length=255)), - ('description', models.CharField(max_length=255)), - ('status', models.CharField(max_length=255)), - ], - options={ - 'abstract': False, - }, - ), - migrations.CreateModel( - name='AnalysisRun', - fields=[ - ('id', models.BigAutoField(primary_key=True, serialize=False)), - ('analysis_run_id', models.CharField(max_length=255, unique=True)), - ('analysis_run_name', models.CharField(max_length=255)), - ('comment', models.CharField(max_length=255)), - ('approval_context', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='approval_context', to='workflow_manager.analysiscontext')), - ('project_context', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='project_context', to='workflow_manager.analysiscontext')), - ], - options={ - 'abstract': False, - }, - ), - migrations.CreateModel( - name='Analysis', - fields=[ - ('id', models.BigAutoField(primary_key=True, serialize=False)), - ('analysis_name', models.CharField(max_length=255)), - ('analysis_version', models.CharField(max_length=255)), - ('description', models.CharField(max_length=255)), - ('status', models.CharField(max_length=255)), - ('workflows', models.ManyToManyField(to='workflow_manager.workflow')), - ('allowed_contexts', models.ManyToManyField(to='workflow_manager.analysiscontext')), - ], - options={ - 'unique_together': {('analysis_name', 'analysis_version')}, - }, - ), - ] diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0003_rename_allowed_contexts_analysis_contexts.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0003_rename_allowed_contexts_analysis_contexts.py deleted file mode 100644 index 936bd4d0c..000000000 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0003_rename_allowed_contexts_analysis_contexts.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 5.1 on 2024-09-07 04:17 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('workflow_manager', '0002_analysiscontext_analysisrun_analysis'), - ] - - operations = [ - migrations.RenameField( - model_name='analysis', - old_name='allowed_contexts', - new_name='contexts', - ), - ] diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0004_analysisrun_analysis_analysisrun_libraries_and_more.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0004_analysisrun_analysis_analysisrun_libraries_and_more.py deleted file mode 100644 index a92c769a9..000000000 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0004_analysisrun_analysis_analysisrun_libraries_and_more.py +++ /dev/null @@ -1,34 +0,0 @@ -# Generated by Django 5.1 on 2024-09-07 06:53 - -import django.db.models.deletion -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('workflow_manager', '0003_rename_allowed_contexts_analysis_contexts'), - ] - - operations = [ - migrations.AddField( - model_name='analysisrun', - name='analysis', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='workflow_manager.analysis'), - ), - migrations.AddField( - model_name='analysisrun', - name='libraries', - field=models.ManyToManyField(to='workflow_manager.library'), - ), - migrations.AddField( - model_name='analysisrun', - name='status', - field=models.CharField(blank=True, max_length=255, null=True), - ), - migrations.AlterField( - model_name='analysisrun', - name='comment', - field=models.CharField(blank=True, max_length=255, null=True), - ), - ] diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0005_workflowrun_analysis_run.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0005_workflowrun_analysis_run.py deleted file mode 100644 index 9b4a44800..000000000 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0005_workflowrun_analysis_run.py +++ /dev/null @@ -1,19 +0,0 @@ -# Generated by Django 5.1 on 2024-09-09 00:23 - -import django.db.models.deletion -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('workflow_manager', '0004_analysisrun_analysis_analysisrun_libraries_and_more'), - ] - - operations = [ - migrations.AddField( - model_name='workflowrun', - name='analysis_run', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='workflow_manager.analysisrun'), - ), - ] diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis.py index 5380fce62..9d3098f2c 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis.py @@ -1,6 +1,7 @@ from django.db import models from workflow_manager.models.base import OrcaBusBaseModel, OrcaBusBaseManager +from workflow_manager.fields import OrcabusIdField from workflow_manager.models.analysis_context import AnalysisContext from workflow_manager.models.workflow import Workflow @@ -13,7 +14,7 @@ class Analysis(OrcaBusBaseModel): class Meta: unique_together = ["analysis_name", "analysis_version"] - id = models.BigAutoField(primary_key=True) + orcabus_id = OrcabusIdField(prefix='ana', primary_key=True) analysis_name = models.CharField(max_length=255) analysis_version = models.CharField(max_length=255) diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_context.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_context.py index e32bb98b3..a8503d2cb 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_context.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_context.py @@ -1,6 +1,7 @@ from django.db import models from workflow_manager.models.base import OrcaBusBaseModel, OrcaBusBaseManager +from workflow_manager.fields import OrcabusIdField class AnalysisContextManager(OrcaBusBaseManager): @@ -9,7 +10,7 @@ class AnalysisContextManager(OrcaBusBaseManager): class AnalysisContext(OrcaBusBaseModel): - orcabus_id = models.CharField(primary_key=True, max_length=255) + orcabus_id = OrcabusIdField(prefix='ctx', primary_key=True) context_id = models.CharField(max_length=255) name = models.CharField(max_length=255) diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_run.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_run.py index f284306dd..995a4ce23 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_run.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_run.py @@ -1,5 +1,6 @@ from django.db import models +from workflow_manager.fields import OrcabusIdField from workflow_manager.models.base import OrcaBusBaseModel, OrcaBusBaseManager from workflow_manager.models.analysis import Analysis from workflow_manager.models.analysis_context import AnalysisContext @@ -11,7 +12,7 @@ class AnalysisRunManager(OrcaBusBaseManager): class AnalysisRun(OrcaBusBaseModel): - id = models.BigAutoField(primary_key=True) + orcabus_id = OrcabusIdField(prefix='anr', primary_key=True) analysis_run_id = models.CharField(max_length=255, unique=True) diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/payload.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/payload.py index 23b2d2097..5e80d9fe2 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/payload.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/payload.py @@ -1,6 +1,7 @@ from django.core.serializers.json import DjangoJSONEncoder from django.db import models +from workflow_manager.fields import OrcabusIdField from workflow_manager.models.base import OrcaBusBaseModel, OrcaBusBaseManager @@ -9,7 +10,7 @@ class PayloadManager(OrcaBusBaseManager): class Payload(OrcaBusBaseModel): - id = models.BigAutoField(primary_key=True) + orcabus_id = OrcabusIdField(prefix='pld', primary_key=True) payload_ref_id = models.CharField(max_length=255, unique=True) version = models.CharField(max_length=255) diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/state.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/state.py index de8c607a5..0e8555fe7 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/state.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/state.py @@ -2,6 +2,8 @@ from enum import Enum from typing import List + +from workflow_manager.fields import OrcabusIdField from workflow_manager.models.base import OrcaBusBaseModel, OrcaBusBaseManager from workflow_manager.models.workflow_run import WorkflowRun from workflow_manager.models.payload import Payload @@ -89,8 +91,7 @@ class State(OrcaBusBaseModel): class Meta: unique_together = ["workflow_run", "status", "timestamp"] - id = models.BigAutoField(primary_key=True) - + orcabus_id = OrcabusIdField(prefix='stt', primary_key=True) # --- mandatory fields workflow_run = models.ForeignKey(WorkflowRun, on_delete=models.CASCADE) status = models.CharField(max_length=255) # TODO: How and where to enforce conventions? diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow.py index 79880a3ec..c87bd8d71 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow.py @@ -1,5 +1,6 @@ from django.db import models +from workflow_manager.fields import OrcabusIdField from workflow_manager.models.base import OrcaBusBaseModel, OrcaBusBaseManager @@ -12,7 +13,7 @@ class Meta: # a combo of this gives us human-readable pipeline id unique_together = ["workflow_name", "workflow_version"] - id = models.BigAutoField(primary_key=True) + orcabus_id = OrcabusIdField(prefix='wfl', primary_key=True) # human choice - how this is being named workflow_name = models.CharField(max_length=255) diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow_run.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow_run.py index 31eef2733..a80752008 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow_run.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow_run.py @@ -1,5 +1,6 @@ from django.db import models +from workflow_manager.fields import OrcabusIdField from workflow_manager.models.base import OrcaBusBaseModel, OrcaBusBaseManager from workflow_manager.models.library import Library from workflow_manager.models.workflow import Workflow @@ -11,8 +12,7 @@ class WorkflowRunManager(OrcaBusBaseManager): class WorkflowRun(OrcaBusBaseModel): - id = models.BigAutoField(primary_key=True) - + orcabus_id = OrcabusIdField(prefix='wfr', primary_key=True) # --- mandatory fields portal_run_id = models.CharField(max_length=255, unique=True) From 76fcb13af9a5fa0d886ca5d44e4f6cafcb99f1a6 Mon Sep 17 00:00:00 2001 From: Florian Reisinger Date: Wed, 2 Oct 2024 14:46:40 +1000 Subject: [PATCH 06/14] Implement Orcabus ID following MM inheritance model --- .../workflow_manager/fields.py | 30 ++++----- .../generate_analysis_for_metadata.py | 21 +++---- .../commands/generate_mock_workflow_run.py | 8 +-- .../migrations/0001_initial.py | 48 +++++++------- .../workflow_manager/models/analysis.py | 11 ++-- .../models/analysis_context.py | 11 ++-- .../workflow_manager/models/analysis_run.py | 11 ++-- .../workflow_manager/models/base.py | 57 +++++++++++++---- .../workflow_manager/models/library.py | 9 +-- .../workflow_manager/models/payload.py | 7 +-- .../workflow_manager/models/state.py | 19 +++--- .../workflow_manager/models/workflow.py | 16 ++--- .../workflow_manager/models/workflow_run.py | 17 ++--- .../workflow_manager/serializers.py | 63 ------------------- .../workflow_manager/serializers/__init__.py | 0 .../workflow_manager/serializers/analysis.py | 45 +++++++++++++ .../serializers/analysis_context.py | 18 ++++++ .../serializers/analysis_run.py | 36 +++++++++++ .../workflow_manager/serializers/base.py | 10 +++ .../workflow_manager/serializers/library.py | 12 ++++ .../workflow_manager/serializers/payload.py | 12 ++++ .../workflow_manager/serializers/state.py | 19 ++++++ .../workflow_manager/serializers/workflow.py | 18 ++++++ .../serializers/workflow_run.py | 35 +++++++++++ .../workflow_manager/tests/factories.py | 3 +- .../workflow_manager/urls/base.py | 29 +++++---- .../workflow_manager/viewsets/analysis.py | 23 +++++++ .../viewsets/analysis_context.py | 21 +++++++ .../workflow_manager/viewsets/analysis_run.py | 23 +++++++ .../workflow_manager/viewsets/base.py | 50 +++++++++++++++ .../workflow_manager/viewsets/library.py | 27 ++++---- .../workflow_manager/viewsets/payload.py | 27 ++++---- .../workflow_manager/viewsets/state.py | 30 ++++----- .../workflow_manager/viewsets/workflow.py | 25 ++++---- .../workflow_manager/viewsets/workflow_run.py | 30 +++++---- .../services/create_workflow_run_state.py | 1 - .../tests/test_create_workflow_run_state.py | 8 +-- 37 files changed, 558 insertions(+), 272 deletions(-) delete mode 100644 lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers.py create mode 100644 lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/__init__.py create mode 100644 lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/analysis.py create mode 100644 lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/analysis_context.py create mode 100644 lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/analysis_run.py create mode 100644 lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/base.py create mode 100644 lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/library.py create mode 100644 lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/payload.py create mode 100644 lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/state.py create mode 100644 lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/workflow.py create mode 100644 lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/workflow_run.py create mode 100644 lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/analysis.py create mode 100644 lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/analysis_context.py create mode 100644 lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/analysis_run.py create mode 100644 lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/base.py diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/fields.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/fields.py index 26f591fb8..fbabd177c 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/fields.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/fields.py @@ -2,32 +2,28 @@ import ulid from django.db import models +from django.core.validators import RegexValidator - -def orcabus_id(prefix: str) -> str: - oid = f"{prefix}.{ulid.new()}" - return oid +orcabus_id_validator = RegexValidator( + regex=r'[\w]{26}$', + message='ULID is expected to be 26 characters long', + code='invalid_orcabus_id' + ) class OrcabusIdField(models.CharField): - description = "An OrcaBus internal ID composed of a 3 letter prefix followed by dot followed by a ULID" + description = "An OrcaBus internal ID (ULID)" def __init__(self, prefix, *args, **kwargs): - self.prefix = prefix - kwargs["max_length"] = 30 # prefix + . + ULID = 3 + 1 + 26 = 30 + kwargs["max_length"] = 26 # ULID length kwargs['unique'] = True + kwargs['editable'] = False + kwargs['blank'] = False + kwargs['null'] = False + kwargs['default'] = ulid.new + kwargs['validators'] = [orcabus_id_validator] super().__init__(*args, **kwargs) - def deconstruct(self): - name, path, args, kwargs = super().deconstruct() - if self.prefix is not None: - kwargs["prefix"] = self.prefix - return name, path, args, kwargs - - def pre_save(self, instance, add): - setattr(instance, self.attname, orcabus_id(self.prefix)) - return super().pre_save(instance, add) - class HashField(models.CharField): description = ( diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/management/commands/generate_analysis_for_metadata.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/management/commands/generate_analysis_for_metadata.py index 950affd76..89ac4562e 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/management/commands/generate_analysis_for_metadata.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/management/commands/generate_analysis_for_metadata.py @@ -111,8 +111,8 @@ def _setup_requirements(): # The contexts information available for analysis clinical_context = AnalysisContext( - context_id="C12345", name="accredited", + usecase="analysis-selection", description="Accredited by NATA", status="ACTIVE", ) @@ -120,8 +120,8 @@ def _setup_requirements(): clinical_context.save() research_context = AnalysisContext( - context_id="C23456", name="research", + usecase="analysis-selection", description="For research use", status="ACTIVE", ) @@ -129,8 +129,8 @@ def _setup_requirements(): research_context.save() internal_context = AnalysisContext( - context_id="C00001", name="internal", + usecase="analysis-selection", description="For internal use", status="ACTIVE", ) @@ -240,12 +240,12 @@ def _setup_requirements(): # Generate a Payload stub and some Libraries generic_payload = PayloadFactory() # Payload content is not important for now - LibraryFactory(orcabus_id="lib.01J5M2JFE1JPYV62RYQEG99CP1", library_id="L000001"), - LibraryFactory(orcabus_id="lib.02J5M2JFE1JPYV62RYQEG99CP2", library_id="L000002"), - LibraryFactory(orcabus_id="lib.03J5M2JFE1JPYV62RYQEG99CP3", library_id="L000003"), - LibraryFactory(orcabus_id="lib.03J5M2JFE1JPYV62RYQEG99CP4", library_id="L000004"), - LibraryFactory(orcabus_id="lib.03J5M2JFE1JPYV62RYQEG99CP5", library_id="L000005"), - LibraryFactory(orcabus_id="lib.04J5M2JFE1JPYV62RYQEG99CP6", library_id="L000006") + LibraryFactory(orcabus_id="01J5M2JFE1JPYV62RYQEG99CP1", library_id="L000001"), + LibraryFactory(orcabus_id="02J5M2JFE1JPYV62RYQEG99CP2", library_id="L000002"), + LibraryFactory(orcabus_id="03J5M2JFE1JPYV62RYQEG99CP3", library_id="L000003"), + LibraryFactory(orcabus_id="03J5M2JFE1JPYV62RYQEG99CP4", library_id="L000004"), + LibraryFactory(orcabus_id="03J5M2JFE1JPYV62RYQEG99CP5", library_id="L000005"), + LibraryFactory(orcabus_id="04J5M2JFE1JPYV62RYQEG99CP6", library_id="L000006") def assign_analysis(libraries: List[dict]) -> List[AnalysisRun]: @@ -278,7 +278,6 @@ def create_qc_analysis(libraries: List[dict]) -> List[AnalysisRun]: if lib['type'] in ['WGS', 'WTS']: # Create QC analysis analysis_run = AnalysisRun( - analysis_run_id=f"ar.{ulid.new().str}", analysis_run_name=f"automated__{analysis_qc.analysis_name}__{lib_record.library_id}", status="DRAFT", approval_context=context_internal, # FIXME: does this matter here? Internal? @@ -324,7 +323,6 @@ def create_wgs_analysis(libraries: List[dict]) -> List[AnalysisRun]: analysis_run_name = f"automated__{analysis.analysis_name}__{context.name}__" + \ f"{tumor_lib_record.library_id}__{normal_lib_record.library_id} " ar_wgs = AnalysisRun( - analysis_run_id=f"ar.{ulid.new().str}", analysis_run_name=analysis_run_name, status="DRAFT", approval_context=context, @@ -354,7 +352,6 @@ def create_cttso_analysis(libraries: List[dict]) -> List[AnalysisRun]: context: AnalysisContext = context_clinical if lib['workflow'] == 'clinical' else context_research analysis_run_name = f"automated__{analysis_cttso_qs.analysis_name}__{context.name}__{lib_record.library_id}" analysis_run = AnalysisRun( - analysis_run_id=f"ar.{ulid.new().str}", analysis_run_name=analysis_run_name, status="DRAFT", approval_context=context, diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/management/commands/generate_mock_workflow_run.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/management/commands/generate_mock_workflow_run.py index 1b1a4c21b..92ecaa0f0 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/management/commands/generate_mock_workflow_run.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/management/commands/generate_mock_workflow_run.py @@ -30,10 +30,10 @@ def handle(self, *args, **options): # Common components: payload and libraries generic_payload = PayloadFactory() # Payload content is not important for now libraries = [ - LibraryFactory(orcabus_id="lib.01J5M2JFE1JPYV62RYQEG99CP1", library_id="L000001"), - LibraryFactory(orcabus_id="lib.02J5M2JFE1JPYV62RYQEG99CP2", library_id="L000002"), - LibraryFactory(orcabus_id="lib.03J5M2JFE1JPYV62RYQEG99CP3", library_id="L000003"), - LibraryFactory(orcabus_id="lib.04J5M2JFE1JPYV62RYQEG99CP4", library_id="L000004") + LibraryFactory(orcabus_id="01J5M2JFE1JPYV62RYQEG99CP1", library_id="L000001"), + LibraryFactory(orcabus_id="02J5M2JFE1JPYV62RYQEG99CP2", library_id="L000002"), + LibraryFactory(orcabus_id="03J5M2JFE1JPYV62RYQEG99CP3", library_id="L000003"), + LibraryFactory(orcabus_id="04J5M2JFE1JPYV62RYQEG99CP4", library_id="L000004") ] # First case: a primary workflow with two executions linked to 4 libraries diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0001_initial.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0001_initial.py index 66eafbcef..b5b9057d0 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0001_initial.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0001_initial.py @@ -1,8 +1,8 @@ -# Generated by Django 5.1 on 2024-09-25 08:06 +# Generated by Django 5.1 on 2024-10-01 05:49 import django.core.serializers.json +import django.core.validators import django.db.models.deletion -import workflow_manager.fields from django.db import migrations, models @@ -14,23 +14,10 @@ class Migration(migrations.Migration): ] operations = [ - migrations.CreateModel( - name='AnalysisContext', - fields=[ - ('orcabus_id', workflow_manager.fields.OrcabusIdField(max_length=30, prefix='ctx', primary_key=True, serialize=False, unique=True)), - ('context_id', models.CharField(max_length=255)), - ('name', models.CharField(max_length=255)), - ('description', models.CharField(max_length=255)), - ('status', models.CharField(max_length=255)), - ], - options={ - 'abstract': False, - }, - ), migrations.CreateModel( name='Library', fields=[ - ('orcabus_id', models.CharField(max_length=255, primary_key=True, serialize=False)), + ('orcabus_id', models.CharField(editable=False, primary_key=True, serialize=False, unique=True, validators=[django.core.validators.RegexValidator(code='invalid_orcabus_id', message='ULID is expected to be 26 characters long', regex='[\\w]{26}$')])), ('library_id', models.CharField(max_length=255)), ], options={ @@ -40,7 +27,7 @@ class Migration(migrations.Migration): migrations.CreateModel( name='Payload', fields=[ - ('orcabus_id', workflow_manager.fields.OrcabusIdField(max_length=30, prefix='pld', primary_key=True, serialize=False, unique=True)), + ('orcabus_id', models.CharField(editable=False, primary_key=True, serialize=False, unique=True, validators=[django.core.validators.RegexValidator(code='invalid_orcabus_id', message='ULID is expected to be 26 characters long', regex='[\\w]{26}$')])), ('payload_ref_id', models.CharField(max_length=255, unique=True)), ('version', models.CharField(max_length=255)), ('data', models.JSONField(encoder=django.core.serializers.json.DjangoJSONEncoder)), @@ -49,10 +36,23 @@ class Migration(migrations.Migration): 'abstract': False, }, ), + migrations.CreateModel( + name='AnalysisContext', + fields=[ + ('orcabus_id', models.CharField(editable=False, primary_key=True, serialize=False, unique=True, validators=[django.core.validators.RegexValidator(code='invalid_orcabus_id', message='ULID is expected to be 26 characters long', regex='[\\w]{26}$')])), + ('name', models.CharField(max_length=255)), + ('usecase', models.CharField(max_length=255)), + ('description', models.CharField(max_length=255)), + ('status', models.CharField(max_length=255)), + ], + options={ + 'unique_together': {('name', 'usecase')}, + }, + ), migrations.CreateModel( name='Analysis', fields=[ - ('orcabus_id', workflow_manager.fields.OrcabusIdField(max_length=30, prefix='ana', primary_key=True, serialize=False, unique=True)), + ('orcabus_id', models.CharField(editable=False, primary_key=True, serialize=False, unique=True, validators=[django.core.validators.RegexValidator(code='invalid_orcabus_id', message='ULID is expected to be 26 characters long', regex='[\\w]{26}$')])), ('analysis_name', models.CharField(max_length=255)), ('analysis_version', models.CharField(max_length=255)), ('description', models.CharField(max_length=255)), @@ -63,8 +63,7 @@ class Migration(migrations.Migration): migrations.CreateModel( name='AnalysisRun', fields=[ - ('orcabus_id', workflow_manager.fields.OrcabusIdField(max_length=30, prefix='anr', primary_key=True, serialize=False, unique=True)), - ('analysis_run_id', models.CharField(max_length=255, unique=True)), + ('orcabus_id', models.CharField(editable=False, primary_key=True, serialize=False, unique=True, validators=[django.core.validators.RegexValidator(code='invalid_orcabus_id', message='ULID is expected to be 26 characters long', regex='[\\w]{26}$')])), ('analysis_run_name', models.CharField(max_length=255)), ('comment', models.CharField(blank=True, max_length=255, null=True)), ('status', models.CharField(blank=True, max_length=255, null=True)), @@ -80,7 +79,7 @@ class Migration(migrations.Migration): migrations.CreateModel( name='LibraryAssociation', fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('orcabus_id', models.CharField(editable=False, primary_key=True, serialize=False, unique=True, validators=[django.core.validators.RegexValidator(code='invalid_orcabus_id', message='ULID is expected to be 26 characters long', regex='[\\w]{26}$')])), ('association_date', models.DateTimeField()), ('status', models.CharField(max_length=255)), ('library', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='workflow_manager.library')), @@ -92,12 +91,11 @@ class Migration(migrations.Migration): migrations.CreateModel( name='Workflow', fields=[ - ('orcabus_id', workflow_manager.fields.OrcabusIdField(max_length=30, prefix='wfl', primary_key=True, serialize=False, unique=True)), + ('orcabus_id', models.CharField(editable=False, primary_key=True, serialize=False, unique=True, validators=[django.core.validators.RegexValidator(code='invalid_orcabus_id', message='ULID is expected to be 26 characters long', regex='[\\w]{26}$')])), ('workflow_name', models.CharField(max_length=255)), ('workflow_version', models.CharField(max_length=255)), ('execution_engine', models.CharField(max_length=255)), ('execution_engine_pipeline_id', models.CharField(max_length=255)), - ('approval_state', models.CharField(max_length=255)), ], options={ 'unique_together': {('workflow_name', 'workflow_version')}, @@ -111,7 +109,7 @@ class Migration(migrations.Migration): migrations.CreateModel( name='WorkflowRun', fields=[ - ('orcabus_id', workflow_manager.fields.OrcabusIdField(max_length=30, prefix='wfr', primary_key=True, serialize=False, unique=True)), + ('orcabus_id', models.CharField(editable=False, primary_key=True, serialize=False, unique=True, validators=[django.core.validators.RegexValidator(code='invalid_orcabus_id', message='ULID is expected to be 26 characters long', regex='[\\w]{26}$')])), ('portal_run_id', models.CharField(max_length=255, unique=True)), ('execution_id', models.CharField(blank=True, max_length=255, null=True)), ('workflow_run_name', models.CharField(blank=True, max_length=255, null=True)), @@ -136,7 +134,7 @@ class Migration(migrations.Migration): migrations.CreateModel( name='State', fields=[ - ('orcabus_id', workflow_manager.fields.OrcabusIdField(max_length=30, prefix='stt', primary_key=True, serialize=False, unique=True)), + ('orcabus_id', models.CharField(editable=False, primary_key=True, serialize=False, unique=True, validators=[django.core.validators.RegexValidator(code='invalid_orcabus_id', message='ULID is expected to be 26 characters long', regex='[\\w]{26}$')])), ('status', models.CharField(max_length=255)), ('timestamp', models.DateTimeField()), ('comment', models.CharField(blank=True, max_length=255, null=True)), diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis.py index 9d3098f2c..149b2ae48 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis.py @@ -1,7 +1,6 @@ from django.db import models from workflow_manager.models.base import OrcaBusBaseModel, OrcaBusBaseManager -from workflow_manager.fields import OrcabusIdField from workflow_manager.models.analysis_context import AnalysisContext from workflow_manager.models.workflow import Workflow @@ -14,7 +13,7 @@ class Analysis(OrcaBusBaseModel): class Meta: unique_together = ["analysis_name", "analysis_version"] - orcabus_id = OrcabusIdField(prefix='ana', primary_key=True) + orcabus_id_prefix = 'ana.' analysis_name = models.CharField(max_length=255) analysis_version = models.CharField(max_length=255) @@ -28,13 +27,13 @@ class Meta: objects = AnalysisManager() def __str__(self): - return f"ID: {self.id}, analysis_name: {self.analysis_name}, analysis_version: {self.analysis_version}" + return f"ID: {self.orcabus_id}, analysis_name: {self.analysis_name}, analysis_version: {self.analysis_version}" def to_dict(self): return { - "id": self.id, - "analysis_name": self.analysis_name, - "analysis_version": self.analysis_version, + "orcabusId": self.orcabus_id, + "analysisName": self.analysis_name, + "analysisVersion": self.analysis_version, "description": self.description, "status": self.status } diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_context.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_context.py index a8503d2cb..8bffa1ce8 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_context.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_context.py @@ -1,7 +1,6 @@ from django.db import models from workflow_manager.models.base import OrcaBusBaseModel, OrcaBusBaseManager -from workflow_manager.fields import OrcabusIdField class AnalysisContextManager(OrcaBusBaseManager): @@ -9,24 +8,26 @@ class AnalysisContextManager(OrcaBusBaseManager): class AnalysisContext(OrcaBusBaseModel): + class Meta: + unique_together = ["name", "usecase"] - orcabus_id = OrcabusIdField(prefix='ctx', primary_key=True) - context_id = models.CharField(max_length=255) + orcabus_id_prefix = 'ctx.' name = models.CharField(max_length=255) + usecase = models.CharField(max_length=255) description = models.CharField(max_length=255) status = models.CharField(max_length=255) objects = AnalysisContextManager() def __str__(self): - return f"ID: {self.orcabus_id}, name: {self.name}, status: {self.status}" + return f"ID: {self.orcabus_id}, name: {self.name}, usecase: {self.usecase}" def to_dict(self): return { "orcabus_id": self.orcabus_id, - "context_id": self.context_id, "name": self.name, + "usecase": self.usecase, "status": self.status, "description": self.description } diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_run.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_run.py index 995a4ce23..298a17ca0 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_run.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_run.py @@ -1,9 +1,8 @@ from django.db import models -from workflow_manager.fields import OrcabusIdField -from workflow_manager.models.base import OrcaBusBaseModel, OrcaBusBaseManager from workflow_manager.models.analysis import Analysis from workflow_manager.models.analysis_context import AnalysisContext +from workflow_manager.models.base import OrcaBusBaseModel, OrcaBusBaseManager from workflow_manager.models.library import Library @@ -12,9 +11,7 @@ class AnalysisRunManager(OrcaBusBaseManager): class AnalysisRun(OrcaBusBaseModel): - orcabus_id = OrcabusIdField(prefix='anr', primary_key=True) - - analysis_run_id = models.CharField(max_length=255, unique=True) + orcabus_id_prefix = 'anr.' analysis_run_name = models.CharField(max_length=255) comment = models.CharField(max_length=255, null=True, blank=True) @@ -30,11 +27,11 @@ class AnalysisRun(OrcaBusBaseModel): objects = AnalysisRunManager() def __str__(self): - return f"ID: {self.analysis_run_id}, analysis_run_name: {self.analysis_run_name}" + return f"ID: {self.orcabus_id}, analysis_run_name: {self.analysis_run_name}" def to_dict(self): return { - "analysis_run_id": self.analysis_run_id, + "orcabusId": self.orcabus_id, "analysis_run_name": self.analysis_run_name, "comment": self.comment, "approval_context": self.approval_context, diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/base.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/base.py index 226bb47ac..d13ad87fd 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/base.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/base.py @@ -1,9 +1,11 @@ import logging import operator +import ulid from functools import reduce from typing import List from django.core.exceptions import FieldError +from django.core.validators import RegexValidator from django.db import models from django.db.models import Q, ManyToManyField, ForeignKey, ForeignObject, OneToOneField, ForeignObjectRel, \ ManyToOneRel, ManyToManyRel, OneToOneRel, QuerySet @@ -13,17 +15,30 @@ logger = logging.getLogger(__name__) +orcabus_id_validator = RegexValidator( + regex=r'^[\w]{26}$', + message='ULID is expected to be 26 characters long', + code='invalid_orcabus_id' + ) class OrcaBusBaseManager(models.Manager): - def get_by_keyword(self, **kwargs) -> QuerySet: - qs: QuerySet = super().get_queryset() + + def get_by_keyword(self, qs=None, **kwargs) -> QuerySet: + if qs is None: + qs = super().get_queryset() return self.get_model_fields_query(qs, **kwargs) @staticmethod def reduce_multi_values_qor(key: str, values: List[str]): - if isinstance(values, (str, int, float,)): + if not isinstance( + values, + list, + ): values = [values] - return reduce(operator.or_, (Q(**{"%s__iexact" % key: value}) for value in values)) + return reduce( + operator.or_, (Q(**{"%s__iexact" % key: value}) + for value in values) + ) def get_model_fields_query(self, qs: QuerySet, **kwargs) -> QuerySet: @@ -31,14 +46,16 @@ def exclude_params(params): for param in params: kwargs.pop(param) if param in kwargs.keys() else None - exclude_params([ - api_settings.SEARCH_PARAM, - api_settings.ORDERING_PARAM, - PaginationConstant.PAGE, - PaginationConstant.ROWS_PER_PAGE, - "sortCol", - "sortAsc", - ]) + exclude_params( + [ + api_settings.SEARCH_PARAM, + api_settings.ORDERING_PARAM, + PaginationConstant.PAGE, + PaginationConstant.ROWS_PER_PAGE, + "sortCol", + "sortAsc", + ] + ) query_string = None @@ -63,6 +80,22 @@ class OrcaBusBaseModel(models.Model): class Meta: abstract = True + orcabus_id = models.CharField( + primary_key=True, + unique=True, + editable=False, + blank=False, + null=False, + validators=[orcabus_id_validator] + ) + + def save(self, *args, **kwargs): + if not self.orcabus_id: + self.orcabus_id = ulid.new().str + self.full_clean() # make sure we are validating + return super(OrcaBusBaseModel, self).save(*args, **kwargs) + + @classmethod def get_fields(cls): return [f.name for f in cls._meta.get_fields()] diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/library.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/library.py index 149947390..f59d47da5 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/library.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/library.py @@ -10,16 +10,17 @@ class LibraryManager(OrcaBusBaseManager): class Library(OrcaBusBaseModel): - orcabus_id = models.CharField(primary_key=True, max_length=255) + orcabus_id_prefix = "lib." + library_id = models.CharField(max_length=255) objects = LibraryManager() def __str__(self): - return f"orcabus_id: {self.orcabus_id}, library_id: {self.library_id}" + return f"ID: {self.orcabus_id}, library_id: {self.library_id}" def to_dict(self): return { - "orcabus_id": self.orcabus_id, - "library_id": self.library_id + "orcabusId": self.orcabus_id, + "libraryId": self.library_id } diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/payload.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/payload.py index 5e80d9fe2..494eae23d 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/payload.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/payload.py @@ -1,7 +1,6 @@ from django.core.serializers.json import DjangoJSONEncoder from django.db import models -from workflow_manager.fields import OrcabusIdField from workflow_manager.models.base import OrcaBusBaseModel, OrcaBusBaseManager @@ -10,7 +9,7 @@ class PayloadManager(OrcaBusBaseManager): class Payload(OrcaBusBaseModel): - orcabus_id = OrcabusIdField(prefix='pld', primary_key=True) + orcabus_id_prefix = 'pld.' payload_ref_id = models.CharField(max_length=255, unique=True) version = models.CharField(max_length=255) @@ -19,11 +18,11 @@ class Payload(OrcaBusBaseModel): objects = PayloadManager() def __str__(self): - return f"ID: {self.id}, payload_ref_id: {self.payload_ref_id}" + return f"ID: {self.orcabus_id}, payload_ref_id: {self.payload_ref_id}" def to_dict(self): return { - "id": self.id, + "orcabusId": self.orcabus_id, "payload_ref_id": self.payload_ref_id, "version": self.version, "data": self.data diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/state.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/state.py index 0e8555fe7..2a1777eae 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/state.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/state.py @@ -1,12 +1,11 @@ -from django.db import models - from enum import Enum from typing import List -from workflow_manager.fields import OrcabusIdField +from django.db import models + from workflow_manager.models.base import OrcaBusBaseModel, OrcaBusBaseManager -from workflow_manager.models.workflow_run import WorkflowRun from workflow_manager.models.payload import Payload +from workflow_manager.models.workflow_run import WorkflowRun class Status(Enum): @@ -91,26 +90,26 @@ class State(OrcaBusBaseModel): class Meta: unique_together = ["workflow_run", "status", "timestamp"] - orcabus_id = OrcabusIdField(prefix='stt', primary_key=True) + orcabus_id_prefix = 'stt.' + # --- mandatory fields - workflow_run = models.ForeignKey(WorkflowRun, on_delete=models.CASCADE) status = models.CharField(max_length=255) # TODO: How and where to enforce conventions? timestamp = models.DateTimeField() - comment = models.CharField(max_length=255, null=True, blank=True) + workflow_run = models.ForeignKey(WorkflowRun, on_delete=models.CASCADE) # Link to workflow run payload data payload = models.ForeignKey(Payload, null=True, blank=True, on_delete=models.SET_NULL) objects = StateManager() def __str__(self): - return f"ID: {self.id}, status: {self.status}" + return f"ID: {self.orcabus_id}, status: {self.status}" def to_dict(self): return { - "id": self.id, - "workflow_run_id": self.workflow_run.id, + "orcabusId": self.orcabus_id, + "workflow_run_id": self.workflow_run.orcabus_id, "status": self.status, "timestamp": str(self.timestamp), "comment": self.comment, diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow.py index c87bd8d71..22ecae61e 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow.py @@ -1,6 +1,5 @@ from django.db import models -from workflow_manager.fields import OrcabusIdField from workflow_manager.models.base import OrcaBusBaseModel, OrcaBusBaseManager @@ -13,33 +12,28 @@ class Meta: # a combo of this gives us human-readable pipeline id unique_together = ["workflow_name", "workflow_version"] - orcabus_id = OrcabusIdField(prefix='wfl', primary_key=True) + orcabus_id_prefix = 'wfl.' - # human choice - how this is being named workflow_name = models.CharField(max_length=255) - - # human choice - how this is being named workflow_version = models.CharField(max_length=255) - - # human choice - how this is being named execution_engine = models.CharField(max_length=255) # definition from an external system (as known to the execution engine) execution_engine_pipeline_id = models.CharField(max_length=255) - approval_state = models.CharField(max_length=255) # FIXME: figure out what states we have and how many + # approval_state = models.CharField(max_length=255) # FIXME: Do we still need this (or just use Analysis)? objects = WorkflowManager() def __str__(self): - return f"ID: {self.id}, workflow_name: {self.workflow_name}, workflow_version: {self.workflow_version}" + return f"ID: {self.orcabus_id}, workflow_name: {self.workflow_name}, workflow_version: {self.workflow_version}" def to_dict(self): return { - "id": self.id, + "orcabusId": self.orcabus_id, "workflow_name": self.workflow_name, "workflow_version": self.workflow_version, "execution_engine": self.execution_engine, "execution_engine_pipeline_id": self.execution_engine_pipeline_id, - "approval_state": self.approval_state + # "approval_state": self.approval_state } diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow_run.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow_run.py index a80752008..062ab5939 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow_run.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow_run.py @@ -1,10 +1,9 @@ from django.db import models -from workflow_manager.fields import OrcabusIdField +from workflow_manager.models.analysis_run import AnalysisRun from workflow_manager.models.base import OrcaBusBaseModel, OrcaBusBaseManager from workflow_manager.models.library import Library from workflow_manager.models.workflow import Workflow -from workflow_manager.models.analysis_run import AnalysisRun class WorkflowRunManager(OrcaBusBaseManager): @@ -12,20 +11,14 @@ class WorkflowRunManager(OrcaBusBaseManager): class WorkflowRun(OrcaBusBaseModel): - orcabus_id = OrcabusIdField(prefix='wfr', primary_key=True) - # --- mandatory fields + orcabus_id_prefix = 'wfr.' portal_run_id = models.CharField(max_length=255, unique=True) - # --- optional fields - - # ID of the external service execution_id = models.CharField(max_length=255, null=True, blank=True) workflow_run_name = models.CharField(max_length=255, null=True, blank=True) comment = models.CharField(max_length=255, null=True, blank=True) - # --- FK link to value objects - # Relationships workflow = models.ForeignKey(Workflow, null=True, blank=True, on_delete=models.SET_NULL) analysis_run = models.ForeignKey(AnalysisRun, null=True, blank=True, on_delete=models.SET_NULL) @@ -34,12 +27,12 @@ class WorkflowRun(OrcaBusBaseModel): objects = WorkflowRunManager() def __str__(self): - return f"ID: {self.id}, portal_run_id: {self.portal_run_id}, workflow_run_name: {self.workflow_run_name}, " \ - f"workflow: {self.workflow.workflow_name} " + return f"ID: {self.orcabus_id}, portal_run_id: {self.portal_run_id}, workflow_run_name: {self.workflow_run_name}, " \ + f"workflowRun: {self.workflow.workflow_name} " def to_dict(self): return { - "id": self.id, + "orcabusId": self.orcabus_id, "portal_run_id": self.portal_run_id, "execution_id": self.execution_id, "workflow_run_name": self.workflow_run_name, diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers.py deleted file mode 100644 index da6ba63f0..000000000 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers.py +++ /dev/null @@ -1,63 +0,0 @@ -from rest_framework import serializers - -from workflow_manager.models import Workflow, WorkflowRun, Payload, Library, State - - -READ_ONLY_SERIALIZER = "READ ONLY SERIALIZER" - - -class LabMetadataSyncSerializer(serializers.Serializer): - sheets = serializers.ListField( - default=["2019", "2020", "2021", "2022", "2023"] - ) # OpenAPI swagger doc hint only - truncate = serializers.BooleanField(default=True) - - def update(self, instance, validated_data): - pass - - def create(self, validated_data): - pass - - -class SubjectIdSerializer(serializers.BaseSerializer): - def to_representation(self, instance): - return instance.subject_id - - def to_internal_value(self, data): - raise NotImplementedError(READ_ONLY_SERIALIZER) - - def update(self, instance, validated_data): - raise NotImplementedError(READ_ONLY_SERIALIZER) - - def create(self, validated_data): - raise NotImplementedError(READ_ONLY_SERIALIZER) - - -class WorkflowModelSerializer(serializers.ModelSerializer): - class Meta: - model = Workflow - fields = '__all__' - - -class WorkflowRunModelSerializer(serializers.ModelSerializer): - class Meta: - model = WorkflowRun - fields = '__all__' - - -class PayloadModelSerializer(serializers.ModelSerializer): - class Meta: - model = Payload - fields = '__all__' - - -class LibraryModelSerializer(serializers.ModelSerializer): - class Meta: - model = Library - fields = '__all__' - - -class StateModelSerializer(serializers.ModelSerializer): - class Meta: - model = State - fields = '__all__' diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/__init__.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/analysis.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/analysis.py new file mode 100644 index 000000000..8aeed96ec --- /dev/null +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/analysis.py @@ -0,0 +1,45 @@ +from workflow_manager.serializers.base import SerializersBase +from workflow_manager.models import Analysis, Workflow, AnalysisContext + + +class AnalysisBaseSerializer(SerializersBase): + prefix = Analysis.orcabus_id_prefix + + +class AnalysisSerializer(AnalysisBaseSerializer): + """ + Serializer to define a default representation of an Analysis record, + mainly used in record listings. + """ + class Meta: + model = Analysis + fields = "__all__" + # exclude = ["contexts", "workflows"] + + def to_representation(self, instance): + representation = super().to_representation(instance) + new_workflow_refs = [] # Rewrite internal OrcaBUs + for item in representation["workflows"]: + new_workflow_refs.append(f"{Workflow.orcabus_id_prefix}{item}") + representation["workflows"] = new_workflow_refs + new_context_refs = [] + for item in representation["contexts"]: + new_context_refs.append(f"{AnalysisContext.orcabus_id_prefix}{item}") + representation["contexts"] = new_context_refs + return representation + + +class AnalysisDetailSerializer(AnalysisBaseSerializer): + """ + Serializer to define a detailed representation of an Analysis record, + mainly used in individual record views. + """ + from .analysis_context import AnalysisContextSerializer + from .workflow import WorkflowMinSerializer + + contexts = AnalysisContextSerializer(many=True, read_only=True) + workflows = WorkflowMinSerializer(many=True, read_only=True) + + class Meta: + model = Analysis + fields = "__all__" diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/analysis_context.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/analysis_context.py new file mode 100644 index 000000000..381738068 --- /dev/null +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/analysis_context.py @@ -0,0 +1,18 @@ +from workflow_manager.serializers.base import SerializersBase +from workflow_manager.models import AnalysisContext + + +class AnalysisContextBaseSerializer(SerializersBase): + prefix = AnalysisContext.orcabus_id_prefix + + +class AnalysisContextMinSerializer(AnalysisContextBaseSerializer): + class Meta: + model = AnalysisContext + fields = ["orcabus_id", "name", "usecase"] + + +class AnalysisContextSerializer(AnalysisContextBaseSerializer): + class Meta: + model = AnalysisContext + fields = "__all__" diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/analysis_run.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/analysis_run.py new file mode 100644 index 000000000..279415800 --- /dev/null +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/analysis_run.py @@ -0,0 +1,36 @@ +from workflow_manager.serializers.base import SerializersBase +from workflow_manager.models import AnalysisRun, Analysis, AnalysisContext + + +class AnalysisRunBaseSerializer(SerializersBase): + prefix = AnalysisRun.orcabus_id_prefix + + +class AnalysisRunSerializer(AnalysisRunBaseSerializer): + class Meta: + model = AnalysisRun + exclude = ["libraries"] + + def to_representation(self, instance): + representation = super().to_representation(instance) + representation['analysis'] = Analysis.orcabus_id_prefix + representation['analysis'] + if representation['project_context']: + representation['project_context'] = AnalysisContext.orcabus_id_prefix + representation['project_context'] + if representation['approval_context']: + representation['approval_context'] = AnalysisContext.orcabus_id_prefix + representation['approval_context'] + return representation + + +class AnalysisRunDetailSerializer(AnalysisRunBaseSerializer): + from .library import LibrarySerializer + from .analysis import AnalysisSerializer + from .analysis_context import AnalysisContextSerializer + + libraries = LibrarySerializer(many=True, read_only=True) + analysis = AnalysisSerializer(read_only=True) + approval_context = AnalysisContextSerializer(read_only=True) + project_context = AnalysisContextSerializer(read_only=True) + + class Meta: + model = AnalysisRun + fields = "__all__" diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/base.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/base.py new file mode 100644 index 000000000..96600774b --- /dev/null +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/base.py @@ -0,0 +1,10 @@ +from rest_framework import serializers + + +class SerializersBase(serializers.ModelSerializer): + prefix = '' + + def to_representation(self, instance): + representation = super().to_representation(instance) + representation['orcabus_id'] = self.prefix + str(representation['orcabus_id']) + return representation diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/library.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/library.py new file mode 100644 index 000000000..9108264b0 --- /dev/null +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/library.py @@ -0,0 +1,12 @@ +from workflow_manager.serializers.base import SerializersBase +from workflow_manager.models import Library + + +class LibraryBaseSerializer(SerializersBase): + prefix = Library.orcabus_id_prefix + + +class LibrarySerializer(LibraryBaseSerializer): + class Meta: + model = Library + fields = "__all__" diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/payload.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/payload.py new file mode 100644 index 000000000..319d17620 --- /dev/null +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/payload.py @@ -0,0 +1,12 @@ +from workflow_manager.serializers.base import SerializersBase +from workflow_manager.models import Payload + + +class PayloadBaseSerializer(SerializersBase): + prefix = Payload.orcabus_id_prefix + + +class PayloadSerializer(PayloadBaseSerializer): + class Meta: + model = Payload + fields = "__all__" diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/state.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/state.py new file mode 100644 index 000000000..88b83de4a --- /dev/null +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/state.py @@ -0,0 +1,19 @@ +from workflow_manager.serializers.base import SerializersBase +from workflow_manager.models import State, WorkflowRun, Payload + + +class StateBaseSerializer(SerializersBase): + prefix = State.orcabus_id_prefix + + +class StateSerializer(StateBaseSerializer): + class Meta: + model = State + fields = "__all__" + + def to_representation(self, instance): + representation = super().to_representation(instance) + representation['workflow_run'] = WorkflowRun.orcabus_id_prefix + representation['workflow_run'] + if representation['payload']: + representation['payload'] = Payload.orcabus_id_prefix + representation['payload'] + return representation diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/workflow.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/workflow.py new file mode 100644 index 000000000..43e7209ca --- /dev/null +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/workflow.py @@ -0,0 +1,18 @@ +from workflow_manager.serializers.base import SerializersBase +from workflow_manager.models import Workflow + + +class WorkflowBaseSerializer(SerializersBase): + prefix = Workflow.orcabus_id_prefix + + +class WorkflowMinSerializer(WorkflowBaseSerializer): + class Meta: + model = Workflow + fields = ["orcabus_id", "workflow_name"] + + +class WorkflowSerializer(WorkflowBaseSerializer): + class Meta: + model = Workflow + fields = "__all__" diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/workflow_run.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/workflow_run.py new file mode 100644 index 000000000..36136f204 --- /dev/null +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/workflow_run.py @@ -0,0 +1,35 @@ +from workflow_manager.serializers.base import SerializersBase +from workflow_manager.models import WorkflowRun, Workflow, AnalysisRun + + +class WorkflowRunBaseSerializer(SerializersBase): + prefix = WorkflowRun.orcabus_id_prefix + + +class WorkflowRunSerializer(WorkflowRunBaseSerializer): + class Meta: + model = WorkflowRun + exclude = ["libraries"] + + def to_representation(self, instance): + representation = super().to_representation(instance) + representation['workflow'] = Workflow.orcabus_id_prefix + representation['workflow'] + if representation['analysis_run']: + representation['analysis_run'] = AnalysisRun.orcabus_id_prefix + representation['analysis_run'] + return representation + + +class WorkflowRunDetailSerializer(WorkflowRunBaseSerializer): + from .library import LibrarySerializer + from .workflow import WorkflowMinSerializer + from .analysis_run import AnalysisRunSerializer + from .state import StateSerializer + + libraries = LibrarySerializer(many=True, read_only=True) + workflow = WorkflowMinSerializer(read_only=True) + analysis_run = AnalysisRunSerializer(read_only=True) + state_set = StateSerializer(many=True, read_only=True) + + class Meta: + model = WorkflowRun + fields = "__all__" diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/tests/factories.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/tests/factories.py index dce8c2842..b271e6b54 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/tests/factories.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/tests/factories.py @@ -19,7 +19,7 @@ class TestConstant(Enum): }, library = { "library_id": "L000001", - "orcabus_id": "lib.01J5M2J44HFJ9424G7074NKTGN" + "orcabus_id": "01J5M2J44HFJ9424G7074NKTGN" } @@ -31,7 +31,6 @@ class Meta: workflow_version = "1.0" execution_engine_pipeline_id = str(uuid.uuid4()) execution_engine = "ICAv2" - approval_state = "NATA" class PayloadFactory(factory.django.DjangoModelFactory): diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/urls/base.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/urls/base.py index 207d2bbce..f020df94d 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/urls/base.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/urls/base.py @@ -1,9 +1,12 @@ from django.urls import path, include from workflow_manager.routers import OptionalSlashDefaultRouter +from workflow_manager.viewsets.analysis import AnalysisViewSet +from workflow_manager.viewsets.analysis_run import AnalysisRunViewSet from workflow_manager.viewsets.workflow import WorkflowViewSet from workflow_manager.viewsets.workflow_run import WorkflowRunViewSet from workflow_manager.viewsets.payload import PayloadViewSet +from workflow_manager.viewsets.analysis_context import AnalysisContextViewSet from workflow_manager.viewsets.state import StateViewSet from workflow_manager.viewsets.library import LibraryViewSet from workflow_manager.settings.base import API_VERSION @@ -13,21 +16,25 @@ api_base = f"{api_namespace}/{api_version}/" router = OptionalSlashDefaultRouter() +router.register(r"analysis", AnalysisViewSet, basename="analysis") +router.register(r"analysisrun", AnalysisRunViewSet, basename="analysisrun") +router.register(r"analysiscontext", AnalysisContextViewSet, basename="analysiscontext") router.register(r"workflow", WorkflowViewSet, basename="workflow") router.register(r"workflowrun", WorkflowRunViewSet, basename="workflowrun") router.register(r"payload", PayloadViewSet, basename="payload") -router.register( - "workflowrun/(?P[^/.]+)/state", - StateViewSet, - basename="workflowrun-state", -) - -router.register( - "workflowrun/(?P[^/.]+)/library", - LibraryViewSet, - basename="workflowrun-library", -) +# may no longer need this as it's currently included in the detail response for an individual WorkflowRun record +# router.register( +# "workflowrun/(?P[^/.]+)/state", +# StateViewSet, +# basename="workflowrun-state", +# ) +# +# router.register( +# "workflowrun/(?P[^/.]+)/library", +# LibraryViewSet, +# basename="workflowrun-library", +# ) urlpatterns = [ path(f"{api_base}", include(router.urls)), diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/analysis.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/analysis.py new file mode 100644 index 000000000..b47e9e077 --- /dev/null +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/analysis.py @@ -0,0 +1,23 @@ +from drf_spectacular.utils import extend_schema + +from workflow_manager.models.analysis import Analysis +from workflow_manager.serializers.analysis import AnalysisDetailSerializer, AnalysisSerializer +from .base import BaseViewSet + + +class AnalysisViewSet(BaseViewSet): + serializer_class = AnalysisDetailSerializer # use detailed serializer as default + search_fields = Analysis.get_base_fields() + queryset = Analysis.objects.prefetch_related("contexts").prefetch_related("workflows").all() + orcabus_id_prefix = Analysis.orcabus_id_prefix + + @extend_schema(parameters=[ + AnalysisSerializer + ]) + def list(self, request, *args, **kwargs): + self.serializer_class = AnalysisSerializer # use simple serializer for list view + return super().list(request, *args, **kwargs) + + def get_queryset(self): + query_params = self.get_query_params() + return Analysis.objects.get_by_keyword(self.queryset, **query_params) diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/analysis_context.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/analysis_context.py new file mode 100644 index 000000000..f4e6f09ab --- /dev/null +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/analysis_context.py @@ -0,0 +1,21 @@ +from drf_spectacular.utils import extend_schema + +from workflow_manager.models.analysis_context import AnalysisContext +from workflow_manager.serializers.analysis_context import AnalysisContextSerializer +from .base import BaseViewSet + + +class AnalysisContextViewSet(BaseViewSet): + serializer_class = AnalysisContextSerializer + search_fields = AnalysisContext.get_base_fields() + orcabus_id_prefix = AnalysisContext.orcabus_id_prefix + + @extend_schema(parameters=[ + AnalysisContextSerializer + ]) + def list(self, request, *args, **kwargs): + return super().list(request, *args, **kwargs) + + def get_queryset(self): + query_params = self.get_query_params() + return AnalysisContext.objects.get_by_keyword(self.queryset, **query_params) diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/analysis_run.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/analysis_run.py new file mode 100644 index 000000000..23f64e67a --- /dev/null +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/analysis_run.py @@ -0,0 +1,23 @@ +from drf_spectacular.utils import extend_schema + +from workflow_manager.models.analysis_run import AnalysisRun +from workflow_manager.serializers.analysis_run import AnalysisRunDetailSerializer, AnalysisRunSerializer +from .base import BaseViewSet + + +class AnalysisRunViewSet(BaseViewSet): + serializer_class = AnalysisRunDetailSerializer # use detailed + search_fields = AnalysisRun.get_base_fields() + queryset = AnalysisRun.objects.prefetch_related("libraries").all() + orcabus_id_prefix = AnalysisRun.orcabus_id_prefix + + @extend_schema(parameters=[ + AnalysisRunSerializer + ]) + def list(self, request, *args, **kwargs): + self.serializer_class = AnalysisRunSerializer # use simple view for record listing + return super().list(request, *args, **kwargs) + + def get_queryset(self): + query_params = self.get_query_params() + return AnalysisRun.objects.get_by_keyword(self.queryset, **query_params) diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/base.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/base.py new file mode 100644 index 000000000..d23b0cc3f --- /dev/null +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/base.py @@ -0,0 +1,50 @@ +from abc import ABC +from rest_framework import filters +from django.shortcuts import get_object_or_404 +from workflow_manager.pagination import StandardResultsSetPagination +from rest_framework.response import Response +from rest_framework.viewsets import ReadOnlyModelViewSet + + +class BaseViewSet(ReadOnlyModelViewSet, ABC): + lookup_value_regex = "[^/]+" # This is to allow for special characters in the URL + orcabus_id_prefix = '' + ordering_fields = "__all__" + ordering = ["-orcabus_id"] + pagination_class = StandardResultsSetPagination + filter_backends = [filters.OrderingFilter, filters.SearchFilter] + + def retrieve(self, request, *args, **kwargs): + """ + Since we have custom orcabus_id prefix for each model, we need to remove the prefix before retrieving it. + """ + pk = self.kwargs.get('pk') + print("DEBUG: pk") + print(pk) + if pk and pk.startswith(self.orcabus_id_prefix): + pk = pk[len(self.orcabus_id_prefix):] + + print("DEBUG: self.queryset") + print(self.queryset) + print(self.get_queryset()) + obj = get_object_or_404(self.get_queryset(), pk=pk) + serializer = self.serializer_class(obj) + return Response(serializer.data) + + def get_query_params(self): + """ + Sanitize query params if needed + e.g. remove prefixes for each orcabus_id + """ + query_params = self.request.query_params.copy() + orcabus_id = query_params.getlist("orcabus_id", None) + if orcabus_id: + id_list = [] + for key in orcabus_id: + if key.startswith(self.orcabus_id_prefix): + id_list.append(key[len(self.orcabus_id_prefix):]) + else: + id_list.append(key) + query_params.setlist('orcabus_id', id_list) + + return query_params diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/library.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/library.py index 04ef9a5a3..f17f90f5c 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/library.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/library.py @@ -1,21 +1,22 @@ -from rest_framework import filters -from rest_framework.viewsets import ReadOnlyModelViewSet +from drf_spectacular.utils import extend_schema from workflow_manager.models.library import Library -from workflow_manager.pagination import StandardResultsSetPagination -from workflow_manager.serializers import LibraryModelSerializer +from workflow_manager.viewsets.base import BaseViewSet +from workflow_manager.serializers.library import LibrarySerializer -class LibraryViewSet(ReadOnlyModelViewSet): - lookup_value_regex = "[^/]+" - serializer_class = LibraryModelSerializer - pagination_class = StandardResultsSetPagination - filter_backends = [filters.OrderingFilter, filters.SearchFilter] - ordering_fields = '__all__' - ordering = ['-orcabus_id'] +class LibraryViewSet(BaseViewSet): + serializer_class = LibrarySerializer search_fields = Library.get_base_fields() + orcabus_id_prefix = Library.orcabus_id_prefix + + @extend_schema(parameters=[ + LibrarySerializer + ]) + def list(self, request, *args, **kwargs): + return super().list(request, *args, **kwargs) def get_queryset(self): + query_params = self.get_query_params() qs = Library.objects.filter(workflowrun=self.kwargs["workflowrun_id"]) - qs = Library.objects.get_model_fields_query(qs, **self.request.query_params) - return qs + return Library.objects.get_by_keyword(qs, **query_params) diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/payload.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/payload.py index c15d6ef34..90a4ff307 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/payload.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/payload.py @@ -1,18 +1,23 @@ -from rest_framework import filters -from rest_framework.viewsets import ReadOnlyModelViewSet +from drf_spectacular.utils import extend_schema from workflow_manager.models.payload import Payload -from workflow_manager.pagination import StandardResultsSetPagination -from workflow_manager.serializers import PayloadModelSerializer +from workflow_manager.serializers.payload import PayloadSerializer +from workflow_manager.viewsets.base import BaseViewSet -class PayloadViewSet(ReadOnlyModelViewSet): - serializer_class = PayloadModelSerializer - pagination_class = StandardResultsSetPagination - filter_backends = [filters.OrderingFilter, filters.SearchFilter] - ordering_fields = '__all__' - ordering = ['-id'] +class PayloadViewSet(BaseViewSet): + serializer_class = PayloadSerializer search_fields = Payload.get_base_fields() + orcabus_id_prefix = Payload.orcabus_id_prefix + + @extend_schema(parameters=[ + PayloadSerializer + ]) + def list(self, request, *args, **kwargs): + return super().list(request, *args, **kwargs) def get_queryset(self): - return Payload.objects.get_by_keyword(**self.request.query_params) + query_params = self.get_query_params() + print("DEBUG: query_params") + print(query_params) + return Payload.objects.get_by_keyword(self.queryset, **query_params) diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/state.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/state.py index 660209330..76c2c6fdf 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/state.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/state.py @@ -1,22 +1,22 @@ -from rest_framework import filters -from rest_framework.viewsets import ReadOnlyModelViewSet +from drf_spectacular.utils import extend_schema from workflow_manager.models import State -from workflow_manager.pagination import StandardResultsSetPagination -from workflow_manager.serializers import StateModelSerializer +from workflow_manager.serializers.state import StateSerializer +from workflow_manager.viewsets.base import BaseViewSet -class StateViewSet(ReadOnlyModelViewSet): - serializer_class = StateModelSerializer - pagination_class = StandardResultsSetPagination - filter_backends = [filters.OrderingFilter, filters.SearchFilter] - ordering_fields = '__all__' - ordering = ['-id'] +class StateViewSet(BaseViewSet): + serializer_class = StateSerializer search_fields = State.get_base_fields() + orcabus_id_prefix = State.orcabus_id_prefix - def get_queryset(self): - qs = State.objects.filter(workflow_run=self.kwargs["workflowrun_id"]) - qs = State.objects.get_model_fields_query(qs, **self.request.query_params) - return qs - + @extend_schema(parameters=[ + StateSerializer + ]) + def list(self, request, *args, **kwargs): + return super().list(request, *args, **kwargs) + def get_queryset(self): + query_params = self.get_query_params() + # qs = State.objects.filter(workflow_run=self.kwargs["workflowrun_id"]) + return State.objects.get_by_keyword(qs, **query_params) diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow.py index bf6758f0b..0306a14f4 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow.py @@ -1,18 +1,21 @@ -from rest_framework import filters -from rest_framework.viewsets import ReadOnlyModelViewSet +from drf_spectacular.utils import extend_schema from workflow_manager.models.workflow import Workflow -from workflow_manager.pagination import StandardResultsSetPagination -from workflow_manager.serializers import WorkflowModelSerializer +from workflow_manager.serializers.workflow import WorkflowSerializer +from workflow_manager.viewsets.base import BaseViewSet -class WorkflowViewSet(ReadOnlyModelViewSet): - serializer_class = WorkflowModelSerializer - pagination_class = StandardResultsSetPagination - filter_backends = [filters.OrderingFilter, filters.SearchFilter] - ordering_fields = '__all__' - ordering = ['-id'] +class WorkflowViewSet(BaseViewSet): + serializer_class = WorkflowSerializer search_fields = Workflow.get_base_fields() + orcabus_id_prefix = Workflow.orcabus_id_prefix + + @extend_schema(parameters=[ + WorkflowSerializer + ]) + def list(self, request, *args, **kwargs): + return super().list(request, *args, **kwargs) def get_queryset(self): - return Workflow.objects.get_by_keyword(**self.request.query_params) + query_params = self.get_query_params() + return Workflow.objects.get_by_keyword(self.queryset, **query_params) diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run.py index 8a3b1d1d4..50ebfc012 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run.py @@ -1,30 +1,36 @@ from django.db.models import Q -from rest_framework import filters +from drf_spectacular.utils import extend_schema from rest_framework.decorators import action -from rest_framework.viewsets import ReadOnlyModelViewSet from workflow_manager.models.workflow_run import WorkflowRun -from workflow_manager.pagination import StandardResultsSetPagination -from workflow_manager.serializers import WorkflowRunModelSerializer +from workflow_manager.serializers.workflow_run import WorkflowRunDetailSerializer, WorkflowRunSerializer +from workflow_manager.viewsets.base import BaseViewSet -class WorkflowRunViewSet(ReadOnlyModelViewSet): - serializer_class = WorkflowRunModelSerializer - pagination_class = StandardResultsSetPagination - filter_backends = [filters.OrderingFilter, filters.SearchFilter] - ordering_fields = '__all__' - ordering = ['-id'] +class WorkflowRunViewSet(BaseViewSet): + serializer_class = WorkflowRunDetailSerializer search_fields = WorkflowRun.get_base_fields() + queryset = WorkflowRun.objects.prefetch_related("state_set").prefetch_related("libraries").all() + orcabus_id_prefix = WorkflowRun.orcabus_id_prefix + + @extend_schema(parameters=[ + WorkflowRunSerializer + ]) + def list(self, request, *args, **kwargs): + self.serializer_class = WorkflowRunSerializer # use simple view for record listing + return super().list(request, *args, **kwargs) def get_queryset(self): - return WorkflowRun.objects.get_by_keyword(**self.request.query_params) + query_params = self.get_query_params() + return WorkflowRun.objects.get_by_keyword(self.queryset, **query_params) @action(detail=False, methods=['GET']) def ongoing(self, request): + self.serializer_class = WorkflowRunSerializer # use simple view for record listing # Get all books marked as favorite print(request) print(self.request.query_params) - ordering = self.request.query_params.get('ordering', '-id') + ordering = self.request.query_params.get('ordering', '-orcabus_id') if "status" in self.request.query_params.keys(): print("found status!") diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager_proc/services/create_workflow_run_state.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager_proc/services/create_workflow_run_state.py index f9fcf8c01..b61f23e14 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager_proc/services/create_workflow_run_state.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager_proc/services/create_workflow_run_state.py @@ -65,7 +65,6 @@ def handler(event, context): workflow_version=srv_wrsc.workflowVersion, execution_engine="Unknown", execution_engine_pipeline_id="Unknown", - approval_state="RESEARCH", ) logger.info("Persisting Workflow record.") workflow.save() diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager_proc/tests/test_create_workflow_run_state.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager_proc/tests/test_create_workflow_run_state.py index 33cc9a9ca..98725b859 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager_proc/tests/test_create_workflow_run_state.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager_proc/tests/test_create_workflow_run_state.py @@ -66,11 +66,11 @@ def test_create_wrsc_library(self): lib_ids = [ { "libraryId": library_ids[0], - "orcabusId": "lib.01J5M2J44HFJ9424G7074NKTGN" + "orcabusId": "01J5M2J44HFJ9424G7074NKTGN" }, { "libraryId": library_ids[1], - "orcabusId": "lib.01J5M2JFE1JPYV62RYQEG99CP5" + "orcabusId": "01J5M2JFE1JPYV62RYQEG99CP5" } ] @@ -129,11 +129,11 @@ def test_create_wrsc_library_exists(self): lib_ids = [ { "libraryId": library_ids[0], - "orcabusId": "lib.01J5M2J44HFJ9424G7074NKTGN" + "orcabusId": "01J5M2J44HFJ9424G7074NKTGN" }, { "libraryId": library_ids[1], - "orcabusId": "lib.01J5M2JFE1JPYV62RYQEG99CP5" + "orcabusId": "01J5M2JFE1JPYV62RYQEG99CP5" } ] for lib_id in lib_ids: From eba68df761792a9bb45f1625ea866858ed12c48c Mon Sep 17 00:00:00 2001 From: Florian Reisinger Date: Wed, 2 Oct 2024 17:04:30 +1000 Subject: [PATCH 07/14] Add initial WorkflowRun state for analysis example data --- .../commands/generate_analysis_for_metadata.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/management/commands/generate_analysis_for_metadata.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/management/commands/generate_analysis_for_metadata.py index 89ac4562e..2be92d1aa 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/management/commands/generate_analysis_for_metadata.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/management/commands/generate_analysis_for_metadata.py @@ -1,13 +1,12 @@ from datetime import datetime, timezone from typing import List -import ulid import uuid from collections import defaultdict from django.core.management import BaseCommand from django.db.models import QuerySet from workflow_manager.models import Workflow, WorkflowRun, Analysis, AnalysisContext, AnalysisRun, \ - Library, LibraryAssociation + Library, LibraryAssociation, State, Status from workflow_manager.tests.factories import PayloadFactory, LibraryFactory # https://docs.djangoproject.com/en/5.0/howto/custom-management-commands/ @@ -98,6 +97,7 @@ def handle(self, *args, **options): runs: List[AnalysisRun] = assign_analysis(libraries) print(runs) + # create WorkflowRun entries (DRAFT) prep_workflow_runs(libraries) print("Done") @@ -413,6 +413,16 @@ def create_workflowrun_for_analysis(analysis_run: AnalysisRun): association_date=datetime.now(timezone.utc), status="ACTIVE", ) + initial_state = State( + workflow_run=wr, + status=Status.DRAFT.convention, + timestamp=datetime.now(timezone.utc), + payload=PayloadFactory( + payload_ref_id=str(uuid.uuid4()), + data={"comment": f"Payload for initial state of wfr.{wr.orcabus_id}"}), + comment="Initial State" + ) + initial_state.save() def create_portal_run_id() -> str: From ad67744d3218829b80f20236cbbae640af520d82 Mon Sep 17 00:00:00 2001 From: Florian Reisinger Date: Wed, 2 Oct 2024 17:05:28 +1000 Subject: [PATCH 08/14] Re-enable workflowrun/{id}/state endpoint --- .../workflow-manager/workflow_manager/urls/base.py | 12 ++++++------ .../workflow_manager/viewsets/state.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/urls/base.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/urls/base.py index f020df94d..119143fe7 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/urls/base.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/urls/base.py @@ -24,12 +24,12 @@ router.register(r"payload", PayloadViewSet, basename="payload") # may no longer need this as it's currently included in the detail response for an individual WorkflowRun record -# router.register( -# "workflowrun/(?P[^/.]+)/state", -# StateViewSet, -# basename="workflowrun-state", -# ) -# +router.register( + "workflowrun/(?P[^/.]+)/state", + StateViewSet, + basename="workflowrun-state", +) + # router.register( # "workflowrun/(?P[^/.]+)/library", # LibraryViewSet, diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/state.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/state.py index 76c2c6fdf..436dd208c 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/state.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/state.py @@ -18,5 +18,5 @@ def list(self, request, *args, **kwargs): def get_queryset(self): query_params = self.get_query_params() - # qs = State.objects.filter(workflow_run=self.kwargs["workflowrun_id"]) + qs = State.objects.filter(workflow_run=self.kwargs["workflowrun_id"]) return State.objects.get_by_keyword(qs, **query_params) From ebde1be82eb1421f43c50ce1c9c9fce9b99d2e7b Mon Sep 17 00:00:00 2001 From: Florian Reisinger Date: Thu, 3 Oct 2024 11:53:05 +1000 Subject: [PATCH 09/14] Add random states to WorkflowRuns --- .../generate_analysis_for_metadata.py | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/management/commands/generate_analysis_for_metadata.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/management/commands/generate_analysis_for_metadata.py index 2be92d1aa..672aac9e9 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/management/commands/generate_analysis_for_metadata.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/management/commands/generate_analysis_for_metadata.py @@ -1,3 +1,4 @@ +import random from datetime import datetime, timezone from typing import List import uuid @@ -423,6 +424,45 @@ def create_workflowrun_for_analysis(analysis_run: AnalysisRun): comment="Initial State" ) initial_state.save() + # add randomly additional states to simulate a state history + if random.random() < 0.8: # ~80% + # create a READY state + ready_state = State( + workflow_run=wr, + status=Status.READY.convention, + timestamp=datetime.now(timezone.utc), + payload=PayloadFactory( + payload_ref_id=str(uuid.uuid4()), + data={"comment": f"Payload for READY state of wfr.{wr.orcabus_id}"}), + comment="READY State" + ) + ready_state.save() + # randomly create another state + if random.random() < 0.7: # ~70% + # create a RUNNING state + ready_state = State( + workflow_run=wr, + status=Status.RUNNING.convention, + timestamp=datetime.now(timezone.utc), + payload=PayloadFactory( + payload_ref_id=str(uuid.uuid4()), + data={"comment": f"Payload for RUNNING state of wfr.{wr.orcabus_id}"}), + comment="RUNNING State" + ) + ready_state.save() + # randomly create another state + if random.random() < 0.6: # ~60% + # create a terminal state + ready_state = State( + workflow_run=wr, + status=random.choice([Status.SUCCEEDED.convention, Status.FAILED.convention]), + timestamp=datetime.now(timezone.utc), + payload=PayloadFactory( + payload_ref_id=str(uuid.uuid4()), + data={"comment": f"Payload for terminal state of wfr.{wr.orcabus_id}"}), + comment="Terminal State" + ) + ready_state.save() def create_portal_run_id() -> str: From b90512dc163b5da4497e1aafeeeaa5f59cf93b9e Mon Sep 17 00:00:00 2001 From: Florian Reisinger Date: Thu, 3 Oct 2024 11:56:28 +1000 Subject: [PATCH 10/14] Update default mock case in Makefile --- lib/workload/stateless/stacks/workflow-manager/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/workload/stateless/stacks/workflow-manager/Makefile b/lib/workload/stateless/stacks/workflow-manager/Makefile index ee07e2122..3eefee21a 100644 --- a/lib/workload/stateless/stacks/workflow-manager/Makefile +++ b/lib/workload/stateless/stacks/workflow-manager/Makefile @@ -31,7 +31,7 @@ start: migrate @python manage.py runserver_plus 0.0.0.0:8000 mock: - @python manage.py generate_mock_workflow_run + @python manage.py generate_analysis_for_metadata run-mock: reset-db migrate mock start From 0fd10f1b93804e72cb2b6c41f0e0efd9ab73bc7c Mon Sep 17 00:00:00 2001 From: Florian Reisinger Date: Fri, 11 Oct 2024 12:11:07 +1100 Subject: [PATCH 11/14] Remove debug print statements --- .../workflow_manager/viewsets/base.py | 5 ----- .../workflow_manager/viewsets/payload.py | 2 -- .../workflow_manager/viewsets/workflow_run.py | 5 ----- .../lambdas/handle_service_wrsc_event.py | 11 +++++++---- .../lambdas/transition_bcm_fastq_copy.py | 6 +++++- .../services/create_analysisruns_for_libraries.py | 7 +++++-- .../services/emit_workflow_run_state_change.py | 12 ++++++++---- .../tests/test_create_workflow_run_state.py | 2 -- 8 files changed, 25 insertions(+), 25 deletions(-) diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/base.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/base.py index d23b0cc3f..d8dbbf103 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/base.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/base.py @@ -19,14 +19,9 @@ def retrieve(self, request, *args, **kwargs): Since we have custom orcabus_id prefix for each model, we need to remove the prefix before retrieving it. """ pk = self.kwargs.get('pk') - print("DEBUG: pk") - print(pk) if pk and pk.startswith(self.orcabus_id_prefix): pk = pk[len(self.orcabus_id_prefix):] - print("DEBUG: self.queryset") - print(self.queryset) - print(self.get_queryset()) obj = get_object_or_404(self.get_queryset(), pk=pk) serializer = self.serializer_class(obj) return Response(serializer.data) diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/payload.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/payload.py index 90a4ff307..48369c912 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/payload.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/payload.py @@ -18,6 +18,4 @@ def list(self, request, *args, **kwargs): def get_queryset(self): query_params = self.get_query_params() - print("DEBUG: query_params") - print(query_params) return Payload.objects.get_by_keyword(self.queryset, **query_params) diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run.py index 00be9c49d..293d659e1 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run.py @@ -102,12 +102,9 @@ def exclude_params(params): def ongoing(self, request): self.serializer_class = WorkflowRunSerializer # use simple view for record listing # Get all books marked as favorite - print(request) - print(self.request.query_params) ordering = self.request.query_params.get('ordering', '-orcabus_id') if "status" in self.request.query_params.keys(): - print("found status!") status = self.request.query_params.get('status') result_set = WorkflowRun.objects.get_by_keyword(states__status=status).order_by(ordering) else: @@ -126,8 +123,6 @@ def ongoing(self, request): @action(detail=False, methods=['GET']) def unresolved(self, request): # Get all books marked as favorite - print(request) - print(self.request.query_params) ordering = self.request.query_params.get('ordering', '-id') result_set = WorkflowRun.objects.get_by_keyword(states__status="FAILED").order_by(ordering) diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager_proc/lambdas/handle_service_wrsc_event.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager_proc/lambdas/handle_service_wrsc_event.py index 59868f4cf..2e6df24a2 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager_proc/lambdas/handle_service_wrsc_event.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager_proc/lambdas/handle_service_wrsc_event.py @@ -6,7 +6,10 @@ import workflow_manager_proc.domain.executionservice.workflowrunstatechange as srv import workflow_manager_proc.domain.workflowmanager.workflowrunstatechange as wfm from workflow_manager_proc.services import emit_workflow_run_state_change, create_workflow_run_state +import logging +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) def handler(event, context): """ @@ -18,7 +21,7 @@ def handler(event, context): - create new State for WorkflowRun if required - relay the state change as WorkflowManager WRSC event if applicable """ - print(f"Processing {event}, {context}") + logger.info(f"Processing {event}, {context}") # remove the AWSEvent wrapper from our WRSC event input_event: srv.AWSEvent = srv.Marshaller.unmarshall(event, srv.AWSEvent) @@ -28,10 +31,10 @@ def handler(event, context): out_wrsc = create_workflow_run_state.handler(srv.Marshaller.marshall(input_wrsc), None) if out_wrsc: # new state resulted in state transition, we can relay the WRSC - print("Emitting WRSC.") + logger.info("Emitting WRSC.") emit_workflow_run_state_change.handler(wfm.Marshaller.marshall(out_wrsc), None) else: # ignore - state has not been updated - print(f"WorkflowRun state not updated. No event to emit.") + logger.info(f"WorkflowRun state not updated. No event to emit.") - print(f"{__name__} done.") + logger.info(f"{__name__} done.") diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager_proc/lambdas/transition_bcm_fastq_copy.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager_proc/lambdas/transition_bcm_fastq_copy.py index bcf91f14f..0be7b318d 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager_proc/lambdas/transition_bcm_fastq_copy.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager_proc/lambdas/transition_bcm_fastq_copy.py @@ -6,11 +6,15 @@ from workflow_manager.models.workflow_run import WorkflowRun import workflow_manager_proc.domain.executionservice.workflowrunstatechange as srv import workflow_manager_proc.domain.workflowmanager.workflowrunstatechange as wfm +import logging + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) def handler(event, context): """event will be a workflowmanager.WorkflowRunStateChange event""" - print(f"Processing {event}, {context}") + logger.info(f"Processing {event}, {context}") input_event: wfm.AWSEvent = wfm.Marshaller.unmarshall(event, wfm.AWSEvent) input_wrsc: wfm.WorkflowRunStateChange = input_event.detail diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager_proc/services/create_analysisruns_for_libraries.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager_proc/services/create_analysisruns_for_libraries.py index e8cb194c4..5e38072a5 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager_proc/services/create_analysisruns_for_libraries.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager_proc/services/create_analysisruns_for_libraries.py @@ -2,7 +2,10 @@ from typing import List from workflow_manager.models import Analysis, AnalysisRun, AnalysisContext, Library +import logging +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) # TODO: get list of libraries (+ required metadata) from list of library IDs # TODO: switch pairing based on subject to pairing based on Case (creating a case is someone else's responsibility) @@ -73,7 +76,7 @@ def create_wgs_analysis(libraries: List[dict]) -> List[AnalysisRun]: normal_lib_record: Library = Library.objects.get(library_id=pairing[sbj]['normal'][0]['library_id']) if not tumor_lib_record or not normal_lib_record: - print("Not a valid pairing.") + logger.info("Not a valid pairing.") break workflow = pairing[sbj]['tumor'][0]['workflow'] @@ -92,7 +95,7 @@ def create_wgs_analysis(libraries: List[dict]) -> List[AnalysisRun]: ar_wgs.libraries.add(normal_lib_record) analysis_runs.append(ar_wgs) else: - print(f"No pairing for {sbj}.") + logger.info(f"No pairing for {sbj}.") return analysis_runs diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager_proc/services/emit_workflow_run_state_change.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager_proc/services/emit_workflow_run_state_change.py index dabd70c5a..e35d8a220 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager_proc/services/emit_workflow_run_state_change.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager_proc/services/emit_workflow_run_state_change.py @@ -3,6 +3,10 @@ import json import workflow_manager_proc.domain.workflowmanager.workflowrunstatechange as wfm from workflow_manager_proc.domain.workflowmanager.workflowrunstatechange import WorkflowRunStateChange +import logging + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) client = boto3.client('events') source = "orcabus.workflowmanager" @@ -13,7 +17,7 @@ def handler(event, context): """ event has to be JSON conform to workflowmanager.WorkflowRunStateChange """ - print(f"Processing {event}, {context}") + logger.info(f"Processing {event}, {context}") response = client.put_events( Entries=[ @@ -26,7 +30,7 @@ def handler(event, context): ], ) - print(f"Sent a WRSC event to event bus {event_bus_name}:") - print(event) - print(f"{__name__} done.") + logger.info(f"Sent a WRSC event to event bus {event_bus_name}:") + logger.info(event) + logger.info(f"{__name__} done.") return response diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager_proc/tests/test_create_workflow_run_state.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager_proc/tests/test_create_workflow_run_state.py index 98725b859..3b11c9791 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager_proc/tests/test_create_workflow_run_state.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager_proc/tests/test_create_workflow_run_state.py @@ -242,7 +242,5 @@ def test_get_last_state(self): t1 = s1.timestamp t2 = s2.timestamp delta = t1 - t2 # = 2 days - print(f"delta sec: {abs(delta.total_seconds())}") window = timedelta(hours=1) - print(f"window sec: {window.total_seconds()}") self.assertTrue(delta > window, "delta > 1h") From 97c351559a892effccfc740d3daf5796f7795922 Mon Sep 17 00:00:00 2001 From: Florian Reisinger Date: Fri, 11 Oct 2024 12:28:26 +1100 Subject: [PATCH 12/14] Fix ordering by orcabus_id --- .../workflow-manager/workflow_manager/viewsets/workflow_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run.py index 293d659e1..9c46234d3 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run.py @@ -123,7 +123,7 @@ def ongoing(self, request): @action(detail=False, methods=['GET']) def unresolved(self, request): # Get all books marked as favorite - ordering = self.request.query_params.get('ordering', '-id') + ordering = self.request.query_params.get('ordering', '-orcabus_id') result_set = WorkflowRun.objects.get_by_keyword(states__status="FAILED").order_by(ordering) From 77887dc29f5b02475cbabb686d5c64dde835b27b Mon Sep 17 00:00:00 2001 From: Florian Reisinger Date: Fri, 11 Oct 2024 12:28:59 +1100 Subject: [PATCH 13/14] Remove unused to_dict methods --- .../workflow_manager/models/analysis.py | 9 --------- .../workflow_manager/models/analysis_context.py | 9 --------- .../workflow_manager/models/analysis_run.py | 9 --------- .../workflow_manager/models/library.py | 6 ------ .../workflow_manager/models/payload.py | 8 -------- .../workflow-manager/workflow_manager/models/state.py | 10 ---------- .../workflow_manager/models/workflow.py | 10 ---------- .../workflow_manager/models/workflow_run.py | 10 ---------- 8 files changed, 71 deletions(-) diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis.py index 149b2ae48..b7c853be7 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis.py @@ -28,12 +28,3 @@ class Meta: def __str__(self): return f"ID: {self.orcabus_id}, analysis_name: {self.analysis_name}, analysis_version: {self.analysis_version}" - - def to_dict(self): - return { - "orcabusId": self.orcabus_id, - "analysisName": self.analysis_name, - "analysisVersion": self.analysis_version, - "description": self.description, - "status": self.status - } diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_context.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_context.py index 8bffa1ce8..a682d450c 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_context.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_context.py @@ -22,12 +22,3 @@ class Meta: def __str__(self): return f"ID: {self.orcabus_id}, name: {self.name}, usecase: {self.usecase}" - - def to_dict(self): - return { - "orcabus_id": self.orcabus_id, - "name": self.name, - "usecase": self.usecase, - "status": self.status, - "description": self.description - } diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_run.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_run.py index 298a17ca0..71a4e27dd 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_run.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_run.py @@ -28,12 +28,3 @@ class AnalysisRun(OrcaBusBaseModel): def __str__(self): return f"ID: {self.orcabus_id}, analysis_run_name: {self.analysis_run_name}" - - def to_dict(self): - return { - "orcabusId": self.orcabus_id, - "analysis_run_name": self.analysis_run_name, - "comment": self.comment, - "approval_context": self.approval_context, - "project_context": self.project_context - } diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/library.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/library.py index f59d47da5..6958e5fe3 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/library.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/library.py @@ -18,9 +18,3 @@ class Library(OrcaBusBaseModel): def __str__(self): return f"ID: {self.orcabus_id}, library_id: {self.library_id}" - - def to_dict(self): - return { - "orcabusId": self.orcabus_id, - "libraryId": self.library_id - } diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/payload.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/payload.py index 494eae23d..1eb32921f 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/payload.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/payload.py @@ -19,11 +19,3 @@ class Payload(OrcaBusBaseModel): def __str__(self): return f"ID: {self.orcabus_id}, payload_ref_id: {self.payload_ref_id}" - - def to_dict(self): - return { - "orcabusId": self.orcabus_id, - "payload_ref_id": self.payload_ref_id, - "version": self.version, - "data": self.data - } diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/state.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/state.py index 8a042d0fe..abfdc4d4e 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/state.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/state.py @@ -104,16 +104,6 @@ class Meta: def __str__(self): return f"ID: {self.orcabus_id}, status: {self.status}" - def to_dict(self): - return { - "orcabusId": self.orcabus_id, - "workflow_run_id": self.workflow_run.orcabus_id, - "status": self.status, - "timestamp": str(self.timestamp), - "comment": self.comment, - "payload": self.payload.to_dict() if (self.payload is not None) else None, - } - def is_terminal(self) -> bool: return Status.is_terminal(str(self.status)) diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow.py index 22ecae61e..530bdfc40 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow.py @@ -27,13 +27,3 @@ class Meta: def __str__(self): return f"ID: {self.orcabus_id}, workflow_name: {self.workflow_name}, workflow_version: {self.workflow_version}" - - def to_dict(self): - return { - "orcabusId": self.orcabus_id, - "workflow_name": self.workflow_name, - "workflow_version": self.workflow_version, - "execution_engine": self.execution_engine, - "execution_engine_pipeline_id": self.execution_engine_pipeline_id, - # "approval_state": self.approval_state - } diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow_run.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow_run.py index cceef65f6..d10d46a6d 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow_run.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow_run.py @@ -30,16 +30,6 @@ def __str__(self): return f"ID: {self.orcabus_id}, portal_run_id: {self.portal_run_id}, workflow_run_name: {self.workflow_run_name}, " \ f"workflowRun: {self.workflow.workflow_name} " - def to_dict(self): - return { - "orcabusId": self.orcabus_id, - "portal_run_id": self.portal_run_id, - "execution_id": self.execution_id, - "workflow_run_name": self.workflow_run_name, - "comment": self.comment, - "workflow": self.workflow.to_dict() if (self.workflow is not None) else None - } - def get_all_states(self): # retrieve all states (DB records rather than a queryset) return list(self.states.all()) # TODO: ensure order by timestamp ? From 46b056cb8e5da25c06dc580b924b669719617a18 Mon Sep 17 00:00:00 2001 From: Florian Reisinger Date: Tue, 15 Oct 2024 10:24:00 +1100 Subject: [PATCH 14/14] Refactor analysis/run contexts --- .../generate_analysis_for_metadata.py | 133 ++++++++++++------ .../migrations/0001_initial.py | 26 ++-- .../0002_alter_state_workflow_run.py | 23 --- ...0003_alter_analysis_orcabus_id_and_more.py | 59 -------- .../workflow_manager/models/analysis_run.py | 8 +- .../serializers/analysis_run.py | 12 +- .../workflow_manager/viewsets/workflow_run.py | 1 + 7 files changed, 114 insertions(+), 148 deletions(-) delete mode 100644 lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0002_alter_state_workflow_run.py delete mode 100644 lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0003_alter_analysis_orcabus_id_and_more.py diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/management/commands/generate_analysis_for_metadata.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/management/commands/generate_analysis_for_metadata.py index 672aac9e9..435b90651 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/management/commands/generate_analysis_for_metadata.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/management/commands/generate_analysis_for_metadata.py @@ -112,30 +112,52 @@ def _setup_requirements(): # The contexts information available for analysis clinical_context = AnalysisContext( - name="accredited", - usecase="analysis-selection", - description="Accredited by NATA", + name="clinical", + usecase="approval", + description="Approved for clinical workloads", status="ACTIVE", ) - print(clinical_context) clinical_context.save() - research_context = AnalysisContext( + clinical_compute_context = AnalysisContext( + name="clinical", + usecase="compute-env", + description="Clinical compute environment", + status="ACTIVE", + ) + clinical_compute_context.save() + + research_compute_context = AnalysisContext( + name="research", + usecase="compute-env", + description="Research compute environment", + status="ACTIVE", + ) + research_compute_context.save() + + clinical_storage_context = AnalysisContext( + name="clinical", + usecase="storage-env", + description="Clinical storage environment", + status="ACTIVE", + ) + clinical_storage_context.save() + + research_storage_context = AnalysisContext( name="research", - usecase="analysis-selection", - description="For research use", + usecase="storage-env", + description="Research storage environment", status="ACTIVE", ) - print(research_context) - research_context.save() + research_storage_context.save() - internal_context = AnalysisContext( - name="internal", - usecase="analysis-selection", - description="For internal use", + temp_storage_context = AnalysisContext( + name="temp", + usecase="storage-env", + description="For internal use, short term storage", status="ACTIVE", ) - internal_context.save() + temp_storage_context.save() # ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- # The workflows that are available to be run @@ -157,8 +179,8 @@ def _setup_requirements(): wgs_workflow.save() cttsov2_workflow = Workflow( - workflow_name="cttsov2", - workflow_version="1.0", + workflow_name="cttso", + workflow_version="2.0", execution_engine="ICAv2", execution_engine_pipeline_id="ica.pipeline.23456", ) @@ -198,8 +220,7 @@ def _setup_requirements(): status="ACTIVE", ) qc_assessment.save() - qc_assessment.contexts.add(clinical_context) - qc_assessment.contexts.add(research_context) + qc_assessment.contexts.add(temp_storage_context) qc_assessment.workflows.add(qc_workflow) wgs_clinical_analysis = Analysis( @@ -210,7 +231,7 @@ def _setup_requirements(): ) wgs_clinical_analysis.save() wgs_clinical_analysis.contexts.add(clinical_context) - wgs_clinical_analysis.contexts.add(research_context) + wgs_clinical_analysis.contexts.add(clinical_context) wgs_clinical_analysis.workflows.add(wgs_workflow) wgs_clinical_analysis.workflows.add(umccrise_workflow) @@ -221,26 +242,24 @@ def _setup_requirements(): status="ACTIVE", ) wgs_research_analysis.save() - wgs_research_analysis.contexts.add(research_context) wgs_research_analysis.workflows.add(wgs_workflow) wgs_research_analysis.workflows.add(oa_wgs_workflow) wgs_research_analysis.workflows.add(sash_workflow) - cttso_research_analysis = Analysis( - analysis_name="ctTSO500", - analysis_version="1.0", + cttso_analysis = Analysis( + analysis_name="ctTSO", + analysis_version="2.0", description="Analysis for ctTSO samples", status="ACTIVE", ) - cttso_research_analysis.save() - cttso_research_analysis.contexts.add(research_context) - cttso_research_analysis.contexts.add(clinical_context) - cttso_research_analysis.workflows.add(cttsov2_workflow) + cttso_analysis.save() + cttso_analysis.contexts.add(clinical_context) + cttso_analysis.contexts.add(clinical_context) + cttso_analysis.workflows.add(cttsov2_workflow) # ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- # Generate a Payload stub and some Libraries - generic_payload = PayloadFactory() # Payload content is not important for now LibraryFactory(orcabus_id="01J5M2JFE1JPYV62RYQEG99CP1", library_id="L000001"), LibraryFactory(orcabus_id="02J5M2JFE1JPYV62RYQEG99CP2", library_id="L000002"), LibraryFactory(orcabus_id="03J5M2JFE1JPYV62RYQEG99CP3", library_id="L000003"), @@ -268,7 +287,11 @@ def assign_analysis(libraries: List[dict]) -> List[AnalysisRun]: def create_qc_analysis(libraries: List[dict]) -> List[AnalysisRun]: analysis_runs: List[AnalysisRun] = [] - context_internal = AnalysisContext.objects.get_by_keyword(name="internal").first() # FIXME + # this is meant to only generate temporary results, we assign it with a temp storage context + # and it does not have to run on a controlled compute env, so we run it on the research compute + research_compute_context = AnalysisContext.objects.get_by_keyword(name="research", usecase="compute-env").first() + temp_storage_context = AnalysisContext.objects.get_by_keyword(name="temp", usecase="storage-env").first() + analysis_qc_qs = Analysis.objects.get_by_keyword(analysis_name='QC_Assessment') analysis_qc = analysis_qc_qs.first() # FIXME: assume there are more than one and select by latest version, etc @@ -281,7 +304,8 @@ def create_qc_analysis(libraries: List[dict]) -> List[AnalysisRun]: analysis_run = AnalysisRun( analysis_run_name=f"automated__{analysis_qc.analysis_name}__{lib_record.library_id}", status="DRAFT", - approval_context=context_internal, # FIXME: does this matter here? Internal? + compute_context=research_compute_context, + storage_context=temp_storage_context, analysis=analysis_qc ) analysis_run.save() @@ -293,10 +317,19 @@ def create_qc_analysis(libraries: List[dict]) -> List[AnalysisRun]: def create_wgs_analysis(libraries: List[dict]) -> List[AnalysisRun]: analysis_runs: List[AnalysisRun] = [] - context_clinical = AnalysisContext.objects.get_by_keyword(name="accredited").first() # FIXME - context_research = AnalysisContext.objects.get_by_keyword(name="research").first() # FIXME - analysis_wgs_clinical: Analysis = Analysis.objects.filter(analysis_name='WGS', contexts=context_clinical).first() # FIXME - analysis_wgs_research: Analysis = Analysis.objects.filter(analysis_name='WGS', contexts=context_research, analysis_version='2.0').first() # FIXME + + # prepare the available compute and storage contexts, to be chosen depending on the actual workload + clinical_compute_context = AnalysisContext.objects.get_by_keyword(name="clinical", usecase="compute-env").first() + clinical_storage_context = AnalysisContext.objects.get_by_keyword(name="clinical", usecase="storage-env").first() + research_compute_context = AnalysisContext.objects.get_by_keyword(name="research", usecase="compute-env").first() + research_storage_context = AnalysisContext.objects.get_by_keyword(name="research", usecase="storage-env").first() + clinical_context = AnalysisContext.objects.get_by_keyword(name="clinical", usecase="approval").first() + + # Find the approved analysis for a wgs workload + # NOTE: for clinical workloads the analysis has to be approved for clinical use, + # for research all are allowed and we choose the "latest" version + analysis_wgs_clinical: Analysis = Analysis.objects.filter(analysis_name='WGS', contexts=clinical_context).first() + analysis_wgs_research: Analysis = Analysis.objects.filter(analysis_name='WGS', analysis_version='2.0').first() # FIXME: better pairing algorithm! pairing = defaultdict(lambda: defaultdict(list)) @@ -318,15 +351,19 @@ def create_wgs_analysis(libraries: List[dict]) -> List[AnalysisRun]: print("Not a valid pairing.") break + # assign the compute and storage contexts based on the metadata annotation ('workflow') for now workflow = pairing[sbj]['tumor'][0]['workflow'] - context = context_clinical if workflow == 'clinical' else context_research + compute_context = clinical_compute_context if workflow == 'clinical' else research_compute_context + storage_context = clinical_storage_context if workflow == 'clinical' else research_storage_context analysis = analysis_wgs_clinical if workflow == 'clinical' else analysis_wgs_research - analysis_run_name = f"automated__{analysis.analysis_name}__{context.name}__" + \ + + analysis_run_name = f"automated__{analysis.analysis_name}__{workflow}__" + \ f"{tumor_lib_record.library_id}__{normal_lib_record.library_id} " ar_wgs = AnalysisRun( analysis_run_name=analysis_run_name, status="DRAFT", - approval_context=context, + compute_context=compute_context, + storage_context=storage_context, analysis=analysis ) ar_wgs.save() @@ -341,21 +378,31 @@ def create_wgs_analysis(libraries: List[dict]) -> List[AnalysisRun]: def create_cttso_analysis(libraries: List[dict]) -> List[AnalysisRun]: analysis_runs: List[AnalysisRun] = [] - context_clinical = AnalysisContext.objects.get_by_keyword(name="accredited").first() # FIXME - context_research = AnalysisContext.objects.get_by_keyword(name="research").first() # FIXME - analysis_cttso_qs = Analysis.objects.get_by_keyword(analysis_name='ctTSO500').first() # FIXME: allow for multiple + + # prepare the available compute and storage contexts, to be chosen depending on the actual workload + clinical_compute_context = AnalysisContext.objects.get_by_keyword(name="clinical", usecase="compute-env").first() + clinical_storage_context = AnalysisContext.objects.get_by_keyword(name="clinical", usecase="storage-env").first() + research_compute_context = AnalysisContext.objects.get_by_keyword(name="research", usecase="compute-env").first() + research_storage_context = AnalysisContext.objects.get_by_keyword(name="research", usecase="storage-env").first() + + # Find the approved analysis for a ctTSO workload + # NOTE: for now we only have one ctTSO analysis + analysis_cttso_qs = Analysis.objects.get_by_keyword(analysis_name='ctTSO').first() for lib in libraries: lib_record: Library = Library.objects.get(library_id=lib['library_id']) # handle QC if lib['type'] in ['ctDNA'] and lib['assay'] in ['ctTSOv2']: - context: AnalysisContext = context_clinical if lib['workflow'] == 'clinical' else context_research - analysis_run_name = f"automated__{analysis_cttso_qs.analysis_name}__{context.name}__{lib_record.library_id}" + workflow = lib['workflow'] + compute_context = clinical_compute_context if workflow == 'clinical' else research_compute_context + storage_context = clinical_storage_context if workflow == 'clinical' else research_storage_context + analysis_run_name = f"automated__{analysis_cttso_qs.analysis_name}__{workflow}__{lib_record.library_id}" analysis_run = AnalysisRun( analysis_run_name=analysis_run_name, status="DRAFT", - approval_context=context, + compute_context=compute_context, + storage_context=storage_context, analysis=analysis_cttso_qs ) analysis_run.save() diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0001_initial.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0001_initial.py index b5b9057d0..753efc753 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0001_initial.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 5.1 on 2024-10-01 05:49 +# Generated by Django 5.1 on 2024-10-14 22:58 import django.core.serializers.json import django.core.validators @@ -17,7 +17,7 @@ class Migration(migrations.Migration): migrations.CreateModel( name='Library', fields=[ - ('orcabus_id', models.CharField(editable=False, primary_key=True, serialize=False, unique=True, validators=[django.core.validators.RegexValidator(code='invalid_orcabus_id', message='ULID is expected to be 26 characters long', regex='[\\w]{26}$')])), + ('orcabus_id', models.CharField(editable=False, primary_key=True, serialize=False, unique=True, validators=[django.core.validators.RegexValidator(code='invalid_orcabus_id', message='ULID is expected to be 26 characters long', regex='^[\\w]{26}$')])), ('library_id', models.CharField(max_length=255)), ], options={ @@ -27,7 +27,7 @@ class Migration(migrations.Migration): migrations.CreateModel( name='Payload', fields=[ - ('orcabus_id', models.CharField(editable=False, primary_key=True, serialize=False, unique=True, validators=[django.core.validators.RegexValidator(code='invalid_orcabus_id', message='ULID is expected to be 26 characters long', regex='[\\w]{26}$')])), + ('orcabus_id', models.CharField(editable=False, primary_key=True, serialize=False, unique=True, validators=[django.core.validators.RegexValidator(code='invalid_orcabus_id', message='ULID is expected to be 26 characters long', regex='^[\\w]{26}$')])), ('payload_ref_id', models.CharField(max_length=255, unique=True)), ('version', models.CharField(max_length=255)), ('data', models.JSONField(encoder=django.core.serializers.json.DjangoJSONEncoder)), @@ -39,7 +39,7 @@ class Migration(migrations.Migration): migrations.CreateModel( name='AnalysisContext', fields=[ - ('orcabus_id', models.CharField(editable=False, primary_key=True, serialize=False, unique=True, validators=[django.core.validators.RegexValidator(code='invalid_orcabus_id', message='ULID is expected to be 26 characters long', regex='[\\w]{26}$')])), + ('orcabus_id', models.CharField(editable=False, primary_key=True, serialize=False, unique=True, validators=[django.core.validators.RegexValidator(code='invalid_orcabus_id', message='ULID is expected to be 26 characters long', regex='^[\\w]{26}$')])), ('name', models.CharField(max_length=255)), ('usecase', models.CharField(max_length=255)), ('description', models.CharField(max_length=255)), @@ -52,7 +52,7 @@ class Migration(migrations.Migration): migrations.CreateModel( name='Analysis', fields=[ - ('orcabus_id', models.CharField(editable=False, primary_key=True, serialize=False, unique=True, validators=[django.core.validators.RegexValidator(code='invalid_orcabus_id', message='ULID is expected to be 26 characters long', regex='[\\w]{26}$')])), + ('orcabus_id', models.CharField(editable=False, primary_key=True, serialize=False, unique=True, validators=[django.core.validators.RegexValidator(code='invalid_orcabus_id', message='ULID is expected to be 26 characters long', regex='^[\\w]{26}$')])), ('analysis_name', models.CharField(max_length=255)), ('analysis_version', models.CharField(max_length=255)), ('description', models.CharField(max_length=255)), @@ -63,13 +63,13 @@ class Migration(migrations.Migration): migrations.CreateModel( name='AnalysisRun', fields=[ - ('orcabus_id', models.CharField(editable=False, primary_key=True, serialize=False, unique=True, validators=[django.core.validators.RegexValidator(code='invalid_orcabus_id', message='ULID is expected to be 26 characters long', regex='[\\w]{26}$')])), + ('orcabus_id', models.CharField(editable=False, primary_key=True, serialize=False, unique=True, validators=[django.core.validators.RegexValidator(code='invalid_orcabus_id', message='ULID is expected to be 26 characters long', regex='^[\\w]{26}$')])), ('analysis_run_name', models.CharField(max_length=255)), ('comment', models.CharField(blank=True, max_length=255, null=True)), ('status', models.CharField(blank=True, max_length=255, null=True)), ('analysis', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='workflow_manager.analysis')), - ('approval_context', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='approval_context', to='workflow_manager.analysiscontext')), - ('project_context', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='project_context', to='workflow_manager.analysiscontext')), + ('compute_context', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='compute_context', to='workflow_manager.analysiscontext')), + ('storage_context', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='storage_context', to='workflow_manager.analysiscontext')), ('libraries', models.ManyToManyField(to='workflow_manager.library')), ], options={ @@ -79,7 +79,7 @@ class Migration(migrations.Migration): migrations.CreateModel( name='LibraryAssociation', fields=[ - ('orcabus_id', models.CharField(editable=False, primary_key=True, serialize=False, unique=True, validators=[django.core.validators.RegexValidator(code='invalid_orcabus_id', message='ULID is expected to be 26 characters long', regex='[\\w]{26}$')])), + ('orcabus_id', models.CharField(editable=False, primary_key=True, serialize=False, unique=True, validators=[django.core.validators.RegexValidator(code='invalid_orcabus_id', message='ULID is expected to be 26 characters long', regex='^[\\w]{26}$')])), ('association_date', models.DateTimeField()), ('status', models.CharField(max_length=255)), ('library', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='workflow_manager.library')), @@ -91,7 +91,7 @@ class Migration(migrations.Migration): migrations.CreateModel( name='Workflow', fields=[ - ('orcabus_id', models.CharField(editable=False, primary_key=True, serialize=False, unique=True, validators=[django.core.validators.RegexValidator(code='invalid_orcabus_id', message='ULID is expected to be 26 characters long', regex='[\\w]{26}$')])), + ('orcabus_id', models.CharField(editable=False, primary_key=True, serialize=False, unique=True, validators=[django.core.validators.RegexValidator(code='invalid_orcabus_id', message='ULID is expected to be 26 characters long', regex='^[\\w]{26}$')])), ('workflow_name', models.CharField(max_length=255)), ('workflow_version', models.CharField(max_length=255)), ('execution_engine', models.CharField(max_length=255)), @@ -109,7 +109,7 @@ class Migration(migrations.Migration): migrations.CreateModel( name='WorkflowRun', fields=[ - ('orcabus_id', models.CharField(editable=False, primary_key=True, serialize=False, unique=True, validators=[django.core.validators.RegexValidator(code='invalid_orcabus_id', message='ULID is expected to be 26 characters long', regex='[\\w]{26}$')])), + ('orcabus_id', models.CharField(editable=False, primary_key=True, serialize=False, unique=True, validators=[django.core.validators.RegexValidator(code='invalid_orcabus_id', message='ULID is expected to be 26 characters long', regex='^[\\w]{26}$')])), ('portal_run_id', models.CharField(max_length=255, unique=True)), ('execution_id', models.CharField(blank=True, max_length=255, null=True)), ('workflow_run_name', models.CharField(blank=True, max_length=255, null=True)), @@ -134,12 +134,12 @@ class Migration(migrations.Migration): migrations.CreateModel( name='State', fields=[ - ('orcabus_id', models.CharField(editable=False, primary_key=True, serialize=False, unique=True, validators=[django.core.validators.RegexValidator(code='invalid_orcabus_id', message='ULID is expected to be 26 characters long', regex='[\\w]{26}$')])), + ('orcabus_id', models.CharField(editable=False, primary_key=True, serialize=False, unique=True, validators=[django.core.validators.RegexValidator(code='invalid_orcabus_id', message='ULID is expected to be 26 characters long', regex='^[\\w]{26}$')])), ('status', models.CharField(max_length=255)), ('timestamp', models.DateTimeField()), ('comment', models.CharField(blank=True, max_length=255, null=True)), ('payload', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='workflow_manager.payload')), - ('workflow_run', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='workflow_manager.workflowrun')), + ('workflow_run', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='states', to='workflow_manager.workflowrun')), ], options={ 'unique_together': {('workflow_run', 'status', 'timestamp')}, diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0002_alter_state_workflow_run.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0002_alter_state_workflow_run.py deleted file mode 100644 index 442c0b3dd..000000000 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0002_alter_state_workflow_run.py +++ /dev/null @@ -1,23 +0,0 @@ -# Generated by Django 5.1 on 2024-09-09 05:55 - -import django.db.models.deletion -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ("workflow_manager", "0001_initial"), - ] - - operations = [ - migrations.AlterField( - model_name="state", - name="workflow_run", - field=models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - related_name="states", - to="workflow_manager.workflowrun", - ), - ), - ] diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0003_alter_analysis_orcabus_id_and_more.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0003_alter_analysis_orcabus_id_and_more.py deleted file mode 100644 index 2041a9101..000000000 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0003_alter_analysis_orcabus_id_and_more.py +++ /dev/null @@ -1,59 +0,0 @@ -# Generated by Django 5.1 on 2024-10-03 00:54 - -import django.core.validators -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('workflow_manager', '0002_alter_state_workflow_run'), - ] - - operations = [ - migrations.AlterField( - model_name='analysis', - name='orcabus_id', - field=models.CharField(editable=False, primary_key=True, serialize=False, unique=True, validators=[django.core.validators.RegexValidator(code='invalid_orcabus_id', message='ULID is expected to be 26 characters long', regex='^[\\w]{26}$')]), - ), - migrations.AlterField( - model_name='analysiscontext', - name='orcabus_id', - field=models.CharField(editable=False, primary_key=True, serialize=False, unique=True, validators=[django.core.validators.RegexValidator(code='invalid_orcabus_id', message='ULID is expected to be 26 characters long', regex='^[\\w]{26}$')]), - ), - migrations.AlterField( - model_name='analysisrun', - name='orcabus_id', - field=models.CharField(editable=False, primary_key=True, serialize=False, unique=True, validators=[django.core.validators.RegexValidator(code='invalid_orcabus_id', message='ULID is expected to be 26 characters long', regex='^[\\w]{26}$')]), - ), - migrations.AlterField( - model_name='library', - name='orcabus_id', - field=models.CharField(editable=False, primary_key=True, serialize=False, unique=True, validators=[django.core.validators.RegexValidator(code='invalid_orcabus_id', message='ULID is expected to be 26 characters long', regex='^[\\w]{26}$')]), - ), - migrations.AlterField( - model_name='libraryassociation', - name='orcabus_id', - field=models.CharField(editable=False, primary_key=True, serialize=False, unique=True, validators=[django.core.validators.RegexValidator(code='invalid_orcabus_id', message='ULID is expected to be 26 characters long', regex='^[\\w]{26}$')]), - ), - migrations.AlterField( - model_name='payload', - name='orcabus_id', - field=models.CharField(editable=False, primary_key=True, serialize=False, unique=True, validators=[django.core.validators.RegexValidator(code='invalid_orcabus_id', message='ULID is expected to be 26 characters long', regex='^[\\w]{26}$')]), - ), - migrations.AlterField( - model_name='state', - name='orcabus_id', - field=models.CharField(editable=False, primary_key=True, serialize=False, unique=True, validators=[django.core.validators.RegexValidator(code='invalid_orcabus_id', message='ULID is expected to be 26 characters long', regex='^[\\w]{26}$')]), - ), - migrations.AlterField( - model_name='workflow', - name='orcabus_id', - field=models.CharField(editable=False, primary_key=True, serialize=False, unique=True, validators=[django.core.validators.RegexValidator(code='invalid_orcabus_id', message='ULID is expected to be 26 characters long', regex='^[\\w]{26}$')]), - ), - migrations.AlterField( - model_name='workflowrun', - name='orcabus_id', - field=models.CharField(editable=False, primary_key=True, serialize=False, unique=True, validators=[django.core.validators.RegexValidator(code='invalid_orcabus_id', message='ULID is expected to be 26 characters long', regex='^[\\w]{26}$')]), - ), - ] diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_run.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_run.py index 71a4e27dd..577b3cf89 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_run.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_run.py @@ -17,10 +17,10 @@ class AnalysisRun(OrcaBusBaseModel): comment = models.CharField(max_length=255, null=True, blank=True) status = models.CharField(max_length=255, null=True, blank=True) - approval_context = models.ForeignKey(AnalysisContext, null=True, blank=True, on_delete=models.SET_NULL, - related_name="approval_context") - project_context = models.ForeignKey(AnalysisContext, null=True, blank=True, on_delete=models.SET_NULL, - related_name="project_context") + compute_context = models.ForeignKey(AnalysisContext, null=True, blank=True, on_delete=models.SET_NULL, + related_name="compute_context") + storage_context = models.ForeignKey(AnalysisContext, null=True, blank=True, on_delete=models.SET_NULL, + related_name="storage_context") analysis = models.ForeignKey(Analysis, null=True, blank=True, on_delete=models.SET_NULL) libraries = models.ManyToManyField(Library) diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/analysis_run.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/analysis_run.py index 279415800..b8815c743 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/analysis_run.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/analysis_run.py @@ -14,10 +14,10 @@ class Meta: def to_representation(self, instance): representation = super().to_representation(instance) representation['analysis'] = Analysis.orcabus_id_prefix + representation['analysis'] - if representation['project_context']: - representation['project_context'] = AnalysisContext.orcabus_id_prefix + representation['project_context'] - if representation['approval_context']: - representation['approval_context'] = AnalysisContext.orcabus_id_prefix + representation['approval_context'] + if representation['storage_context']: + representation['storage_context'] = AnalysisContext.orcabus_id_prefix + representation['storage_context'] + if representation['compute_context']: + representation['compute_context'] = AnalysisContext.orcabus_id_prefix + representation['compute_context'] return representation @@ -28,8 +28,8 @@ class AnalysisRunDetailSerializer(AnalysisRunBaseSerializer): libraries = LibrarySerializer(many=True, read_only=True) analysis = AnalysisSerializer(read_only=True) - approval_context = AnalysisContextSerializer(read_only=True) - project_context = AnalysisContextSerializer(read_only=True) + storage_context = AnalysisContextSerializer(read_only=True) + compute_context = AnalysisContextSerializer(read_only=True) class Meta: model = AnalysisRun diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run.py index 9c46234d3..8ff925fa0 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run.py @@ -122,6 +122,7 @@ def ongoing(self, request): @action(detail=False, methods=['GET']) def unresolved(self, request): + self.serializer_class = WorkflowRunSerializer # use simple view for record listing # Get all books marked as favorite ordering = self.request.query_params.get('ordering', '-orcabus_id')