From 4ba8dd976be4c7e71fdc1c5f4b781f751c7079f4 Mon Sep 17 00:00:00 2001 From: william Date: Tue, 17 Dec 2024 14:54:10 +1100 Subject: [PATCH 1/6] MM refactor --- .../stacks/metadata-manager/app/fields.py | 74 ++++++++++--------- .../0003_alter_contact_orcabus_id_and_more.py | 74 +++++++++++++++++++ .../metadata-manager/app/models/base.py | 32 +++----- .../metadata-manager/app/models/contact.py | 4 +- .../metadata-manager/app/models/individual.py | 3 +- .../metadata-manager/app/models/library.py | 3 +- .../metadata-manager/app/models/project.py | 3 +- .../metadata-manager/app/models/sample.py | 3 +- .../metadata-manager/app/models/subject.py | 4 +- .../metadata-manager/app/serializers/base.py | 11 --- .../app/serializers/contact.py | 16 +--- .../app/serializers/individual.py | 15 ++-- .../app/serializers/library.py | 24 +----- .../app/serializers/project.py | 13 +--- .../app/serializers/sample.py | 13 ++-- .../app/serializers/subject.py | 17 +---- .../metadata-manager/app/viewsets/base.py | 34 +-------- .../metadata-manager/app/viewsets/contact.py | 4 +- .../app/viewsets/individual.py | 4 +- .../metadata-manager/app/viewsets/library.py | 4 +- .../metadata-manager/app/viewsets/project.py | 3 +- .../metadata-manager/app/viewsets/sample.py | 3 +- .../metadata-manager/app/viewsets/subject.py | 13 +--- .../deps/requirements-slim.txt | 2 +- 24 files changed, 178 insertions(+), 198 deletions(-) create mode 100644 lib/workload/stateless/stacks/metadata-manager/app/migrations/0003_alter_contact_orcabus_id_and_more.py delete mode 100644 lib/workload/stateless/stacks/metadata-manager/app/serializers/base.py diff --git a/lib/workload/stateless/stacks/metadata-manager/app/fields.py b/lib/workload/stateless/stacks/metadata-manager/app/fields.py index ba79fa4f7..a183a63f2 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/fields.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/fields.py @@ -1,48 +1,56 @@ -import hashlib - +import ulid +from django.core.validators import RegexValidator from django.db import models +ULID_REGEX_STR = r"[0123456789ABCDEFGHJKMNPQRSTVWXYZ]{26}" +ulid_validator = RegexValidator(regex=ULID_REGEX_STR, + message='ULID is expected to be 26 characters long', + code='invalid_orcabus_id') + + +def get_ulid() -> str: + return ulid.new().str -class HashField(models.CharField): - description = ( - "HashField is related to some base fields (other columns) in a model and" - "stores its hashed value for better indexing performance." - ) - def __init__(self, base_fields, *args, **kwargs): - """ - :param base_fields: name of fields storing the value to be hashed - """ - self.base_fields = base_fields - kwargs["max_length"] = 64 - super(HashField, self).__init__(*args, **kwargs) +class UlidField(models.CharField): + description = "An OrcaBus internal ID (ULID)" + + def __init__(self, *args, **kwargs): + kwargs['max_length'] = 26 # ULID length + kwargs['validators'] = [ulid_validator] + kwargs['default'] = get_ulid + super().__init__(*args, **kwargs) def deconstruct(self): name, path, args, kwargs = super().deconstruct() del kwargs["max_length"] - if self.base_fields is not None: - kwargs["base_fields"] = self.base_fields + del kwargs['validators'] + del kwargs['default'] return name, path, args, kwargs - def pre_save(self, instance, add): - self.calculate_hash(instance) - return super(HashField, self).pre_save(instance, add) - def calculate_hash(self, instance): - sha256 = hashlib.sha256() - for field in self.base_fields: - value = getattr(instance, field) - sha256.update(value.encode("utf-8")) - setattr(instance, self.attname, sha256.hexdigest()) +class OrcaBusIdField(UlidField): + description = "An OrcaBus internal ID (based on ULID)" + + def __init__(self, prefix='', *args, **kwargs): + self.prefix = prefix + super().__init__(*args, **kwargs) + @property + def non_db_attrs(self): + return super().non_db_attrs + ("prefix",) -class HashFieldHelper(object): - def __init__(self): - self.__sha256 = hashlib.sha256() + def from_db_value(self, value, expression, connection): + # print('hello from_db_value', value) + if value and self.prefix != '': + return f"{self.prefix}.{value}" + else: + return value - def add(self, value): - self.__sha256.update(value.encode("utf-8")) - return self + def to_python(self, value): + # This will be called when the function + return self.get_prep_value(value) - def calculate_hash(self): - return self.__sha256.hexdigest() + def get_prep_value(self, value): + # We just want the last 26 characters which is the ULID (ignoring any prefix) when dealing with the database + return value[-26:] diff --git a/lib/workload/stateless/stacks/metadata-manager/app/migrations/0003_alter_contact_orcabus_id_and_more.py b/lib/workload/stateless/stacks/metadata-manager/app/migrations/0003_alter_contact_orcabus_id_and_more.py new file mode 100644 index 000000000..d68ab4c12 --- /dev/null +++ b/lib/workload/stateless/stacks/metadata-manager/app/migrations/0003_alter_contact_orcabus_id_and_more.py @@ -0,0 +1,74 @@ +# Generated by Django 5.1.4 on 2024-12-17 01:44 + +import app.fields +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('app', '0002_remove_historicalcontact_history_user_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='contact', + name='orcabus_id', + field=app.fields.OrcaBusIdField(primary_key=True, serialize=False), + ), + migrations.AlterField( + model_name='historicalcontact', + name='orcabus_id', + field=app.fields.OrcaBusIdField(db_index=True), + ), + migrations.AlterField( + model_name='historicalindividual', + name='orcabus_id', + field=app.fields.OrcaBusIdField(db_index=True), + ), + migrations.AlterField( + model_name='historicallibrary', + name='orcabus_id', + field=app.fields.OrcaBusIdField(db_index=True), + ), + migrations.AlterField( + model_name='historicalproject', + name='orcabus_id', + field=app.fields.OrcaBusIdField(db_index=True), + ), + migrations.AlterField( + model_name='historicalsample', + name='orcabus_id', + field=app.fields.OrcaBusIdField(db_index=True), + ), + migrations.AlterField( + model_name='historicalsubject', + name='orcabus_id', + field=app.fields.OrcaBusIdField(db_index=True), + ), + migrations.AlterField( + model_name='individual', + name='orcabus_id', + field=app.fields.OrcaBusIdField(primary_key=True, serialize=False), + ), + migrations.AlterField( + model_name='library', + name='orcabus_id', + field=app.fields.OrcaBusIdField(primary_key=True, serialize=False), + ), + migrations.AlterField( + model_name='project', + name='orcabus_id', + field=app.fields.OrcaBusIdField(primary_key=True, serialize=False), + ), + migrations.AlterField( + model_name='sample', + name='orcabus_id', + field=app.fields.OrcaBusIdField(primary_key=True, serialize=False), + ), + migrations.AlterField( + model_name='subject', + name='orcabus_id', + field=app.fields.OrcaBusIdField(primary_key=True, serialize=False), + ), + ] diff --git a/lib/workload/stateless/stacks/metadata-manager/app/models/base.py b/lib/workload/stateless/stacks/metadata-manager/app/models/base.py index 5e47881b7..dc970724f 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/models/base.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/models/base.py @@ -23,6 +23,7 @@ from simple_history.models import HistoricalRecords from rest_framework.settings import api_settings + from app.pagination import PaginationConstant logger = logging.getLogger(__name__) @@ -42,7 +43,9 @@ def reduce_multi_values_qor(key: str, values: List[str]): ): values = [values] return reduce( - operator.or_, (Q(**{"%s__iexact" % key: value}) + # Apparently the `get_prep_value` from the custom fields.py is not called prior hitting the Db but, + # the regular `__exact` still execute that function. + operator.or_, (Q(**{"%s__exact" % key: value}) for value in values) ) @@ -74,6 +77,7 @@ def exclude_params(params): try: if query_string: + print('query_string', query_string) qs = qs.filter(query_string) except FieldError as e: logger.debug(e) @@ -102,7 +106,6 @@ def update_or_create_if_needed(self, search_key: dict, data: dict, user_id: str """ is_created = False is_updated = False - try: obj = self.get(**search_key) for key, value in data.items(): @@ -125,27 +128,16 @@ class BaseModel(models.Model): class Meta: abstract = True - orcabus_id = models.CharField( - primary_key=True, - unique=True, - editable=False, - blank=False, - null=False, - validators=[ - RegexValidator( - regex=r'[\w]{26}$', - message='ULID is expected to be 26 characters long', - code='invalid_orcabus_id' - )] - - ) - def save(self, *args, **kwargs): - if not self.orcabus_id: - self.orcabus_id = ulid.new().str + # To make django validate the constraint before saving it self.full_clean() - return super(BaseModel, self).save(*args, **kwargs) + super(BaseModel, self).save(*args, **kwargs) + + # Reload the object from the database to ensure custom fields like OrcaBusIdField + # invoke the `from_db_value` method (which provides the annotation) after saving. + self.refresh_from_db() + @classmethod def get_fields(cls): diff --git a/lib/workload/stateless/stacks/metadata-manager/app/models/contact.py b/lib/workload/stateless/stacks/metadata-manager/app/models/contact.py index ad3750de9..4e6de60fb 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/models/contact.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/models/contact.py @@ -1,16 +1,16 @@ from django.db import models +from app.fields import OrcaBusIdField from app.models.base import BaseModel, BaseManager, BaseHistoricalRecords - class ContactManager(BaseManager): pass class Contact(BaseModel): - orcabus_id_prefix = 'ctc.' objects = ContactManager() + orcabus_id = OrcaBusIdField(primary_key=True, prefix='ctc') contact_id = models.CharField( unique=True, blank=True, diff --git a/lib/workload/stateless/stacks/metadata-manager/app/models/individual.py b/lib/workload/stateless/stacks/metadata-manager/app/models/individual.py index 4d904bb10..1cb9a1a40 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/models/individual.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/models/individual.py @@ -1,5 +1,6 @@ from django.db import models +from app.fields import OrcaBusIdField from app.models.base import BaseModel, BaseManager, BaseHistoricalRecords @@ -8,9 +9,9 @@ class IndividualManager(BaseManager): class Individual(BaseModel): - orcabus_id_prefix = 'idv.' objects = IndividualManager() + orcabus_id = OrcaBusIdField(primary_key=True, prefix='idv') individual_id = models.CharField( unique=True, blank=True, diff --git a/lib/workload/stateless/stacks/metadata-manager/app/models/library.py b/lib/workload/stateless/stacks/metadata-manager/app/models/library.py index 3010f0e7f..02cfcf159 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/models/library.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/models/library.py @@ -2,6 +2,7 @@ from django.db import models +from app.fields import OrcaBusIdField from app.models.base import BaseManager, BaseModel, BaseHistoricalRecords from app.models.subject import Subject from app.models.sample import Sample @@ -64,9 +65,9 @@ class LibraryProjectLink(models.Model): class Library(BaseModel): - orcabus_id_prefix = 'lib.' objects = LibraryManager() + orcabus_id = OrcaBusIdField(primary_key=True, prefix='lib') library_id = models.CharField( unique=True, blank=True, diff --git a/lib/workload/stateless/stacks/metadata-manager/app/models/project.py b/lib/workload/stateless/stacks/metadata-manager/app/models/project.py index 95e6bb3d4..99b7758d2 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/models/project.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/models/project.py @@ -1,5 +1,6 @@ from django.db import models +from app.fields import OrcaBusIdField from app.models.contact import Contact from app.models.base import BaseModel, BaseManager, BaseHistoricalRecords @@ -19,9 +20,9 @@ class ProjectContactLink(models.Model): class Project(BaseModel): - orcabus_id_prefix = 'prj.' objects = ProjectManager() + orcabus_id = OrcaBusIdField(primary_key=True, prefix='prj') project_id = models.CharField( unique=True, blank=True, diff --git a/lib/workload/stateless/stacks/metadata-manager/app/models/sample.py b/lib/workload/stateless/stacks/metadata-manager/app/models/sample.py index 2a74e3369..00fe1f71f 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/models/sample.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/models/sample.py @@ -1,6 +1,7 @@ import ulid from django.db import models +from app.fields import OrcaBusIdField from app.models.base import BaseModel, BaseManager, BaseHistoricalRecords @@ -31,9 +32,9 @@ class SampleManager(BaseManager): class Sample(BaseModel): - orcabus_id_prefix = 'smp.' objects = SampleManager() + orcabus_id = OrcaBusIdField(primary_key=True, prefix='smp') sample_id = models.CharField( unique=True, blank=True, diff --git a/lib/workload/stateless/stacks/metadata-manager/app/models/subject.py b/lib/workload/stateless/stacks/metadata-manager/app/models/subject.py index 8f2873bf9..bd4688758 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/models/subject.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/models/subject.py @@ -1,5 +1,6 @@ from django.db import models +from app.fields import OrcaBusIdField from app.models.base import BaseModel, BaseManager, BaseHistoricalRecords @@ -17,8 +18,9 @@ class SubjectIndividualLink(models.Model): class Subject(BaseModel): - orcabus_id_prefix = 'sbj.' objects = SubjectManager() + + orcabus_id = OrcaBusIdField(primary_key=True, prefix='sbj') subject_id = models.CharField( unique=True, blank=True, diff --git a/lib/workload/stateless/stacks/metadata-manager/app/serializers/base.py b/lib/workload/stateless/stacks/metadata-manager/app/serializers/base.py deleted file mode 100644 index d372527a6..000000000 --- a/lib/workload/stateless/stacks/metadata-manager/app/serializers/base.py +++ /dev/null @@ -1,11 +0,0 @@ -from rest_framework import serializers - - -class SerializersBase(serializers.ModelSerializer): - prefix = '' - - def to_representation(self, instance): - representation = super().to_representation(instance) - representation['orcabus_id'] = self.prefix + str(representation['orcabus_id']) - return representation - diff --git a/lib/workload/stateless/stacks/metadata-manager/app/serializers/contact.py b/lib/workload/stateless/stacks/metadata-manager/app/serializers/contact.py index 5a69ce920..911811c51 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/serializers/contact.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/serializers/contact.py @@ -1,22 +1,14 @@ -from abc import ABC - -from rest_framework import serializers - +from rest_framework.serializers import ModelSerializer from app.models import Contact -from .base import SerializersBase - - -class ContactBaseSerializer(SerializersBase): - prefix = Contact.orcabus_id_prefix -class ContactSerializer(ContactBaseSerializer): +class ContactSerializer(ModelSerializer): class Meta: model = Contact fields = "__all__" -class ContactDetailSerializer(ContactBaseSerializer): +class ContactDetailSerializer(ModelSerializer): from .project import ProjectSerializer project_set = ProjectSerializer(many=True, read_only=True) @@ -26,7 +18,7 @@ class Meta: fields = "__all__" -class ContactHistorySerializer(ContactBaseSerializer): +class ContactHistorySerializer(ModelSerializer): class Meta: model = Contact.history.model fields = "__all__" diff --git a/lib/workload/stateless/stacks/metadata-manager/app/serializers/individual.py b/lib/workload/stateless/stacks/metadata-manager/app/serializers/individual.py index d66c553da..8b25f7b04 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/serializers/individual.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/serializers/individual.py @@ -1,22 +1,25 @@ -from app.models import Individual -from .base import SerializersBase +from rest_framework.serializers import ModelSerializer +from app.models import Individual -class IndividualSerializer(SerializersBase): - prefix = Individual.orcabus_id_prefix +class IndividualSerializer(ModelSerializer): class Meta: model = Individual fields = '__all__' -class IndividualDetailSerializer(IndividualSerializer): +class IndividualDetailSerializer(ModelSerializer): from .subject import SubjectSerializer + class Meta: + model = Individual + fields = '__all__' + subject_set = SubjectSerializer(many=True, read_only=True) -class IndividualHistorySerializer(IndividualSerializer): +class IndividualHistorySerializer(ModelSerializer): class Meta: model = Individual.history.model fields = "__all__" diff --git a/lib/workload/stateless/stacks/metadata-manager/app/serializers/library.py b/lib/workload/stateless/stacks/metadata-manager/app/serializers/library.py index 2b8ce8e2a..8392c5f6d 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/serializers/library.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/serializers/library.py @@ -1,31 +1,16 @@ -from abc import ABC - from rest_framework import serializers +from rest_framework.serializers import ModelSerializer from app.models import Library, Sample, Subject, Project -from .base import SerializersBase - - -class LibraryBaseSerializer(SerializersBase): - prefix = Library.orcabus_id_prefix -class LibrarySerializer(LibraryBaseSerializer): +class LibrarySerializer(ModelSerializer): class Meta: model = Library exclude = ["project_set"] - def to_representation(self, instance): - representation = super().to_representation(instance) - if representation.get('sample', None): - representation['sample'] = Sample.orcabus_id_prefix + representation['sample'] - if representation.get('subject', None): - representation['subject'] = Subject.orcabus_id_prefix + representation['subject'] - return representation - - -class LibraryDetailSerializer(LibraryBaseSerializer): +class LibraryDetailSerializer(ModelSerializer): from .sample import SampleSerializer from .project import ProjectSerializer from .subject import SubjectSerializer @@ -45,9 +30,6 @@ class ProjectOrcabusIdSet(serializers.StringRelatedField): def to_internal_value(self, data): raise NotImplementedError() - def to_representation(self, value): - return Project.orcabus_id_prefix + value.project.orcabus_id - class Meta: model = Library.history.model fields = "__all__" diff --git a/lib/workload/stateless/stacks/metadata-manager/app/serializers/project.py b/lib/workload/stateless/stacks/metadata-manager/app/serializers/project.py index 0418fa5fe..bc2d258b1 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/serializers/project.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/serializers/project.py @@ -1,20 +1,16 @@ from rest_framework import serializers +from rest_framework.serializers import ModelSerializer -from .base import SerializersBase from app.models import Project, Contact -class ProjectBaseSerializer(SerializersBase): - prefix = Project.orcabus_id_prefix - - -class ProjectSerializer(ProjectBaseSerializer): +class ProjectSerializer(ModelSerializer): class Meta: model = Project exclude = ["contact_set"] -class ProjectDetailSerializer(ProjectBaseSerializer): +class ProjectDetailSerializer(ModelSerializer): from .contact import ContactSerializer contact_set = ContactSerializer(many=True, read_only=True) @@ -24,7 +20,7 @@ class Meta: fields = "__all__" -class ProjectHistorySerializer(ProjectBaseSerializer): +class ProjectHistorySerializer(ModelSerializer): class ContactOrcabusIdSet(serializers.StringRelatedField): def to_internal_value(self, data): @@ -38,4 +34,3 @@ class Meta: fields = "__all__" contact_set = ContactOrcabusIdSet(many=True, read_only=True) - diff --git a/lib/workload/stateless/stacks/metadata-manager/app/serializers/sample.py b/lib/workload/stateless/stacks/metadata-manager/app/serializers/sample.py index 381aeed59..118d039d5 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/serializers/sample.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/serializers/sample.py @@ -1,18 +1,15 @@ -from .base import SerializersBase -from app.models import Sample - +from rest_framework.serializers import ModelSerializer -class SampleBaseSerializer(SerializersBase): - prefix = Sample.orcabus_id_prefix +from app.models import Sample -class SampleSerializer(SampleBaseSerializer): +class SampleSerializer(ModelSerializer): class Meta: model = Sample fields = "__all__" -class SampleDetailSerializer(SampleBaseSerializer): +class SampleDetailSerializer(ModelSerializer): from .library import LibrarySerializer class Meta: @@ -22,7 +19,7 @@ class Meta: library_set = LibrarySerializer(many=True, read_only=True) -class SampleHistorySerializer(SampleBaseSerializer): +class SampleHistorySerializer(ModelSerializer): class Meta: model = Sample.history.model diff --git a/lib/workload/stateless/stacks/metadata-manager/app/serializers/subject.py b/lib/workload/stateless/stacks/metadata-manager/app/serializers/subject.py index cdef28fa5..d9d0c007e 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/serializers/subject.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/serializers/subject.py @@ -1,22 +1,16 @@ from rest_framework import serializers +from rest_framework.serializers import ModelSerializer from app.models import Subject, Individual -from .base import SerializersBase -class SubjectBaseSerializer(SerializersBase): - prefix = Subject.orcabus_id_prefix - - -class SubjectSerializer(SubjectBaseSerializer): - prefix = Subject.orcabus_id_prefix - +class SubjectSerializer(ModelSerializer): class Meta: model = Subject exclude = ["individual_set"] -class SubjectDetailSerializer(SubjectBaseSerializer): +class SubjectDetailSerializer(ModelSerializer): from .individual import IndividualSerializer from .library import LibrarySerializer @@ -28,14 +22,11 @@ class Meta: library_set = LibrarySerializer(many=True, read_only=True) -class SubjectHistorySerializer(SubjectBaseSerializer): +class SubjectHistorySerializer(ModelSerializer): class IndividualOrcabusIdSet(serializers.StringRelatedField): def to_internal_value(self, data): raise NotImplementedError() - def to_representation(self, value): - return Individual.orcabus_id_prefix + value.individual.orcabus_id - class Meta: model = Subject.history.model fields = "__all__" diff --git a/lib/workload/stateless/stacks/metadata-manager/app/viewsets/base.py b/lib/workload/stateless/stacks/metadata-manager/app/viewsets/base.py index a5443faa7..c57da7b31 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/viewsets/base.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/viewsets/base.py @@ -16,43 +16,12 @@ class BaseViewSet(ModelViewSet, ABC): lookup_value_regex = "[^/]+" # This is to allow for special characters in the URL - orcabus_id_prefix = '' ordering_fields = "__all__" ordering = ["-orcabus_id"] pagination_class = StandardResultsSetPagination filter_backends = [filters.OrderingFilter, filters.SearchFilter] http_method_names = ['get', 'patch', 'delete'] - def retrieve(self, request, *args, **kwargs): - """ - Since we have custom orcabus_id prefix for each model, we need to remove the prefix before retrieving it. - """ - pk = self.kwargs.get('pk') - if pk and pk.startswith(self.orcabus_id_prefix): - pk = pk[len(self.orcabus_id_prefix):] - - obj = get_object_or_404(self.queryset, pk=pk) - serializer = self.serializer_class(obj) - return Response(serializer.data) - - def get_query_params(self): - """ - Sanitize query params if needed - e.g. remove prefixes for each orcabus_id - """ - query_params = self.request.query_params.copy() - orcabus_id = query_params.getlist("orcabus_id", None) - if orcabus_id: - id_list = [] - for key in orcabus_id: - if key.startswith(self.orcabus_id_prefix): - id_list.append(key[len(self.orcabus_id_prefix):]) - else: - id_list.append(key) - query_params.setlist('orcabus_id', id_list) - - return query_params - def retrieve_history(self, history_serializer): """ To use this as API routes, you need to call it from the child class and put the appropriate decorator. @@ -72,8 +41,7 @@ def retrieve_history(self, request, *args, **kwargs): # Grab the PK object from the queryset pk = self.kwargs.get('pk') - if pk and pk.startswith(self.orcabus_id_prefix): - pk = pk[len(self.orcabus_id_prefix):] + obj = get_object_or_404(self.queryset, pk=pk) history_qs = obj.history.all() diff --git a/lib/workload/stateless/stacks/metadata-manager/app/viewsets/contact.py b/lib/workload/stateless/stacks/metadata-manager/app/viewsets/contact.py index 36189c5b9..ab59838b8 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/viewsets/contact.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/viewsets/contact.py @@ -11,10 +11,9 @@ class ContactViewSet(BaseViewSet): serializer_class = ContactSerializer search_fields = Contact.get_base_fields() queryset = Contact.objects.all() - orcabus_id_prefix = Contact.orcabus_id_prefix def get_queryset(self): - query_params = super().get_query_params() + query_params = self.request.query_params.copy() return Contact.objects.get_by_keyword(**query_params) @extend_schema(responses=ContactDetailSerializer(many=False)) @@ -34,7 +33,6 @@ def list(self, request, *args, **kwargs): self.queryset = Contact.objects.prefetch_related('project_set').all() return super().list(request, *args, **kwargs) - @extend_schema(responses=ContactHistorySerializer(many=True), description="Retrieve the history of this model") @action(detail=True, methods=['get'], url_name='history', url_path='history') def retrieve_history(self, request, *args, **kwargs): diff --git a/lib/workload/stateless/stacks/metadata-manager/app/viewsets/individual.py b/lib/workload/stateless/stacks/metadata-manager/app/viewsets/individual.py index aed759204..58ca37c99 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/viewsets/individual.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/viewsets/individual.py @@ -11,7 +11,6 @@ class IndividualViewSet(BaseViewSet): serializer_class = IndividualSerializer search_fields = Individual.get_base_fields() queryset = Individual.objects.all() - orcabus_id_prefix = Individual.orcabus_id_prefix @extend_schema(responses=IndividualDetailSerializer(many=False)) def retrieve(self, request, *args, **kwargs): @@ -29,11 +28,10 @@ def list(self, request, *args, **kwargs): return super().list(request, *args, **kwargs) def get_queryset(self): - query_params = self.get_query_params() + query_params = self.request.query_params.copy() return Individual.objects.get_by_keyword(self.queryset, **query_params) @extend_schema(responses=IndividualHistorySerializer(many=True), description="Retrieve the history of this model") @action(detail=True, methods=['get'], url_name='history', url_path='history') def retrieve_history(self, request, *args, **kwargs): return super().retrieve_history(IndividualHistorySerializer) - diff --git a/lib/workload/stateless/stacks/metadata-manager/app/viewsets/library.py b/lib/workload/stateless/stacks/metadata-manager/app/viewsets/library.py index a5f5f665b..1d6bf0e66 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/viewsets/library.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/viewsets/library.py @@ -12,11 +12,11 @@ class LibraryViewSet(BaseViewSet): detail_serializer_class = LibraryDetailSerializer search_fields = Library.get_base_fields() queryset = Library.objects.all() - orcabus_id_prefix = Library.orcabus_id_prefix + def get_queryset(self): qs = self.queryset - query_params = self.get_query_params() + query_params = self.request.query_params.copy() coverage__lte = query_params.get("coverage[lte]", None) if coverage__lte: diff --git a/lib/workload/stateless/stacks/metadata-manager/app/viewsets/project.py b/lib/workload/stateless/stacks/metadata-manager/app/viewsets/project.py index d86ed1119..f92320bba 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/viewsets/project.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/viewsets/project.py @@ -11,7 +11,6 @@ class ProjectViewSet(BaseViewSet): serializer_class = ProjectSerializer search_fields = Project.get_base_fields() queryset = Project.objects.all() - orcabus_id_prefix = Project.orcabus_id_prefix @extend_schema(responses=ProjectDetailSerializer(many=False)) def retrieve(self, request, *args, **kwargs): @@ -31,7 +30,7 @@ def list(self, request, *args, **kwargs): return super().list(request, *args, **kwargs) def get_queryset(self): - query_params = self.get_query_params() + query_params = self.request.query_params.copy() return Project.objects.get_by_keyword(self.queryset, **query_params) @extend_schema(responses=ProjectHistorySerializer(many=True), description="Retrieve the history of this model") diff --git a/lib/workload/stateless/stacks/metadata-manager/app/viewsets/sample.py b/lib/workload/stateless/stacks/metadata-manager/app/viewsets/sample.py index 351fc8aa4..5aed741dd 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/viewsets/sample.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/viewsets/sample.py @@ -11,11 +11,10 @@ class SampleViewSet(BaseViewSet): serializer_class = SampleSerializer search_fields = Sample.get_base_fields() queryset = Sample.objects.all() - orcabus_id_prefix = Sample.orcabus_id_prefix def get_queryset(self): qs = self.queryset - query_params = self.get_query_params() + query_params = self.request.query_params.copy() is_library_none = query_params.getlist("is_library_none", None) if is_library_none: diff --git a/lib/workload/stateless/stacks/metadata-manager/app/viewsets/subject.py b/lib/workload/stateless/stacks/metadata-manager/app/viewsets/subject.py index df88a583e..20c4f9ca7 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/viewsets/subject.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/viewsets/subject.py @@ -10,28 +10,17 @@ class SubjectViewSet(BaseViewSet): serializer_class = SubjectSerializer search_fields = Subject.get_base_fields() queryset = Subject.objects.all() - orcabus_id_prefix = Subject.orcabus_id_prefix def get_queryset(self): qs = self.queryset - query_params = self.get_query_params() + query_params = self.request.query_params.copy() library_id = query_params.get("library_id", None) if library_id: query_params.pop("library_id") qs = qs.filter(library__library_id=library_id) - library_orcabus_id = query_params.get("library_orcabus_id", None) - if library_orcabus_id: - query_params.pop("library_orcabus_id") - - # Remove '.lib' prefix if present - if library_orcabus_id.startswith(Library.orcabus_id_prefix): - library_orcabus_id = library_orcabus_id[len(self.orcabus_id_prefix):] - - qs = qs.filter(library__orcabus_id=library_orcabus_id) - is_library_none = query_params.getlist("is_library_none", None) if is_library_none: query_params.pop("is_library_none") diff --git a/lib/workload/stateless/stacks/metadata-manager/deps/requirements-slim.txt b/lib/workload/stateless/stacks/metadata-manager/deps/requirements-slim.txt index 0e7a873b2..f3d80af4d 100644 --- a/lib/workload/stateless/stacks/metadata-manager/deps/requirements-slim.txt +++ b/lib/workload/stateless/stacks/metadata-manager/deps/requirements-slim.txt @@ -4,7 +4,7 @@ aws-xray-sdk # intentionally leave out version, the daily release of this dep is ok djangorestframework==3.15.2 djangorestframework-camel-case==1.4.2 -Django==5.1.2 +Django==5.1.4 django-cors-headers==4.5.0 django-environ==0.11.2 django-simple-history==3.7.0 From b8f1a2899c038dea4f0defe6e39e066616f80c8e Mon Sep 17 00:00:00 2001 From: william Date: Tue, 17 Dec 2024 15:48:51 +1100 Subject: [PATCH 2/6] (MM): print removal --- .../stateless/stacks/metadata-manager/app/models/base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/workload/stateless/stacks/metadata-manager/app/models/base.py b/lib/workload/stateless/stacks/metadata-manager/app/models/base.py index dc970724f..66b8a5bb2 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/models/base.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/models/base.py @@ -77,7 +77,6 @@ def exclude_params(params): try: if query_string: - print('query_string', query_string) qs = qs.filter(query_string) except FieldError as e: logger.debug(e) From eafc80246e41ecc6c13f54f67f4ff8066258e135 Mon Sep 17 00:00:00 2001 From: william Date: Tue, 17 Dec 2024 19:33:36 +1100 Subject: [PATCH 3/6] wfm: OrcaBusId Refactor --- .../stacks/metadata-manager/app/fields.py | 1 - .../workflow_manager/fields.py | 87 ++++++++----------- ...0003_alter_analysis_orcabus_id_and_more.py | 64 ++++++++++++++ .../workflow_manager/models/analysis.py | 4 +- .../models/analysis_context.py | 4 +- .../workflow_manager/models/analysis_run.py | 3 +- .../workflow_manager/models/base.py | 37 +++----- .../workflow_manager/models/library.py | 4 +- .../workflow_manager/models/payload.py | 4 +- .../workflow_manager/models/state.py | 4 +- .../workflow_manager/models/workflow.py | 4 +- .../workflow_manager/models/workflow_run.py | 5 +- .../models/workflow_run_comment.py | 9 +- .../workflow_manager/serializers/analysis.py | 20 ++--- .../serializers/analysis_context.py | 5 +- .../serializers/analysis_run.py | 11 ++- .../workflow_manager/serializers/base.py | 9 -- .../workflow_manager/serializers/library.py | 2 +- .../workflow_manager/serializers/payload.py | 4 +- .../workflow_manager/serializers/state.py | 9 +- .../workflow_manager/serializers/workflow.py | 4 +- .../serializers/workflow_run.py | 17 ++-- .../serializers/workflow_run_action.py | 4 +- .../serializers/workflow_run_comment.py | 12 ++- .../workflow_manager/viewsets/analysis.py | 3 +- .../viewsets/analysis_context.py | 3 +- .../workflow_manager/viewsets/analysis_run.py | 3 +- .../workflow_manager/viewsets/base.py | 31 ------- .../workflow_manager/viewsets/library.py | 3 +- .../workflow_manager/viewsets/payload.py | 3 +- .../workflow_manager/viewsets/state.py | 1 - .../workflow_manager/viewsets/workflow.py | 6 +- .../workflow_manager/viewsets/workflow_run.py | 1 - .../viewsets/workflow_run_action.py | 7 +- .../viewsets/workflow_run_comment.py | 1 - 35 files changed, 182 insertions(+), 207 deletions(-) create mode 100644 lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0003_alter_analysis_orcabus_id_and_more.py diff --git a/lib/workload/stateless/stacks/metadata-manager/app/fields.py b/lib/workload/stateless/stacks/metadata-manager/app/fields.py index a183a63f2..c141c92bb 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/fields.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/fields.py @@ -41,7 +41,6 @@ def non_db_attrs(self): return super().non_db_attrs + ("prefix",) def from_db_value(self, value, expression, connection): - # print('hello from_db_value', value) if value and self.prefix != '': return f"{self.prefix}.{value}" else: diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/fields.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/fields.py index fbabd177c..c141c92bb 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/fields.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/fields.py @@ -1,70 +1,55 @@ -import hashlib import ulid - -from django.db import models from django.core.validators import RegexValidator +from django.db import models -orcabus_id_validator = RegexValidator( - regex=r'[\w]{26}$', - message='ULID is expected to be 26 characters long', - code='invalid_orcabus_id' - ) - +ULID_REGEX_STR = r"[0123456789ABCDEFGHJKMNPQRSTVWXYZ]{26}" +ulid_validator = RegexValidator(regex=ULID_REGEX_STR, + message='ULID is expected to be 26 characters long', + code='invalid_orcabus_id') -class OrcabusIdField(models.CharField): - description = "An OrcaBus internal ID (ULID)" - def __init__(self, prefix, *args, **kwargs): - kwargs["max_length"] = 26 # ULID length - kwargs['unique'] = True - kwargs['editable'] = False - kwargs['blank'] = False - kwargs['null'] = False - kwargs['default'] = ulid.new - kwargs['validators'] = [orcabus_id_validator] - super().__init__(*args, **kwargs) +def get_ulid() -> str: + return ulid.new().str -class HashField(models.CharField): - description = ( - "HashField is related to some base fields (other columns) in a model and" - "stores its hashed value for better indexing performance." - ) +class UlidField(models.CharField): + description = "An OrcaBus internal ID (ULID)" - def __init__(self, base_fields, *args, **kwargs): - """ - :param base_fields: name of fields storing the value to be hashed - """ - self.base_fields = base_fields - kwargs["max_length"] = 64 - super(HashField, self).__init__(*args, **kwargs) + def __init__(self, *args, **kwargs): + kwargs['max_length'] = 26 # ULID length + kwargs['validators'] = [ulid_validator] + kwargs['default'] = get_ulid + super().__init__(*args, **kwargs) def deconstruct(self): name, path, args, kwargs = super().deconstruct() del kwargs["max_length"] - if self.base_fields is not None: - kwargs["base_fields"] = self.base_fields + del kwargs['validators'] + del kwargs['default'] return name, path, args, kwargs - def pre_save(self, instance, add): - self.calculate_hash(instance) - return super(HashField, self).pre_save(instance, add) - def calculate_hash(self, instance): - sha256 = hashlib.sha256() - for field in self.base_fields: - value = getattr(instance, field) - sha256.update(value.encode("utf-8")) - setattr(instance, self.attname, sha256.hexdigest()) +class OrcaBusIdField(UlidField): + description = "An OrcaBus internal ID (based on ULID)" + + def __init__(self, prefix='', *args, **kwargs): + self.prefix = prefix + super().__init__(*args, **kwargs) + @property + def non_db_attrs(self): + return super().non_db_attrs + ("prefix",) -class HashFieldHelper(object): - def __init__(self): - self.__sha256 = hashlib.sha256() + def from_db_value(self, value, expression, connection): + if value and self.prefix != '': + return f"{self.prefix}.{value}" + else: + return value - def add(self, value): - self.__sha256.update(value.encode("utf-8")) - return self + def to_python(self, value): + # This will be called when the function + return self.get_prep_value(value) - def calculate_hash(self): - return self.__sha256.hexdigest() + def get_prep_value(self, value): + # We just want the last 26 characters which is the ULID (ignoring any prefix) when dealing with the database + return value[-26:] diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0003_alter_analysis_orcabus_id_and_more.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0003_alter_analysis_orcabus_id_and_more.py new file mode 100644 index 000000000..780642ee1 --- /dev/null +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/migrations/0003_alter_analysis_orcabus_id_and_more.py @@ -0,0 +1,64 @@ +# Generated by Django 5.1.2 on 2024-12-17 08:04 + +import workflow_manager.fields +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('workflow_manager', '0002_workflowruncomment'), + ] + + operations = [ + migrations.AlterField( + model_name='analysis', + name='orcabus_id', + field=workflow_manager.fields.OrcaBusIdField(primary_key=True, serialize=False), + ), + migrations.AlterField( + model_name='analysiscontext', + name='orcabus_id', + field=workflow_manager.fields.OrcaBusIdField(primary_key=True, serialize=False), + ), + migrations.AlterField( + model_name='analysisrun', + name='orcabus_id', + field=workflow_manager.fields.OrcaBusIdField(primary_key=True, serialize=False), + ), + migrations.AlterField( + model_name='library', + name='orcabus_id', + field=workflow_manager.fields.OrcaBusIdField(primary_key=True, serialize=False), + ), + migrations.AlterField( + model_name='libraryassociation', + name='orcabus_id', + field=workflow_manager.fields.OrcaBusIdField(primary_key=True, serialize=False), + ), + migrations.AlterField( + model_name='payload', + name='orcabus_id', + field=workflow_manager.fields.OrcaBusIdField(primary_key=True, serialize=False), + ), + migrations.AlterField( + model_name='state', + name='orcabus_id', + field=workflow_manager.fields.OrcaBusIdField(primary_key=True, serialize=False), + ), + migrations.AlterField( + model_name='workflow', + name='orcabus_id', + field=workflow_manager.fields.OrcaBusIdField(primary_key=True, serialize=False), + ), + migrations.AlterField( + model_name='workflowrun', + name='orcabus_id', + field=workflow_manager.fields.OrcaBusIdField(primary_key=True, serialize=False), + ), + migrations.AlterField( + model_name='workflowruncomment', + name='orcabus_id', + field=workflow_manager.fields.OrcaBusIdField(primary_key=True, serialize=False), + ), + ] diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis.py index b7c853be7..e112854d3 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis.py @@ -1,5 +1,6 @@ from django.db import models +from workflow_manager.fields import OrcaBusIdField from workflow_manager.models.base import OrcaBusBaseModel, OrcaBusBaseManager from workflow_manager.models.analysis_context import AnalysisContext from workflow_manager.models.workflow import Workflow @@ -13,8 +14,7 @@ class Analysis(OrcaBusBaseModel): class Meta: unique_together = ["analysis_name", "analysis_version"] - orcabus_id_prefix = 'ana.' - + orcabus_id = OrcaBusIdField(primary_key=True, prefix='ana') analysis_name = models.CharField(max_length=255) analysis_version = models.CharField(max_length=255) description = models.CharField(max_length=255) diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_context.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_context.py index a682d450c..355c96958 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_context.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_context.py @@ -1,5 +1,6 @@ from django.db import models +from workflow_manager.fields import OrcaBusIdField from workflow_manager.models.base import OrcaBusBaseModel, OrcaBusBaseManager @@ -11,8 +12,7 @@ class AnalysisContext(OrcaBusBaseModel): class Meta: unique_together = ["name", "usecase"] - orcabus_id_prefix = 'ctx.' - + orcabus_id = OrcaBusIdField(primary_key=True, prefix='ctx') name = models.CharField(max_length=255) usecase = models.CharField(max_length=255) description = models.CharField(max_length=255) diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_run.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_run.py index 577b3cf89..c38863cfa 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_run.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/analysis_run.py @@ -1,5 +1,6 @@ from django.db import models +from workflow_manager.fields import OrcaBusIdField from workflow_manager.models.analysis import Analysis from workflow_manager.models.analysis_context import AnalysisContext from workflow_manager.models.base import OrcaBusBaseModel, OrcaBusBaseManager @@ -11,8 +12,8 @@ class AnalysisRunManager(OrcaBusBaseManager): class AnalysisRun(OrcaBusBaseModel): - orcabus_id_prefix = 'anr.' + orcabus_id = OrcaBusIdField(primary_key=True, prefix='anr') analysis_run_name = models.CharField(max_length=255) comment = models.CharField(max_length=255, null=True, blank=True) status = models.CharField(max_length=255, null=True, blank=True) diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/base.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/base.py index 4f72b47f6..3df07b2a4 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/base.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/base.py @@ -11,15 +11,11 @@ ManyToOneRel, ManyToManyRel, OneToOneRel, QuerySet from rest_framework.settings import api_settings +from workflow_manager.fields import OrcaBusIdField from workflow_manager.pagination import PaginationConstant logger = logging.getLogger(__name__) -orcabus_id_validator = RegexValidator( - regex=r'^[\w]{26}$', - message='ULID is expected to be 26 characters long', - code='invalid_orcabus_id' - ) class OrcaBusBaseManager(models.Manager): @@ -36,7 +32,9 @@ def reduce_multi_values_qor(key: str, values: List[str]): ): values = [values] return reduce( - operator.or_, (Q(**{"%s__iexact" % key: value}) + # Apparently the `get_prep_value` from the custom fields.py is not called prior hitting the Db but, + # the regular `__exact` still execute that function. + operator.or_, (Q(**{"%s__exact" % key: value}) for value in values) ) @@ -80,32 +78,17 @@ class OrcaBusBaseModel(models.Model): class Meta: abstract = True - orcabus_id_prefix = None + orcabus_prefix = '' - orcabus_id = models.CharField( - primary_key=True, - unique=True, - editable=False, - blank=False, - null=False, - validators=[orcabus_id_validator] - ) + orcabus_id = OrcaBusIdField(primary_key=True, prefix=orcabus_prefix) def save(self, *args, **kwargs): - # handle the OrcaBus ID - if not self.orcabus_id: - # if no OrcaBus ID was provided, then generate one - self.orcabus_id = ulid.new().str - else: - # check provided OrcaBus ID - if len(self.orcabus_id) > 26: - # assume the OrcaBus ID carries the prefix - # we strip it off and continue to the validation - l = len(self.orcabus_id_prefix) - self.orcabus_id = str(self.orcabus_id)[l:] self.full_clean() # make sure we are validating the inputs (especially the OrcaBus ID) - return super(OrcaBusBaseModel, self).save(*args, **kwargs) + super(OrcaBusBaseModel, self).save(*args, **kwargs) + # Reload the object from the database to ensure custom fields like OrcaBusIdField + # invoke the `from_db_value` method (which provides the annotation) after saving. + self.refresh_from_db() @classmethod def get_fields(cls): diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/library.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/library.py index 6958e5fe3..0da70e38c 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/library.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/library.py @@ -1,6 +1,7 @@ from django.core.serializers.json import DjangoJSONEncoder from django.db import models +from workflow_manager.fields import OrcaBusIdField from workflow_manager.models.base import OrcaBusBaseModel, OrcaBusBaseManager @@ -10,8 +11,7 @@ class LibraryManager(OrcaBusBaseManager): class Library(OrcaBusBaseModel): - orcabus_id_prefix = "lib." - + orcabus_id = OrcaBusIdField(primary_key=True, prefix='lib') library_id = models.CharField(max_length=255) objects = LibraryManager() diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/payload.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/payload.py index 1eb32921f..6da66b406 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/payload.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/payload.py @@ -1,6 +1,7 @@ from django.core.serializers.json import DjangoJSONEncoder from django.db import models +from workflow_manager.fields import OrcaBusIdField from workflow_manager.models.base import OrcaBusBaseModel, OrcaBusBaseManager @@ -9,8 +10,7 @@ class PayloadManager(OrcaBusBaseManager): class Payload(OrcaBusBaseModel): - orcabus_id_prefix = 'pld.' - + orcabus_id = OrcaBusIdField(primary_key=True, prefix='pld') payload_ref_id = models.CharField(max_length=255, unique=True) version = models.CharField(max_length=255) data = models.JSONField(encoder=DjangoJSONEncoder) diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/state.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/state.py index abfdc4d4e..e1148332b 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/state.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/state.py @@ -3,6 +3,7 @@ from django.db import models +from workflow_manager.fields import OrcaBusIdField from workflow_manager.models.base import OrcaBusBaseModel, OrcaBusBaseManager from workflow_manager.models.payload import Payload from workflow_manager.models.workflow_run import WorkflowRun @@ -88,9 +89,8 @@ class State(OrcaBusBaseModel): class Meta: unique_together = ["workflow_run", "status", "timestamp"] - orcabus_id_prefix = 'stt.' - # --- mandatory fields + orcabus_id = OrcaBusIdField(primary_key=True, prefix='stt') status = models.CharField(max_length=255) # TODO: How and where to enforce conventions? timestamp = models.DateTimeField() comment = models.CharField(max_length=255, null=True, blank=True) diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow.py index 530bdfc40..caaf5cb79 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow.py @@ -1,5 +1,6 @@ from django.db import models +from workflow_manager.fields import OrcaBusIdField from workflow_manager.models.base import OrcaBusBaseModel, OrcaBusBaseManager @@ -12,8 +13,7 @@ class Meta: # a combo of this gives us human-readable pipeline id unique_together = ["workflow_name", "workflow_version"] - orcabus_id_prefix = 'wfl.' - + orcabus_id = OrcaBusIdField(primary_key=True, prefix='wfl') workflow_name = models.CharField(max_length=255) workflow_version = models.CharField(max_length=255) execution_engine = models.CharField(max_length=255) diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow_run.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow_run.py index d10d46a6d..055b0139a 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow_run.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow_run.py @@ -1,5 +1,6 @@ from django.db import models +from workflow_manager.fields import OrcaBusIdField from workflow_manager.models.analysis_run import AnalysisRun from workflow_manager.models.base import OrcaBusBaseModel, OrcaBusBaseManager from workflow_manager.models.library import Library @@ -11,8 +12,7 @@ class WorkflowRunManager(OrcaBusBaseManager): class WorkflowRun(OrcaBusBaseModel): - orcabus_id_prefix = 'wfr.' - + orcabus_id = OrcaBusIdField(primary_key=True, prefix='wfr') portal_run_id = models.CharField(max_length=255, unique=True) execution_id = models.CharField(max_length=255, null=True, blank=True) @@ -38,6 +38,7 @@ def get_latest_state(self): # retrieve all related states and get the latest one return self.states.order_by('-timestamp').first() + class LibraryAssociationManager(OrcaBusBaseManager): pass diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow_run_comment.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow_run_comment.py index 216db1ec4..ab8e99925 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow_run_comment.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow_run_comment.py @@ -1,14 +1,17 @@ from django.db import models +from workflow_manager.fields import OrcaBusIdField from workflow_manager.models.base import OrcaBusBaseModel, OrcaBusBaseManager from workflow_manager.models.workflow_run import WorkflowRun + class WorkflowRunCommentManager(OrcaBusBaseManager): pass + class WorkflowRunComment(OrcaBusBaseModel): - orcabus_id_prefix = 'cmt.' - + + orcabus_id = OrcaBusIdField(primary_key=True, prefix='cmt') workflow_run = models.ForeignKey(WorkflowRun, related_name="comments", on_delete=models.CASCADE) comment = models.TextField() created_at = models.DateTimeField(auto_now_add=True) @@ -19,4 +22,4 @@ class WorkflowRunComment(OrcaBusBaseModel): objects = WorkflowRunCommentManager() def __str__(self): - return f"ID: {self.orcabus_id}, workflow_run: {self.workflow_run}, comment: {self.comment}" \ No newline at end of file + return f"ID: {self.orcabus_id}, workflow_run: {self.workflow_run}, comment: {self.comment}" diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/analysis.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/analysis.py index bcc27e04b..30421d0b1 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/analysis.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/analysis.py @@ -1,40 +1,34 @@ from workflow_manager.serializers.base import SerializersBase, OptionalFieldsMixin from workflow_manager.models import Analysis, Workflow, AnalysisContext + class AnalysisBaseSerializer(SerializersBase): - prefix = Analysis.orcabus_id_prefix + pass + -class AnalysisListParamSerializer( OptionalFieldsMixin, AnalysisBaseSerializer): +class AnalysisListParamSerializer(OptionalFieldsMixin, AnalysisBaseSerializer): class Meta: model = Analysis fields = "__all__" + class AnalysisMinSerializer(AnalysisBaseSerializer): class Meta: model = Analysis fields = ["orcabus_id", "analysis_name", "analysis_version", 'status'] + class AnalysisSerializer(AnalysisBaseSerializer): """ Serializer to define a default representation of an Analysis record, mainly used in record listings. """ + class Meta: model = Analysis fields = "__all__" # exclude = ["contexts", "workflows"] - def to_representation(self, instance): - representation = super().to_representation(instance) - new_workflow_refs = [] # Rewrite internal OrcaBUs - for item in representation["workflows"]: - new_workflow_refs.append(f"{Workflow.orcabus_id_prefix}{item}") - representation["workflows"] = new_workflow_refs - new_context_refs = [] - for item in representation["contexts"]: - new_context_refs.append(f"{AnalysisContext.orcabus_id_prefix}{item}") - representation["contexts"] = new_context_refs - return representation class AnalysisDetailSerializer(AnalysisBaseSerializer): diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/analysis_context.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/analysis_context.py index 3579a5c3f..046348abf 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/analysis_context.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/analysis_context.py @@ -3,14 +3,15 @@ class AnalysisContextBaseSerializer(SerializersBase): - prefix = AnalysisContext.orcabus_id_prefix + pass -class AnalysisContextListParamSerializer( OptionalFieldsMixin, AnalysisContextBaseSerializer): +class AnalysisContextListParamSerializer(OptionalFieldsMixin, AnalysisContextBaseSerializer): class Meta: model = AnalysisContext fields = "__all__" + class AnalysisContextMinSerializer(AnalysisContextBaseSerializer): class Meta: model = AnalysisContext diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/analysis_run.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/analysis_run.py index 338bbcca0..3aa895cd0 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/analysis_run.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/analysis_run.py @@ -1,27 +1,30 @@ from workflow_manager.serializers.base import SerializersBase, OptionalFieldsMixin from workflow_manager.models import AnalysisRun + class AnalysisRunBaseSerializer(SerializersBase): - prefix = AnalysisRun.orcabus_id_prefix + pass + -class AnalysisRunListParamSerializer( OptionalFieldsMixin, AnalysisRunBaseSerializer,): +class AnalysisRunListParamSerializer(OptionalFieldsMixin, AnalysisRunBaseSerializer, ): class Meta: model = AnalysisRun fields = "__all__" + class AnalysisRunSerializer(AnalysisRunBaseSerializer): from .analysis import AnalysisMinSerializer from .analysis_context import AnalysisContextMinSerializer - + analysis = AnalysisMinSerializer(read_only=True) storage_context = AnalysisContextMinSerializer(read_only=True) compute_context = AnalysisContextMinSerializer(read_only=True) + class Meta: model = AnalysisRun exclude = ["libraries"] - class AnalysisRunDetailSerializer(AnalysisRunBaseSerializer): from .library import LibrarySerializer from .analysis import AnalysisDetailSerializer diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/base.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/base.py index 2b57a52a5..a0d59eb8c 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/base.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/base.py @@ -8,20 +8,11 @@ def to_camel_case(snake_str): class SerializersBase(serializers.ModelSerializer): - prefix = '' def __init__(self, *args, camel_case_data=False, **kwargs): super().__init__(*args, **kwargs) self.use_camel_case = camel_case_data - def to_representation(self, instance): - representation = super().to_representation(instance) - representation['orcabus_id'] = self.prefix + str(representation['orcabus_id']) - - if self.use_camel_case: - return {to_camel_case(key): value for key, value in representation.items()} - return representation - class OptionalFieldsMixin: def make_fields_optional(self): diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/library.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/library.py index 29b27140e..bf0ad9a39 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/library.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/library.py @@ -3,7 +3,7 @@ class LibraryBaseSerializer(SerializersBase): - prefix = Library.orcabus_id_prefix + pass class LibraryListParamSerializer(OptionalFieldsMixin, LibraryBaseSerializer): diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/payload.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/payload.py index 0aa8ef1a0..3eb428f76 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/payload.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/payload.py @@ -3,13 +3,15 @@ class PayloadBaseSerializer(SerializersBase): - prefix = Payload.orcabus_id_prefix + pass + class PayloadListParamSerializer(OptionalFieldsMixin, PayloadBaseSerializer): class Meta: model = Payload fields = "__all__" + class PayloadSerializer(PayloadBaseSerializer): class Meta: model = Payload diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/state.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/state.py index 9225402ee..5960c522a 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/state.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/state.py @@ -3,7 +3,7 @@ class StateBaseSerializer(SerializersBase): - prefix = State.orcabus_id_prefix + pass class StateMinSerializer(StateBaseSerializer): @@ -16,10 +16,3 @@ class StateSerializer(StateBaseSerializer): class Meta: model = State fields = "__all__" - - def to_representation(self, instance): - representation = super().to_representation(instance) - representation['workflow_run'] = WorkflowRun.orcabus_id_prefix + representation['workflow_run'] - if representation['payload']: - representation['payload'] = Payload.orcabus_id_prefix + representation['payload'] - return representation diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/workflow.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/workflow.py index 7b4fe5a59..28aec9d17 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/workflow.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/workflow.py @@ -3,13 +3,15 @@ class WorkflowBaseSerializer(SerializersBase): - prefix = Workflow.orcabus_id_prefix + pass + class WorkflowListParamSerializer(OptionalFieldsMixin, WorkflowBaseSerializer): class Meta: model = Workflow fields = "__all__" + class WorkflowMinSerializer(WorkflowBaseSerializer): class Meta: model = Workflow diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/workflow_run.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/workflow_run.py index a8fce4aa2..166cbb3d0 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/workflow_run.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/workflow_run.py @@ -4,9 +4,8 @@ from workflow_manager.models import WorkflowRun, AnalysisRun from workflow_manager.serializers.state import StateMinSerializer -class WorkflowRunBaseSerializer(SerializersBase): - prefix = WorkflowRun.orcabus_id_prefix +class WorkflowRunBaseSerializer(SerializersBase): # we only want to include the current state # all states are available via a dedicated endpoint current_state = serializers.SerializerMethodField() @@ -19,23 +18,19 @@ def get_current_state(self, obj) -> dict: class WorkflowRunListParamSerializer(OptionalFieldsMixin, WorkflowRunBaseSerializer): class Meta: model = WorkflowRun - fields = ["orcabus_id", "workflow", "analysis_run", "workflow_run_name", "portal_run_id", "execution_id", "comment",] + fields = ["orcabus_id", "workflow", "analysis_run", "workflow_run_name", "portal_run_id", "execution_id", + "comment", ] + class WorkflowRunSerializer(WorkflowRunBaseSerializer): from .workflow import WorkflowMinSerializer - + workflow = WorkflowMinSerializer(read_only=True) + class Meta: model = WorkflowRun exclude = ["libraries"] - def to_representation(self, instance): - representation = super().to_representation(instance) - # representation['workflow'] = Workflow.orcabus_id_prefix + representation['workflow'] - if representation['analysis_run']: - representation['analysis_run'] = AnalysisRun.orcabus_id_prefix + representation['analysis_run'] - return representation - class WorkflowRunDetailSerializer(WorkflowRunBaseSerializer): from .library import LibrarySerializer diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/workflow_run_action.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/workflow_run_action.py index be317906f..21d21b7a0 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/workflow_run_action.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/workflow_run_action.py @@ -6,12 +6,14 @@ class AllowedRerunWorkflow(StrEnum): RNASUM = "rnasum" - + + class AllowedRerunWorkflowSerializer(serializers.Serializer): is_valid = serializers.BooleanField() allowed_dataset_choice = serializers.ListField(child=serializers.CharField()) valid_workflows = serializers.ListField(child=serializers.CharField()) + class BaseRerunInputSerializer(serializers.Serializer): def update(self, instance, validated_data): diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/workflow_run_comment.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/workflow_run_comment.py index b21a4ec99..57f4d05a8 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/workflow_run_comment.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/serializers/workflow_run_comment.py @@ -1,20 +1,18 @@ from workflow_manager.serializers.base import SerializersBase from workflow_manager.models import WorkflowRunComment, WorkflowRun + class WorkflowRunCommentBaseSerializer(SerializersBase): - prefix = WorkflowRunComment.orcabus_id_prefix - + pass + + class WorkflowRunCommentMinSerializer(WorkflowRunCommentBaseSerializer): class Meta: model = WorkflowRunComment fields = ["orcabus_id", "comment", "timestamp"] + class WorkflowRunCommentSerializer(WorkflowRunCommentBaseSerializer): class Meta: model = WorkflowRunComment fields = "__all__" - - def to_representation(self, instance): - representation = super().to_representation(instance) - representation['workflow_run'] = WorkflowRun.orcabus_id_prefix + representation['workflow_run'] - return representation \ No newline at end of file diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/analysis.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/analysis.py index 2da744175..a69bd41cf 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/analysis.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/analysis.py @@ -9,7 +9,6 @@ class AnalysisViewSet(BaseViewSet): serializer_class = AnalysisDetailSerializer # use detailed serializer as default search_fields = Analysis.get_base_fields() queryset = Analysis.objects.prefetch_related("contexts").prefetch_related("workflows").all() - orcabus_id_prefix = Analysis.orcabus_id_prefix @extend_schema(parameters=[ AnalysisListParamSerializer @@ -19,5 +18,5 @@ def list(self, request, *args, **kwargs): return super().list(request, *args, **kwargs) def get_queryset(self): - query_params = self.get_query_params() + query_params = self.request.query_params.copy() return Analysis.objects.get_by_keyword(self.queryset, **query_params) diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/analysis_context.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/analysis_context.py index 5e89bb579..0e997a98f 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/analysis_context.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/analysis_context.py @@ -8,7 +8,6 @@ class AnalysisContextViewSet(BaseViewSet): serializer_class = AnalysisContextSerializer search_fields = AnalysisContext.get_base_fields() - orcabus_id_prefix = AnalysisContext.orcabus_id_prefix @extend_schema(parameters=[ AnalysisContextListParamSerializer @@ -17,5 +16,5 @@ def list(self, request, *args, **kwargs): return super().list(request, *args, **kwargs) def get_queryset(self): - query_params = self.get_query_params() + query_params = self.request.query_params.copy() return AnalysisContext.objects.get_by_keyword(self.queryset, **query_params) diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/analysis_run.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/analysis_run.py index fb7a6b8ec..964e5153a 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/analysis_run.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/analysis_run.py @@ -9,7 +9,6 @@ class AnalysisRunViewSet(BaseViewSet): serializer_class = AnalysisRunDetailSerializer # use detailed search_fields = AnalysisRun.get_base_fields() queryset = AnalysisRun.objects.prefetch_related("libraries").all() - orcabus_id_prefix = AnalysisRun.orcabus_id_prefix @extend_schema(parameters=[ AnalysisRunListParamSerializer, @@ -19,5 +18,5 @@ def list(self, request, *args, **kwargs): return super().list(request, *args, **kwargs) def get_queryset(self): - query_params = self.get_query_params() + query_params =self.request.query_params.copy() return AnalysisRun.objects.get_by_keyword(self.queryset, **query_params) diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/base.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/base.py index d8dbbf103..78d49c978 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/base.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/base.py @@ -8,38 +8,7 @@ class BaseViewSet(ReadOnlyModelViewSet, ABC): lookup_value_regex = "[^/]+" # This is to allow for special characters in the URL - orcabus_id_prefix = '' ordering_fields = "__all__" ordering = ["-orcabus_id"] pagination_class = StandardResultsSetPagination filter_backends = [filters.OrderingFilter, filters.SearchFilter] - - def retrieve(self, request, *args, **kwargs): - """ - Since we have custom orcabus_id prefix for each model, we need to remove the prefix before retrieving it. - """ - pk = self.kwargs.get('pk') - if pk and pk.startswith(self.orcabus_id_prefix): - pk = pk[len(self.orcabus_id_prefix):] - - obj = get_object_or_404(self.get_queryset(), pk=pk) - serializer = self.serializer_class(obj) - return Response(serializer.data) - - def get_query_params(self): - """ - Sanitize query params if needed - e.g. remove prefixes for each orcabus_id - """ - query_params = self.request.query_params.copy() - orcabus_id = query_params.getlist("orcabus_id", None) - if orcabus_id: - id_list = [] - for key in orcabus_id: - if key.startswith(self.orcabus_id_prefix): - id_list.append(key[len(self.orcabus_id_prefix):]) - else: - id_list.append(key) - query_params.setlist('orcabus_id', id_list) - - return query_params diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/library.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/library.py index 6911d63ea..cd0a51dae 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/library.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/library.py @@ -8,7 +8,6 @@ class LibraryViewSet(BaseViewSet): serializer_class = LibrarySerializer search_fields = Library.get_base_fields() - orcabus_id_prefix = Library.orcabus_id_prefix @extend_schema(parameters=[ LibrarySerializer @@ -17,6 +16,6 @@ def list(self, request, *args, **kwargs): return super().list(request, *args, **kwargs) def get_queryset(self): - query_params = self.get_query_params() + query_params = self.request.query_params.copy() qs = Library.objects.filter(workflowrun=self.kwargs["workflowrun_id"]) return Library.objects.get_by_keyword(qs, **query_params) diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/payload.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/payload.py index c26da8cf5..cbbeb5c2d 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/payload.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/payload.py @@ -8,7 +8,6 @@ class PayloadViewSet(BaseViewSet): serializer_class = PayloadSerializer search_fields = Payload.get_base_fields() - orcabus_id_prefix = Payload.orcabus_id_prefix @extend_schema(parameters=[ PayloadListParamSerializer @@ -17,5 +16,5 @@ def list(self, request, *args, **kwargs): return super().list(request, *args, **kwargs) def get_queryset(self): - query_params = self.get_query_params() + query_params = self.request.query_params.copy() return Payload.objects.get_by_keyword(self.queryset, **query_params) diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/state.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/state.py index 93bd9d631..dbb53441a 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/state.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/state.py @@ -13,7 +13,6 @@ class StateViewSet(mixins.CreateModelMixin, mixins.UpdateModelMixin, mixins.ListModelMixin, GenericViewSet): serializer_class = StateSerializer search_fields = State.get_base_fields() - orcabus_id_prefix = State.orcabus_id_prefix http_method_names = ['get', 'post', 'patch'] pagination_class = None diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow.py index c8f4a7412..dbb8020e5 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow.py @@ -4,17 +4,17 @@ from workflow_manager.serializers.workflow import WorkflowSerializer, WorkflowListParamSerializer from workflow_manager.viewsets.base import BaseViewSet + class WorkflowViewSet(BaseViewSet): serializer_class = WorkflowSerializer search_fields = Workflow.get_base_fields() - orcabus_id_prefix = Workflow.orcabus_id_prefix @extend_schema(parameters=[ WorkflowListParamSerializer ]) def list(self, request, *args, **kwargs): return super().list(request, *args, **kwargs) - + def get_queryset(self): - query_params = self.get_query_params() + query_params = self.request.query_params.copy() return Workflow.objects.get_by_keyword(self.queryset, **query_params) diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run.py index 25fa8aa5f..3ee63ff33 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run.py @@ -11,7 +11,6 @@ class WorkflowRunViewSet(BaseViewSet): serializer_class = WorkflowRunDetailSerializer search_fields = WorkflowRun.get_base_fields() queryset = WorkflowRun.objects.prefetch_related("libraries").all() - orcabus_id_prefix = WorkflowRun.orcabus_id_prefix @extend_schema(parameters=[WorkflowRunListParamSerializer]) def list(self, request, *args, **kwargs): diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run_action.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run_action.py index ec6c26450..3af89fc63 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run_action.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run_action.py @@ -25,7 +25,6 @@ class WorkflowRunActionViewSet(ViewSet): lookup_value_regex = "[^/]+" # to allow orcabus id prefix queryset = WorkflowRun.objects.prefetch_related('states').all() - orcabus_id_prefix = WorkflowRun.orcabus_id_prefix @extend_schema(responses=AllowedRerunWorkflowSerializer, description="Allowed rerun workflows") @action(detail=True, methods=['get'], url_name='validate_rerun_workflows', url_path='validate_rerun_workflows') @@ -39,12 +38,12 @@ def validate_rerun_workflows(self, request, *args, **kwargs): if wfl_name == AllowedRerunWorkflow.RNASUM.value: allowed_dataset_choice = RERUN_INPUT_SERIALIZERS[wfl_name].allowed_dataset_choice - reponse = { + response = { 'is_valid': is_valid, 'allowed_dataset_choice': allowed_dataset_choice, 'valid_workflows': AllowedRerunWorkflow, } - return Response(reponse, status=status.HTTP_200_OK) + return Response(response, status=status.HTTP_200_OK) @extend_schema( request=PolymorphicProxySerializer( @@ -67,8 +66,6 @@ def rerun(self, request, *args, **kwargs): rerun from existing workflow run """ pk = self.kwargs.get('pk') - if pk and pk.startswith(self.orcabus_id_prefix): - pk = pk[len(self.orcabus_id_prefix):] wfl_run = get_object_or_404(self.queryset, pk=pk) # Only approved workflow_name is allowed diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run_comment.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run_comment.py index a8c2843f9..e350a7606 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run_comment.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run_comment.py @@ -13,7 +13,6 @@ class WorkflowRunCommentViewSet(mixins.CreateModelMixin, mixins.UpdateModelMixin serializer_class = WorkflowRunCommentSerializer search_fields = WorkflowRunComment.get_base_fields() http_method_names = ['get', 'post', 'patch', 'delete'] - orcabus_id_prefix = WorkflowRunComment.orcabus_id_prefix pagination_class = None def get_queryset(self): From dc3d327f2366fcf08521fcae652d802df8735fc2 Mon Sep 17 00:00:00 2001 From: william Date: Wed, 18 Dec 2024 16:25:23 +1100 Subject: [PATCH 4/6] srm: Django OrcaBusId refactor --- .../sequence_run_manager/fields.py | 73 ++++++++++--------- ...s_id_alter_sequence_orcabus_id_and_more.py | 29 ++++++++ .../sequence_run_manager/models/base.py | 72 +++++++----------- .../sequence_run_manager/models/comment.py | 10 +-- .../sequence_run_manager/models/sequence.py | 11 ++- .../sequence_run_manager/models/state.py | 10 ++- .../sequence_run_manager/serializers/base.py | 1 - .../serializers/comment.py | 2 +- .../serializers/sequence.py | 2 +- .../sequence_run_manager/serializers/state.py | 9 +-- .../sequence_run_manager/settings/local.py | 1 - .../sequence_run_manager/urls/base.py | 4 +- .../sequence_run_manager/viewsets/base.py | 31 -------- .../sequence_run_manager/viewsets/comment.py | 22 +++--- .../sequence_run_manager/viewsets/sequence.py | 10 +-- .../sequence_run_manager/viewsets/state.py | 1 - .../domain/sequence.py | 4 +- .../workflow_manager/models/base.py | 8 -- .../workflow_manager/urls/base.py | 6 +- .../viewsets/workflow_run_comment.py | 24 +++--- 20 files changed, 154 insertions(+), 176 deletions(-) create mode 100644 lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/migrations/0003_alter_comment_orcabus_id_alter_sequence_orcabus_id_and_more.py diff --git a/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/fields.py b/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/fields.py index ba79fa4f7..c141c92bb 100644 --- a/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/fields.py +++ b/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/fields.py @@ -1,48 +1,55 @@ -import hashlib - +import ulid +from django.core.validators import RegexValidator from django.db import models +ULID_REGEX_STR = r"[0123456789ABCDEFGHJKMNPQRSTVWXYZ]{26}" +ulid_validator = RegexValidator(regex=ULID_REGEX_STR, + message='ULID is expected to be 26 characters long', + code='invalid_orcabus_id') + + +def get_ulid() -> str: + return ulid.new().str -class HashField(models.CharField): - description = ( - "HashField is related to some base fields (other columns) in a model and" - "stores its hashed value for better indexing performance." - ) - def __init__(self, base_fields, *args, **kwargs): - """ - :param base_fields: name of fields storing the value to be hashed - """ - self.base_fields = base_fields - kwargs["max_length"] = 64 - super(HashField, self).__init__(*args, **kwargs) +class UlidField(models.CharField): + description = "An OrcaBus internal ID (ULID)" + + def __init__(self, *args, **kwargs): + kwargs['max_length'] = 26 # ULID length + kwargs['validators'] = [ulid_validator] + kwargs['default'] = get_ulid + super().__init__(*args, **kwargs) def deconstruct(self): name, path, args, kwargs = super().deconstruct() del kwargs["max_length"] - if self.base_fields is not None: - kwargs["base_fields"] = self.base_fields + del kwargs['validators'] + del kwargs['default'] return name, path, args, kwargs - def pre_save(self, instance, add): - self.calculate_hash(instance) - return super(HashField, self).pre_save(instance, add) - def calculate_hash(self, instance): - sha256 = hashlib.sha256() - for field in self.base_fields: - value = getattr(instance, field) - sha256.update(value.encode("utf-8")) - setattr(instance, self.attname, sha256.hexdigest()) +class OrcaBusIdField(UlidField): + description = "An OrcaBus internal ID (based on ULID)" + + def __init__(self, prefix='', *args, **kwargs): + self.prefix = prefix + super().__init__(*args, **kwargs) + @property + def non_db_attrs(self): + return super().non_db_attrs + ("prefix",) -class HashFieldHelper(object): - def __init__(self): - self.__sha256 = hashlib.sha256() + def from_db_value(self, value, expression, connection): + if value and self.prefix != '': + return f"{self.prefix}.{value}" + else: + return value - def add(self, value): - self.__sha256.update(value.encode("utf-8")) - return self + def to_python(self, value): + # This will be called when the function + return self.get_prep_value(value) - def calculate_hash(self): - return self.__sha256.hexdigest() + def get_prep_value(self, value): + # We just want the last 26 characters which is the ULID (ignoring any prefix) when dealing with the database + return value[-26:] diff --git a/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/migrations/0003_alter_comment_orcabus_id_alter_sequence_orcabus_id_and_more.py b/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/migrations/0003_alter_comment_orcabus_id_alter_sequence_orcabus_id_and_more.py new file mode 100644 index 000000000..e1cf47605 --- /dev/null +++ b/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/migrations/0003_alter_comment_orcabus_id_alter_sequence_orcabus_id_and_more.py @@ -0,0 +1,29 @@ +# Generated by Django 5.1.2 on 2024-12-18 01:29 + +import sequence_run_manager.fields +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('sequence_run_manager', '0002_alter_sequence_run_folder_path'), + ] + + operations = [ + migrations.AlterField( + model_name='comment', + name='orcabus_id', + field=sequence_run_manager.fields.OrcaBusIdField(primary_key=True, serialize=False), + ), + migrations.AlterField( + model_name='sequence', + name='orcabus_id', + field=sequence_run_manager.fields.OrcaBusIdField(primary_key=True, serialize=False), + ), + migrations.AlterField( + model_name='state', + name='orcabus_id', + field=sequence_run_manager.fields.OrcaBusIdField(primary_key=True, serialize=False), + ), + ] diff --git a/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/models/base.py b/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/models/base.py index 276c1d338..8c446e6a7 100644 --- a/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/models/base.py +++ b/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/models/base.py @@ -26,22 +26,22 @@ logger = logging.getLogger(__name__) orcabus_id_validator = RegexValidator( - regex=r'^[\w]{26}$', - message='ULID is expected to be 26 characters long', - code='invalid_orcabus_id' - ) + regex=r'^[\w]{26}$', + message='ULID is expected to be 26 characters long', + code='invalid_orcabus_id' +) class OrcaBusBaseManager(models.Manager): @staticmethod def reduce_multi_values_qor(key: str, values: List[str]): if isinstance( - values, - ( - str, - int, - float, - ), + values, + ( + str, + int, + float, + ), ): values = [values] return reduce( @@ -86,32 +86,14 @@ def exclude_params(params): class OrcaBusBaseModel(models.Model): class Meta: abstract = True - - orcabus_id_prefix = None - - orcabus_id = models.CharField( - primary_key=True, - unique=True, - editable=False, - blank=False, - null=False, - validators=[orcabus_id_validator] - ) - + def save(self, *args, **kwargs): - # handle the OrcaBus ID - if not self.orcabus_id: - # if no OrcaBus ID was provided, then generate one - self.orcabus_id = ulid.new().str - else: - # check provided OrcaBus ID - if len(self.orcabus_id) > 26: - # assume the OrcaBus ID carries the prefix - # we strip it off and continue to the validation - l = len(self.orcabus_id_prefix) - self.orcabus_id = str(self.orcabus_id)[l:] self.full_clean() # make sure we are validating the inputs (especially the OrcaBus ID) - return super(OrcaBusBaseModel, self).save(*args, **kwargs) + super(OrcaBusBaseModel, self).save(*args, **kwargs) + + # Reload the object from the database to ensure custom fields like OrcaBusIdField + # invoke the `from_db_value` method (which provides the annotation) after saving. + self.refresh_from_db() @classmethod def get_fields(cls): @@ -122,17 +104,17 @@ def get_base_fields(cls): base_fields = set() for f in cls._meta.get_fields(): if isinstance( - f, - ( - ForeignKey, - ForeignObject, - OneToOneField, - ManyToManyField, - ForeignObjectRel, - ManyToOneRel, - ManyToManyRel, - OneToOneRel, - ), + f, + ( + ForeignKey, + ForeignObject, + OneToOneField, + ManyToManyField, + ForeignObjectRel, + ManyToOneRel, + ManyToManyRel, + OneToOneRel, + ), ): continue base_fields.add(f.name) diff --git a/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/models/comment.py b/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/models/comment.py index 5e7963d2a..e70ad1d2d 100644 --- a/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/models/comment.py +++ b/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/models/comment.py @@ -3,19 +3,19 @@ from django.db import models from sequence_run_manager.models.base import OrcaBusBaseModel, OrcaBusBaseManager +from sequence_run_manager.fields import OrcaBusIdField logger = logging.getLogger(__name__) + class CommentManager(OrcaBusBaseManager): pass class Comment(OrcaBusBaseModel): - # primary key - orcabus_id_prefix = 'cmt.' - + orcabus_id = OrcaBusIdField(primary_key=True, prefix='cmt') comment = models.TextField(null=False, blank=False) - association_id = models.CharField(max_length=255, null=False, blank=False) # comment association object id + association_id = models.CharField(max_length=255, null=False, blank=False) # comment association object id created_at = models.DateTimeField(auto_now_add=True) created_by = models.CharField(max_length=255, null=False, blank=False) updated_at = models.DateTimeField(auto_now=True) @@ -24,4 +24,4 @@ class Comment(OrcaBusBaseModel): objects = CommentManager() def __str__(self): - return f"ID: {self.orcabus_id}, comment: {self.comment}, from {self.created_by}, for {self.association_id}" \ No newline at end of file + return f"ID: {self.orcabus_id}, comment: {self.comment}, from {self.created_by}, for {self.association_id}" diff --git a/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/models/sequence.py b/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/models/sequence.py index 0da275586..5eb0b6351 100644 --- a/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/models/sequence.py +++ b/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/models/sequence.py @@ -4,6 +4,7 @@ from django.db.models import QuerySet from sequence_run_manager.models.base import OrcaBusBaseModel, OrcaBusBaseManager +from sequence_run_manager.fields import OrcaBusIdField logger = logging.getLogger(__name__) @@ -61,17 +62,19 @@ def get_by_keyword(self, **kwargs) -> QuerySet: class Sequence(OrcaBusBaseModel): - # primary key - orcabus_id_prefix = 'seq.' - # must have (run_folder_path) or (v1pre3_id and ica_project_id and api_url) # NOTE: we use this to retrieve further details for icav2 bssh event # for reference: https://github.com/umccr/orcabus/pull/748#issuecomment-2516246960 class Meta: constraints = [ - models.CheckConstraint(check=models.Q(run_folder_path__isnull=False) | models.Q(v1pre3_id__isnull=False, ica_project_id__isnull=False, api_url__isnull=False), name='check_run_folder_path_or_bssh_keys_not_null') + models.CheckConstraint(check=models.Q(run_folder_path__isnull=False) | models.Q(v1pre3_id__isnull=False, + ica_project_id__isnull=False, + api_url__isnull=False), + name='check_run_folder_path_or_bssh_keys_not_null') ] + orcabus_id = OrcaBusIdField(primary_key=True, prefix='seq') + # mandatory non-nullable base fields instrument_run_id = models.CharField( unique=True, max_length=255, null=False, blank=False diff --git a/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/models/state.py b/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/models/state.py index 2beb3fe16..8ef91fe0a 100644 --- a/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/models/state.py +++ b/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/models/state.py @@ -4,23 +4,25 @@ from sequence_run_manager.models.base import OrcaBusBaseModel, OrcaBusBaseManager from sequence_run_manager.models.sequence import Sequence +from sequence_run_manager.fields import OrcaBusIdField logger = logging.getLogger(__name__) + class StateManager(OrcaBusBaseManager): pass class State(OrcaBusBaseModel): - orcabus_id_prefix = 'sqs.' - + orcabus_id = OrcaBusIdField(primary_key=True, prefix='sqs') + status = models.CharField(max_length=255, null=False, blank=False) timestamp = models.DateTimeField() comment = models.CharField(max_length=255, null=True, blank=True) - + sequence = models.ForeignKey(Sequence, on_delete=models.CASCADE, related_name='states') objects = StateManager() - + def __str__(self): return f"ID: {self.orcabus_id}, status: {self.status}, for {self.sequence}" diff --git a/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/serializers/base.py b/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/serializers/base.py index 2b57a52a5..69eadc021 100644 --- a/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/serializers/base.py +++ b/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/serializers/base.py @@ -16,7 +16,6 @@ def __init__(self, *args, camel_case_data=False, **kwargs): def to_representation(self, instance): representation = super().to_representation(instance) - representation['orcabus_id'] = self.prefix + str(representation['orcabus_id']) if self.use_camel_case: return {to_camel_case(key): value for key, value in representation.items()} diff --git a/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/serializers/comment.py b/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/serializers/comment.py index d1f68eec9..f9de58300 100644 --- a/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/serializers/comment.py +++ b/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/serializers/comment.py @@ -4,7 +4,7 @@ from sequence_run_manager.serializers.base import SerializersBase, OptionalFieldsMixin class CommentBaseSerializer(SerializersBase): - orcabus_id_prefix = Comment.orcabus_id_prefix + pass class CommentSerializer(CommentBaseSerializer): class Meta: diff --git a/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/serializers/sequence.py b/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/serializers/sequence.py index d87dfca5f..d9e5ea919 100644 --- a/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/serializers/sequence.py +++ b/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/serializers/sequence.py @@ -5,7 +5,7 @@ class SequenceBaseSerializer(SerializersBase): - orcabus_id_prefix = Sequence.orcabus_id_prefix + pass class SequenceListParamSerializer(OptionalFieldsMixin, SequenceBaseSerializer): diff --git a/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/serializers/state.py b/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/serializers/state.py index ce9aef77b..18dc8ae82 100644 --- a/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/serializers/state.py +++ b/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/serializers/state.py @@ -3,15 +3,12 @@ from sequence_run_manager.models import State, Sequence from sequence_run_manager.serializers.base import SerializersBase, OptionalFieldsMixin + class StateBaseSerializer(SerializersBase): - orcabus_id_prefix = State.orcabus_id_prefix + pass + class StateSerializer(StateBaseSerializer): class Meta: model = State fields = "__all__" - - def to_representation(self, instance): - representation = super().to_representation(instance) - representation['sequence'] = Sequence.orcabus_id_prefix + representation['sequence'] - return representation diff --git a/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/settings/local.py b/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/settings/local.py index 08c870530..1d5c462ad 100644 --- a/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/settings/local.py +++ b/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/settings/local.py @@ -21,7 +21,6 @@ } } - INSTALLED_APPS += ( "django_extensions", "drf_spectacular", diff --git a/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/urls/base.py b/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/urls/base.py index 924d83dac..97b385c3b 100644 --- a/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/urls/base.py +++ b/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/urls/base.py @@ -13,8 +13,8 @@ router = OptionalSlashDefaultRouter() router.register(r"sequence", SequenceViewSet, basename="sequence") -router.register("sequence/(?P[^/.]+)/comment", CommentViewSet, basename="sequence-comment") -router.register("sequence/(?P[^/.]+)/state", StateViewSet, basename="sequence-states") +router.register("sequence/(?P[^/]+)/comment", CommentViewSet, basename="sequence-comment") +router.register("sequence/(?P[^/]+)/state", StateViewSet, basename="sequence-states") urlpatterns = [ path(f"{api_base}", include(router.urls)), diff --git a/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/viewsets/base.py b/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/viewsets/base.py index 701a96685..58f700a43 100644 --- a/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/viewsets/base.py +++ b/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/viewsets/base.py @@ -8,38 +8,7 @@ class BaseViewSet(ReadOnlyModelViewSet, ABC): lookup_value_regex = "[^/]+" # This is to allow for special characters in the URL - orcabus_id_prefix = '' ordering_fields = "__all__" ordering = ["-orcabus_id"] pagination_class = StandardResultsSetPagination filter_backends = [filters.OrderingFilter, filters.SearchFilter] - - def retrieve(self, request, *args, **kwargs): - """ - Since we have custom orcabus_id prefix for each model, we need to remove the prefix before retrieving it. - """ - pk = self.kwargs.get('pk') - if pk and pk.startswith(self.orcabus_id_prefix): - pk = pk[len(self.orcabus_id_prefix):] - - obj = get_object_or_404(self.get_queryset(), pk=pk) - serializer = self.serializer_class(obj) - return Response(serializer.data) - - def get_query_params(self): - """ - Sanitize query params if needed - e.g. remove prefixes for each orcabus_id - """ - query_params = self.request.query_params.copy() - orcabus_id = query_params.getlist("orcabus_id", None) - if orcabus_id: - id_list = [] - for key in orcabus_id: - if key.startswith(self.orcabus_id_prefix): - id_list.append(key[len(self.orcabus_id_prefix):]) - else: - id_list.append(key) - query_params.setlist('orcabus_id', id_list) - - return query_params diff --git a/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/viewsets/comment.py b/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/viewsets/comment.py index 8cef5277e..3be718e16 100644 --- a/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/viewsets/comment.py +++ b/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/viewsets/comment.py @@ -9,10 +9,10 @@ from sequence_run_manager.models.sequence import Sequence from sequence_run_manager.serializers.comment import CommentSerializer + class CommentViewSet(mixins.CreateModelMixin, mixins.UpdateModelMixin, mixins.ListModelMixin, GenericViewSet): serializer_class = CommentSerializer search_fields = Comment.get_base_fields() - orcabus_id_prefix = Comment.orcabus_id_prefix http_method_names = ['get', 'post', 'patch', 'delete'] pagination_class = None @@ -27,13 +27,13 @@ def perform_create(self, serializer): def create(self, request, *args, **kwargs): seq_orcabus_id = self.kwargs["orcabus_id"] - + # Check if the SequenceRun exists try: Sequence.objects.get(orcabus_id=seq_orcabus_id) except Sequence.DoesNotExist: return Response({"detail": "SequenceRun not found."}, status=status.HTTP_404_NOT_FOUND) - + # Check if created_by and comment are provided if not request.data.get('created_by') or not request.data.get('comment'): return Response({"detail": "created_by and comment are required."}, status=status.HTTP_400_BAD_REQUEST) @@ -41,7 +41,7 @@ def create(self, request, *args, **kwargs): # Add workflow_run_id to the request data mutable_data = request.data.copy() mutable_data['association_id'] = seq_orcabus_id - + serializer = self.get_serializer(data=mutable_data) serializer.is_valid(raise_exception=True) self.perform_create(serializer) @@ -54,11 +54,11 @@ def perform_create(self, serializer): def update(self, request, *args, **kwargs): partial = kwargs.pop('partial', False) instance = self.get_object() - + # Check if the user updating the comment is the same as the one who created it if instance.created_by != request.data.get('created_by'): raise PermissionDenied("You don't have permission to update this comment.") - + # Ensure only the comment field can be updated if set(request.data.keys()) - {'comment', 'created_by'}: return Response({"detail": "Only the comment field can be updated."}, @@ -72,16 +72,16 @@ def update(self, request, *args, **kwargs): def perform_update(self, serializer): serializer.save() - + @action(detail=True, methods=['delete']) def soft_delete(self, request, *args, **kwargs): instance = self.get_object() - + # Check if the user deleting the comment is the same as the one who created it if instance.created_by != request.data.get('created_by'): raise PermissionDenied("You don't have permission to delete this comment.") - + instance.is_deleted = True instance.save() - - return Response({"detail": "Comment successfully marked as deleted."}, status=status.HTTP_204_NO_CONTENT) \ No newline at end of file + + return Response({"detail": "Comment successfully marked as deleted."}, status=status.HTTP_204_NO_CONTENT) diff --git a/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/viewsets/sequence.py b/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/viewsets/sequence.py index 6041ae7f6..f10ca47a2 100644 --- a/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/viewsets/sequence.py +++ b/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/viewsets/sequence.py @@ -1,21 +1,21 @@ - from drf_spectacular.utils import extend_schema from sequence_run_manager.viewsets.base import BaseViewSet from sequence_run_manager.models.sequence import Sequence -from sequence_run_manager.serializers.sequence import SequenceSerializer, SequenceListParamSerializer, SequenceMinSerializer +from sequence_run_manager.serializers.sequence import SequenceSerializer, SequenceListParamSerializer, \ + SequenceMinSerializer + class SequenceViewSet(BaseViewSet): serializer_class = SequenceSerializer search_fields = Sequence.get_base_fields() - orcabus_id_prefix = Sequence.orcabus_id_prefix def get_queryset(self): return Sequence.objects.get_by_keyword(**self.request.query_params) - + @extend_schema(parameters=[ SequenceListParamSerializer ]) def list(self, request, *args, **kwargs): self.serializer_class = SequenceMinSerializer - return super().list(request, *args, **kwargs) + return super().list(request, *args, **kwargs) diff --git a/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/viewsets/state.py b/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/viewsets/state.py index 3ec62824b..109e0f3ff 100644 --- a/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/viewsets/state.py +++ b/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager/viewsets/state.py @@ -8,7 +8,6 @@ class StateViewSet(mixins.ListModelMixin, GenericViewSet): serializer_class = StateSerializer search_fields = State.get_base_fields() - orcabus_id_prefix = State.orcabus_id_prefix pagination_class = None def get_queryset(self): diff --git a/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager_proc/domain/sequence.py b/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager_proc/domain/sequence.py index a1c7a4930..d8320327d 100644 --- a/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager_proc/domain/sequence.py +++ b/lib/workload/stateless/stacks/sequence-run-manager/sequence_run_manager_proc/domain/sequence.py @@ -49,7 +49,7 @@ def to_event(self) -> SequenceRunStateChange: raise SequenceRuleError("Sequence status is null or not loaded yet") return SequenceRunStateChange( - id=Sequence.orcabus_id_prefix + self.sequence.orcabus_id, + id=self.sequence.orcabus_id, instrumentRunId=self.sequence.instrument_run_id, runVolumeName=self.sequence.run_volume_name, runFolderPath=self.sequence.run_folder_path, @@ -69,7 +69,7 @@ def to_event_with_envelope(self) -> AWSEvent: ) def to_put_events_request_entry( - self, event_bus_name: str, trace_header: str = "" + self, event_bus_name: str, trace_header: str = "" ) -> dict: """Convert Domain event with envelope to Entry dict struct of PutEvent API""" domain_event_with_envelope = self.to_event_with_envelope() diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/base.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/base.py index 3df07b2a4..1e8fa5d8d 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/base.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/base.py @@ -1,17 +1,13 @@ import logging import operator -import ulid from functools import reduce from typing import List from django.core.exceptions import FieldError -from django.core.validators import RegexValidator from django.db import models from django.db.models import Q, ManyToManyField, ForeignKey, ForeignObject, OneToOneField, ForeignObjectRel, \ ManyToOneRel, ManyToManyRel, OneToOneRel, QuerySet from rest_framework.settings import api_settings - -from workflow_manager.fields import OrcaBusIdField from workflow_manager.pagination import PaginationConstant logger = logging.getLogger(__name__) @@ -78,10 +74,6 @@ class OrcaBusBaseModel(models.Model): class Meta: abstract = True - orcabus_prefix = '' - - orcabus_id = OrcaBusIdField(primary_key=True, prefix=orcabus_prefix) - def save(self, *args, **kwargs): self.full_clean() # make sure we are validating the inputs (especially the OrcaBus ID) super(OrcaBusBaseModel, self).save(*args, **kwargs) diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/urls/base.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/urls/base.py index c83584485..937bc24ba 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/urls/base.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/urls/base.py @@ -30,19 +30,19 @@ router.register(r"payload", PayloadViewSet, basename="payload") router.register( - "workflowrun/(?P[^/.]+)/state", + "workflowrun/(?P[^/]+)/state", StateViewSet, basename="workflowrun-state", ) # router.register( -# "workflowrun/(?P[^/.]+)/library", +# "workflowrun/(?P[^/]+)/library", # LibraryViewSet, # basename="workflowrun-library", # ) router.register( - "workflowrun/(?P[^/.]+)/comment", + "workflowrun/(?P[^/]+)/comment", WorkflowRunCommentViewSet, basename="workflowrun-comment", ) diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run_comment.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run_comment.py index e350a7606..25eb58e06 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run_comment.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/viewsets/workflow_run_comment.py @@ -1,4 +1,3 @@ - from rest_framework import mixins, status from rest_framework.viewsets import GenericViewSet from rest_framework.response import Response @@ -9,7 +8,8 @@ from workflow_manager.serializers.workflow_run_comment import WorkflowRunCommentSerializer -class WorkflowRunCommentViewSet(mixins.CreateModelMixin, mixins.UpdateModelMixin, mixins.ListModelMixin, GenericViewSet): +class WorkflowRunCommentViewSet(mixins.CreateModelMixin, mixins.UpdateModelMixin, mixins.ListModelMixin, + GenericViewSet): serializer_class = WorkflowRunCommentSerializer search_fields = WorkflowRunComment.get_base_fields() http_method_names = ['get', 'post', 'patch', 'delete'] @@ -23,13 +23,13 @@ def get_queryset(self): def create(self, request, *args, **kwargs): wfr_orcabus_id = self.kwargs["orcabus_id"] - + # Check if the WorkflowRun exists try: WorkflowRun.objects.get(orcabus_id=wfr_orcabus_id) except WorkflowRun.DoesNotExist: return Response({"detail": "WorkflowRun not found."}, status=status.HTTP_404_NOT_FOUND) - + # Check if created_by and comment are provided if not request.data.get('created_by') or not request.data.get('comment'): return Response({"detail": "created_by and comment are required."}, status=status.HTTP_400_BAD_REQUEST) @@ -37,7 +37,7 @@ def create(self, request, *args, **kwargs): # Add workflow_run_id to the request data mutable_data = request.data.copy() mutable_data['workflow_run'] = wfr_orcabus_id - + serializer = self.get_serializer(data=mutable_data) serializer.is_valid(raise_exception=True) self.perform_create(serializer) @@ -50,11 +50,11 @@ def perform_create(self, serializer): def update(self, request, *args, **kwargs): partial = kwargs.pop('partial', False) instance = self.get_object() - + # Check if the user updating the comment is the same as the one who created it if instance.created_by != request.data.get('created_by'): raise PermissionDenied("You don't have permission to update this comment.") - + # Ensure only the comment field can be updated if set(request.data.keys()) - {'comment', 'created_by'}: return Response({"detail": "Only the comment field can be updated."}, @@ -68,16 +68,16 @@ def update(self, request, *args, **kwargs): def perform_update(self, serializer): serializer.save() - + @action(detail=True, methods=['delete']) def soft_delete(self, request, *args, **kwargs): instance = self.get_object() - + # Check if the user deleting the comment is the same as the one who created it if instance.created_by != request.data.get('created_by'): raise PermissionDenied("You don't have permission to delete this comment.") - + instance.is_deleted = True instance.save() - - return Response({"detail": "Comment successfully marked as deleted."}, status=status.HTTP_204_NO_CONTENT) \ No newline at end of file + + return Response({"detail": "Comment successfully marked as deleted."}, status=status.HTTP_204_NO_CONTENT) From cd21258b475e02aa85f290c758742157df8dbe95 Mon Sep 17 00:00:00 2001 From: william Date: Wed, 18 Dec 2024 17:58:18 +1100 Subject: [PATCH 5/6] skipped commit --- .../workflow-manager/workflow_manager/models/workflow_run.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow_run.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow_run.py index 055b0139a..fe9d69a12 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow_run.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow_run.py @@ -44,6 +44,7 @@ class LibraryAssociationManager(OrcaBusBaseManager): class LibraryAssociation(OrcaBusBaseModel): + orcabus_id = OrcaBusIdField(primary_key=True, prefix='lib') workflow_run = models.ForeignKey(WorkflowRun, on_delete=models.CASCADE) library = models.ForeignKey(Library, on_delete=models.CASCADE) association_date = models.DateTimeField() From ce7751f5a3c5791c9d5a52051434efd59fd6343c Mon Sep 17 00:00:00 2001 From: william Date: Wed, 18 Dec 2024 17:59:39 +1100 Subject: [PATCH 6/6] Update workflow_run.py --- .../workflow-manager/workflow_manager/models/workflow_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow_run.py b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow_run.py index fe9d69a12..5bbced5d6 100644 --- a/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow_run.py +++ b/lib/workload/stateless/stacks/workflow-manager/workflow_manager/models/workflow_run.py @@ -44,7 +44,7 @@ class LibraryAssociationManager(OrcaBusBaseManager): class LibraryAssociation(OrcaBusBaseModel): - orcabus_id = OrcaBusIdField(primary_key=True, prefix='lib') + orcabus_id = OrcaBusIdField(primary_key=True) workflow_run = models.ForeignKey(WorkflowRun, on_delete=models.CASCADE) library = models.ForeignKey(Library, on_delete=models.CASCADE) association_date = models.DateTimeField()