From 9cacc93cd360aaad425cdcefe1ab867297a86c82 Mon Sep 17 00:00:00 2001 From: william Date: Tue, 15 Oct 2024 12:24:20 +1100 Subject: [PATCH 1/4] add history testing --- .../metadata-manager/app/tests/test_models.py | 62 ++++++++++++++++++- .../metadata-manager/app/tests/utils.py | 2 +- .../deps/requirements-test.txt | 1 + .../proc/tests/test_tracking_sheet_srv.py | 10 ++- 4 files changed, 71 insertions(+), 4 deletions(-) diff --git a/lib/workload/stateless/stacks/metadata-manager/app/tests/test_models.py b/lib/workload/stateless/stacks/metadata-manager/app/tests/test_models.py index 4e06ef7dd..2ca58ee1f 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/tests/test_models.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/tests/test_models.py @@ -1,6 +1,7 @@ import logging -from unittest.mock import MagicMock, patch +from unittest.mock import patch + from django.test import TestCase from app.models import Subject, Sample, Library, Contact, Project, Individual @@ -73,7 +74,7 @@ def test_metadata_model_relationship(self): def test_upsert_method(self): """ - python manage.py test app.tests.test_models.MetadataTestCase.test_upsert_method + python manage.py test app.tests.test_models.ModelTestCase.test_upsert_method """ # Test function with updating existing record @@ -119,3 +120,60 @@ def test_upsert_method(self): self.assertFalse(is_created, "object should not be created") self.assertFalse(is_updated, "object should not be updated") + def test_model_history(self): + """ + python manage.py test app.tests.test_models.ModelTestCase.test_model_history + """ + + # Test function with updating existing record + updated_spc_data = { + "sample_id": SAMPLE_1['sample_id'], + "source": 'skin', + } + smp, is_created, is_updated = Sample.objects.update_or_create_if_needed( + {"sample_id": updated_spc_data["sample_id"]}, + updated_spc_data + ) + + new_smp_record, old_smp_record = smp.history.all() + smp_delta = new_smp_record.diff_against(old_smp_record) + for change in smp_delta.changes: + self.assertEqual(change.field, 'source', "incorrect field changed") + self.assertEqual(change.old, SAMPLE_1['source'], "incorrect old value") + self.assertEqual(change.new, updated_spc_data['source'], "incorrect new value") + + # Test function with library + lib_one = Library.objects.get(library_id=LIBRARY_1['library_id']) + + # Create new Project record + new_prj_data = { + "project_id": 'prj-02', + "name": 'test_project_2' + } + prj_two = Project.objects.create(**new_prj_data) + + # Test addition of project create a new history from library records + lib_one.project_set.add(prj_two) + lib_history = lib_one.history.all() + self.assertGreaterEqual(len(lib_history), 2, 'no history found for library') + + def find_new_prj(relationship_array): + for relationship in relationship_array: + if relationship['project'] == prj_two.orcabus_id: + return True + return False + lib_delta = lib_history[0].diff_against(lib_history[1]) + for change in lib_delta.changes: + self.assertTrue(find_new_prj(change.new), 'new project not found in relationship history') + self.assertFalse(find_new_prj(change.old), 'old project found in relationship history') + + # Test deletion of project recorded in history + lib_one.project_set.remove(prj_two) + # Test addition of project create a new history from library records + lib_history = lib_one.history.all() + self.assertGreaterEqual(len(lib_history), 2, 'no history found for library') + + lib_delta = lib_history[0].diff_against(lib_history[1]) + for change in lib_delta.changes: + self.assertFalse(find_new_prj(change.new), 'new project history should be up to date with current') + self.assertTrue(find_new_prj(change.old), 'old project found should still contain project_two') \ No newline at end of file diff --git a/lib/workload/stateless/stacks/metadata-manager/app/tests/utils.py b/lib/workload/stateless/stacks/metadata-manager/app/tests/utils.py index 2875b4395..f17478a99 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/tests/utils.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/tests/utils.py @@ -30,8 +30,8 @@ def insert_mock_1(): project.contact_set.add(contact) library.sample = sample library.subject = subject - library.project_set.add(project) library.save() + library.project_set.add(project) subject.individual_set.add(individual) subject.save() diff --git a/lib/workload/stateless/stacks/metadata-manager/deps/requirements-test.txt b/lib/workload/stateless/stacks/metadata-manager/deps/requirements-test.txt index 3d55cb2e1..065df13d5 100644 --- a/lib/workload/stateless/stacks/metadata-manager/deps/requirements-test.txt +++ b/lib/workload/stateless/stacks/metadata-manager/deps/requirements-test.txt @@ -7,3 +7,4 @@ pytest==8.3.2 factory_boy==3.3.0 pytz==2024.1 mockito==1.5.0 +coverage==7.6.3 diff --git a/lib/workload/stateless/stacks/metadata-manager/proc/tests/test_tracking_sheet_srv.py b/lib/workload/stateless/stacks/metadata-manager/proc/tests/test_tracking_sheet_srv.py index ecc462584..b3bcf2181 100644 --- a/lib/workload/stateless/stacks/metadata-manager/proc/tests/test_tracking_sheet_srv.py +++ b/lib/workload/stateless/stacks/metadata-manager/proc/tests/test_tracking_sheet_srv.py @@ -108,12 +108,20 @@ def test_persist_lab_metadata(self): """ python manage.py test proc.tests.test_tracking_sheet_srv.TrackingSheetSrvUnitTests.test_persist_lab_metadata """ - mock_sheet_data = [RECORD_1, RECORD_2, RECORD_3] + mock_sheet_data = [RECORD_1] metadata_pd = pd.json_normalize(mock_sheet_data) metadata_pd = sanitize_lab_metadata_df(metadata_pd) result = persist_lab_metadata(metadata_pd, SHEET_YEAR) + lib_one = Library.objects.get(library_id=RECORD_1.get("LibraryID")) + print(lib_one.history.all()) + new_record, old_record = lib_one.history.all() + delta = new_record.diff_against(old_record) + print(delta) + for change in delta.changes: + print(f"'{change.field}' changed from '{change.old}' to '{change.new}'") + return # Stats check self.assertEqual(result.get("invalid_record_count"), 0, "non invalid record should exist") self.assertEqual(result.get("library").get("create_count"), 3, "3 new library should be created") From 19a89dad08e82532950cf0d28865a766f9ea2581 Mon Sep 17 00:00:00 2001 From: william Date: Tue, 15 Oct 2024 23:34:59 +1100 Subject: [PATCH 2/4] Some history API --- .../management/commands/insert_mock_data.py | 2 +- .../app/serializers/contact.py | 10 +++++- .../app/serializers/individual.py | 5 +++ .../app/serializers/library.py | 28 +++++++++++++-- .../app/serializers/project.py | 21 ++++++++++- .../app/serializers/sample.py | 7 ++++ .../app/serializers/subject.py | 19 +++++++++- .../metadata-manager/app/tests/test_models.py | 3 +- .../metadata-manager/app/viewsets/base.py | 36 +++++++++++++++++-- .../metadata-manager/app/viewsets/contact.py | 8 ++++- .../app/viewsets/individual.py | 9 ++++- .../metadata-manager/app/viewsets/library.py | 9 ++++- .../metadata-manager/app/viewsets/project.py | 8 ++++- .../metadata-manager/app/viewsets/sample.py | 8 ++++- .../metadata-manager/app/viewsets/subject.py | 8 ++++- 15 files changed, 165 insertions(+), 16 deletions(-) diff --git a/lib/workload/stateless/stacks/metadata-manager/app/management/commands/insert_mock_data.py b/lib/workload/stateless/stacks/metadata-manager/app/management/commands/insert_mock_data.py index 1dc1df82c..8da594cdd 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/management/commands/insert_mock_data.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/management/commands/insert_mock_data.py @@ -16,7 +16,7 @@ def handle(self, *args, **options): metadata_pd = pd.json_normalize(mock_sheet_data) metadata_pd = sanitize_lab_metadata_df(metadata_pd) - result = persist_lab_metadata(metadata_pd, SHEET_YEAR) + result = persist_lab_metadata(metadata_pd, SHEET_YEAR, is_emit_eb_events=False) print(json.dumps(result, indent=4)) print("insert mock data completed") diff --git a/lib/workload/stateless/stacks/metadata-manager/app/serializers/contact.py b/lib/workload/stateless/stacks/metadata-manager/app/serializers/contact.py index 9188960d9..5a69ce920 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/serializers/contact.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/serializers/contact.py @@ -1,3 +1,7 @@ +from abc import ABC + +from rest_framework import serializers + from app.models import Contact from .base import SerializersBase @@ -7,7 +11,6 @@ class ContactBaseSerializer(SerializersBase): class ContactSerializer(ContactBaseSerializer): - class Meta: model = Contact fields = "__all__" @@ -22,3 +25,8 @@ class Meta: model = Contact fields = "__all__" + +class ContactHistorySerializer(ContactBaseSerializer): + class Meta: + model = Contact.history.model + fields = "__all__" diff --git a/lib/workload/stateless/stacks/metadata-manager/app/serializers/individual.py b/lib/workload/stateless/stacks/metadata-manager/app/serializers/individual.py index ad91fe26d..d66c553da 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/serializers/individual.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/serializers/individual.py @@ -15,3 +15,8 @@ class IndividualDetailSerializer(IndividualSerializer): subject_set = SubjectSerializer(many=True, read_only=True) + +class IndividualHistorySerializer(IndividualSerializer): + class Meta: + model = Individual.history.model + fields = "__all__" diff --git a/lib/workload/stateless/stacks/metadata-manager/app/serializers/library.py b/lib/workload/stateless/stacks/metadata-manager/app/serializers/library.py index b281e047c..95cc5aeda 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/serializers/library.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/serializers/library.py @@ -1,4 +1,8 @@ -from app.models import Library, Sample, Subject +from abc import ABC + +from rest_framework import serializers + +from app.models import Library, Sample, Subject, Project from .base import SerializersBase @@ -13,8 +17,11 @@ class Meta: def to_representation(self, instance): representation = super().to_representation(instance) - representation['sample'] = Sample.orcabus_id_prefix + representation['sample'] - representation['subject'] = Subject.orcabus_id_prefix + representation['subject'] + + if representation.get('sample', None): + representation['sample'] = Sample.orcabus_id_prefix + representation['sample'] + if representation.get('subject', None): + representation['subject'] = Subject.orcabus_id_prefix + representation['subject'] return representation @@ -31,3 +38,18 @@ class LibraryDetailSerializer(LibraryBaseSerializer): class Meta: model = Library fields = "__all__" + + +class LibraryHistorySerializer(LibrarySerializer): + class ProjectOrcabusIdSet(serializers.RelatedField): + def to_internal_value(self, data): + return None + + def to_representation(self, value): + return Project.orcabus_id_prefix + value.project.orcabus_id + + class Meta: + model = Library.history.model + fields = "__all__" + + project_set = ProjectOrcabusIdSet(many=True, read_only=True) diff --git a/lib/workload/stateless/stacks/metadata-manager/app/serializers/project.py b/lib/workload/stateless/stacks/metadata-manager/app/serializers/project.py index 51752357c..6f7b42ef9 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/serializers/project.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/serializers/project.py @@ -1,5 +1,7 @@ +from rest_framework import serializers + from .base import SerializersBase -from app.models import Project +from app.models import Project, Contact class ProjectBaseSerializer(SerializersBase): @@ -22,3 +24,20 @@ class ProjectDetailSerializer(ProjectBaseSerializer): class Meta: model = Project fields = "__all__" + + +class ProjectHistorySerializer(ProjectBaseSerializer): + class ContactOrcabusIdSet(serializers.RelatedField): + + def to_internal_value(self, data): + return None + + def to_representation(self, value): + return Contact.orcabus_id_prefix + value.contact.orcabus_id + + class Meta: + model = Project.history.model + fields = "__all__" + + contact_set = ContactOrcabusIdSet(many=True, read_only=True) + diff --git a/lib/workload/stateless/stacks/metadata-manager/app/serializers/sample.py b/lib/workload/stateless/stacks/metadata-manager/app/serializers/sample.py index 9a3e5a8bf..381aeed59 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/serializers/sample.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/serializers/sample.py @@ -20,3 +20,10 @@ class Meta: fields = '__all__' library_set = LibrarySerializer(many=True, read_only=True) + + +class SampleHistorySerializer(SampleBaseSerializer): + + class Meta: + model = Sample.history.model + fields = "__all__" diff --git a/lib/workload/stateless/stacks/metadata-manager/app/serializers/subject.py b/lib/workload/stateless/stacks/metadata-manager/app/serializers/subject.py index 498daf100..9480a3fa7 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/serializers/subject.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/serializers/subject.py @@ -1,4 +1,6 @@ -from app.models import Subject +from rest_framework import serializers + +from app.models import Subject, Individual from .base import SerializersBase @@ -24,3 +26,18 @@ class Meta: individual_set = IndividualSerializer(many=True, read_only=True) library_set = LibrarySerializer(many=True, read_only=True) + + +class SubjectHistorySerializer(SubjectBaseSerializer): + class IndividualOrcabusIdSet(serializers.RelatedField): + def to_internal_value(self, data): + return None + + def to_representation(self, value): + return Individual.orcabus_id_prefix + value.individual.orcabus_id + + class Meta: + model = Subject.history.model + fields = "__all__" + + individual_set = IndividualOrcabusIdSet(many=True, read_only=True) diff --git a/lib/workload/stateless/stacks/metadata-manager/app/tests/test_models.py b/lib/workload/stateless/stacks/metadata-manager/app/tests/test_models.py index 2ca58ee1f..a138b5f29 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/tests/test_models.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/tests/test_models.py @@ -2,6 +2,7 @@ from unittest.mock import patch +from django.core import serializers from django.test import TestCase from app.models import Subject, Sample, Library, Contact, Project, Individual @@ -176,4 +177,4 @@ def find_new_prj(relationship_array): lib_delta = lib_history[0].diff_against(lib_history[1]) for change in lib_delta.changes: self.assertFalse(find_new_prj(change.new), 'new project history should be up to date with current') - self.assertTrue(find_new_prj(change.old), 'old project found should still contain project_two') \ No newline at end of file + self.assertTrue(find_new_prj(change.old), 'old project found should still contain project_two') diff --git a/lib/workload/stateless/stacks/metadata-manager/app/viewsets/base.py b/lib/workload/stateless/stacks/metadata-manager/app/viewsets/base.py index 248ae688f..0fe52fe61 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/viewsets/base.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/viewsets/base.py @@ -1,7 +1,10 @@ from abc import ABC -from rest_framework import filters -from django.shortcuts import get_object_or_404 + from app.pagination import StandardResultsSetPagination + +from django.shortcuts import get_object_or_404 + +from rest_framework import filters from rest_framework.response import Response from rest_framework.viewsets import ReadOnlyModelViewSet @@ -43,3 +46,32 @@ def get_query_params(self): query_params.setlist('orcabus_id', id_list) return query_params + + def retrieve_history(self, history_serializer): + """ + To use this as API routes, you need to call it from the child class and put the appropriate decorator. + + e.g. + @extend_schema(responses=LibraryHistorySerializer(many=True)) + @action(detail=True, methods=['get'], url_name='history', url_path='history') + def retrieve_history(self, request, *args, **kwargs): + return super().retrieve_history(LibraryHistorySerializer) + + Args: + history_serializer (serializers.Serializer, optional): The serializer for the history data. + + Returns: + Response: A Response with the paginated, serialized history data. + """ + + # Grab the PK object from the queryset + pk = self.kwargs.get('pk') + if pk and pk.startswith(self.orcabus_id_prefix): + pk = pk[len(self.orcabus_id_prefix):] + obj = get_object_or_404(self.queryset, pk=pk) + + history_qs = obj.history.all() + page = self.paginate_queryset(history_qs) + serializer = history_serializer(page, many=True) + + return self.get_paginated_response(serializer.data) diff --git a/lib/workload/stateless/stacks/metadata-manager/app/viewsets/contact.py b/lib/workload/stateless/stacks/metadata-manager/app/viewsets/contact.py index aace47451..6f5f60e4b 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/viewsets/contact.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/viewsets/contact.py @@ -1,7 +1,8 @@ from drf_spectacular.utils import extend_schema +from rest_framework.decorators import action from app.models import Contact -from app.serializers.contact import ContactSerializer, ContactDetailSerializer +from app.serializers.contact import ContactSerializer, ContactDetailSerializer, ContactHistorySerializer from .base import BaseViewSet @@ -21,3 +22,8 @@ def list(self, request, *args, **kwargs): def get_queryset(self): query_params = self.get_query_params() return Contact.objects.get_by_keyword(**query_params) + + @extend_schema(responses=ContactHistorySerializer(many=True)) + @action(detail=True, methods=['get'], url_name='history', url_path='history') + def retrieve_history(self, request, *args, **kwargs): + return super().retrieve_history(ContactHistorySerializer) diff --git a/lib/workload/stateless/stacks/metadata-manager/app/viewsets/individual.py b/lib/workload/stateless/stacks/metadata-manager/app/viewsets/individual.py index c214efe84..8fed7619b 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/viewsets/individual.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/viewsets/individual.py @@ -1,7 +1,8 @@ from drf_spectacular.utils import extend_schema +from rest_framework.decorators import action from app.models import Individual -from app.serializers.individual import IndividualDetailSerializer +from app.serializers.individual import IndividualDetailSerializer, IndividualHistorySerializer from .base import BaseViewSet @@ -21,3 +22,9 @@ def list(self, request, *args, **kwargs): def get_queryset(self): query_params = self.get_query_params() return Individual.objects.get_by_keyword(self.queryset, **query_params) + + @extend_schema(responses=IndividualHistorySerializer(many=True)) + @action(detail=True, methods=['get'], url_name='history', url_path='history') + def retrieve_history(self, request, *args, **kwargs): + return super().retrieve_history(IndividualHistorySerializer) + diff --git a/lib/workload/stateless/stacks/metadata-manager/app/viewsets/library.py b/lib/workload/stateless/stacks/metadata-manager/app/viewsets/library.py index 5bc831af5..3d18927a3 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/viewsets/library.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/viewsets/library.py @@ -1,6 +1,8 @@ from drf_spectacular.utils import extend_schema +from rest_framework.decorators import action + from app.models import Library -from app.serializers.library import LibrarySerializer, LibraryDetailSerializer +from app.serializers.library import LibrarySerializer, LibraryDetailSerializer, LibraryHistorySerializer from .base import BaseViewSet @@ -33,3 +35,8 @@ def get_queryset(self): ]) def list(self, request, *args, **kwargs): return super().list(request, *args, **kwargs) + + @extend_schema(responses=LibraryHistorySerializer(many=True)) + @action(detail=True, methods=['get'], url_name='history', url_path='history') + def retrieve_history(self, request, *args, **kwargs): + return super().retrieve_history(LibraryHistorySerializer) diff --git a/lib/workload/stateless/stacks/metadata-manager/app/viewsets/project.py b/lib/workload/stateless/stacks/metadata-manager/app/viewsets/project.py index be751197b..8f1c9f7a4 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/viewsets/project.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/viewsets/project.py @@ -1,7 +1,8 @@ from drf_spectacular.utils import extend_schema +from rest_framework.decorators import action from app.models import Project -from app.serializers.project import ProjectDetailSerializer, ProjectSerializer +from app.serializers.project import ProjectDetailSerializer, ProjectSerializer, ProjectHistorySerializer from .base import BaseViewSet @@ -21,3 +22,8 @@ def list(self, request, *args, **kwargs): def get_queryset(self): query_params = self.get_query_params() return Project.objects.get_by_keyword(self.queryset, **query_params) + + @extend_schema(responses=ProjectHistorySerializer(many=True)) + @action(detail=True, methods=['get'], url_name='history', url_path='history') + def retrieve_history(self, request, *args, **kwargs): + return super().retrieve_history(ProjectHistorySerializer) diff --git a/lib/workload/stateless/stacks/metadata-manager/app/viewsets/sample.py b/lib/workload/stateless/stacks/metadata-manager/app/viewsets/sample.py index af0033845..faa9d00ef 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/viewsets/sample.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/viewsets/sample.py @@ -1,7 +1,8 @@ from drf_spectacular.utils import extend_schema +from rest_framework.decorators import action from app.models import Sample -from app.serializers.sample import SampleSerializer, SampleDetailSerializer +from app.serializers.sample import SampleSerializer, SampleDetailSerializer, SampleHistorySerializer from .base import BaseViewSet @@ -21,3 +22,8 @@ def list(self, request, *args, **kwargs): def get_queryset(self): query_params = self.get_query_params() return Sample.objects.get_by_keyword(**query_params) + + @extend_schema(responses=SampleHistorySerializer(many=True)) + @action(detail=True, methods=['get'], url_name='history', url_path='history') + def retrieve_history(self, request, *args, **kwargs): + return super().retrieve_history(SampleHistorySerializer) diff --git a/lib/workload/stateless/stacks/metadata-manager/app/viewsets/subject.py b/lib/workload/stateless/stacks/metadata-manager/app/viewsets/subject.py index fa252a42e..de8b55c7f 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/viewsets/subject.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/viewsets/subject.py @@ -1,7 +1,8 @@ from drf_spectacular.utils import extend_schema, OpenApiParameter +from rest_framework.decorators import action from app.models import Subject, Library -from app.serializers.subject import SubjectSerializer, SubjectDetailSerializer +from app.serializers.subject import SubjectSerializer, SubjectDetailSerializer, SubjectHistorySerializer from .base import BaseViewSet @@ -46,3 +47,8 @@ def get_queryset(self): ]) def list(self, request, *args, **kwargs): return super().list(request, *args, **kwargs) + + @extend_schema(responses=SubjectHistorySerializer(many=True)) + @action(detail=True, methods=['get'], url_name='history', url_path='history') + def retrieve_history(self, request, *args, **kwargs): + return super().retrieve_history(SubjectHistorySerializer) From 537cd03adb59ad9107bdd7cbae949a56ebc67a12 Mon Sep 17 00:00:00 2001 From: william Date: Wed, 16 Oct 2024 15:29:15 +1100 Subject: [PATCH 3/4] coverage --- .../stateless/stacks/metadata-manager/Makefile | 4 ++++ .../stateless/stacks/metadata-manager/README.md | 6 ++++++ .../stacks/metadata-manager/app/serializers/library.py | 2 +- .../stacks/metadata-manager/app/serializers/project.py | 2 +- .../stacks/metadata-manager/app/serializers/subject.py | 2 +- .../stacks/metadata-manager/app/tests/utils.py | 2 +- .../stacks/metadata-manager/app/viewsets/base.py | 4 ++-- .../stacks/metadata-manager/app/viewsets/contact.py | 2 +- .../stacks/metadata-manager/app/viewsets/individual.py | 2 +- .../stacks/metadata-manager/app/viewsets/library.py | 2 +- .../stacks/metadata-manager/app/viewsets/project.py | 2 +- .../stacks/metadata-manager/app/viewsets/sample.py | 2 +- .../stacks/metadata-manager/app/viewsets/subject.py | 2 +- .../proc/tests/test_tracking_sheet_srv.py | 10 +--------- 14 files changed, 23 insertions(+), 21 deletions(-) diff --git a/lib/workload/stateless/stacks/metadata-manager/Makefile b/lib/workload/stateless/stacks/metadata-manager/Makefile index 10cb5fa4b..61dfc0999 100644 --- a/lib/workload/stateless/stacks/metadata-manager/Makefile +++ b/lib/workload/stateless/stacks/metadata-manager/Makefile @@ -78,5 +78,9 @@ insert-data: suite: @python manage.py test --parallel +coverage: + @coverage run manage.py test + @coverage report -m + # full mock suite test pipeline - install deps, bring up compose stack, run suite, bring down compose stack test: install up suite down diff --git a/lib/workload/stateless/stacks/metadata-manager/README.md b/lib/workload/stateless/stacks/metadata-manager/README.md index 7018fde4b..0cb748b60 100644 --- a/lib/workload/stateless/stacks/metadata-manager/README.md +++ b/lib/workload/stateless/stacks/metadata-manager/README.md @@ -261,6 +261,12 @@ To stop the running server, simply use the `make stop` command To run the test from scratch use `make test`, but if you want to test with a running database you could use `make suite` . +Coverage test + +```bash +make coverage +``` + ### Development #### Migrations diff --git a/lib/workload/stateless/stacks/metadata-manager/app/serializers/library.py b/lib/workload/stateless/stacks/metadata-manager/app/serializers/library.py index 95cc5aeda..6479e5fe4 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/serializers/library.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/serializers/library.py @@ -43,7 +43,7 @@ class Meta: class LibraryHistorySerializer(LibrarySerializer): class ProjectOrcabusIdSet(serializers.RelatedField): def to_internal_value(self, data): - return None + raise NotImplementedError() def to_representation(self, value): return Project.orcabus_id_prefix + value.project.orcabus_id diff --git a/lib/workload/stateless/stacks/metadata-manager/app/serializers/project.py b/lib/workload/stateless/stacks/metadata-manager/app/serializers/project.py index 6f7b42ef9..3ccc787c5 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/serializers/project.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/serializers/project.py @@ -30,7 +30,7 @@ class ProjectHistorySerializer(ProjectBaseSerializer): class ContactOrcabusIdSet(serializers.RelatedField): def to_internal_value(self, data): - return None + raise NotImplementedError() def to_representation(self, value): return Contact.orcabus_id_prefix + value.contact.orcabus_id diff --git a/lib/workload/stateless/stacks/metadata-manager/app/serializers/subject.py b/lib/workload/stateless/stacks/metadata-manager/app/serializers/subject.py index 9480a3fa7..49a3674c9 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/serializers/subject.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/serializers/subject.py @@ -31,7 +31,7 @@ class Meta: class SubjectHistorySerializer(SubjectBaseSerializer): class IndividualOrcabusIdSet(serializers.RelatedField): def to_internal_value(self, data): - return None + raise NotImplementedError() def to_representation(self, value): return Individual.orcabus_id_prefix + value.individual.orcabus_id diff --git a/lib/workload/stateless/stacks/metadata-manager/app/tests/utils.py b/lib/workload/stateless/stacks/metadata-manager/app/tests/utils.py index f17478a99..c7c855099 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/tests/utils.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/tests/utils.py @@ -4,7 +4,7 @@ def clear_all_data(): - """This function clear all existing models objcet""" + """This function clear all existing models object""" Library.objects.all().delete() Sample.objects.all().delete() Subject.objects.all().delete() diff --git a/lib/workload/stateless/stacks/metadata-manager/app/viewsets/base.py b/lib/workload/stateless/stacks/metadata-manager/app/viewsets/base.py index 0fe52fe61..ce54bc5dc 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/viewsets/base.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/viewsets/base.py @@ -52,13 +52,13 @@ def retrieve_history(self, history_serializer): To use this as API routes, you need to call it from the child class and put the appropriate decorator. e.g. - @extend_schema(responses=LibraryHistorySerializer(many=True)) + @extend_schema(responses=LibraryHistorySerializer(many=True), description="Retrieve the history of this model") @action(detail=True, methods=['get'], url_name='history', url_path='history') def retrieve_history(self, request, *args, **kwargs): return super().retrieve_history(LibraryHistorySerializer) Args: - history_serializer (serializers.Serializer, optional): The serializer for the history data. + history_serializer (serializers.Serializer): The serializer for the history data. Returns: Response: A Response with the paginated, serialized history data. diff --git a/lib/workload/stateless/stacks/metadata-manager/app/viewsets/contact.py b/lib/workload/stateless/stacks/metadata-manager/app/viewsets/contact.py index 6f5f60e4b..52f59498a 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/viewsets/contact.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/viewsets/contact.py @@ -23,7 +23,7 @@ def get_queryset(self): query_params = self.get_query_params() return Contact.objects.get_by_keyword(**query_params) - @extend_schema(responses=ContactHistorySerializer(many=True)) + @extend_schema(responses=ContactHistorySerializer(many=True), description="Retrieve the history of this model") @action(detail=True, methods=['get'], url_name='history', url_path='history') def retrieve_history(self, request, *args, **kwargs): return super().retrieve_history(ContactHistorySerializer) diff --git a/lib/workload/stateless/stacks/metadata-manager/app/viewsets/individual.py b/lib/workload/stateless/stacks/metadata-manager/app/viewsets/individual.py index 8fed7619b..a0ae40b2e 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/viewsets/individual.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/viewsets/individual.py @@ -23,7 +23,7 @@ def get_queryset(self): query_params = self.get_query_params() return Individual.objects.get_by_keyword(self.queryset, **query_params) - @extend_schema(responses=IndividualHistorySerializer(many=True)) + @extend_schema(responses=IndividualHistorySerializer(many=True), description="Retrieve the history of this model") @action(detail=True, methods=['get'], url_name='history', url_path='history') def retrieve_history(self, request, *args, **kwargs): return super().retrieve_history(IndividualHistorySerializer) diff --git a/lib/workload/stateless/stacks/metadata-manager/app/viewsets/library.py b/lib/workload/stateless/stacks/metadata-manager/app/viewsets/library.py index 3d18927a3..c74d893ca 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/viewsets/library.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/viewsets/library.py @@ -36,7 +36,7 @@ def get_queryset(self): def list(self, request, *args, **kwargs): return super().list(request, *args, **kwargs) - @extend_schema(responses=LibraryHistorySerializer(many=True)) + @extend_schema(responses=LibraryHistorySerializer(many=True), description="Retrieve the history of this model") @action(detail=True, methods=['get'], url_name='history', url_path='history') def retrieve_history(self, request, *args, **kwargs): return super().retrieve_history(LibraryHistorySerializer) diff --git a/lib/workload/stateless/stacks/metadata-manager/app/viewsets/project.py b/lib/workload/stateless/stacks/metadata-manager/app/viewsets/project.py index 8f1c9f7a4..af1abff29 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/viewsets/project.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/viewsets/project.py @@ -23,7 +23,7 @@ def get_queryset(self): query_params = self.get_query_params() return Project.objects.get_by_keyword(self.queryset, **query_params) - @extend_schema(responses=ProjectHistorySerializer(many=True)) + @extend_schema(responses=ProjectHistorySerializer(many=True), description="Retrieve the history of this model") @action(detail=True, methods=['get'], url_name='history', url_path='history') def retrieve_history(self, request, *args, **kwargs): return super().retrieve_history(ProjectHistorySerializer) diff --git a/lib/workload/stateless/stacks/metadata-manager/app/viewsets/sample.py b/lib/workload/stateless/stacks/metadata-manager/app/viewsets/sample.py index faa9d00ef..562bd187c 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/viewsets/sample.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/viewsets/sample.py @@ -23,7 +23,7 @@ def get_queryset(self): query_params = self.get_query_params() return Sample.objects.get_by_keyword(**query_params) - @extend_schema(responses=SampleHistorySerializer(many=True)) + @extend_schema(responses=SampleHistorySerializer(many=True), description="Retrieve the history of this model") @action(detail=True, methods=['get'], url_name='history', url_path='history') def retrieve_history(self, request, *args, **kwargs): return super().retrieve_history(SampleHistorySerializer) diff --git a/lib/workload/stateless/stacks/metadata-manager/app/viewsets/subject.py b/lib/workload/stateless/stacks/metadata-manager/app/viewsets/subject.py index de8b55c7f..e12c6f8c8 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/viewsets/subject.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/viewsets/subject.py @@ -48,7 +48,7 @@ def get_queryset(self): def list(self, request, *args, **kwargs): return super().list(request, *args, **kwargs) - @extend_schema(responses=SubjectHistorySerializer(many=True)) + @extend_schema(responses=SubjectHistorySerializer(many=True), description="Retrieve the history of this model") @action(detail=True, methods=['get'], url_name='history', url_path='history') def retrieve_history(self, request, *args, **kwargs): return super().retrieve_history(SubjectHistorySerializer) diff --git a/lib/workload/stateless/stacks/metadata-manager/proc/tests/test_tracking_sheet_srv.py b/lib/workload/stateless/stacks/metadata-manager/proc/tests/test_tracking_sheet_srv.py index b3bcf2181..ecc462584 100644 --- a/lib/workload/stateless/stacks/metadata-manager/proc/tests/test_tracking_sheet_srv.py +++ b/lib/workload/stateless/stacks/metadata-manager/proc/tests/test_tracking_sheet_srv.py @@ -108,20 +108,12 @@ def test_persist_lab_metadata(self): """ python manage.py test proc.tests.test_tracking_sheet_srv.TrackingSheetSrvUnitTests.test_persist_lab_metadata """ - mock_sheet_data = [RECORD_1] + mock_sheet_data = [RECORD_1, RECORD_2, RECORD_3] metadata_pd = pd.json_normalize(mock_sheet_data) metadata_pd = sanitize_lab_metadata_df(metadata_pd) result = persist_lab_metadata(metadata_pd, SHEET_YEAR) - lib_one = Library.objects.get(library_id=RECORD_1.get("LibraryID")) - print(lib_one.history.all()) - new_record, old_record = lib_one.history.all() - delta = new_record.diff_against(old_record) - print(delta) - for change in delta.changes: - print(f"'{change.field}' changed from '{change.old}' to '{change.new}'") - return # Stats check self.assertEqual(result.get("invalid_record_count"), 0, "non invalid record should exist") self.assertEqual(result.get("library").get("create_count"), 3, "3 new library should be created") From 2a3584d5b19d9b2f7835cec511b91632e69d9eda Mon Sep 17 00:00:00 2001 From: william Date: Wed, 16 Oct 2024 22:42:10 +1100 Subject: [PATCH 4/4] sync mock data wfm <=> mm --- .../stacks/metadata-manager/Makefile | 2 +- .../app/management/commands/clean_db.py | 13 ++ .../management/commands/insert_mock_data.py | 158 +++++++++++++++++- 3 files changed, 165 insertions(+), 8 deletions(-) create mode 100644 lib/workload/stateless/stacks/metadata-manager/app/management/commands/clean_db.py diff --git a/lib/workload/stateless/stacks/metadata-manager/Makefile b/lib/workload/stateless/stacks/metadata-manager/Makefile index 61dfc0999..cd8c7fed9 100644 --- a/lib/workload/stateless/stacks/metadata-manager/Makefile +++ b/lib/workload/stateless/stacks/metadata-manager/Makefile @@ -72,7 +72,7 @@ makemigrations: migrate: @python manage.py migrate -insert-data: +mock: reset-db migrate @python manage.py insert_mock_data suite: diff --git a/lib/workload/stateless/stacks/metadata-manager/app/management/commands/clean_db.py b/lib/workload/stateless/stacks/metadata-manager/app/management/commands/clean_db.py new file mode 100644 index 000000000..a43a5d1d1 --- /dev/null +++ b/lib/workload/stateless/stacks/metadata-manager/app/management/commands/clean_db.py @@ -0,0 +1,13 @@ +from django.core.management import BaseCommand + +from app.tests.utils import clear_all_data + + +# https://docs.djangoproject.com/en/5.0/howto/custom-management-commands/ +class Command(BaseCommand): + help = "Delete all DB data" + + def handle(self, *args, **options): + clear_all_data() + + print("Done") diff --git a/lib/workload/stateless/stacks/metadata-manager/app/management/commands/insert_mock_data.py b/lib/workload/stateless/stacks/metadata-manager/app/management/commands/insert_mock_data.py index 8da594cdd..29f65554f 100644 --- a/lib/workload/stateless/stacks/metadata-manager/app/management/commands/insert_mock_data.py +++ b/lib/workload/stateless/stacks/metadata-manager/app/management/commands/insert_mock_data.py @@ -1,22 +1,166 @@ import json import pandas as pd +from django.core.exceptions import ObjectDoesNotExist from django.core.management import BaseCommand +from app.models import Subject, Library, Sample, Individual, Project, Contact +from app.tests.utils import clear_all_data from proc.service.tracking_sheet_srv import sanitize_lab_metadata_df, persist_lab_metadata from proc.tests.test_tracking_sheet_srv import RECORD_1, RECORD_2, RECORD_3, SHEET_YEAR class Command(BaseCommand): + """ + python manage.py insert_mock_data + """ help = "Generate mock Metadata into database for local development and testing" def handle(self, *args, **options): - print("insert data from proc service test") + clear_all_data() + load_mock_from_wfm() - mock_sheet_data = [RECORD_1, RECORD_2, RECORD_3] - metadata_pd = pd.json_normalize(mock_sheet_data) - metadata_pd = sanitize_lab_metadata_df(metadata_pd) - result = persist_lab_metadata(metadata_pd, SHEET_YEAR, is_emit_eb_events=False) +def load_mock_from_proc(): + """Not in use for now, as loading data from wfm is preferred to sync data""" + mock_sheet_data = [RECORD_1, RECORD_2, RECORD_3] - print(json.dumps(result, indent=4)) - print("insert mock data completed") + metadata_pd = pd.json_normalize(mock_sheet_data) + metadata_pd = sanitize_lab_metadata_df(metadata_pd) + result = persist_lab_metadata(metadata_pd, SHEET_YEAR, is_emit_eb_events=False) + + print(json.dumps(result, indent=4)) + print("insert mock data completed") + + +def load_mock_from_wfm(): + # The libraries are taken from WFM as of 16/10/2024 + # Will sync this so test data sync across MM <=> WFM + libraries = [ + { + "orcabus_id": "01J5M2JFE1JPYV62RYQEG99CP1", + "phenotype": "tumor", + "library_id": "L000001", + "assay": "TsqNano", + "type": "WGS", + "subject": "SBJ00001", + "workflow": "clinical" + }, + { + "orcabus_id": "02J5M2JFE1JPYV62RYQEG99CP2", + "phenotype": "normal", + "library_id": "L000002", + "assay": "TsqNano", + "type": "WGS", + "subject": "SBJ00001", + "workflow": "clinical" + }, + { + "orcabus_id": "03J5M2JFE1JPYV62RYQEG99CP3", + "phenotype": "tumor", + "library_id": "L000003", + "assay": "TsqNano", + "type": "WGS", + "subject": "SBJ00002", + "workflow": "research" + }, + { + "orcabus_id": "04J5M2JFE1JPYV62RYQEG99CP4", + "phenotype": "normal", + "library_id": "L000004", + "assay": "TsqNano", + "type": "WGS", + "subject": "SBJ00002", + "workflow": "research" + }, + { + "orcabus_id": "05J5M2JFE1JPYV62RYQEG99CP5", + "phenotype": "tumor", + "library_id": "L000005", + "assay": "ctTSOv2", + "type": "ctDNA", + "subject": "SBJ00003", + "workflow": "clinical" + }, + { + "orcabus_id": "06J5M2JFE1JPYV62RYQEG99CP6", + "phenotype": "tumor", + "library_id": "L000006", + "assay": "ctTSOv2", + "type": "ctDNA", + "subject": "SBJ00003", + "workflow": "research" + }, + ] + + for lib in libraries: + idv, is_idv_created, is_idv_updated = Individual.objects.update_or_create_if_needed( + search_key={ + "individual_id": 'IDV0001', + "source": "lab" + }, + data={ + "individual_id": 'IDV0001', + "source": "lab" + } + ) + + subject, is_sub_created, is_sub_updated = Subject.objects.update_or_create_if_needed( + search_key={"subject_id": lib["subject"]}, + data={ + "subject_id": lib["subject"], + } + ) + + try: + subject.individual_set.get(orcabus_id=idv.orcabus_id) + except ObjectDoesNotExist: + subject.individual_set.add(idv) + + sample, is_smp_created, is_smp_updated = Sample.objects.update_or_create_if_needed( + search_key={"sample_id": f"""smp-{lib["library_id"]}"""}, + data={ + "sample_id": f"""smp-{lib["library_id"]}""", + "external_sample_id": f"""ext-smp-{lib["library_id"]}""", + "source": "blood", + } + ) + + contact, is_ctc_created, is_ctc_updated = Contact.objects.update_or_create_if_needed( + search_key={"contact_id": 'ctc-1'}, + data={ + "contact_id": 'ctc-1', + } + ) + + project, is_prj_created, is_prj_updated = Project.objects.update_or_create_if_needed( + search_key={"project_id": 'prj-1'}, + data={ + "project_id": 'prj-1', + } + ) + + try: + project.contact_set.get(orcabus_id=contact.orcabus_id) + except ObjectDoesNotExist: + project.contact_set.add(contact) + + library, is_lib_created, is_lib_updated = Library.objects.update_or_create_if_needed( + search_key={'library_id': lib["library_id"]}, + data={ + "orcabus_id": lib["orcabus_id"], + "library_id": lib["library_id"], + "phenotype": lib["phenotype"], + "assay": lib["assay"], + "type": lib["type"], + "workflow": lib["workflow"], + + "subject_id": subject.orcabus_id, + "sample_id": sample.orcabus_id, + } + + ) + + try: + library.project_set.get(orcabus_id=project.orcabus_id) + except ObjectDoesNotExist: + library.project_set.add(project)