From 40156db8bf6ad218806812743918e824b8a54515 Mon Sep 17 00:00:00 2001 From: Anne Mickan Date: Fri, 6 Dec 2024 10:44:53 +0100 Subject: [PATCH 01/18] Add AlgorithmInterface model (#3720) First PR for https://github.com/DIAGNijmegen/rse-roadmap/issues/153 This adds the `AlgorithmInterface` model, as well as the necessary through table, and removes the inputs and outputs fields from the algorithm form. The following restrictions apply: - AlgorithmInterfaces cannot be deleted (protection on model level and admin form level) - AlgorithmInterfaces cannot be updated (protection on admin form level) - AlgorithmInterfaces are unique (i.e. input and output combinations are unique, we check for that in the form) - An algorithm can only have 1 default algorithm interface - There can only be one entry per algorithm and interface in the through table. --------- Co-authored-by: James Meakin <12661555+jmsmkn@users.noreply.github.com> --- app/grandchallenge/algorithms/admin.py | 64 ++++++- app/grandchallenge/algorithms/forms.py | 98 ++++------- ...rface_algorithminterfaceoutput_and_more.py | 162 ++++++++++++++++++ .../0064_create_algorithm_interfaces.py | 41 +++++ app/grandchallenge/algorithms/models.py | 101 +++++++++++ app/grandchallenge/algorithms/views.py | 19 +- app/tests/algorithms_tests/factories.py | 22 ++- app/tests/algorithms_tests/test_admin.py | 12 -- app/tests/algorithms_tests/test_forms.py | 41 +++-- app/tests/algorithms_tests/test_models.py | 118 +++++++++++++ app/tests/algorithms_tests/test_views.py | 25 --- 11 files changed, 569 insertions(+), 134 deletions(-) create mode 100644 app/grandchallenge/algorithms/migrations/0063_algorithminterface_algorithminterfaceoutput_and_more.py create mode 100644 app/grandchallenge/algorithms/migrations/0064_create_algorithm_interfaces.py delete mode 100644 app/tests/algorithms_tests/test_admin.py diff --git a/app/grandchallenge/algorithms/admin.py b/app/grandchallenge/algorithms/admin.py index b9ac6ff3fc..044393a2dc 100644 --- a/app/grandchallenge/algorithms/admin.py +++ b/app/grandchallenge/algorithms/admin.py @@ -2,19 +2,21 @@ from django.contrib import admin from django.contrib.admin import ModelAdmin -from django.core.exceptions import ObjectDoesNotExist +from django.core.exceptions import ObjectDoesNotExist, ValidationError from django.db.models import Count, Sum from django.forms import ModelForm from django.utils.html import format_html from guardian.admin import GuardedModelAdmin -from grandchallenge.algorithms.forms import AlgorithmIOValidationMixin +from grandchallenge.algorithms.forms import AlgorithmInterfaceBaseForm from grandchallenge.algorithms.models import ( Algorithm, + AlgorithmAlgorithmInterface, AlgorithmGroupObjectPermission, AlgorithmImage, AlgorithmImageGroupObjectPermission, AlgorithmImageUserObjectPermission, + AlgorithmInterface, AlgorithmModel, AlgorithmModelGroupObjectPermission, AlgorithmModelUserObjectPermission, @@ -36,12 +38,13 @@ UserObjectPermissionAdmin, ) from grandchallenge.core.templatetags.costs import millicents_to_euro +from grandchallenge.core.templatetags.remove_whitespace import oxford_comma from grandchallenge.core.utils.grand_challenge_forge import ( get_forge_algorithm_template_context, ) -class AlgorithmAdminForm(AlgorithmIOValidationMixin, ModelForm): +class AlgorithmAdminForm(ModelForm): class Meta: model = Algorithm fields = "__all__" @@ -49,11 +52,12 @@ class Meta: @admin.register(Algorithm) class AlgorithmAdmin(GuardedModelAdmin): - readonly_fields = ("algorithm_forge_json",) + readonly_fields = ("algorithm_forge_json", "inputs", "outputs") list_display = ( "title", "created", "public", + "default_io", "time_limit", "job_requires_gpu_type", "job_requires_memory_gb", @@ -75,6 +79,11 @@ def algorithm_forge_json(obj): "
{json_desc}
", json_desc=json.dumps(json_desc, indent=2) ) + def default_io(self, obj): + return AlgorithmAlgorithmInterface.objects.get( + algorithm=obj, is_default=True + ) + def get_queryset(self, request): queryset = super().get_queryset(request) queryset = queryset.annotate( @@ -235,6 +244,53 @@ class AlgorithmModelAdmin(GuardedModelAdmin): readonly_fields = ("creator", "algorithm", "sha256", "size_in_storage") +class AlgorithmInterfaceAdminForm(AlgorithmInterfaceBaseForm): + def clean(self): + cleaned_data = super().clean() + if cleaned_data["existing_io"]: + raise ValidationError( + "An AlgorithmIO with the same combination of inputs and outputs already exists." + ) + return cleaned_data + + +@admin.register(AlgorithmInterface) +class AlgorithmInterfaceAdmin(GuardedModelAdmin): + list_display = ( + "pk", + "algorithm_inputs", + "algorithm_outputs", + ) + search_fields = ( + "inputs__slug", + "outputs__slug", + ) + form = AlgorithmInterfaceAdminForm + + def algorithm_inputs(self, obj): + return oxford_comma(obj.inputs.all()) + + def algorithm_outputs(self, obj): + return oxford_comma(obj.outputs.all()) + + def has_change_permission(self, request, obj=None): + return False + + def has_delete_permission(self, request, obj=None): + return False + + +@admin.register(AlgorithmAlgorithmInterface) +class AlgorithmAlgorithmInterfaceAdmin(GuardedModelAdmin): + list_display = ( + "pk", + "interface", + "is_default", + "algorithm", + ) + list_filter = ("is_default", "algorithm") + + admin.site.register(AlgorithmUserObjectPermission, UserObjectPermissionAdmin) admin.site.register(AlgorithmGroupObjectPermission, GroupObjectPermissionAdmin) admin.site.register(AlgorithmImage, ComponentImageAdmin) diff --git a/app/grandchallenge/algorithms/forms.py b/app/grandchallenge/algorithms/forms.py index a61df6de75..9d873979ee 100644 --- a/app/grandchallenge/algorithms/forms.py +++ b/app/grandchallenge/algorithms/forms.py @@ -30,7 +30,6 @@ HiddenInput, ModelChoiceField, ModelForm, - ModelMultipleChoiceField, Select, TextInput, URLField, @@ -45,9 +44,11 @@ from grandchallenge.algorithms.models import ( Algorithm, AlgorithmImage, + AlgorithmInterface, AlgorithmModel, AlgorithmPermissionRequest, Job, + get_existing_interface_for_inputs_and_outputs, ) from grandchallenge.algorithms.serializers import ( AlgorithmImageSerializer, @@ -220,60 +221,11 @@ def reformat_inputs(self, *, cleaned_data): ] -class AlgorithmIOValidationMixin: - def clean(self): - cleaned_data = super().clean() - - duplicate_interfaces = {*cleaned_data.get("inputs", [])}.intersection( - {*cleaned_data.get("outputs", [])} - ) - - if duplicate_interfaces: - raise ValidationError( - f"The sets of Inputs and Outputs must be unique: " - f"{oxford_comma(duplicate_interfaces)} present in both" - ) - - return cleaned_data - - class AlgorithmForm( - AlgorithmIOValidationMixin, WorkstationUserFilterMixin, SaveFormInitMixin, ModelForm, ): - inputs = ModelMultipleChoiceField( - queryset=ComponentInterface.objects.exclude( - slug__in=[*NON_ALGORITHM_INTERFACES, "results-json-file"] - ), - widget=Select2MultipleWidget, - help_text=format_lazy( - ( - "The inputs to this algorithm. " - 'See the list of interfaces for more ' - "information about each interface. " - "Please contact support if your desired input is missing." - ), - reverse_lazy("components:component-interface-list-algorithms"), - ), - ) - outputs = ModelMultipleChoiceField( - queryset=ComponentInterface.objects.exclude( - slug__in=NON_ALGORITHM_INTERFACES - ), - widget=Select2MultipleWidget, - help_text=format_lazy( - ( - "The outputs to this algorithm. " - 'See the list of interfaces for more ' - "information about each interface. " - "Please contact support if your desired output is missing." - ), - reverse_lazy("components:component-interface-list-algorithms"), - ), - ) - class Meta: model = Algorithm fields = ( @@ -293,8 +245,6 @@ class Meta: "hanging_protocol", "optional_hanging_protocols", "view_content", - "inputs", - "outputs", "minimum_credits_per_job", "job_requires_gpu_type", "job_requires_memory_gb", @@ -638,15 +588,6 @@ def __init__(self, *args, **kwargs): ) -class AlgorithmUpdateForm(AlgorithmForm): - def __init__(self, *args, interfaces_editable, **kwargs): - super().__init__(*args, **kwargs) - - if not interfaces_editable: - for field_key in ("inputs", "outputs"): - self.fields.pop(field_key) - - class AlgorithmImageForm(ContainerImageForm): algorithm = ModelChoiceField(widget=HiddenInput(), queryset=None) @@ -1289,6 +1230,7 @@ def __init__( ) if hide_algorithm_model_input: + self.fields["algorithm_model"].widget = HiddenInput() self.helper = FormHelper(self) if activate: @@ -1317,3 +1259,37 @@ def clean_algorithm_model(self): raise ValidationError("Model updating already in progress.") return algorithm_model + + +class AlgorithmInterfaceBaseForm(SaveFormInitMixin, ModelForm): + class Meta: + model = AlgorithmInterface + fields = ("inputs", "outputs") + + def clean(self): + cleaned_data = super().clean() + + inputs = cleaned_data.get("inputs", []) + outputs = cleaned_data.get("outputs", []) + + if not inputs or not outputs: + raise ValidationError( + "You must provide at least 1 input and 1 output." + ) + + duplicate_interfaces = {*inputs}.intersection({*outputs}) + + if duplicate_interfaces: + raise ValidationError( + f"The sets of Inputs and Outputs must be unique: " + f"{oxford_comma(duplicate_interfaces)} present in both" + ) + + cleaned_data["existing_io"] = ( + get_existing_interface_for_inputs_and_outputs( + inputs=inputs, + outputs=outputs, + ) + ) + + return cleaned_data diff --git a/app/grandchallenge/algorithms/migrations/0063_algorithminterface_algorithminterfaceoutput_and_more.py b/app/grandchallenge/algorithms/migrations/0063_algorithminterface_algorithminterfaceoutput_and_more.py new file mode 100644 index 0000000000..9a0997b11b --- /dev/null +++ b/app/grandchallenge/algorithms/migrations/0063_algorithminterface_algorithminterfaceoutput_and_more.py @@ -0,0 +1,162 @@ +# Generated by Django 4.2.16 on 2024-12-04 14:06 + +import uuid + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("components", "0022_alter_componentinterface_kind_and_more"), + ("algorithms", "0062_algorithmusercredit"), + ] + + operations = [ + migrations.CreateModel( + name="AlgorithmInterface", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("created", models.DateTimeField(auto_now_add=True)), + ("modified", models.DateTimeField(auto_now=True)), + ], + options={ + "abstract": False, + }, + ), + migrations.CreateModel( + name="AlgorithmInterfaceOutput", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "interface", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="algorithms.algorithminterface", + ), + ), + ( + "output", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="components.componentinterface", + ), + ), + ], + ), + migrations.CreateModel( + name="AlgorithmInterfaceInput", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "input", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="components.componentinterface", + ), + ), + ( + "interface", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="algorithms.algorithminterface", + ), + ), + ], + ), + migrations.AddField( + model_name="algorithminterface", + name="inputs", + field=models.ManyToManyField( + related_name="inputs", + through="algorithms.AlgorithmInterfaceInput", + to="components.componentinterface", + ), + ), + migrations.AddField( + model_name="algorithminterface", + name="outputs", + field=models.ManyToManyField( + related_name="outputs", + through="algorithms.AlgorithmInterfaceOutput", + to="components.componentinterface", + ), + ), + migrations.CreateModel( + name="AlgorithmAlgorithmInterface", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("is_default", models.BooleanField(default=False)), + ( + "algorithm", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="algorithms.algorithm", + ), + ), + ( + "interface", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="algorithms.algorithminterface", + ), + ), + ], + ), + migrations.AddField( + model_name="algorithm", + name="interfaces", + field=models.ManyToManyField( + related_name="algorithm_interfaces", + through="algorithms.AlgorithmAlgorithmInterface", + to="algorithms.algorithminterface", + ), + ), + migrations.AddConstraint( + model_name="algorithmalgorithminterface", + constraint=models.UniqueConstraint( + condition=models.Q(("is_default", True)), + fields=("algorithm",), + name="unique_default_interface_per_algorithm", + ), + ), + migrations.AddConstraint( + model_name="algorithmalgorithminterface", + constraint=models.UniqueConstraint( + fields=("algorithm", "interface"), + name="unique_algorithm_interface_combination", + ), + ), + ] diff --git a/app/grandchallenge/algorithms/migrations/0064_create_algorithm_interfaces.py b/app/grandchallenge/algorithms/migrations/0064_create_algorithm_interfaces.py new file mode 100644 index 0000000000..0f3bb8e8dd --- /dev/null +++ b/app/grandchallenge/algorithms/migrations/0064_create_algorithm_interfaces.py @@ -0,0 +1,41 @@ +from django.db import migrations + +from grandchallenge.algorithms.models import ( + get_existing_interface_for_inputs_and_outputs, +) + + +def create_algorithm_interfaces(apps, _schema_editor): + Algorithm = apps.get_model("algorithms", "Algorithm") # noqa: N806 + AlgorithmInterface = apps.get_model( # noqa: N806 + "algorithms", "AlgorithmInterface" + ) + + for algorithm in Algorithm.objects.prefetch_related( + "inputs", "outputs" + ).all(): + inputs = algorithm.inputs.all() + outputs = algorithm.outputs.all() + + io = get_existing_interface_for_inputs_and_outputs( + model=AlgorithmInterface, inputs=inputs, outputs=outputs + ) + if not io: + io = AlgorithmInterface.objects.create() + io.inputs.set(inputs) + io.outputs.set(outputs) + + algorithm.interfaces.add(io, through_defaults={"is_default": True}) + + +class Migration(migrations.Migration): + dependencies = [ + ( + "algorithms", + "0063_algorithminterface_algorithminterfaceoutput_and_more", + ), + ] + + operations = [ + migrations.RunPython(create_algorithm_interfaces, elidable=True), + ] diff --git a/app/grandchallenge/algorithms/models.py b/app/grandchallenge/algorithms/models.py index 1e03a57f7a..981189c82f 100644 --- a/app/grandchallenge/algorithms/models.py +++ b/app/grandchallenge/algorithms/models.py @@ -69,6 +69,80 @@ JINJA_ENGINE = sandbox.ImmutableSandboxedEnvironment() +class AlgorithmInterfaceManager(models.Manager): + + def create( + self, + *, + inputs, + outputs, + **kwargs, + ): + obj = get_existing_interface_for_inputs_and_outputs( + inputs=inputs, outputs=outputs + ) + if not obj: + obj = super().create(**kwargs) + obj.inputs.set(inputs) + obj.outputs.set(outputs) + + return obj + + def delete(self): + raise NotImplementedError("Bulk delete is not allowed.") + + +class AlgorithmInterface(UUIDModel): + inputs = models.ManyToManyField( + to=ComponentInterface, + related_name="inputs", + through="algorithms.AlgorithmInterfaceInput", + ) + outputs = models.ManyToManyField( + to=ComponentInterface, + related_name="outputs", + through="algorithms.AlgorithmInterfaceOutput", + ) + + objects = AlgorithmInterfaceManager() + + def delete(self, *args, **kwargs): + raise ValidationError("AlgorithmInterfaces cannot be deleted.") + + +class AlgorithmInterfaceInput(models.Model): + input = models.ForeignKey(ComponentInterface, on_delete=models.CASCADE) + interface = models.ForeignKey(AlgorithmInterface, on_delete=models.CASCADE) + + +class AlgorithmInterfaceOutput(models.Model): + output = models.ForeignKey(ComponentInterface, on_delete=models.CASCADE) + interface = models.ForeignKey(AlgorithmInterface, on_delete=models.CASCADE) + + +def get_existing_interface_for_inputs_and_outputs( + *, inputs, outputs, model=AlgorithmInterface +): + try: + return model.objects.annotate( + input_count=Count("inputs", distinct=True), + output_count=Count("outputs", distinct=True), + relevant_input_count=Count( + "inputs", filter=Q(inputs__in=inputs), distinct=True + ), + relevant_output_count=Count( + "outputs", filter=Q(outputs__in=outputs), distinct=True + ), + ).get( + relevant_input_count=len(inputs), + relevant_output_count=len(outputs), + input_count=len(inputs), + output_count=len(outputs), + ) + except ObjectDoesNotExist: + return None + + class Algorithm(UUIDModel, TitleSlugDescriptionModel, HangingProtocolMixin): editors_group = models.OneToOneField( Group, @@ -148,6 +222,11 @@ class Algorithm(UUIDModel, TitleSlugDescriptionModel, HangingProtocolMixin): "{% endfor %}" ), ) + interfaces = models.ManyToManyField( + to=AlgorithmInterface, + related_name="algorithm_interfaces", + through="algorithms.AlgorithmAlgorithmInterface", + ) inputs = models.ManyToManyField( to=ComponentInterface, related_name="algorithm_inputs", blank=False ) @@ -521,6 +600,28 @@ def form_field_label(self): return title +class AlgorithmAlgorithmInterface(models.Model): + algorithm = models.ForeignKey(Algorithm, on_delete=models.CASCADE) + interface = models.ForeignKey(AlgorithmInterface, on_delete=models.CASCADE) + is_default = models.BooleanField(default=False) + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["algorithm"], + condition=Q(is_default=True), + name="unique_default_interface_per_algorithm", + ), + models.UniqueConstraint( + fields=["algorithm", "interface"], + name="unique_algorithm_interface_combination", + ), + ] + + def __str__(self): + return str(self.interface) + + class AlgorithmUserObjectPermission(UserObjectPermissionBase): content_object = models.ForeignKey(Algorithm, on_delete=models.CASCADE) diff --git a/app/grandchallenge/algorithms/views.py b/app/grandchallenge/algorithms/views.py index 842f1ffc5f..8005f8c76c 100644 --- a/app/grandchallenge/algorithms/views.py +++ b/app/grandchallenge/algorithms/views.py @@ -52,7 +52,6 @@ AlgorithmPermissionRequestUpdateForm, AlgorithmPublishForm, AlgorithmRepoForm, - AlgorithmUpdateForm, DisplaySetFromJobForm, ImageActivateForm, JobCreateForm, @@ -253,26 +252,10 @@ class AlgorithmUpdate( UpdateView, ): model = Algorithm - form_class = AlgorithmUpdateForm + form_class = AlgorithmForm permission_required = "algorithms.change_algorithm" raise_exception = True - def get_form_kwargs(self): - kwargs = super().get_form_kwargs() - - # Only users with the add_algorithm permission can change - # the input and output interfaces, other users must use - # the interfaces pre-set by the Phase - kwargs.update( - { - "interfaces_editable": self.request.user.has_perm( - "algorithms.add_algorithm" - ) - } - ) - - return kwargs - class AlgorithmDescriptionUpdate( LoginRequiredMixin, diff --git a/app/tests/algorithms_tests/factories.py b/app/tests/algorithms_tests/factories.py index d198d83ae4..c3027e72de 100644 --- a/app/tests/algorithms_tests/factories.py +++ b/app/tests/algorithms_tests/factories.py @@ -3,13 +3,17 @@ from grandchallenge.algorithms.models import ( Algorithm, AlgorithmImage, + AlgorithmInterface, AlgorithmModel, AlgorithmPermissionRequest, AlgorithmUserCredit, Job, ) from grandchallenge.components.models import GPUTypeChoices -from tests.components_tests.factories import ComponentInterfaceValueFactory +from tests.components_tests.factories import ( + ComponentInterfaceFactory, + ComponentInterfaceValueFactory, +) from tests.factories import ( ImageFactory, UserFactory, @@ -84,3 +88,19 @@ class Meta: user = factory.SubFactory(UserFactory) algorithm = factory.SubFactory(AlgorithmFactory) + + +class AlgorithmInterfaceFactory(factory.django.DjangoModelFactory): + class Meta: + model = AlgorithmInterface + + @classmethod + def _create(cls, model_class, *args, **kwargs): + manager = cls._get_manager(model_class) + inputs = kwargs.pop("inputs", None) + outputs = kwargs.pop("outputs", None) + if not inputs: + inputs = [ComponentInterfaceFactory()] + if not outputs: + outputs = [ComponentInterfaceFactory()] + return manager.create(*args, inputs=inputs, outputs=outputs, **kwargs) diff --git a/app/tests/algorithms_tests/test_admin.py b/app/tests/algorithms_tests/test_admin.py deleted file mode 100644 index 0b33043456..0000000000 --- a/app/tests/algorithms_tests/test_admin.py +++ /dev/null @@ -1,12 +0,0 @@ -import pytest - -from grandchallenge.algorithms.admin import AlgorithmAdmin -from tests.components_tests.factories import ComponentInterfaceFactory - - -@pytest.mark.django_db -def test_disjoint_interfaces(): - i = ComponentInterfaceFactory() - form = AlgorithmAdmin.form(data={"inputs": [i.pk], "outputs": [i.pk]}) - assert form.is_valid() is False - assert "The sets of Inputs and Outputs must be unique" in str(form.errors) diff --git a/app/tests/algorithms_tests/test_forms.py b/app/tests/algorithms_tests/test_forms.py index 87ad0a2bf5..a16d38b695 100644 --- a/app/tests/algorithms_tests/test_forms.py +++ b/app/tests/algorithms_tests/test_forms.py @@ -5,6 +5,7 @@ from django.core.validators import MaxValueValidator, MinValueValidator from factory.django import ImageField +from grandchallenge.algorithms.admin import AlgorithmInterfaceAdminForm from grandchallenge.algorithms.forms import ( AlgorithmForm, AlgorithmForPhaseForm, @@ -35,6 +36,7 @@ from tests.algorithms_tests.factories import ( AlgorithmFactory, AlgorithmImageFactory, + AlgorithmInterfaceFactory, AlgorithmJobFactory, AlgorithmModelFactory, AlgorithmPermissionRequestFactory, @@ -155,7 +157,6 @@ def test_algorithm_create(client, uploaded_image): VerificationFactory(user=creator, is_verified=True) ws = WorkstationFactory() - ci = ComponentInterface.objects.get(slug="generic-medical-image") def try_create_algorithm(): return get_view_for_user( @@ -166,8 +167,7 @@ def try_create_algorithm(): "title": "foo bar", "logo": uploaded_image(), "workstation": ws.pk, - "inputs": [ci.pk], - "outputs": [ComponentInterfaceFactory().pk], + "interfaces": AlgorithmInterfaceFactory(), "minimum_credits_per_job": 20, "job_requires_gpu_type": GPUTypeChoices.NO_GPU, "job_requires_memory_gb": 4, @@ -407,16 +407,6 @@ def create_algorithm_with_input(slug): return alg, creator -@pytest.mark.django_db -def test_disjoint_interfaces(): - i = ComponentInterfaceFactory() - form = AlgorithmForm( - user=UserFactory(), data={"inputs": [i.pk], "outputs": [i.pk]} - ) - assert form.is_valid() is False - assert "The sets of Inputs and Outputs must be unique" in str(form.errors) - - @pytest.mark.django_db def test_publish_algorithm(): algorithm = AlgorithmFactory() @@ -889,3 +879,28 @@ def test_algorithm_for_phase_form_memory_limited(): ) assert max_validator is not None assert max_validator.limit_value == 32 + + +@pytest.mark.django_db +def test_algorithm_interface_check_for_uniqueness(): + interface = AlgorithmInterfaceFactory() + ci1, ci2 = ComponentInterfaceFactory.create_batch(2) + interface.inputs.set([ci1]) + interface.outputs.set([ci2]) + + form = AlgorithmInterfaceAdminForm( + data={"inputs": [ci1], "outputs": [ci2]} + ) + assert form.is_valid() is False + assert ( + "An AlgorithmIO with the same combination of inputs and outputs already exists." + in str(form.errors) + ) + + +@pytest.mark.django_db +def test_algorithm_interface_disjoint_interfaces(): + ci = ComponentInterfaceFactory() + form = AlgorithmInterfaceAdminForm(data={"inputs": [ci], "outputs": [ci]}) + assert form.is_valid() is False + assert "The sets of Inputs and Outputs must be unique" in str(form.errors) diff --git a/app/tests/algorithms_tests/test_models.py b/app/tests/algorithms_tests/test_models.py index 8ca3879413..f1c82b1b8a 100644 --- a/app/tests/algorithms_tests/test_models.py +++ b/app/tests/algorithms_tests/test_models.py @@ -6,14 +6,18 @@ from django.contrib.auth import get_user_model from django.core.exceptions import ObjectDoesNotExist, ValidationError from django.core.files.base import ContentFile +from django.db import IntegrityError, transaction from django.db.models import ProtectedError from django.test import TestCase from django.utils.timezone import now from grandchallenge.algorithms.models import ( Algorithm, + AlgorithmAlgorithmInterface, + AlgorithmInterface, AlgorithmUserCredit, Job, + get_existing_interface_for_inputs_and_outputs, ) from grandchallenge.components.models import ( CIVData, @@ -24,6 +28,7 @@ from tests.algorithms_tests.factories import ( AlgorithmFactory, AlgorithmImageFactory, + AlgorithmInterfaceFactory, AlgorithmJobFactory, AlgorithmModelFactory, AlgorithmUserCreditFactory, @@ -1379,3 +1384,116 @@ def test_active_credits_with_spent_credits(self): algorithm_image.get_remaining_non_complimentary_jobs(user=user) == 5 ) + + +@pytest.mark.django_db +def test_algorithm_interface_cannot_be_deleted(): + interface, _, _ = AlgorithmInterfaceFactory.create_batch(3) + + with pytest.raises(ValidationError): + interface.delete() + + with pytest.raises(NotImplementedError): + AlgorithmInterface.objects.delete() + + +@pytest.mark.django_db +def test_algorithmalgorithminterface_unique_constraints(): + interface1, interface2 = AlgorithmInterfaceFactory.create_batch(2) + algorithm = AlgorithmFactory() + + io = AlgorithmAlgorithmInterface.objects.create( + interface=interface1, algorithm=algorithm, is_default=True + ) + + # cannot add a second default interface to an algorithm + with pytest.raises(IntegrityError): + with transaction.atomic(): + AlgorithmAlgorithmInterface.objects.create( + interface=interface2, algorithm=algorithm, is_default=True + ) + + # cannot add a second time the same interface for the same algorithm + with pytest.raises(IntegrityError): + with transaction.atomic(): + AlgorithmAlgorithmInterface.objects.create( + interface=interface1, algorithm=algorithm, is_default=False + ) + + # but you can update an existing entry from default to non-default + io.is_default = False + io.save() + assert AlgorithmAlgorithmInterface.objects.count() == 1 + assert not AlgorithmAlgorithmInterface.objects.get().is_default + + +@pytest.mark.parametrize( + "inputs, outputs, expected_output", + ( + ([1], [2], 1), + ([1, 2], [3, 4], 2), + ([3, 4, 5], [6], 3), + ([1], [3, 4, 6], 4), + ([5, 6], [2], 5), + ([1], [3], None), + ([1], [3, 4], None), + ([2], [6], None), + ([2], [1], None), + ([3, 4, 5], [1], None), + ([1], [3, 4], None), + ([1, 3], [4], None), + ), +) +@pytest.mark.django_db +def test_get_existing_interface_for_inputs_and_outputs( + inputs, outputs, expected_output +): + io1, io2, io3, io4, io5, io6 = AlgorithmInterfaceFactory.create_batch(6) + ci1, ci2, ci3, ci4, ci5, ci6 = ComponentInterfaceFactory.create_batch(6) + + interfaces = [io1, io2, io3, io4, io5, io6] + cis = [ci1, ci2, ci3, ci4, ci5, ci6] + + io1.inputs.set([ci1]) + io2.inputs.set([ci1, ci2]) + io3.inputs.set([ci3, ci4, ci5]) + io4.inputs.set([ci1]) + io5.inputs.set([ci5, ci6]) + io6.inputs.set([ci1, ci2]) + + io1.outputs.set([ci2]) + io2.outputs.set([ci3, ci4]) + io3.outputs.set([ci6]) + io4.outputs.set([ci3, ci4, ci6]) + io5.outputs.set([ci2]) + io6.outputs.set([ci4]) + + inputs = [cis[i - 1] for i in inputs] + outputs = [cis[i - 1] for i in outputs] + + existing_interface = get_existing_interface_for_inputs_and_outputs( + inputs=inputs, outputs=outputs + ) + + if expected_output: + assert existing_interface == interfaces[expected_output - 1] + else: + assert not existing_interface + + +@pytest.mark.django_db +def test_algorithminterface_create(): + inputs = [ComponentInterfaceFactory(), ComponentInterfaceFactory()] + outputs = [ComponentInterfaceFactory(), ComponentInterfaceFactory()] + + with pytest.raises(TypeError) as e: + AlgorithmInterface.objects.create() + + assert ( + "AlgorithmInterfaceManager.create() missing 2 required keyword-only arguments: 'inputs' and 'outputs'" + in str(e) + ) + + io = AlgorithmInterface.objects.create(inputs=inputs, outputs=outputs) + assert list(io.inputs.all()) == inputs + assert list(io.outputs.all()) == outputs diff --git a/app/tests/algorithms_tests/test_views.py b/app/tests/algorithms_tests/test_views.py index 2f1d3ee1d1..220ad5959c 100644 --- a/app/tests/algorithms_tests/test_views.py +++ b/app/tests/algorithms_tests/test_views.py @@ -1635,31 +1635,6 @@ def test_algorithm_image_activate( assert i2.is_in_registry -@pytest.mark.django_db -@pytest.mark.parametrize("interfaces_editable", (True, False)) -def test_algorithm_interfaces_editable(client, interfaces_editable): - creator = UserFactory() - VerificationFactory(user=creator, is_verified=True) - - if interfaces_editable: - assign_perm("algorithms.add_algorithm", creator) - - alg = AlgorithmFactory() - alg.add_editor(user=creator) - - response = get_view_for_user( - viewname="algorithms:update", - client=client, - reverse_kwargs={"slug": alg.slug}, - user=creator, - ) - - assert ("inputs" in response.context["form"].fields) is interfaces_editable - assert ( - "outputs" in response.context["form"].fields - ) is interfaces_editable - - @pytest.mark.django_db def test_job_time_limit(client): algorithm = AlgorithmFactory(time_limit=600) From 2f9a6b15eb9487869ecdd6de992afe356d3f11a1 Mon Sep 17 00:00:00 2001 From: Anne Mickan Date: Thu, 12 Dec 2024 13:41:27 +0100 Subject: [PATCH 02/18] Add create and list views for algorithm interfaces (#3735) Part of https://github.com/DIAGNijmegen/rse-roadmap/issues/153 This adds a create and list view for algorithm interfaces that is accessible to algorithm editors with the global `add_algorithm` permission. Create view: ![image](https://github.com/user-attachments/assets/4af87e6b-ef54-4a79-91e5-6df9f4ffb8c3) List view: ![image](https://github.com/user-attachments/assets/221b9cc4-bf01-43e3-833c-db6a082106cb) --------- Co-authored-by: James Meakin <12661555+jmsmkn@users.noreply.github.com> --- app/grandchallenge/algorithms/admin.py | 32 ++-- app/grandchallenge/algorithms/forms.py | 68 ++++++-- app/grandchallenge/algorithms/models.py | 9 ++ .../algorithms/algorithm_detail.html | 4 + .../algorithmalgorithminterface_list.html | 52 ++++++ .../algorithms/algorithminterface_form.html | 42 +++++ app/grandchallenge/algorithms/urls.py | 12 ++ app/grandchallenge/algorithms/views.py | 60 +++++++ app/tests/algorithms_tests/test_forms.py | 152 +++++++++++++++--- app/tests/algorithms_tests/test_models.py | 13 ++ app/tests/algorithms_tests/test_views.py | 107 +++++++++++- 11 files changed, 502 insertions(+), 49 deletions(-) create mode 100644 app/grandchallenge/algorithms/templates/algorithms/algorithmalgorithminterface_list.html create mode 100644 app/grandchallenge/algorithms/templates/algorithms/algorithminterface_form.html diff --git a/app/grandchallenge/algorithms/admin.py b/app/grandchallenge/algorithms/admin.py index 044393a2dc..966a4af6f1 100644 --- a/app/grandchallenge/algorithms/admin.py +++ b/app/grandchallenge/algorithms/admin.py @@ -2,13 +2,12 @@ from django.contrib import admin from django.contrib.admin import ModelAdmin -from django.core.exceptions import ObjectDoesNotExist, ValidationError +from django.core.exceptions import ObjectDoesNotExist from django.db.models import Count, Sum from django.forms import ModelForm from django.utils.html import format_html from guardian.admin import GuardedModelAdmin -from grandchallenge.algorithms.forms import AlgorithmInterfaceBaseForm from grandchallenge.algorithms.models import ( Algorithm, AlgorithmAlgorithmInterface, @@ -52,12 +51,17 @@ class Meta: @admin.register(Algorithm) class AlgorithmAdmin(GuardedModelAdmin): - readonly_fields = ("algorithm_forge_json", "inputs", "outputs") + readonly_fields = ( + "algorithm_forge_json", + "default_interface", + "inputs", + "outputs", + ) list_display = ( "title", "created", "public", - "default_io", + "default_interface", "time_limit", "job_requires_gpu_type", "job_requires_memory_gb", @@ -79,11 +83,6 @@ def algorithm_forge_json(obj): "
{json_desc}
", json_desc=json.dumps(json_desc, indent=2) ) - def default_io(self, obj): - return AlgorithmAlgorithmInterface.objects.get( - algorithm=obj, is_default=True - ) - def get_queryset(self, request): queryset = super().get_queryset(request) queryset = queryset.annotate( @@ -244,16 +243,6 @@ class AlgorithmModelAdmin(GuardedModelAdmin): readonly_fields = ("creator", "algorithm", "sha256", "size_in_storage") -class AlgorithmInterfaceAdminForm(AlgorithmInterfaceBaseForm): - def clean(self): - cleaned_data = super().clean() - if cleaned_data["existing_io"]: - raise ValidationError( - "An AlgorithmIO with the same combination of inputs and outputs already exists." - ) - return cleaned_data - - @admin.register(AlgorithmInterface) class AlgorithmInterfaceAdmin(GuardedModelAdmin): list_display = ( @@ -262,10 +251,10 @@ class AlgorithmInterfaceAdmin(GuardedModelAdmin): "algorithm_outputs", ) search_fields = ( + "pk", "inputs__slug", "outputs__slug", ) - form = AlgorithmInterfaceAdminForm def algorithm_inputs(self, obj): return oxford_comma(obj.inputs.all()) @@ -279,6 +268,9 @@ def has_change_permission(self, request, obj=None): def has_delete_permission(self, request, obj=None): return False + def has_add_permission(self, request, obj=None): + return False + @admin.register(AlgorithmAlgorithmInterface) class AlgorithmAlgorithmInterfaceAdmin(GuardedModelAdmin): diff --git a/app/grandchallenge/algorithms/forms.py b/app/grandchallenge/algorithms/forms.py index 8c3b7935b2..8fd8070ec8 100644 --- a/app/grandchallenge/algorithms/forms.py +++ b/app/grandchallenge/algorithms/forms.py @@ -26,11 +26,13 @@ from django.db.models import Count, Exists, Max, OuterRef, Q from django.db.transaction import on_commit from django.forms import ( + BooleanField, CharField, Form, HiddenInput, ModelChoiceField, ModelForm, + ModelMultipleChoiceField, Select, TextInput, URLField, @@ -44,12 +46,12 @@ from grandchallenge.algorithms.models import ( Algorithm, + AlgorithmAlgorithmInterface, AlgorithmImage, AlgorithmInterface, AlgorithmModel, AlgorithmPermissionRequest, Job, - get_existing_interface_for_inputs_and_outputs, ) from grandchallenge.algorithms.serializers import ( AlgorithmImageSerializer, @@ -1345,10 +1347,35 @@ def clean_algorithm_model(self): return algorithm_model -class AlgorithmInterfaceBaseForm(SaveFormInitMixin, ModelForm): +class AlgorithmInterfaceForm(SaveFormInitMixin, ModelForm): + inputs = ModelMultipleChoiceField( + queryset=ComponentInterface.objects.exclude( + slug__in=[*NON_ALGORITHM_INTERFACES, "results-json-file"] + ), + widget=Select2MultipleWidget, + ) + outputs = ModelMultipleChoiceField( + queryset=ComponentInterface.objects.exclude( + slug__in=[*NON_ALGORITHM_INTERFACES, "results-json-file"] + ), + widget=Select2MultipleWidget, + ) + set_as_default = BooleanField(required=False) + class Meta: model = AlgorithmInterface - fields = ("inputs", "outputs") + fields = ( + "inputs", + "outputs", + "set_as_default", + ) + + def __init__(self, *args, algorithm, **kwargs): + super().__init__(*args, **kwargs) + self._algorithm = algorithm + + if not self._algorithm.default_interface: + self.fields["set_as_default"].initial = True def clean(self): cleaned_data = super().clean() @@ -1369,11 +1396,34 @@ def clean(self): f"{oxford_comma(duplicate_interfaces)} present in both" ) - cleaned_data["existing_io"] = ( - get_existing_interface_for_inputs_and_outputs( - inputs=inputs, - outputs=outputs, - ) + return cleaned_data + + def save(self): + interface = AlgorithmInterface.objects.create( + inputs=self.cleaned_data["inputs"], + outputs=self.cleaned_data["outputs"], ) - return cleaned_data + if self.cleaned_data["set_as_default"]: + AlgorithmAlgorithmInterface.objects.filter( + algorithm=self._algorithm + ).update(is_default=False) + + matched_rows = AlgorithmAlgorithmInterface.objects.filter( + algorithm=self._algorithm, interface=interface + ).update(is_default=self.cleaned_data["set_as_default"]) + + if matched_rows == 0: + self._algorithm.interfaces.add( + interface, + through_defaults={ + "is_default": self.cleaned_data["set_as_default"] + }, + ) + elif matched_rows > 1: + raise RuntimeError( + "This algorithm and interface are associated " + "with each other more than once." + ) + + return interface diff --git a/app/grandchallenge/algorithms/models.py b/app/grandchallenge/algorithms/models.py index 162276b8c1..bb98a22e05 100644 --- a/app/grandchallenge/algorithms/models.py +++ b/app/grandchallenge/algorithms/models.py @@ -504,6 +504,15 @@ def default_workstation(self): return w + @cached_property + def default_interface(self): + try: + return self.interfaces.get( + algorithmalgorithminterface__is_default=True + ) + except ObjectDoesNotExist: + return None + def is_editor(self, user): return user.groups.filter(pk=self.editors_group.pk).exists() diff --git a/app/grandchallenge/algorithms/templates/algorithms/algorithm_detail.html b/app/grandchallenge/algorithms/templates/algorithms/algorithm_detail.html index 408501fd94..5180ab6365 100644 --- a/app/grandchallenge/algorithms/templates/algorithms/algorithm_detail.html +++ b/app/grandchallenge/algorithms/templates/algorithms/algorithm_detail.html @@ -36,6 +36,10 @@ {% if "change_algorithm" in algorithm_perms %} {% if perms.algorithms.add_algorithm %} +  Interfaces + +
  • Algorithms +
    • +
  • {{ algorithm.title }} + +
    • +
  • Interfaces +
    • + +{% endblock %} + +{% block content %} +

    Algorithm Interfaces for {{ algorithm }}

    + +

    + The following interfaces are configured for your algorithm: +

    +

    Add new interface

    + +
    + + + + + + + + {% for object in object_list %} + + + + + + {% empty %} + + {% endfor %} + +
    InputsOutputsDefault
    {{ object.interface.inputs.all|oxford_comma }}{{ object.interface.outputs.all|oxford_comma }}{% if object.is_default %}{% else %}{% endif %}
    This algorithm does not have any interfaces defined yet.
    +
    +{% endblock %} diff --git a/app/grandchallenge/algorithms/templates/algorithms/algorithminterface_form.html b/app/grandchallenge/algorithms/templates/algorithms/algorithminterface_form.html new file mode 100644 index 0000000000..356412ffa0 --- /dev/null +++ b/app/grandchallenge/algorithms/templates/algorithms/algorithminterface_form.html @@ -0,0 +1,42 @@ +{% extends "base.html" %} +{% load crispy_forms_tags %} + +{% block title %} + Create Interface - {{ algorithm.title }} - {{ block.super }} +{% endblock %} + +{% block breadcrumbs %} +
      +
    1. Algorithms +
      2. +
    2. {{ algorithm.title }} +
      3. +
    3. Interfaces +
      4. +
    4. Create Interface +
      5. +
    +{% endblock %} + +{% block content %} + +

    Create An Algorithm Interface

    +
    +

    + Create an interface for your algorithm: define any combination of inputs and outputs, and optionally mark the interface as default for the algorithm. +

    +

    + Please see the list of input options and the + list of output options + for more information and examples. +

    +

    + If you cannot find suitable inputs or outputs, please contact support@grand-challenge.org. +

    + + {% crispy form %} + +{% endblock %} diff --git a/app/grandchallenge/algorithms/urls.py b/app/grandchallenge/algorithms/urls.py index 339c1c9207..72d406efbf 100644 --- a/app/grandchallenge/algorithms/urls.py +++ b/app/grandchallenge/algorithms/urls.py @@ -12,6 +12,8 @@ AlgorithmImageTemplate, AlgorithmImageUpdate, AlgorithmImportView, + AlgorithmInterfaceForAlgorithmCreate, + AlgorithmInterfacesForAlgorithmList, AlgorithmList, AlgorithmModelCreate, AlgorithmModelDetail, @@ -50,6 +52,16 @@ AlgorithmDescriptionUpdate.as_view(), name="description-update", ), + path( + "/interfaces/", + AlgorithmInterfacesForAlgorithmList.as_view(), + name="interface-list", + ), + path( + "/interfaces/create/", + AlgorithmInterfaceForAlgorithmCreate.as_view(), + name="interface-create", + ), path( "/repository/", AlgorithmRepositoryUpdate.as_view(), diff --git a/app/grandchallenge/algorithms/views.py b/app/grandchallenge/algorithms/views.py index 8005f8c76c..db9f837d6f 100644 --- a/app/grandchallenge/algorithms/views.py +++ b/app/grandchallenge/algorithms/views.py @@ -5,6 +5,7 @@ from django.conf import settings from django.contrib import messages from django.contrib.auth.mixins import ( + AccessMixin, PermissionRequiredMixin, UserPassesTestMixin, ) @@ -46,6 +47,7 @@ AlgorithmImageForm, AlgorithmImageUpdateForm, AlgorithmImportForm, + AlgorithmInterfaceForm, AlgorithmModelForm, AlgorithmModelUpdateForm, AlgorithmModelVersionControlForm, @@ -61,7 +63,9 @@ ) from grandchallenge.algorithms.models import ( Algorithm, + AlgorithmAlgorithmInterface, AlgorithmImage, + AlgorithmInterface, AlgorithmModel, AlgorithmPermissionRequest, Job, @@ -1091,3 +1095,59 @@ def get(self, *_, **__): filename=f"{algorithm.slug}-template.zip", content_type="application/zip", ) + + +class AlgorithmInterfacePermissionMixin(AccessMixin): + @property + def algorithm(self): + return get_object_or_404(Algorithm, slug=self.kwargs["slug"]) + + def dispatch(self, request, *args, **kwargs): + if request.user.has_perm( + "change_algorithm", self.algorithm + ) and request.user.has_perm("algorithms.add_algorithm"): + return super().dispatch(request, *args, **kwargs) + else: + return self.handle_no_permission() + + +class AlgorithmInterfaceForAlgorithmCreate( + AlgorithmInterfacePermissionMixin, CreateView +): + model = AlgorithmInterface + form_class = AlgorithmInterfaceForm + success_message = "Algorithm interface successfully added" + + def get_success_url(self): + return reverse( + "algorithms:interface-list", kwargs={"slug": self.algorithm.slug} + ) + + def get_context_data(self, *args, **kwargs): + context = super().get_context_data(*args, **kwargs) + context.update({"algorithm": self.algorithm}) + return context + + def get_form_kwargs(self): + kwargs = super().get_form_kwargs() + kwargs.update({"algorithm": self.algorithm}) + return kwargs + + +class AlgorithmInterfacesForAlgorithmList( + AlgorithmInterfacePermissionMixin, ListView +): + model = AlgorithmAlgorithmInterface + + def get_queryset(self): + qs = super().get_queryset() + return qs.filter(algorithm=self.algorithm) + + def get_context_data(self, *args, **kwargs): + context = super().get_context_data(*args, **kwargs) + context.update( + { + "algorithm": self.algorithm, + } + ) + return context diff --git a/app/tests/algorithms_tests/test_forms.py b/app/tests/algorithms_tests/test_forms.py index 825caf2caa..07b9bbeca2 100644 --- a/app/tests/algorithms_tests/test_forms.py +++ b/app/tests/algorithms_tests/test_forms.py @@ -5,10 +5,10 @@ from django.core.validators import MaxValueValidator, MinValueValidator from factory.django import ImageField -from grandchallenge.algorithms.admin import AlgorithmInterfaceAdminForm from grandchallenge.algorithms.forms import ( AlgorithmForm, AlgorithmForPhaseForm, + AlgorithmInterfaceForm, AlgorithmModelForm, AlgorithmModelVersionControlForm, AlgorithmPublishForm, @@ -18,6 +18,7 @@ ) from grandchallenge.algorithms.models import ( Algorithm, + AlgorithmAlgorithmInterface, AlgorithmPermissionRequest, Job, ) @@ -1297,27 +1298,12 @@ def test_algorithm_for_phase_form_memory_limited(): assert max_validator.limit_value == 32 -@pytest.mark.django_db -def test_algorithm_interface_check_for_uniqueness(): - interface = AlgorithmInterfaceFactory() - ci1, ci2 = ComponentInterfaceFactory.create_batch(2) - interface.inputs.set([ci1]) - interface.outputs.set([ci2]) - - form = AlgorithmInterfaceAdminForm( - data={"inputs": [ci1], "outputs": [ci2]} - ) - assert form.is_valid() is False - assert ( - "An AlgorithmIO with the same combination of inputs and outputs already exists." - in str(form.errors) - ) - - @pytest.mark.django_db def test_algorithm_interface_disjoint_interfaces(): ci = ComponentInterfaceFactory() - form = AlgorithmInterfaceAdminForm(data={"inputs": [ci], "outputs": [ci]}) + form = AlgorithmInterfaceForm( + algorithm=AlgorithmFactory(), data={"inputs": [ci], "outputs": [ci]} + ) assert form.is_valid() is False assert "The sets of Inputs and Outputs must be unique" in str(form.errors) @@ -1347,3 +1333,131 @@ def test_algorithm_for_phase_form_memory(): ) assert max_validator is not None assert max_validator.limit_value == 42 + + +class TestAlgorithmInterfaceForm: + @pytest.mark.django_db + def test_set_as_default_initial_value(self): + alg = AlgorithmFactory() + + form = AlgorithmInterfaceForm( + algorithm=alg, + ) + assert form.fields["set_as_default"].initial + + alg.interfaces.add( + AlgorithmInterfaceFactory(), through_defaults={"is_default": True} + ) + + del alg.default_interface + + form = AlgorithmInterfaceForm( + algorithm=alg, + ) + assert not form.fields["set_as_default"].initial + + @pytest.mark.django_db + def test_existing_io_is_reused(self): + inp = ComponentInterfaceFactory() + out = ComponentInterfaceFactory() + io = AlgorithmInterfaceFactory() + io.inputs.set([inp]) + io.outputs.set([out]) + + alg = AlgorithmFactory() + + form = AlgorithmInterfaceForm( + algorithm=alg, + data={ + "inputs": [inp.pk], + "outputs": [out.pk], + "set_as_default": False, + }, + ) + assert form.is_valid() + new_io = form.save() + + assert io == new_io + + @pytest.mark.django_db + def test_new_default_interface_updates_related_interfaces(self): + ci_1, ci_2 = ComponentInterfaceFactory.create_batch(2) + alg = AlgorithmFactory() + io = AlgorithmInterfaceFactory() + alg.interfaces.add(io, through_defaults={"is_default": True}) + + old_iot = AlgorithmAlgorithmInterface.objects.get() + + form = AlgorithmInterfaceForm( + algorithm=alg, + data={ + "inputs": [ci_1.pk], + "outputs": [ci_2.pk], + "set_as_default": True, + }, + ) + form.is_valid() + new_io = form.save() + new_iot = AlgorithmAlgorithmInterface.objects.get(interface=new_io) + old_iot.refresh_from_db() + + assert new_io != io + assert new_iot.is_default + assert not old_iot.is_default + + @pytest.mark.django_db + def test_default_interface_for_algorithm_not_updated_when_adding_new_non_default_interface( + self, + ): + alg = AlgorithmFactory() + io = AlgorithmInterfaceFactory() + alg.interfaces.add(io, through_defaults={"is_default": True}) + old_iot = AlgorithmAlgorithmInterface.objects.get() + + ci_1, ci_2 = ComponentInterfaceFactory.create_batch(2) + + form = AlgorithmInterfaceForm( + algorithm=alg, + data={ + "inputs": [ci_1.pk], + "outputs": [ci_2.pk], + "set_as_default": False, + }, + ) + form.is_valid() + new_io = form.save() + old_iot.refresh_from_db() + new_iot = AlgorithmAlgorithmInterface.objects.get(interface=new_io) + + assert new_io != io + assert not new_iot.is_default + assert old_iot.is_default + + @pytest.mark.django_db + def test_is_default_is_updated_when_adding_an_already_existing_interface( + self, + ): + alg = AlgorithmFactory() + ci_1, ci_2 = ComponentInterfaceFactory.create_batch(2) + + io = AlgorithmInterfaceFactory() + io.inputs.set([ci_1]) + io.outputs.set([ci_2]) + + alg.interfaces.add(io, through_defaults={"is_default": True}) + old_iot = AlgorithmAlgorithmInterface.objects.get() + + form = AlgorithmInterfaceForm( + algorithm=alg, + data={ + "inputs": [ci_1.pk], + "outputs": [ci_2.pk], + "set_as_default": False, + }, + ) + form.is_valid() + new_io = form.save() + old_iot.refresh_from_db() + + assert new_io == io + assert not old_iot.is_default diff --git a/app/tests/algorithms_tests/test_models.py b/app/tests/algorithms_tests/test_models.py index 2de0cdeac0..6c7d8028be 100644 --- a/app/tests/algorithms_tests/test_models.py +++ b/app/tests/algorithms_tests/test_models.py @@ -1497,3 +1497,16 @@ def test_algorithminterface_create(): io = AlgorithmInterface.objects.create(inputs=inputs, outputs=outputs) assert list(io.inputs.all()) == inputs assert list(io.outputs.all()) == outputs + + +@pytest.mark.django_db +def test_has_default_interface(): + alg1, alg2, alg3 = AlgorithmFactory.create_batch(3) + io1, io2 = AlgorithmInterfaceFactory.create_batch(2) + + alg1.interfaces.add(io1, through_defaults={"is_default": True}) + alg2.interfaces.add(io2) + + assert alg1.default_interface == io1 + assert not alg2.default_interface + assert not alg3.default_interface diff --git a/app/tests/algorithms_tests/test_views.py b/app/tests/algorithms_tests/test_views.py index 28073abbfa..822110fbec 100644 --- a/app/tests/algorithms_tests/test_views.py +++ b/app/tests/algorithms_tests/test_views.py @@ -15,7 +15,13 @@ from guardian.shortcuts import assign_perm, remove_perm from requests import put -from grandchallenge.algorithms.models import Algorithm, AlgorithmImage, Job +from grandchallenge.algorithms.models import ( + Algorithm, + AlgorithmAlgorithmInterface, + AlgorithmImage, + AlgorithmInterface, + Job, +) from grandchallenge.algorithms.views import JobsList from grandchallenge.components.models import ( ComponentInterface, @@ -29,6 +35,7 @@ from tests.algorithms_tests.factories import ( AlgorithmFactory, AlgorithmImageFactory, + AlgorithmInterfaceFactory, AlgorithmJobFactory, AlgorithmModelFactory, AlgorithmPermissionRequestFactory, @@ -2138,3 +2145,101 @@ def test_algorithm_template_download(client): assert ( file_name in zip_file.namelist() ), f"{file_name} is in the ZIP file" + + +@pytest.mark.parametrize( + "viewname", ["algorithms:interface-list", "algorithms:interface-create"] +) +@pytest.mark.django_db +def test_algorithm_interface_view_permission(client, viewname): + ( + user_with_alg_add_perm, + user_without_alg_add_perm, + algorithm_editor_with_alg_add, + algorithm_editor_without_alg_add, + ) = UserFactory.create_batch(4) + assign_perm("algorithms.add_algorithm", user_with_alg_add_perm) + assign_perm("algorithms.add_algorithm", algorithm_editor_with_alg_add) + + alg = AlgorithmFactory() + alg.add_editor(algorithm_editor_with_alg_add) + alg.add_editor(algorithm_editor_without_alg_add) + + for user, status in [ + [user_with_alg_add_perm, 403], + [user_without_alg_add_perm, 403], + [algorithm_editor_with_alg_add, 200], + [algorithm_editor_without_alg_add, 403], + ]: + response = get_view_for_user( + viewname=viewname, + client=client, + reverse_kwargs={"slug": alg.slug}, + user=user, + ) + assert response.status_code == status + + +@pytest.mark.django_db +def test_algorithm_interface_create(client): + user = UserFactory() + assign_perm("algorithms.add_algorithm", user) + alg = AlgorithmFactory() + alg.add_editor(user) + + ci_1 = ComponentInterfaceFactory() + ci_2 = ComponentInterfaceFactory() + + response = get_view_for_user( + viewname="algorithms:interface-create", + client=client, + method=client.post, + reverse_kwargs={"slug": alg.slug}, + data={ + "inputs": [ci_1.pk], + "outputs": [ci_2.pk], + "set_as_default": True, + }, + user=user, + ) + assert response.status_code == 302 + + assert AlgorithmInterface.objects.count() == 1 + io = AlgorithmInterface.objects.get() + assert io.inputs.get() == ci_1 + assert io.outputs.get() == ci_2 + + assert AlgorithmAlgorithmInterface.objects.count() == 1 + io_through = AlgorithmAlgorithmInterface.objects.get() + assert io_through.algorithm == alg + assert io_through.interface == io + assert io_through.is_default + + +@pytest.mark.django_db +def test_algorithm_interfaces_list_queryset(client): + user = UserFactory() + assign_perm("algorithms.add_algorithm", user) + alg, alg2 = AlgorithmFactory.create_batch(2) + alg.add_editor(user) + VerificationFactory(user=user, is_verified=True) + + io1, io2, io3, io4 = AlgorithmInterfaceFactory.create_batch(4) + + alg.interfaces.set([io1, io2]) + alg2.interfaces.set([io3, io4]) + + iots = AlgorithmAlgorithmInterface.objects.order_by("id").all() + + response = get_view_for_user( + viewname="algorithms:interface-list", + client=client, + reverse_kwargs={"slug": alg.slug}, + user=user, + ) + assert response.status_code == 200 + assert response.context["object_list"].count() == 2 + assert iots[0] in response.context["object_list"] + assert iots[1] in response.context["object_list"] + assert iots[2] not in response.context["object_list"] + assert iots[3] not in response.context["object_list"] From 6e2da3d8f27d30aac5c6729aa6a01131d5d156fa Mon Sep 17 00:00:00 2001 From: Anne Mickan Date: Wed, 18 Dec 2024 15:30:11 +0100 Subject: [PATCH 03/18] Require default interface for algorithm (#3759) Adds a check to both the form and the model that ensures that an algorithm has a default interface. We're not using the through model directly to create an interface for an algorithm through the UI, hence the check in 2 places. --- app/grandchallenge/algorithms/forms.py | 8 ++++++++ app/grandchallenge/algorithms/models.py | 6 ++++++ app/tests/algorithms_tests/test_forms.py | 26 +++++++++++++++++++++++- 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/app/grandchallenge/algorithms/forms.py b/app/grandchallenge/algorithms/forms.py index bc37bc1503..070a0cdd9c 100644 --- a/app/grandchallenge/algorithms/forms.py +++ b/app/grandchallenge/algorithms/forms.py @@ -1377,6 +1377,14 @@ def __init__(self, *args, algorithm, **kwargs): if not self._algorithm.default_interface: self.fields["set_as_default"].initial = True + def clean_set_as_default(self): + set_as_default = self.cleaned_data["set_as_default"] + + if not set_as_default and not self._algorithm.default_interface: + raise ValidationError("Your algorithm needs a default interface.") + + return set_as_default + def clean(self): cleaned_data = super().clean() diff --git a/app/grandchallenge/algorithms/models.py b/app/grandchallenge/algorithms/models.py index bb98a22e05..56c2d95595 100644 --- a/app/grandchallenge/algorithms/models.py +++ b/app/grandchallenge/algorithms/models.py @@ -634,6 +634,12 @@ class Meta: def __str__(self): return str(self.interface) + def clean(self): + super().clean() + + if not self.is_default and not self.algorithm.default_interface: + raise ValidationError("This algorithm needs a default interface.") + class AlgorithmUserObjectPermission(UserObjectPermissionBase): content_object = models.ForeignKey(Algorithm, on_delete=models.CASCADE) diff --git a/app/tests/algorithms_tests/test_forms.py b/app/tests/algorithms_tests/test_forms.py index 580ed73e33..fa023fcd24 100644 --- a/app/tests/algorithms_tests/test_forms.py +++ b/app/tests/algorithms_tests/test_forms.py @@ -1364,6 +1364,30 @@ def test_algorithm_interface_disjoint_interfaces(): assert "The sets of Inputs and Outputs must be unique" in str(form.errors) +@pytest.mark.django_db +def test_algorithm_interface_default_interface_required(): + ci1, ci2 = ComponentInterfaceFactory.create_batch(2) + alg = AlgorithmFactory() + form = AlgorithmInterfaceForm( + algorithm=alg, + data={"inputs": [ci1], "outputs": [ci2], "set_as_default": False}, + ) + assert form.is_valid() is False + assert "Your algorithm needs a default interface" in str( + form.errors["set_as_default"] + ) + + alg.interfaces.add( + AlgorithmInterfaceFactory(), through_defaults={"is_default": True} + ) + del alg.default_interface + form = AlgorithmInterfaceForm( + algorithm=alg, + data={"inputs": [ci1], "outputs": [ci2], "set_as_default": False}, + ) + assert form.is_valid() + + def test_algorithm_for_phase_form_memory(): form = AlgorithmForPhaseForm( workstation_config=WorkstationConfigFactory.build(), @@ -1427,7 +1451,7 @@ def test_existing_io_is_reused(self): data={ "inputs": [inp.pk], "outputs": [out.pk], - "set_as_default": False, + "set_as_default": True, }, ) assert form.is_valid() From f9e5d5ad4c601e47638d8b1d3339056ab6f7e8fb Mon Sep 17 00:00:00 2001 From: Anne Mickan Date: Thu, 16 Jan 2025 16:12:59 +0100 Subject: [PATCH 04/18] Enable interface selection for algorithm job (#3753) This enables selecting an interface when trying out an algorithm. Interface selection happens in a separate form and view, but is unnecessary if an algorithm has only 1 interface. ![image](https://github.com/user-attachments/assets/ee3c3f99-1b71-4e26-9db3-223d8625d5f6) Part of https://github.com/DIAGNijmegen/rse-roadmap/issues/153 --- app/grandchallenge/algorithms/admin.py | 3 +- app/grandchallenge/algorithms/forms.py | 57 ++- ...rface_algorithminterfaceoutput_and_more.py | 10 + .../0065_create_interfaces_for_jobs.py | 27 ++ app/grandchallenge/algorithms/models.py | 9 +- .../algorithms/algorithm_detail.html | 6 +- .../templates/algorithms/job_detail.html | 2 +- .../templates/algorithms/job_form_create.html | 99 ++--- .../templates/algorithms/job_list.html | 2 +- .../algorithms/job_progress_detail.html | 2 +- app/grandchallenge/algorithms/urls.py | 12 +- app/grandchallenge/algorithms/views.py | 79 +++- app/tests/algorithms_tests/test_api.py | 1 + app/tests/algorithms_tests/test_forms.py | 91 +++-- app/tests/algorithms_tests/test_models.py | 67 +++- .../algorithms_tests/test_permissions.py | 12 +- .../algorithms_tests/test_serializers.py | 4 + app/tests/algorithms_tests/test_tasks.py | 132 +++++-- app/tests/algorithms_tests/test_views.py | 346 +++++++++--------- app/tests/conftest.py | 6 +- 20 files changed, 642 insertions(+), 325 deletions(-) create mode 100644 app/grandchallenge/algorithms/migrations/0065_create_interfaces_for_jobs.py diff --git a/app/grandchallenge/algorithms/admin.py b/app/grandchallenge/algorithms/admin.py index 966a4af6f1..4b1de23ec4 100644 --- a/app/grandchallenge/algorithms/admin.py +++ b/app/grandchallenge/algorithms/admin.py @@ -61,7 +61,6 @@ class AlgorithmAdmin(GuardedModelAdmin): "title", "created", "public", - "default_interface", "time_limit", "job_requires_gpu_type", "job_requires_memory_gb", @@ -216,6 +215,7 @@ class JobAdmin(GuardedModelAdmin): "task_on_success", "task_on_failure", "runtime_metrics", + "algorithm_interface", ) search_fields = ( "creator__username", @@ -245,6 +245,7 @@ class AlgorithmModelAdmin(GuardedModelAdmin): @admin.register(AlgorithmInterface) class AlgorithmInterfaceAdmin(GuardedModelAdmin): + readonly_fields = ("algorithm_inputs", "algorithm_outputs") list_display = ( "pk", "algorithm_inputs", diff --git a/app/grandchallenge/algorithms/forms.py b/app/grandchallenge/algorithms/forms.py index 070a0cdd9c..8ad5fc4ee3 100644 --- a/app/grandchallenge/algorithms/forms.py +++ b/app/grandchallenge/algorithms/forms.py @@ -37,7 +37,11 @@ TextInput, URLField, ) -from django.forms.widgets import MultipleHiddenInput, PasswordInput +from django.forms.widgets import ( + MultipleHiddenInput, + PasswordInput, + RadioSelect, +) from django.urls import Resolver404, resolve from django.utils.functional import cached_property from django.utils.html import format_html @@ -116,10 +120,18 @@ class JobCreateForm(SaveFormInitMixin, Form): creator = ModelChoiceField( queryset=None, disabled=True, required=False, widget=HiddenInput ) + algorithm_interface = ModelChoiceField( + queryset=None, + disabled=True, + required=True, + widget=HiddenInput, + ) - def __init__(self, *args, algorithm, user, **kwargs): + def __init__(self, *args, algorithm, user, interface, **kwargs): super().__init__(*args, **kwargs) + self._algorithm = algorithm + self.helper = FormHelper() self._user = user @@ -128,7 +140,11 @@ def __init__(self, *args, algorithm, user, **kwargs): ) self.fields["creator"].initial = self._user - self._algorithm = algorithm + self.fields["algorithm_interface"].queryset = ( + self._algorithm.interfaces.all() + ) + self.fields["algorithm_interface"].initial = interface + self._algorithm_image = self._algorithm.active_image active_model = self._algorithm.active_model @@ -145,7 +161,7 @@ def __init__(self, *args, algorithm, user, **kwargs): ) self.fields["algorithm_model"].initial = active_model - for inp in self._algorithm.inputs.all(): + for inp in interface.inputs.all(): prefixed_interface_slug = ( f"{INTERFACE_FORM_FIELD_PREFIX}{inp.slug}" ) @@ -1435,3 +1451,36 @@ def save(self): ) return interface + + +class JobInterfaceSelectForm(SaveFormInitMixin, Form): + algorithm_interface = ModelChoiceField( + queryset=None, + required=True, + help_text="Select an input-output combination to use for this job.", + widget=RadioSelect, + ) + + def __init__(self, *args, algorithm, **kwargs): + super().__init__(*args, **kwargs) + + self._algorithm = algorithm + + self.fields["algorithm_interface"].queryset = ( + self._algorithm.interfaces.all() + ) + self.fields["algorithm_interface"].initial = ( + self._algorithm.default_interface + ) + self.fields["algorithm_interface"].widget.choices = { + ( + interface.pk, + format_html( + "
    Inputs: {inputs}
    " + "
    Outputs: {outputs}
    ", + inputs=oxford_comma(interface.inputs.all()), + outputs=oxford_comma(interface.outputs.all()), + ), + ) + for interface in self._algorithm.interfaces.all() + } diff --git a/app/grandchallenge/algorithms/migrations/0063_algorithminterface_algorithminterfaceoutput_and_more.py b/app/grandchallenge/algorithms/migrations/0063_algorithminterface_algorithminterfaceoutput_and_more.py index 9a0997b11b..ffdcabea83 100644 --- a/app/grandchallenge/algorithms/migrations/0063_algorithminterface_algorithminterfaceoutput_and_more.py +++ b/app/grandchallenge/algorithms/migrations/0063_algorithminterface_algorithminterfaceoutput_and_more.py @@ -159,4 +159,14 @@ class Migration(migrations.Migration): name="unique_algorithm_interface_combination", ), ), + migrations.AddField( + model_name="job", + name="algorithm_interface", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.PROTECT, + to="algorithms.algorithminterface", + ), + ), ] diff --git a/app/grandchallenge/algorithms/migrations/0065_create_interfaces_for_jobs.py b/app/grandchallenge/algorithms/migrations/0065_create_interfaces_for_jobs.py new file mode 100644 index 0000000000..f1e116af3d --- /dev/null +++ b/app/grandchallenge/algorithms/migrations/0065_create_interfaces_for_jobs.py @@ -0,0 +1,27 @@ +from django.db import migrations + + +def add_algorithm_interfaces_to_jobs(apps, _schema_editor): + Algorithm = apps.get_model("algorithms", "Algorithm") # noqa: N806 + Job = apps.get_model("algorithms", "Job") # noqa: N806 + + for algorithm in Algorithm.objects.prefetch_related("interfaces").all(): + default_interface = algorithm.interfaces.get( + algorithmalgorithminterface__is_default=True + ) + Job.objects.filter(algorithm_image__algorithm=algorithm).update( + algorithm_interface=default_interface + ) + + +class Migration(migrations.Migration): + dependencies = [ + ( + "algorithms", + "0064_create_algorithm_interfaces", + ), + ] + + operations = [ + migrations.RunPython(add_algorithm_interfaces_to_jobs, elidable=True), + ] diff --git a/app/grandchallenge/algorithms/models.py b/app/grandchallenge/algorithms/models.py index 56c2d95595..6e5550cbc7 100644 --- a/app/grandchallenge/algorithms/models.py +++ b/app/grandchallenge/algorithms/models.py @@ -973,7 +973,7 @@ def get_jobs_with_same_inputs( unique_kwargs = { "algorithm_image": algorithm_image, } - input_interface_count = algorithm_image.algorithm.inputs.count() + input_interface_count = len(inputs) if algorithm_model: unique_kwargs["algorithm_model"] = algorithm_model @@ -1078,6 +1078,9 @@ class Job(CIVForObjectMixin, ComponentJob): algorithm_model = models.ForeignKey( AlgorithmModel, on_delete=models.PROTECT, null=True, blank=True ) + algorithm_interface = models.ForeignKey( + AlgorithmInterface, on_delete=models.PROTECT, null=True, blank=True + ) creator = models.ForeignKey( settings.AUTH_USER_MODEL, null=True, on_delete=models.SET_NULL ) @@ -1124,14 +1127,14 @@ def container(self): @property def output_interfaces(self): - return self.algorithm_image.algorithm.outputs + return self.algorithm_interface.outputs.all() @cached_property def inputs_complete(self): # check if all inputs are present and if they all have a value return { civ.interface for civ in self.inputs.all() if civ.has_value - } == {*self.algorithm_image.algorithm.inputs.all()} + } == {*self.algorithm_interface.inputs.all()} @cached_property def rendered_result_text(self) -> str: diff --git a/app/grandchallenge/algorithms/templates/algorithms/algorithm_detail.html b/app/grandchallenge/algorithms/templates/algorithms/algorithm_detail.html index 5180ab6365..31c63c5e87 100644 --- a/app/grandchallenge/algorithms/templates/algorithms/algorithm_detail.html +++ b/app/grandchallenge/algorithms/templates/algorithms/algorithm_detail.html @@ -79,7 +79,7 @@ {% if "execute_algorithm" in algorithm_perms and object.active_image %} + href="{% url 'algorithms:job-interface-select' slug=object.slug %}">  Try-out Algorithm {% endif %} @@ -118,7 +118,7 @@ {% if object.public and not object.public_test_case %} {% endif %} {% endif %} @@ -618,7 +618,7 @@
    Make this algorithm public
    Update Settings Update Description - Add Test Case + Add Test Case
    Publish algorithm diff --git a/app/grandchallenge/algorithms/templates/algorithms/job_detail.html b/app/grandchallenge/algorithms/templates/algorithms/job_detail.html index 2eb9edf587..f96b87de04 100644 --- a/app/grandchallenge/algorithms/templates/algorithms/job_detail.html +++ b/app/grandchallenge/algorithms/templates/algorithms/job_detail.html @@ -55,7 +55,7 @@ {% if object.status == object.SUCCESS and perms.reader_studies.add_readerstudy %} + href="{% url "algorithms:display-set-from-job-create" slug=object.algorithm_image.algorithm.slug interface_pk=object.pk %}">  Edit in Reader Study {% endif %} diff --git a/app/grandchallenge/algorithms/templates/algorithms/job_form_create.html b/app/grandchallenge/algorithms/templates/algorithms/job_form_create.html index 1914cf8adb..47527a4902 100644 --- a/app/grandchallenge/algorithms/templates/algorithms/job_form_create.html +++ b/app/grandchallenge/algorithms/templates/algorithms/job_form_create.html @@ -36,59 +36,66 @@ {% block content %}

    Try-out Algorithm

    - {{ algorithm.job_create_page_markdown|md2html }} - {% get_obj_perms request.user for algorithm as "algorithm_perms" %} - {% if not algorithm.active_image %} -

    - This algorithm is not ready to be used. - {% if 'change_algorithm' in algorithm_perms %} - Please upload a valid container image for this algorithm. - {% endif %} -

    - {% elif form.jobs_limit < 1 %} -

    - You have run out of credits to try this algorithm. - You can request more credits by sending an e-mail to - - support@grand-challenge.org. -

    + {% if not algorithm.interfaces.all and 'change_algorithm' in algorithm_perms %} +

    Your algorithm does not have any interfaces yet. You need to define at least one interface (i.e. input - output combination) before you can try it out.

    +

    To define an interface, navigate here.

    + {% elif not algorithm.interfaces.all %} +

    This algorithm is not fully configured yet and hence cannot be used.

    {% else %} -

    - Select the data that you would like to run the algorithm on. -

    -

    - {% if 'change_algorithm' in algorithm_perms %} - As an editor for this algorithm you can test and debug your algorithm in total {{ editors_job_limit }} times per unique algorithm image. - You share these credits with all other editors of this algorithm. - Once you have reached the limit, any extra jobs will be deducted from your personal algorithm credits, - of which you get {{ request.user.user_credit.credits }} per month. - {% else %} - You receive {{ request.user.user_credit.credits }} credits per month. - {% endif %} - Using this algorithm requires {{ algorithm.credits_per_job }} - credit{{ algorithm.credits_per_job|pluralize }} per job. - You can currently create up to {{ form.jobs_limit }} job{{ form.jobs_limit|pluralize }} for this algorithm. -

    + {{ algorithm.job_create_page_markdown|md2html }} + + {% if not algorithm.active_image %} +

    + This algorithm is not ready to be used. + {% if 'change_algorithm' in algorithm_perms %} + Please upload a valid container image for this algorithm. + {% endif %} +

    + {% elif form.jobs_limit < 1 %} +

    + You have run out of credits to try this algorithm. + You can request more credits by sending an e-mail to + + support@grand-challenge.org. +

    + {% else %} +

    + Select the data that you would like to run the algorithm on. +

    +

    + {% if 'change_algorithm' in algorithm_perms %} + As an editor for this algorithm you can test and debug your algorithm in total {{ editors_job_limit }} times per unique algorithm image. + You share these credits with all other editors of this algorithm. + Once you have reached the limit, any extra jobs will be deducted from your personal algorithm credits, + of which you get {{ request.user.user_credit.credits }} per month. + {% else %} + You receive {{ request.user.user_credit.credits }} credits per month. + {% endif %} + Using this algorithm requires {{ algorithm.credits_per_job }} + credit{{ algorithm.credits_per_job|pluralize }} per job. + You can currently create up to {{ form.jobs_limit }} job{{ form.jobs_limit|pluralize }} for this algorithm. +

    -
    - {% csrf_token %} - {{ form|crispy }} - -
    +
    + {% csrf_token %} + {{ form|crispy }} + +
    -

    - By running this algorithm you agree to the - General - Terms of Service{% if algorithm.additional_terms_markdown %}, - as well as this algorithm's specific Terms of Service: - {% else %}. - {% endif %} -

    +

    + By running this algorithm you agree to the + General + Terms of Service{% if algorithm.additional_terms_markdown %}, + as well as this algorithm's specific Terms of Service: + {% else %}. + {% endif %} +

    - {{ algorithm.additional_terms_markdown|md2html }} + {{ algorithm.additional_terms_markdown|md2html }} + {% endif %} {% endif %} {% endblock %} diff --git a/app/grandchallenge/algorithms/templates/algorithms/job_list.html b/app/grandchallenge/algorithms/templates/algorithms/job_list.html index 5f216ec98c..ab3d5657b2 100644 --- a/app/grandchallenge/algorithms/templates/algorithms/job_list.html +++ b/app/grandchallenge/algorithms/templates/algorithms/job_list.html @@ -23,7 +23,7 @@

    Results for {{ algorithm.title }}

    {% if "execute_algorithm" in algorithm_perms and algorithm.active_image %}

    + href="{% url 'algorithms:job-interface-select' slug=algorithm.slug %}">  Try-out Algorithm

    diff --git a/app/grandchallenge/algorithms/templates/algorithms/job_progress_detail.html b/app/grandchallenge/algorithms/templates/algorithms/job_progress_detail.html index 2d8f2d8d28..39e4351db5 100644 --- a/app/grandchallenge/algorithms/templates/algorithms/job_progress_detail.html +++ b/app/grandchallenge/algorithms/templates/algorithms/job_progress_detail.html @@ -35,7 +35,7 @@
    Creating inputs
    - diff --git a/app/grandchallenge/algorithms/urls.py b/app/grandchallenge/algorithms/urls.py index 72d406efbf..7b424a6e6e 100644 --- a/app/grandchallenge/algorithms/urls.py +++ b/app/grandchallenge/algorithms/urls.py @@ -30,6 +30,7 @@ EditorsUpdate, JobCreate, JobDetail, + JobInterfaceSelect, JobProgressDetail, JobsList, JobStatusDetail, @@ -133,7 +134,16 @@ name="model-update", ), path("/jobs/", JobsList.as_view(), name="job-list"), - path("/jobs/create/", JobCreate.as_view(), name="job-create"), + path( + "/jobs/interface-select/", + JobInterfaceSelect.as_view(), + name="job-interface-select", + ), + path( + "//jobs/create/", + JobCreate.as_view(), + name="job-create", + ), path("/jobs//", JobDetail.as_view(), name="job-detail"), path( "/jobs//status/", diff --git a/app/grandchallenge/algorithms/views.py b/app/grandchallenge/algorithms/views.py index db9f837d6f..6d947c0663 100644 --- a/app/grandchallenge/algorithms/views.py +++ b/app/grandchallenge/algorithms/views.py @@ -58,6 +58,7 @@ ImageActivateForm, JobCreateForm, JobForm, + JobInterfaceSelectForm, UsersForm, ViewersForm, ) @@ -478,28 +479,84 @@ def get_success_url(self): return self.algorithm.get_absolute_url() +class JobCreatePermissionMixin( + LoginRequiredMixin, VerificationRequiredMixin, AccessMixin +): + @cached_property + def algorithm(self) -> Algorithm: + return get_object_or_404(Algorithm, slug=self.kwargs["slug"]) + + def dispatch(self, request, *args, **kwargs): + if not request.user.has_perm( + "algorithms.execute_algorithm", self.algorithm + ): + return self.handle_no_permission() + return super().dispatch(request, *args, **kwargs) + + +class JobInterfaceSelect( + JobCreatePermissionMixin, + FormView, +): + form_class = JobInterfaceSelectForm + template_name = "algorithms/job_form_create.html" + selected_interface = None + + def get(self, request, *args, **kwargs): + if self.algorithm.interfaces.count() == 1: + self.selected_interface = self.algorithm.default_interface + return HttpResponseRedirect(self.get_success_url()) + else: + return super().get(request, *args, **kwargs) + + def get_form_kwargs(self): + kwargs = super().get_form_kwargs() + kwargs.update({"algorithm": self.algorithm}) + return kwargs + + def get_context_data(self, *args, **kwargs): + context = super().get_context_data(*args, **kwargs) + context.update( + { + "algorithm": self.algorithm, + "editors_job_limit": settings.ALGORITHM_IMAGES_COMPLIMENTARY_EDITOR_JOBS, + } + ) + return context + + def form_valid(self, form): + self.selected_interface = form.cleaned_data["algorithm_interface"] + return super().form_valid(form) + + def get_success_url(self): + return reverse( + "algorithms:job-create", + kwargs={ + "slug": self.algorithm.slug, + "interface_pk": self.selected_interface.pk, + }, + ) + + class JobCreate( - LoginRequiredMixin, - ObjectPermissionRequiredMixin, - VerificationRequiredMixin, + JobCreatePermissionMixin, UserFormKwargsMixin, FormView, ): form_class = JobCreateForm template_name = "algorithms/job_form_create.html" - permission_required = "algorithms.execute_algorithm" - raise_exception = True @cached_property - def algorithm(self) -> Algorithm: - return get_object_or_404(Algorithm, slug=self.kwargs["slug"]) - - def get_permission_object(self): - return self.algorithm + def interface(self): + return get_object_or_404( + AlgorithmInterface, pk=self.kwargs["interface_pk"] + ) def get_form_kwargs(self): kwargs = super().get_form_kwargs() - kwargs.update({"algorithm": self.algorithm}) + kwargs.update( + {"algorithm": self.algorithm, "interface": self.interface} + ) return kwargs def get_context_data(self, *args, **kwargs): diff --git a/app/tests/algorithms_tests/test_api.py b/app/tests/algorithms_tests/test_api.py index 7478f75081..2eb04c248c 100644 --- a/app/tests/algorithms_tests/test_api.py +++ b/app/tests/algorithms_tests/test_api.py @@ -92,6 +92,7 @@ def test_job_list_view_num_queries( assert len(response.json()["results"]) == num_jobs +@pytest.mark.xfail(reason="Still to be addressed for optional inputs pitch") @pytest.mark.django_db class TestJobCreationThroughAPI: diff --git a/app/tests/algorithms_tests/test_forms.py b/app/tests/algorithms_tests/test_forms.py index fa023fcd24..5ff451df71 100644 --- a/app/tests/algorithms_tests/test_forms.py +++ b/app/tests/algorithms_tests/test_forms.py @@ -321,7 +321,10 @@ def test_create_job_input_fields( response = get_view_for_user( viewname="algorithms:job-create", client=client, - reverse_kwargs={"slug": alg.slug}, + reverse_kwargs={ + "slug": alg.slug, + "interface_pk": alg.default_interface.pk, + }, follow=True, user=creator, ) @@ -353,7 +356,10 @@ def test_create_job_json_input_field_validation( response = get_view_for_user( viewname="algorithms:job-create", client=client, - reverse_kwargs={"slug": alg.slug}, + reverse_kwargs={ + "slug": alg.slug, + "interface_pk": alg.default_interface.pk, + }, method=client.post, follow=True, user=creator, @@ -384,7 +390,10 @@ def test_create_job_simple_input_field_validation( response = get_view_for_user( viewname="algorithms:job-create", client=client, - reverse_kwargs={"slug": alg.slug}, + reverse_kwargs={ + "slug": alg.slug, + "interface_pk": alg.default_interface.pk, + }, method=client.post, follow=True, user=creator, @@ -400,7 +409,11 @@ def create_algorithm_with_input(slug): VerificationFactory(user=creator, is_verified=True) alg = AlgorithmFactory() alg.add_editor(user=creator) - alg.inputs.set([ComponentInterface.objects.get(slug=slug)]) + interface = AlgorithmInterfaceFactory( + inputs=[ComponentInterface.objects.get(slug=slug)], + outputs=[ComponentInterfaceFactory()], + ) + alg.interfaces.add(interface, through_defaults={"is_default": True}) AlgorithmImageFactory( algorithm=alg, is_manifest_valid=True, @@ -500,13 +513,34 @@ def test_only_publish_successful_jobs(): @pytest.mark.django_db class TestJobCreateLimits: + + def create_form(self, algorithm, user, algorithm_image=None): + ci = ComponentInterfaceFactory(kind=ComponentInterface.Kind.STRING) + interface = AlgorithmInterfaceFactory(inputs=[ci]) + algorithm.interfaces.add(interface) + + algorithm_image_kwargs = {} + if algorithm_image: + algorithm_image_kwargs = { + "algorithm_image": str(algorithm_image.pk) + } + + return JobCreateForm( + algorithm=algorithm, + user=user, + interface=interface, + data={ + **algorithm_image_kwargs, + **get_interface_form_data(interface_slug=ci.slug, data="Foo"), + }, + ) + def test_form_invalid_without_enough_credits(self, settings): algorithm = AlgorithmFactory( minimum_credits_per_job=( settings.ALGORITHMS_GENERAL_CREDITS_PER_MONTH_PER_USER + 1 ), ) - algorithm.inputs.clear() user = UserFactory() AlgorithmImageFactory( algorithm=algorithm, @@ -514,13 +548,11 @@ def test_form_invalid_without_enough_credits(self, settings): is_in_registry=True, is_desired_version=True, ) - - form = JobCreateForm(algorithm=algorithm, user=user, data={}) - + form = self.create_form(algorithm=algorithm, user=user) assert not form.is_valid() - assert form.errors == { - "__all__": ["You have run out of algorithm credits"], - } + assert "You have run out of algorithm credits" in str( + form.errors["__all__"] + ) def test_form_valid_for_editor(self, settings): algorithm = AlgorithmFactory( @@ -528,7 +560,6 @@ def test_form_valid_for_editor(self, settings): settings.ALGORITHMS_GENERAL_CREDITS_PER_MONTH_PER_USER + 1 ), ) - algorithm.inputs.clear() algorithm_image = AlgorithmImageFactory( algorithm=algorithm, is_manifest_valid=True, @@ -539,17 +570,15 @@ def test_form_valid_for_editor(self, settings): algorithm.add_editor(user=user) - form = JobCreateForm( + form = self.create_form( algorithm=algorithm, user=user, - data={"algorithm_image": str(algorithm_image.pk)}, + algorithm_image=algorithm_image, ) - assert form.is_valid() def test_form_valid_with_credits(self): algorithm = AlgorithmFactory(minimum_credits_per_job=1) - algorithm.inputs.clear() algorithm_image = AlgorithmImageFactory( algorithm=algorithm, is_manifest_valid=True, @@ -558,12 +587,11 @@ def test_form_valid_with_credits(self): ) user = UserFactory() - form = JobCreateForm( + form = self.create_form( algorithm=algorithm, user=user, - data={"algorithm_image": str(algorithm_image.pk)}, + algorithm_image=algorithm_image, ) - assert form.is_valid() @@ -710,7 +738,12 @@ def test_creator_queryset( ): algorithm = algorithm_with_image_and_model_and_two_inputs.algorithm editor = algorithm.editors_group.user_set.first() - form = JobCreateForm(algorithm=algorithm, user=editor, data={}) + form = JobCreateForm( + algorithm=algorithm, + user=editor, + interface=algorithm.default_interface, + data={}, + ) assert list(form.fields["creator"].queryset.all()) == [editor] assert form.fields["creator"].initial == editor @@ -726,7 +759,12 @@ def test_algorithm_image_queryset( is_manifest_valid=True, is_in_registry=True, ) - form = JobCreateForm(algorithm=algorithm, user=editor, data={}) + form = JobCreateForm( + algorithm=algorithm, + user=editor, + interface=algorithm.default_interface, + data={}, + ) ai_qs = form.fields["algorithm_image"].queryset.all() assert algorithm.active_image in ai_qs assert inactive_image not in ai_qs @@ -745,12 +783,14 @@ def test_cannot_create_job_with_same_inputs_twice( algorithm_model=algorithm.active_model, status=Job.SUCCESS, time_limit=123, + algorithm_interface=algorithm.default_interface, ) job.inputs.set(civs) form = JobCreateForm( algorithm=algorithm, user=editor, + interface=algorithm.default_interface, data={ "algorithm_image": algorithm.active_image, "algorithm_model": algorithm.active_model, @@ -775,13 +815,18 @@ def test_all_inputs_required_on_job_creation(algorithm_with_multiple_inputs): kind=InterfaceKind.InterfaceKindChoices.ANY, store_in_database=True, ) - algorithm_with_multiple_inputs.algorithm.inputs.add( - ci_json_in_db_without_schema + interface = AlgorithmInterfaceFactory( + inputs=[ci_json_in_db_without_schema], + outputs=[ComponentInterfaceFactory()], + ) + algorithm_with_multiple_inputs.algorithm.interfaces.add( + interface, through_defaults={"is_default": True} ) form = JobCreateForm( algorithm=algorithm_with_multiple_inputs.algorithm, user=algorithm_with_multiple_inputs.editor, + interface=interface, data={}, ) diff --git a/app/tests/algorithms_tests/test_models.py b/app/tests/algorithms_tests/test_models.py index 6c7d8028be..b80a048e53 100644 --- a/app/tests/algorithms_tests/test_models.py +++ b/app/tests/algorithms_tests/test_models.py @@ -656,7 +656,9 @@ def test_job_with_same_image_different_model( data = self.get_civ_data(civs=civs) j = AlgorithmJobFactory( - algorithm_image=alg.active_image, time_limit=10 + algorithm_image=alg.active_image, + time_limit=10, + algorithm_interface=alg.default_interface, ) j.inputs.set(civs) @@ -678,6 +680,7 @@ def test_job_with_same_model_different_image( algorithm_image=AlgorithmImageFactory(), algorithm_model=alg.active_model, time_limit=10, + algorithm_interface=alg.default_interface, ) j.inputs.set(civs) jobs = Job.objects.get_jobs_with_same_inputs( @@ -698,6 +701,7 @@ def test_job_with_same_model_and_image( algorithm_model=alg.active_model, algorithm_image=alg.active_image, time_limit=10, + algorithm_interface=alg.default_interface, ) j.inputs.set(civs) jobs = Job.objects.get_jobs_with_same_inputs( @@ -719,6 +723,7 @@ def test_job_with_different_image_and_model( algorithm_model=AlgorithmModelFactory(), algorithm_image=AlgorithmImageFactory(), time_limit=10, + algorithm_interface=alg.default_interface, ) j.inputs.set(civs) jobs = Job.objects.get_jobs_with_same_inputs( @@ -739,10 +744,13 @@ def test_job_with_same_image_no_model_provided( algorithm_model=alg.active_model, algorithm_image=alg.active_image, time_limit=10, + algorithm_interface=alg.default_interface, ) j.inputs.set(civs) jobs = Job.objects.get_jobs_with_same_inputs( - inputs=data, algorithm_image=alg.active_image, algorithm_model=None + inputs=data, + algorithm_image=alg.active_image, + algorithm_model=None, ) assert len(jobs) == 0 @@ -754,11 +762,15 @@ def test_job_with_same_image_and_without_model( data = self.get_civ_data(civs=civs) j = AlgorithmJobFactory( - algorithm_image=alg.active_image, time_limit=10 + algorithm_image=alg.active_image, + time_limit=10, + algorithm_interface=alg.default_interface, ) j.inputs.set(civs) jobs = Job.objects.get_jobs_with_same_inputs( - inputs=data, algorithm_image=alg.active_image, algorithm_model=None + inputs=data, + algorithm_image=alg.active_image, + algorithm_model=None, ) assert j in jobs assert len(jobs) == 1 @@ -771,7 +783,9 @@ def test_job_with_different_input( data = self.get_civ_data(civs=civs) j = AlgorithmJobFactory( - algorithm_image=alg.active_image, time_limit=10 + algorithm_image=alg.active_image, + time_limit=10, + algorithm_interface=alg.default_interface, ) j.inputs.set( [ @@ -780,7 +794,34 @@ def test_job_with_different_input( ] ) jobs = Job.objects.get_jobs_with_same_inputs( - inputs=data, algorithm_image=alg.active_image, algorithm_model=None + inputs=data, + algorithm_image=alg.active_image, + algorithm_model=None, + ) + assert len(jobs) == 0 + + def test_job_with_partially_overlapping_input( + self, algorithm_with_image_and_model_and_two_inputs + ): + alg = algorithm_with_image_and_model_and_two_inputs.algorithm + civs = algorithm_with_image_and_model_and_two_inputs.civs + data = self.get_civ_data(civs=civs) + + j = AlgorithmJobFactory( + algorithm_image=alg.active_image, + time_limit=10, + algorithm_interface=alg.default_interface, + ) + j.inputs.set( + [ + civs[0], + ComponentInterfaceValueFactory(), + ] + ) + jobs = Job.objects.get_jobs_with_same_inputs( + inputs=data, + algorithm_image=alg.active_image, + algorithm_model=None, ) assert len(jobs) == 0 @@ -982,8 +1023,15 @@ def test_inputs_complete(): ci1, ci2, ci3 = ComponentInterfaceFactory.create_batch( 3, kind=ComponentInterface.Kind.STRING ) - alg.inputs.set([ci1, ci2, ci3]) - job = AlgorithmJobFactory(algorithm_image__algorithm=alg, time_limit=10) + interface = AlgorithmInterfaceFactory( + inputs=[ci1, ci2, ci3], outputs=[ComponentInterfaceFactory()] + ) + alg.interfaces.add(interface, through_defaults={"is_default": True}) + job = AlgorithmJobFactory( + algorithm_image__algorithm=alg, + time_limit=10, + algorithm_interface=alg.default_interface, + ) civ_with_value_1 = ComponentInterfaceValueFactory( interface=ci1, value="Foo" ) @@ -1505,7 +1553,8 @@ def test_has_default_interface(): io1, io2 = AlgorithmInterfaceFactory.create_batch(2) alg1.interfaces.add(io1, through_defaults={"is_default": True}) - alg2.interfaces.add(io2) + alg1.interfaces.add(io2) + alg2.interfaces.add(io1) assert alg1.default_interface == io1 assert not alg2.default_interface diff --git a/app/tests/algorithms_tests/test_permissions.py b/app/tests/algorithms_tests/test_permissions.py index 4349bd47e8..3a0ff8951c 100644 --- a/app/tests/algorithms_tests/test_permissions.py +++ b/app/tests/algorithms_tests/test_permissions.py @@ -18,6 +18,7 @@ from tests.algorithms_tests.factories import ( AlgorithmFactory, AlgorithmImageFactory, + AlgorithmInterfaceFactory, AlgorithmJobFactory, ) from tests.algorithms_tests.utils import TwoAlgorithms @@ -313,7 +314,12 @@ def test_job_permissions_from_template(self, client): kind=InterfaceKind.InterfaceKindChoices.ANY, store_in_database=True, ) - algorithm_image.algorithm.inputs.set([ci]) + interface = AlgorithmInterfaceFactory( + inputs=[ci], outputs=[ComponentInterfaceFactory()] + ) + algorithm_image.algorithm.interfaces.add( + interface, through_defaults={"is_default": True} + ) response = get_view_for_user( viewname="algorithms:job-create", @@ -321,6 +327,7 @@ def test_job_permissions_from_template(self, client): method=client.post, reverse_kwargs={ "slug": algorithm_image.algorithm.slug, + "interface_pk": algorithm_image.algorithm.default_interface.pk, }, user=user, follow=True, @@ -338,6 +345,9 @@ def test_job_permissions_from_template(self, client): algorithm_image=algorithm_image, job=job, user=user ) + @pytest.mark.xfail( + reason="Still to be addressed for optional inputs pitch" + ) def test_job_permissions_from_api(self, rf): # setup user = UserFactory() diff --git a/app/tests/algorithms_tests/test_serializers.py b/app/tests/algorithms_tests/test_serializers.py index 1f4c88a891..eb5cd14412 100644 --- a/app/tests/algorithms_tests/test_serializers.py +++ b/app/tests/algorithms_tests/test_serializers.py @@ -35,6 +35,7 @@ def test_algorithm_relations_on_job_serializer(rf): ) +@pytest.mark.xfail(reason="Still to be addressed for optional inputs pitch") @pytest.mark.django_db @pytest.mark.parametrize( "title, " @@ -183,6 +184,7 @@ def test_algorithm_job_post_serializer_validations( assert len(job.inputs.all()) == 2 +@pytest.mark.xfail(reason="Still to be addressed for optional inputs pitch") @pytest.mark.django_db def test_algorithm_job_post_serializer_create( rf, settings, django_capture_on_commit_callbacks @@ -240,6 +242,7 @@ def test_algorithm_job_post_serializer_create( assert len(job.inputs.all()) == 3 +@pytest.mark.xfail(reason="Still to be addressed for optional inputs pitch") @pytest.mark.django_db class TestJobCreateLimits: def test_form_invalid_without_enough_credits(self, rf, settings): @@ -380,6 +383,7 @@ def test_reformat_inputs(rf): ) +@pytest.mark.xfail(reason="Still to be addressed for optional inputs pitch") @pytest.mark.django_db def test_algorithm_post_serializer_image_and_time_limit_fixed(rf): request = rf.get("/foo") diff --git a/app/tests/algorithms_tests/test_tasks.py b/app/tests/algorithms_tests/test_tasks.py index 2e81afe8bf..3f2baa705e 100644 --- a/app/tests/algorithms_tests/test_tasks.py +++ b/app/tests/algorithms_tests/test_tasks.py @@ -5,6 +5,7 @@ from actstream.models import Follow from django.core.exceptions import ObjectDoesNotExist from django.core.files.base import ContentFile +from guardian.shortcuts import assign_perm from grandchallenge.algorithms.models import Job from grandchallenge.algorithms.tasks import ( @@ -27,6 +28,7 @@ from tests.algorithms_tests.factories import ( AlgorithmFactory, AlgorithmImageFactory, + AlgorithmInterfaceFactory, AlgorithmJobFactory, AlgorithmModelFactory, ) @@ -36,6 +38,7 @@ ComponentInterfaceFactory, ComponentInterfaceValueFactory, ) +from tests.conftest import get_interface_form_data from tests.factories import ( GroupFactory, ImageFactory, @@ -44,6 +47,7 @@ UserFactory, ) from tests.utils import get_view_for_user, recurse_callbacks +from tests.verification_tests.factories import VerificationFactory @pytest.mark.django_db @@ -186,7 +190,7 @@ def test_jobs_workflow(django_capture_on_commit_callbacks): @pytest.mark.flaky(reruns=3) @pytest.mark.django_db def test_algorithm( - algorithm_image, settings, django_capture_on_commit_callbacks + algorithm_image, client, settings, django_capture_on_commit_callbacks ): # Override the celery settings settings.task_eager_propagates = (True,) @@ -201,6 +205,10 @@ def test_algorithm( with open(algorithm_image, "rb") as f: ai.image.save(algorithm_image, ContentFile(f.read())) + user = UserFactory() + ai.algorithm.add_editor(user) + VerificationFactory(user=user, is_verified=True) + recurse_callbacks( callbacks=callbacks, django_capture_on_commit_callbacks=django_capture_on_commit_callbacks, @@ -211,6 +219,7 @@ def test_algorithm( image_file = ImageFileFactory( file__from_path=Path(__file__).parent / "resources" / "input_file.tif" ) + assign_perm("cases.view_image", user, image_file.image) input_interface = ComponentInterface.objects.get( slug="generic-medical-image" @@ -219,20 +228,31 @@ def test_algorithm( slug="results-json-file" ) heatmap_interface = ComponentInterface.objects.get(slug="generic-overlay") - ai.algorithm.inputs.set([input_interface]) - ai.algorithm.outputs.set([json_result_interface, heatmap_interface]) - - civ = ComponentInterfaceValueFactory( - image=image_file.image, interface=input_interface, file=None + interface = AlgorithmInterfaceFactory( + inputs=[input_interface], + outputs=[json_result_interface, heatmap_interface], + ) + ai.algorithm.interfaces.add( + interface, through_defaults={"is_default": True} ) with django_capture_on_commit_callbacks() as callbacks: - create_algorithm_jobs( - algorithm_image=ai, - civ_sets=[{civ}], - time_limit=ai.algorithm.time_limit, - requires_gpu_type=ai.algorithm.job_requires_gpu_type, - requires_memory_gb=ai.algorithm.job_requires_memory_gb, + get_view_for_user( + viewname="algorithms:job-create", + client=client, + method=client.post, + user=user, + reverse_kwargs={ + "slug": ai.algorithm.slug, + "interface_pk": interface.pk, + }, + follow=True, + data={ + **get_interface_form_data( + interface_slug=input_interface.slug, + data=image_file.image.pk, + ), + }, ) recurse_callbacks( @@ -273,22 +293,37 @@ def test_algorithm( slug="detection-json-file", kind=ComponentInterface.Kind.ANY, ) - ai.algorithm.outputs.add(detection_interface) - ai.save() + interface2 = AlgorithmInterfaceFactory( + inputs=[input_interface], + outputs=[ + json_result_interface, + heatmap_interface, + detection_interface, + ], + ) + ai.algorithm.interfaces.add(interface2) image_file = ImageFileFactory( file__from_path=Path(__file__).parent / "resources" / "input_file.tif" ) - civ = ComponentInterfaceValueFactory( - image=image_file.image, interface=input_interface, file=None - ) + assign_perm("cases.view_image", user, image_file.image) with django_capture_on_commit_callbacks() as callbacks: - create_algorithm_jobs( - algorithm_image=ai, - civ_sets=[{civ}], - time_limit=ai.algorithm.time_limit, - requires_gpu_type=ai.algorithm.job_requires_gpu_type, - requires_memory_gb=ai.algorithm.job_requires_memory_gb, + get_view_for_user( + viewname="algorithms:job-create", + client=client, + method=client.post, + user=user, + reverse_kwargs={ + "slug": ai.algorithm.slug, + "interface_pk": interface2.pk, + }, + follow=True, + data={ + **get_interface_form_data( + interface_slug=input_interface.slug, + data=image_file.image.pk, + ), + }, ) recurse_callbacks( @@ -312,7 +347,7 @@ def test_algorithm( @pytest.mark.django_db def test_algorithm_with_invalid_output( - algorithm_image, settings, django_capture_on_commit_callbacks + algorithm_image, client, settings, django_capture_on_commit_callbacks ): # Override the celery settings settings.task_eager_propagates = (True,) @@ -327,6 +362,10 @@ def test_algorithm_with_invalid_output( with open(algorithm_image, "rb") as f: ai.image.save(algorithm_image, ContentFile(f.read())) + user = UserFactory() + ai.algorithm.add_editor(user) + VerificationFactory(user=user, is_verified=True) + recurse_callbacks( callbacks=callbacks, django_capture_on_commit_callbacks=django_capture_on_commit_callbacks, @@ -343,26 +382,37 @@ def test_algorithm_with_invalid_output( slug="detection-json-file", kind=ComponentInterface.Kind.ANY, ) - ai.algorithm.inputs.add(input_interface) - ai.algorithm.outputs.add(detection_interface) - ai.save() + interface = AlgorithmInterfaceFactory( + inputs=[input_interface], outputs=[detection_interface] + ) + ai.algorithm.interfaces.add( + interface, through_defaults={"is_default": True} + ) image_file = ImageFileFactory( file__from_path=Path(__file__).parent / "resources" / "input_file.tif" ) - - civ = ComponentInterfaceValueFactory( - image=image_file.image, interface=input_interface, file=None - ) + assign_perm("cases.view_image", user, image_file.image) with django_capture_on_commit_callbacks() as callbacks: - create_algorithm_jobs( - algorithm_image=ai, - civ_sets=[{civ}], - time_limit=ai.algorithm.time_limit, - requires_gpu_type=ai.algorithm.job_requires_gpu_type, - requires_memory_gb=ai.algorithm.job_requires_memory_gb, + get_view_for_user( + viewname="algorithms:job-create", + client=client, + method=client.post, + user=user, + reverse_kwargs={ + "slug": ai.algorithm.slug, + "interface_pk": interface.pk, + }, + follow=True, + data={ + **get_interface_form_data( + interface_slug=input_interface.slug, + data=image_file.image.pk, + ), + }, ) + recurse_callbacks( callbacks=callbacks, django_capture_on_commit_callbacks=django_capture_on_commit_callbacks, @@ -422,11 +472,17 @@ def test_execute_algorithm_job_for_missing_inputs(settings): # create the job without value for the ComponentInterfaceValues ci = ComponentInterface.objects.get(slug="generic-medical-image") ComponentInterfaceValue.objects.create(interface=ci) - alg.algorithm.inputs.add(ci) + interface = AlgorithmInterfaceFactory( + inputs=[ci], outputs=[ComponentInterfaceFactory()] + ) + alg.algorithm.interfaces.add( + interface, through_defaults={"is_default": True} + ) job = AlgorithmJobFactory( creator=creator, algorithm_image=alg, time_limit=alg.algorithm.time_limit, + algorithm_interface=interface, ) execute_algorithm_job_for_inputs(job_pk=job.pk) diff --git a/app/tests/algorithms_tests/test_views.py b/app/tests/algorithms_tests/test_views.py index 822110fbec..b164a27add 100644 --- a/app/tests/algorithms_tests/test_views.py +++ b/app/tests/algorithms_tests/test_views.py @@ -1,7 +1,6 @@ import datetime import io import json -import tempfile import zipfile from pathlib import Path from unittest.mock import patch @@ -53,7 +52,7 @@ UserUploadFactory, create_upload_from_file, ) -from tests.utils import get_view_for_user, recurse_callbacks +from tests.utils import get_view_for_user from tests.verification_tests.factories import VerificationFactory @@ -355,6 +354,13 @@ def test_permission_required_views(self, client): time_limit=ai.algorithm.time_limit, ) p = AlgorithmPermissionRequestFactory(algorithm=ai.algorithm) + interface = AlgorithmInterfaceFactory( + inputs=[ComponentInterfaceFactory()], + outputs=[ComponentInterfaceFactory()], + ) + ai.algorithm.interfaces.add( + interface, through_defaults={"is_default": True} + ) VerificationFactory(user=u, is_verified=True) @@ -421,7 +427,10 @@ def test_permission_required_views(self, client): ), ( "job-create", - {"slug": ai.algorithm.slug}, + { + "slug": ai.algorithm.slug, + "interface_pk": ai.algorithm.default_interface.pk, + }, "execute_algorithm", ai.algorithm, None, @@ -926,158 +935,6 @@ def list_algorithm_images(self, **__): assert image_interface.kind == ComponentInterface.Kind.IMAGE -@pytest.mark.django_db -def test_create_job_with_json_file( - client, settings, algorithm_io_image, django_capture_on_commit_callbacks -): - settings.task_eager_propagates = (True,) - settings.task_always_eager = (True,) - - with django_capture_on_commit_callbacks() as callbacks: - ai = AlgorithmImageFactory(image__from_path=algorithm_io_image) - recurse_callbacks( - callbacks=callbacks, - django_capture_on_commit_callbacks=django_capture_on_commit_callbacks, - ) - - editor = UserFactory() - VerificationFactory(user=editor, is_verified=True) - ai.algorithm.add_editor(editor) - ci = ComponentInterfaceFactory( - kind=InterfaceKind.InterfaceKindChoices.ANY, store_in_database=False - ) - ai.algorithm.inputs.set([ci]) - - with tempfile.NamedTemporaryFile(mode="w+", suffix=".json") as file: - json.dump('{"Foo": "bar"}', file) - file.seek(0) - upload = create_upload_from_file( - creator=editor, file_path=Path(file.name) - ) - with django_capture_on_commit_callbacks(execute=True): - with django_capture_on_commit_callbacks(execute=True): - response = get_view_for_user( - viewname="algorithms:job-create", - client=client, - method=client.post, - reverse_kwargs={ - "slug": ai.algorithm.slug, - }, - user=editor, - follow=True, - data={ - **get_interface_form_data( - interface_slug=ci.slug, data=upload.pk - ) - }, - ) - assert response.status_code == 200 - assert ( - file.name.split("/")[-1] - in Job.objects.get().inputs.first().file.name - ) - - -@pytest.mark.django_db -def test_algorithm_job_create_with_image_input( - settings, client, algorithm_io_image, django_capture_on_commit_callbacks -): - settings.task_eager_propagates = (True,) - settings.task_always_eager = (True,) - - with django_capture_on_commit_callbacks() as callbacks: - ai = AlgorithmImageFactory(image__from_path=algorithm_io_image) - recurse_callbacks( - callbacks=callbacks, - django_capture_on_commit_callbacks=django_capture_on_commit_callbacks, - ) - - editor = UserFactory() - VerificationFactory(user=editor, is_verified=True) - ai.algorithm.add_editor(editor) - ci = ComponentInterfaceFactory( - kind=InterfaceKind.InterfaceKindChoices.IMAGE, store_in_database=False - ) - ai.algorithm.inputs.set([ci]) - - image1, image2 = ImageFactory.create_batch(2) - assign_perm("cases.view_image", editor, image1) - assign_perm("cases.view_image", editor, image2) - - civ = ComponentInterfaceValueFactory(interface=ci, image=image1) - with django_capture_on_commit_callbacks(execute=True): - with django_capture_on_commit_callbacks(execute=True): - response = get_view_for_user( - viewname="algorithms:job-create", - client=client, - method=client.post, - reverse_kwargs={ - "slug": ai.algorithm.slug, - }, - user=editor, - follow=True, - data={ - **get_interface_form_data( - interface_slug=ci.slug, - data=image1.pk, - existing_data=True, - ) - }, - ) - assert response.status_code == 200 - assert str(Job.objects.get().inputs.first().image.pk) == str(image1.pk) - # same civ reused - assert Job.objects.get().inputs.first() == civ - - with django_capture_on_commit_callbacks(execute=True): - with django_capture_on_commit_callbacks(execute=True): - response = get_view_for_user( - viewname="algorithms:job-create", - client=client, - method=client.post, - reverse_kwargs={ - "slug": ai.algorithm.slug, - }, - user=editor, - follow=True, - data={ - **get_interface_form_data( - interface_slug=ci.slug, - data=image2.pk, - existing_data=True, - ) - }, - ) - assert response.status_code == 200 - assert str(Job.objects.last().inputs.first().image.pk) == str(image2.pk) - assert Job.objects.last().inputs.first() != civ - - upload = create_upload_from_file( - file_path=RESOURCE_PATH / "image10x10x10.mha", - creator=editor, - ) - with django_capture_on_commit_callbacks(execute=True): - with django_capture_on_commit_callbacks(execute=True): - response = get_view_for_user( - viewname="algorithms:job-create", - client=client, - method=client.post, - reverse_kwargs={ - "slug": ai.algorithm.slug, - }, - user=editor, - follow=True, - data={ - **get_interface_form_data( - interface_slug=ci.slug, data=upload.pk - ) - }, - ) - assert response.status_code == 200 - assert Job.objects.last().inputs.first().image.name == "image10x10x10.mha" - assert Job.objects.last().inputs.first() != civ - - @pytest.mark.django_db class TestJobCreateView: @@ -1103,6 +960,7 @@ def create_job( user=user, reverse_kwargs={ "slug": algorithm.slug, + "interface_pk": algorithm.default_interface.pk, }, follow=True, data=inputs, @@ -1141,15 +999,19 @@ def test_create_job_with_multiple_new_inputs( algorithm_with_multiple_inputs, ): # configure multiple inputs - algorithm_with_multiple_inputs.algorithm.inputs.set( - [ + interface = AlgorithmInterfaceFactory( + inputs=[ algorithm_with_multiple_inputs.ci_json_in_db_with_schema, algorithm_with_multiple_inputs.ci_existing_img, algorithm_with_multiple_inputs.ci_str, algorithm_with_multiple_inputs.ci_bool, algorithm_with_multiple_inputs.ci_json_file, algorithm_with_multiple_inputs.ci_img_upload, - ] + ], + outputs=[ComponentInterfaceFactory()], + ) + algorithm_with_multiple_inputs.algorithm.interfaces.add( + interface, through_defaults={"is_default": True} ) assert ComponentInterfaceValue.objects.count() == 0 @@ -1206,7 +1068,7 @@ def test_create_job_with_multiple_new_inputs( assert sorted( [ int.pk - for int in algorithm_with_multiple_inputs.algorithm.inputs.all() + for int in algorithm_with_multiple_inputs.algorithm.default_interface.inputs.all() ] ) == sorted([civ.interface.pk for civ in job.inputs.all()]) @@ -1233,14 +1095,18 @@ def test_create_job_with_existing_inputs( algorithm_with_multiple_inputs, ): # configure multiple inputs - algorithm_with_multiple_inputs.algorithm.inputs.set( - [ + interface = AlgorithmInterfaceFactory( + inputs=[ algorithm_with_multiple_inputs.ci_json_in_db_with_schema, algorithm_with_multiple_inputs.ci_existing_img, algorithm_with_multiple_inputs.ci_str, algorithm_with_multiple_inputs.ci_bool, algorithm_with_multiple_inputs.ci_json_file, - ] + ], + outputs=[ComponentInterfaceFactory()], + ) + algorithm_with_multiple_inputs.algorithm.interfaces.add( + interface, through_defaults={"is_default": True} ) civ1, civ2, civ3, civ4, civ5 = self.create_existing_civs( @@ -1307,13 +1173,17 @@ def test_create_job_is_idempotent( algorithm_with_multiple_inputs, ): # configure multiple inputs - algorithm_with_multiple_inputs.algorithm.inputs.set( - [ + interface = AlgorithmInterfaceFactory( + inputs=[ algorithm_with_multiple_inputs.ci_str, algorithm_with_multiple_inputs.ci_bool, algorithm_with_multiple_inputs.ci_existing_img, algorithm_with_multiple_inputs.ci_json_in_db_with_schema, - ] + ], + outputs=[ComponentInterfaceFactory()], + ) + algorithm_with_multiple_inputs.algorithm.interfaces.add( + interface, through_defaults={"is_default": True} ) civ1, civ2, civ3, civ4, civ5 = self.create_existing_civs( interface_data=algorithm_with_multiple_inputs @@ -1371,8 +1241,12 @@ def test_create_job_with_faulty_file_input( algorithm_with_multiple_inputs, ): # configure file input - algorithm_with_multiple_inputs.algorithm.inputs.set( - [algorithm_with_multiple_inputs.ci_json_file] + interface = AlgorithmInterfaceFactory( + inputs=[algorithm_with_multiple_inputs.ci_json_file], + outputs=[ComponentInterfaceFactory()], + ) + algorithm_with_multiple_inputs.algorithm.interfaces.add( + interface, through_defaults={"is_default": True} ) file_upload = UserUploadFactory( filename="file.json", creator=algorithm_with_multiple_inputs.editor @@ -1418,8 +1292,12 @@ def test_create_job_with_faulty_json_input( django_capture_on_commit_callbacks, algorithm_with_multiple_inputs, ): - algorithm_with_multiple_inputs.algorithm.inputs.set( - [algorithm_with_multiple_inputs.ci_json_in_db_with_schema] + interface = AlgorithmInterfaceFactory( + inputs=[algorithm_with_multiple_inputs.ci_json_in_db_with_schema], + outputs=[ComponentInterfaceFactory()], + ) + algorithm_with_multiple_inputs.algorithm.interfaces.add( + interface, through_defaults={"is_default": True} ) response = self.create_job( client=client, @@ -1448,8 +1326,12 @@ def test_create_job_with_faulty_image_input( django_capture_on_commit_callbacks, algorithm_with_multiple_inputs, ): - algorithm_with_multiple_inputs.algorithm.inputs.set( - [algorithm_with_multiple_inputs.ci_img_upload] + interface = AlgorithmInterfaceFactory( + inputs=[algorithm_with_multiple_inputs.ci_img_upload], + outputs=[ComponentInterfaceFactory()], + ) + algorithm_with_multiple_inputs.algorithm.interfaces.add( + interface, through_defaults={"is_default": True} ) user_upload = create_upload_from_file( creator=algorithm_with_multiple_inputs.editor, @@ -1501,11 +1383,11 @@ def test_create_job_with_multiple_faulty_existing_image_inputs( ] ci.save() - algorithm_with_multiple_inputs.algorithm.inputs.set( - [ - ci1, - ci2, - ] + interface = AlgorithmInterfaceFactory( + inputs=[ci1, ci2], outputs=[ComponentInterfaceFactory()] + ) + algorithm_with_multiple_inputs.algorithm.interfaces.add( + interface, through_defaults={"is_default": True} ) assert ComponentInterfaceValue.objects.count() == 0 @@ -1658,7 +1540,10 @@ def test_job_time_limit(client): ci = ComponentInterfaceFactory( kind=InterfaceKind.InterfaceKindChoices.ANY, store_in_database=True ) - algorithm.inputs.set([ci]) + interface = AlgorithmInterfaceFactory( + inputs=[ci], outputs=[ComponentInterfaceFactory()] + ) + algorithm.interfaces.add(interface, through_defaults={"is_default": True}) response = get_view_for_user( viewname="algorithms:job-create", @@ -1666,6 +1551,7 @@ def test_job_time_limit(client): method=client.post, reverse_kwargs={ "slug": algorithm.slug, + "interface_pk": algorithm.default_interface.pk, }, user=user, follow=True, @@ -1705,7 +1591,10 @@ def test_job_gpu_type_set(client, settings): ci = ComponentInterfaceFactory( kind=InterfaceKind.InterfaceKindChoices.ANY, store_in_database=True ) - algorithm.inputs.set([ci]) + interface = AlgorithmInterfaceFactory( + inputs=[ci], outputs=[ComponentInterfaceFactory()] + ) + algorithm.interfaces.add(interface, through_defaults={"is_default": True}) response = get_view_for_user( viewname="algorithms:job-create", @@ -1713,6 +1602,7 @@ def test_job_gpu_type_set(client, settings): method=client.post, reverse_kwargs={ "slug": algorithm.slug, + "interface_pk": algorithm.default_interface.pk, }, user=user, follow=True, @@ -1734,6 +1624,7 @@ def test_job_gpu_type_set(client, settings): assert algorithm.credits_per_job == 190 +@pytest.mark.xfail(reason="Still to be addressed for optional inputs pitch") @pytest.mark.django_db def test_job_gpu_type_set_with_api(client, settings): settings.COMPONENTS_DEFAULT_BACKEND = "grandchallenge.components.backends.amazon_sagemaker_training.AmazonSageMakerTrainingExecutor" @@ -1795,9 +1686,18 @@ def test_job_create_view_for_verified_users_only(client): alg.add_user(user) alg.add_user(editor) + interface = AlgorithmInterfaceFactory( + inputs=[ComponentInterfaceFactory()], + outputs=[ComponentInterfaceFactory()], + ) + alg.interfaces.add(interface, through_defaults={"is_default": True}) + response = get_view_for_user( viewname="algorithms:job-create", - reverse_kwargs={"slug": alg.slug}, + reverse_kwargs={ + "slug": alg.slug, + "interface_pk": alg.default_interface.pk, + }, client=client, user=user, ) @@ -1805,7 +1705,10 @@ def test_job_create_view_for_verified_users_only(client): response2 = get_view_for_user( viewname="algorithms:job-create", - reverse_kwargs={"slug": alg.slug}, + reverse_kwargs={ + "slug": alg.slug, + "interface_pk": alg.default_interface.pk, + }, client=client, user=editor, ) @@ -1903,7 +1806,10 @@ def test_job_create_denied_for_same_input_model_and_image(client): ci = ComponentInterfaceFactory( kind=InterfaceKind.InterfaceKindChoices.IMAGE ) - alg.inputs.set([ci]) + interface = AlgorithmInterfaceFactory( + inputs=[ci], outputs=[ComponentInterfaceFactory()] + ) + alg.interfaces.add(interface, through_defaults={"is_default": True}) ai = AlgorithmImageFactory( algorithm=alg, is_manifest_valid=True, @@ -1926,6 +1832,7 @@ def test_job_create_denied_for_same_input_model_and_image(client): method=client.post, reverse_kwargs={ "slug": alg.slug, + "interface_pk": alg.default_interface.pk, }, user=creator, data={ @@ -2243,3 +2150,80 @@ def test_algorithm_interfaces_list_queryset(client): assert iots[1] in response.context["object_list"] assert iots[2] not in response.context["object_list"] assert iots[3] not in response.context["object_list"] + + +@pytest.mark.django_db +def test_interface_select_for_job_view_permission(client): + verified_user, unverified_user = UserFactory.create_batch(2) + VerificationFactory(user=verified_user, is_verified=True) + alg = AlgorithmFactory() + alg.add_user(verified_user) + alg.add_user(unverified_user) + + interface1 = AlgorithmInterfaceFactory( + inputs=[ComponentInterfaceFactory()], + outputs=[ComponentInterfaceFactory()], + ) + interface2 = AlgorithmInterfaceFactory( + inputs=[ComponentInterfaceFactory()], + outputs=[ComponentInterfaceFactory()], + ) + alg.interfaces.add(interface1, through_defaults={"is_default": True}) + alg.interfaces.add(interface2) + + response = get_view_for_user( + viewname="algorithms:job-interface-select", + reverse_kwargs={"slug": alg.slug}, + client=client, + user=unverified_user, + ) + assert response.status_code == 403 + + response = get_view_for_user( + viewname="algorithms:job-interface-select", + reverse_kwargs={"slug": alg.slug}, + client=client, + user=verified_user, + ) + assert response.status_code == 200 + + +@pytest.mark.django_db +def test_interface_select_automatic_redirect(client): + verified_user = UserFactory() + VerificationFactory(user=verified_user, is_verified=True) + alg = AlgorithmFactory() + alg.add_user(verified_user) + + interface = AlgorithmInterfaceFactory( + inputs=[ComponentInterfaceFactory()], + outputs=[ComponentInterfaceFactory()], + ) + alg.interfaces.add(interface, through_defaults={"is_default": True}) + + # with just 1 interface, user gets redirected immediately + response = get_view_for_user( + viewname="algorithms:job-interface-select", + reverse_kwargs={"slug": alg.slug}, + client=client, + user=verified_user, + ) + assert response.status_code == 302 + assert response.url == reverse( + "algorithms:job-create", + kwargs={"slug": alg.slug, "interface_pk": interface.pk}, + ) + + # with more than 1 interfaces, user has to choose the interface + interface2 = AlgorithmInterfaceFactory( + inputs=[ComponentInterfaceFactory()], + outputs=[ComponentInterfaceFactory()], + ) + alg.interfaces.add(interface2) + response = get_view_for_user( + viewname="algorithms:job-interface-select", + reverse_kwargs={"slug": alg.slug}, + client=client, + user=verified_user, + ) + assert response.status_code == 200 diff --git a/app/tests/conftest.py b/app/tests/conftest.py index 14ecdcdd37..a46b213262 100644 --- a/app/tests/conftest.py +++ b/app/tests/conftest.py @@ -21,6 +21,7 @@ from tests.algorithms_tests.factories import ( AlgorithmFactory, AlgorithmImageFactory, + AlgorithmInterfaceFactory, AlgorithmModelFactory, ) from tests.cases_tests import RESOURCE_PATH @@ -480,7 +481,10 @@ def algorithm_with_image_and_model_and_two_inputs(): ci1, ci2 = ComponentInterfaceFactory.create_batch( 2, kind=ComponentInterface.Kind.STRING ) - alg.inputs.set([ci1, ci2]) + interface = AlgorithmInterfaceFactory( + inputs=[ci1, ci2], outputs=[ComponentInterfaceFactory()] + ) + alg.interfaces.add(interface, through_defaults={"is_default": True}) civs = [ ComponentInterfaceValueFactory(interface=ci1, value="foo"), ComponentInterfaceValueFactory(interface=ci2, value="bar"), From 86b0396d2636d9072a4c4e03109ef9f96cd6b2a2 Mon Sep 17 00:00:00 2001 From: amickan Date: Fri, 17 Jan 2025 09:13:16 +0100 Subject: [PATCH 05/18] Redo migrations --- ..._algorithminterface_algorithminterfaceoutput_and_more.py} | 5 ++++- ...thm_interfaces.py => 0065_create_algorithm_interfaces.py} | 2 +- ...rfaces_for_jobs.py => 0066_create_interfaces_for_jobs.py} | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) rename app/grandchallenge/algorithms/migrations/{0063_algorithminterface_algorithminterfaceoutput_and_more.py => 0064_algorithminterface_algorithminterfaceoutput_and_more.py} (97%) rename app/grandchallenge/algorithms/migrations/{0064_create_algorithm_interfaces.py => 0065_create_algorithm_interfaces.py} (91%) rename app/grandchallenge/algorithms/migrations/{0065_create_interfaces_for_jobs.py => 0066_create_interfaces_for_jobs.py} (91%) diff --git a/app/grandchallenge/algorithms/migrations/0063_algorithminterface_algorithminterfaceoutput_and_more.py b/app/grandchallenge/algorithms/migrations/0064_algorithminterface_algorithminterfaceoutput_and_more.py similarity index 97% rename from app/grandchallenge/algorithms/migrations/0063_algorithminterface_algorithminterfaceoutput_and_more.py rename to app/grandchallenge/algorithms/migrations/0064_algorithminterface_algorithminterfaceoutput_and_more.py index ffdcabea83..82c19b6d33 100644 --- a/app/grandchallenge/algorithms/migrations/0063_algorithminterface_algorithminterfaceoutput_and_more.py +++ b/app/grandchallenge/algorithms/migrations/0064_algorithminterface_algorithminterfaceoutput_and_more.py @@ -9,7 +9,10 @@ class Migration(migrations.Migration): dependencies = [ ("components", "0022_alter_componentinterface_kind_and_more"), - ("algorithms", "0062_algorithmusercredit"), + ( + "algorithms", + "0063_alter_optionalhangingprotocolalgorithm_unique_together", + ), ] operations = [ diff --git a/app/grandchallenge/algorithms/migrations/0064_create_algorithm_interfaces.py b/app/grandchallenge/algorithms/migrations/0065_create_algorithm_interfaces.py similarity index 91% rename from app/grandchallenge/algorithms/migrations/0064_create_algorithm_interfaces.py rename to app/grandchallenge/algorithms/migrations/0065_create_algorithm_interfaces.py index 0f3bb8e8dd..8e74cc9cd8 100644 --- a/app/grandchallenge/algorithms/migrations/0064_create_algorithm_interfaces.py +++ b/app/grandchallenge/algorithms/migrations/0065_create_algorithm_interfaces.py @@ -32,7 +32,7 @@ class Migration(migrations.Migration): dependencies = [ ( "algorithms", - "0063_algorithminterface_algorithminterfaceoutput_and_more", + "0064_algorithminterface_algorithminterfaceoutput_and_more", ), ] diff --git a/app/grandchallenge/algorithms/migrations/0065_create_interfaces_for_jobs.py b/app/grandchallenge/algorithms/migrations/0066_create_interfaces_for_jobs.py similarity index 91% rename from app/grandchallenge/algorithms/migrations/0065_create_interfaces_for_jobs.py rename to app/grandchallenge/algorithms/migrations/0066_create_interfaces_for_jobs.py index f1e116af3d..550a4066b9 100644 --- a/app/grandchallenge/algorithms/migrations/0065_create_interfaces_for_jobs.py +++ b/app/grandchallenge/algorithms/migrations/0066_create_interfaces_for_jobs.py @@ -18,7 +18,7 @@ class Migration(migrations.Migration): dependencies = [ ( "algorithms", - "0064_create_algorithm_interfaces", + "0065_create_algorithm_interfaces", ), ] From 50793e69b54cdb7b47f9d263953e4000639f1c83 Mon Sep 17 00:00:00 2001 From: amickan Date: Fri, 17 Jan 2025 11:10:27 +0100 Subject: [PATCH 06/18] Fix new tests after merging main --- app/tests/algorithms_tests/test_admin.py | 1 + app/tests/algorithms_tests/test_views.py | 18 ++++++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/app/tests/algorithms_tests/test_admin.py b/app/tests/algorithms_tests/test_admin.py index 5b09b4f177..8c90afbdfb 100644 --- a/app/tests/algorithms_tests/test_admin.py +++ b/app/tests/algorithms_tests/test_admin.py @@ -16,6 +16,7 @@ from tests.utils import recurse_callbacks +@pytest.mark.xfail(reason="Still to be addressed for optional inputs pitch") @pytest.mark.django_db def test_job_updated_start_and_complete_times_after_admin_requeue( algorithm_image, settings, django_capture_on_commit_callbacks diff --git a/app/tests/algorithms_tests/test_views.py b/app/tests/algorithms_tests/test_views.py index aaf0dba230..2e1f57e49f 100644 --- a/app/tests/algorithms_tests/test_views.py +++ b/app/tests/algorithms_tests/test_views.py @@ -957,7 +957,12 @@ def test_create_job_with_json_file( ci = ComponentInterfaceFactory( kind=InterfaceKind.InterfaceKindChoices.ANY, store_in_database=False ) - ai.algorithm.inputs.set([ci]) + interface = AlgorithmInterfaceFactory( + inputs=[ci], + ) + ai.algorithm.interfaces.add( + interface, through_defaults={"is_default": True} + ) with tempfile.NamedTemporaryFile(mode="w+", suffix=".json") as file: json.dump('{"Foo": "bar"}', file) @@ -973,6 +978,7 @@ def test_create_job_with_json_file( method=client.post, reverse_kwargs={ "slug": ai.algorithm.slug, + "interface_pk": interface.pk, }, user=editor, follow=True, @@ -1010,7 +1016,12 @@ def test_algorithm_job_create_with_image_input( ci = ComponentInterfaceFactory( kind=InterfaceKind.InterfaceKindChoices.IMAGE, store_in_database=False ) - ai.algorithm.inputs.set([ci]) + interface = AlgorithmInterfaceFactory( + inputs=[ci], + ) + ai.algorithm.interfaces.add( + interface, through_defaults={"is_default": True} + ) image1, image2 = ImageFactory.create_batch(2) assign_perm("cases.view_image", editor, image1) @@ -1025,6 +1036,7 @@ def test_algorithm_job_create_with_image_input( method=client.post, reverse_kwargs={ "slug": ai.algorithm.slug, + "interface_pk": interface.pk, }, user=editor, follow=True, @@ -1049,6 +1061,7 @@ def test_algorithm_job_create_with_image_input( method=client.post, reverse_kwargs={ "slug": ai.algorithm.slug, + "interface_pk": interface.pk, }, user=editor, follow=True, @@ -1076,6 +1089,7 @@ def test_algorithm_job_create_with_image_input( method=client.post, reverse_kwargs={ "slug": ai.algorithm.slug, + "interface_pk": interface.pk, }, user=editor, follow=True, From a7ff28f4ec585e1f9399d2780d8fc354017a21dc Mon Sep 17 00:00:00 2001 From: Anne Mickan Date: Thu, 23 Jan 2025 11:36:43 +0100 Subject: [PATCH 07/18] Algorithm interface delete view (#3789) This enables users with the `add_algorithm` permission to remove interfaces from their algorithm. Part of https://github.com/DIAGNijmegen/rse-roadmap/issues/153 --- ...ithmalgorithminterface_confirm_delete.html | 45 +++++++++ .../algorithmalgorithminterface_list.html | 7 ++ app/grandchallenge/algorithms/urls.py | 6 ++ app/grandchallenge/algorithms/views.py | 37 +++++++- app/tests/algorithms_tests/test_views.py | 91 +++++++++++++++++++ 5 files changed, 185 insertions(+), 1 deletion(-) create mode 100644 app/grandchallenge/algorithms/templates/algorithms/algorithmalgorithminterface_confirm_delete.html diff --git a/app/grandchallenge/algorithms/templates/algorithms/algorithmalgorithminterface_confirm_delete.html b/app/grandchallenge/algorithms/templates/algorithms/algorithmalgorithminterface_confirm_delete.html new file mode 100644 index 0000000000..58ca9ae6a2 --- /dev/null +++ b/app/grandchallenge/algorithms/templates/algorithms/algorithmalgorithminterface_confirm_delete.html @@ -0,0 +1,45 @@ +{% extends "base.html" %} +{% load remove_whitespace %} + +{% block title %} + Delete Interface - {{ algorithm.title }} - {{ block.super }} +{% endblock %} + +{% block breadcrumbs %} +
      +
    1. Algorithms +
      2. +
    2. {{ algorithm.title }} +
      3. +
    3. Interfaces +
      4. +
    4. Delete Interface +
      5. +
    +{% endblock %} + +{% block content %} + +

    Confirm Algorithm Interface Deletion

    +
    + {% csrf_token %} +

    Are you sure that you want to delete the following algorithm interface from your algorithm?

    +
      +
    • Inputs: {{ object.interface.inputs.all|oxford_comma }}
      • +
    • Outputs: {{ object.interface.outputs.all|oxford_comma }}
      • +
    +

    + WARNING: You are not able to undo this action. Once the interface is deleted, it is deleted forever. +

    + Cancel + +
    + +{% endblock %} diff --git a/app/grandchallenge/algorithms/templates/algorithms/algorithmalgorithminterface_list.html b/app/grandchallenge/algorithms/templates/algorithms/algorithmalgorithminterface_list.html index 4dccf4ca9b..360db2f1fd 100644 --- a/app/grandchallenge/algorithms/templates/algorithms/algorithmalgorithminterface_list.html +++ b/app/grandchallenge/algorithms/templates/algorithms/algorithmalgorithminterface_list.html @@ -35,6 +35,7 @@

    Algorithm Interfaces for {{ algorithm }}

    Inputs Outputs Default + Delete {% for object in object_list %} @@ -42,6 +43,12 @@

    Algorithm Interfaces for {{ algorithm }}

    {{ object.interface.inputs.all|oxford_comma }} {{ object.interface.outputs.all|oxford_comma }} {% if object.is_default %}{% else %}{% endif %} + + + + + {% empty %} This algorithm does not have any interfaces defined yet. diff --git a/app/grandchallenge/algorithms/urls.py b/app/grandchallenge/algorithms/urls.py index 7b424a6e6e..83cd4e6c44 100644 --- a/app/grandchallenge/algorithms/urls.py +++ b/app/grandchallenge/algorithms/urls.py @@ -13,6 +13,7 @@ AlgorithmImageUpdate, AlgorithmImportView, AlgorithmInterfaceForAlgorithmCreate, + AlgorithmInterfaceForAlgorithmDelete, AlgorithmInterfacesForAlgorithmList, AlgorithmList, AlgorithmModelCreate, @@ -63,6 +64,11 @@ AlgorithmInterfaceForAlgorithmCreate.as_view(), name="interface-create", ), + path( + "/interfaces//delete/", + AlgorithmInterfaceForAlgorithmDelete.as_view(), + name="interface-delete", + ), path( "/repository/", AlgorithmRepositoryUpdate.as_view(), diff --git a/app/grandchallenge/algorithms/views.py b/app/grandchallenge/algorithms/views.py index b7fe1ba5d9..de0c7c5618 100644 --- a/app/grandchallenge/algorithms/views.py +++ b/app/grandchallenge/algorithms/views.py @@ -22,6 +22,7 @@ from django.utils.safestring import mark_safe from django.views.generic import ( CreateView, + DeleteView, DetailView, FormView, ListView, @@ -1174,7 +1175,8 @@ class AlgorithmInterfaceForAlgorithmCreate( def get_success_url(self): return reverse( - "algorithms:interface-list", kwargs={"slug": self.algorithm.slug} + "algorithms:interface-list", + kwargs={"slug": self.algorithm.slug}, ) def get_context_data(self, *args, **kwargs): @@ -1205,3 +1207,36 @@ def get_context_data(self, *args, **kwargs): } ) return context + + +class AlgorithmInterfaceForAlgorithmDelete( + AlgorithmInterfacePermissionMixin, DeleteView +): + model = AlgorithmAlgorithmInterface + + @property + def algorithm_interface(self): + return get_object_or_404( + klass=AlgorithmAlgorithmInterface, + algorithm=self.algorithm, + interface__pk=self.kwargs["interface_pk"], + is_default=False, + ) + + def get_object(self, queryset=None): + return self.algorithm_interface + + def get_success_url(self): + return reverse( + "algorithms:interface-list", + kwargs={"slug": self.algorithm.slug}, + ) + + def get_context_data(self, *args, **kwargs): + context = super().get_context_data(*args, **kwargs) + context.update( + { + "algorithm": self.algorithm, + } + ) + return context diff --git a/app/tests/algorithms_tests/test_views.py b/app/tests/algorithms_tests/test_views.py index 2e1f57e49f..cd2fcaeea5 100644 --- a/app/tests/algorithms_tests/test_views.py +++ b/app/tests/algorithms_tests/test_views.py @@ -2260,6 +2260,55 @@ def test_algorithm_interface_view_permission(client, viewname): assert response.status_code == status +@pytest.mark.django_db +def test_algorithm_interface_delete_permission(client): + ( + user_with_alg_add_perm, + user_without_alg_add_perm, + algorithm_editor_with_alg_add, + algorithm_editor_without_alg_add, + ) = UserFactory.create_batch(4) + assign_perm("algorithms.add_algorithm", user_with_alg_add_perm) + assign_perm("algorithms.add_algorithm", algorithm_editor_with_alg_add) + + alg = AlgorithmFactory() + alg.add_editor(algorithm_editor_with_alg_add) + alg.add_editor(algorithm_editor_without_alg_add) + + int1, int2 = AlgorithmInterfaceFactory.create_batch(2) + alg.interfaces.add(int1, through_defaults={"is_default": True}) + alg.interfaces.add(int2) + + for user, status1, status2 in [ + [user_with_alg_add_perm, 403, 403], + [user_without_alg_add_perm, 403, 403], + [algorithm_editor_with_alg_add, 200, 404], + [algorithm_editor_without_alg_add, 403, 403], + ]: + response = get_view_for_user( + viewname="algorithms:interface-delete", + client=client, + reverse_kwargs={ + "slug": alg.slug, + "interface_pk": int2.pk, + }, + user=user, + ) + assert response.status_code == status1 + + # default interface cannot be deleted + response = get_view_for_user( + viewname="algorithms:interface-delete", + client=client, + reverse_kwargs={ + "slug": alg.slug, + "interface_pk": int1.pk, + }, + user=user, + ) + assert response.status_code == status2 + + @pytest.mark.django_db def test_algorithm_interface_create(client): user = UserFactory() @@ -2325,6 +2374,48 @@ def test_algorithm_interfaces_list_queryset(client): assert iots[3] not in response.context["object_list"] +@pytest.mark.django_db +def test_algorithm_interface_delete(client): + user = UserFactory() + assign_perm("algorithms.add_algorithm", user) + alg = AlgorithmFactory() + alg.add_editor(user) + + int1, int2 = AlgorithmInterfaceFactory.create_batch(2) + alg.interfaces.add(int1, through_defaults={"is_default": True}) + alg.interfaces.add(int2) + + response = get_view_for_user( + viewname="algorithms:interface-delete", + client=client, + method=client.post, + reverse_kwargs={ + "slug": alg.slug, + "interface_pk": int1.pk, + }, + user=user, + ) + assert response.status_code == 404 + + response = get_view_for_user( + viewname="algorithms:interface-delete", + client=client, + method=client.post, + reverse_kwargs={ + "slug": alg.slug, + "interface_pk": int2.pk, + }, + user=user, + ) + assert response.status_code == 302 + # no interface was deleted + assert AlgorithmInterface.objects.count() == 2 + # only the relation between interface and algorithm was deleted + assert AlgorithmAlgorithmInterface.objects.count() == 1 + assert alg.interfaces.count() == 1 + assert alg.interfaces.get() == int1 + + @pytest.mark.django_db def test_interface_select_for_job_view_permission(client): verified_user, unverified_user = UserFactory.create_batch(2) From a8cfa597bf980706c4fa763ca93647736e98a104 Mon Sep 17 00:00:00 2001 From: Anne Mickan Date: Mon, 27 Jan 2025 11:50:51 +0100 Subject: [PATCH 08/18] Unique input sets per algorithm (#3797) Adds restriction that interfaces for an algorithm need to have unique sets of inputs, discussed [here](https://github.com/DIAGNijmegen/rse-roadmap/issues/153#issuecomment-2602182907). Also fixes an oversights from an earlier PR. --- app/grandchallenge/algorithms/admin.py | 11 +++++ app/grandchallenge/algorithms/forms.py | 60 ++++++++++++++++++----- app/grandchallenge/algorithms/models.py | 21 ++++++-- app/tests/algorithms_tests/test_forms.py | 46 +++++++---------- app/tests/algorithms_tests/test_models.py | 12 +++++ 5 files changed, 105 insertions(+), 45 deletions(-) diff --git a/app/grandchallenge/algorithms/admin.py b/app/grandchallenge/algorithms/admin.py index 4b1de23ec4..669d270ff7 100644 --- a/app/grandchallenge/algorithms/admin.py +++ b/app/grandchallenge/algorithms/admin.py @@ -264,12 +264,15 @@ def algorithm_outputs(self, obj): return oxford_comma(obj.outputs.all()) def has_change_permission(self, request, obj=None): + # interfaces cannot be modified return False def has_delete_permission(self, request, obj=None): + # interfaces cannot be deleted return False def has_add_permission(self, request, obj=None): + # interfaces should only be created through the UI return False @@ -283,6 +286,14 @@ class AlgorithmAlgorithmInterfaceAdmin(GuardedModelAdmin): ) list_filter = ("is_default", "algorithm") + def has_add_permission(self, request, obj=None): + # through table entries should only be created through the UI + return False + + def has_change_permission(self, request, obj=None): + # through table entries should only be updated through the UI + return False + admin.site.register(AlgorithmUserObjectPermission, UserObjectPermissionAdmin) admin.site.register(AlgorithmGroupObjectPermission, GroupObjectPermissionAdmin) diff --git a/app/grandchallenge/algorithms/forms.py b/app/grandchallenge/algorithms/forms.py index 8ad5fc4ee3..4154a48c67 100644 --- a/app/grandchallenge/algorithms/forms.py +++ b/app/grandchallenge/algorithms/forms.py @@ -1401,24 +1401,60 @@ def clean_set_as_default(self): return set_as_default - def clean(self): - cleaned_data = super().clean() + def clean_inputs(self): + inputs = self.cleaned_data.get("inputs", []) - inputs = cleaned_data.get("inputs", []) - outputs = cleaned_data.get("outputs", []) + if not inputs: + raise ValidationError("You must provide at least 1 input.") - if not inputs or not outputs: + if ( + AlgorithmAlgorithmInterface.objects.filter( + algorithm=self._algorithm + ) + .annotate( + input_count=Count("interface__inputs", distinct=True), + relevant_input_count=Count( + "interface__inputs", + filter=Q(interface__inputs__in=inputs), + distinct=True, + ), + ) + .filter(input_count=len(inputs), relevant_input_count=len(inputs)) + .exists() + ): raise ValidationError( - "You must provide at least 1 input and 1 output." + "An AlgorithmInterface for this algorithm with the " + "same inputs already exists. " + "Algorithm interfaces need to have unique sets of inputs." ) + return inputs - duplicate_interfaces = {*inputs}.intersection({*outputs}) + def clean_outputs(self): + outputs = self.cleaned_data.get("outputs", []) - if duplicate_interfaces: - raise ValidationError( - f"The sets of Inputs and Outputs must be unique: " - f"{oxford_comma(duplicate_interfaces)} present in both" - ) + if not outputs: + raise ValidationError("You must provide at least 1 output.") + + return outputs + + def clean(self): + cleaned_data = super().clean() + + # there should always be at least one input and one output, + # this is checked in the individual fields clean methods + inputs = cleaned_data.get("inputs") + outputs = cleaned_data.get("outputs") + + # if either of the two fields is not provided, no need to check for + # duplicates here + if inputs and outputs: + duplicate_interfaces = {*inputs}.intersection({*outputs}) + + if duplicate_interfaces: + raise ValidationError( + f"The sets of Inputs and Outputs must be unique: " + f"{oxford_comma(duplicate_interfaces)} present in both" + ) return cleaned_data diff --git a/app/grandchallenge/algorithms/models.py b/app/grandchallenge/algorithms/models.py index 17bc7acf5b..d28a12a93f 100644 --- a/app/grandchallenge/algorithms/models.py +++ b/app/grandchallenge/algorithms/models.py @@ -78,6 +78,11 @@ def create( outputs, **kwargs, ): + if not inputs or not outputs: + raise ValidationError( + "An interface must have at least one input and one output." + ) + obj = get_existing_interface_for_inputs_and_outputs( inputs=inputs, outputs=outputs ) @@ -980,14 +985,22 @@ def get_jobs_with_same_inputs( else: unique_kwargs["algorithm_model__isnull"] = True + # annotate the number of inputs and the number of inputs that match + # the existing civs and filter on both counts so as to not include jobs + # with partially overlapping inputs + # or jobs with more inputs than the existing civs existing_jobs = ( Job.objects.filter(**unique_kwargs) .annotate( - inputs_match_count=Count( - "inputs", filter=Q(inputs__in=existing_civs) - ) + input_count=Count("inputs", distinct=True), + input_match_count=Count( + "inputs", filter=Q(inputs__in=existing_civs), distinct=True + ), + ) + .filter( + input_count=input_interface_count, + input_match_count=input_interface_count, ) - .filter(inputs_match_count=input_interface_count) ) return existing_jobs diff --git a/app/tests/algorithms_tests/test_forms.py b/app/tests/algorithms_tests/test_forms.py index 5ff451df71..52a111de12 100644 --- a/app/tests/algorithms_tests/test_forms.py +++ b/app/tests/algorithms_tests/test_forms.py @@ -1433,6 +1433,23 @@ def test_algorithm_interface_default_interface_required(): assert form.is_valid() +@pytest.mark.django_db +def test_algorithm_interface_unique_inputs_required(): + ci1, ci2 = ComponentInterfaceFactory.create_batch(2) + alg = AlgorithmFactory() + interface = AlgorithmInterfaceFactory(inputs=[ci1]) + alg.interfaces.add(interface, through_defaults={"is_default": True}) + + form = AlgorithmInterfaceForm( + algorithm=alg, data={"inputs": [ci1], "outputs": [ci2]} + ) + assert form.is_valid() is False + assert ( + "An AlgorithmInterface for this algorithm with the same inputs already exists" + in str(form.errors) + ) + + def test_algorithm_for_phase_form_memory(): form = AlgorithmForPhaseForm( workstation_config=WorkstationConfigFactory.build(), @@ -1557,32 +1574,3 @@ def test_default_interface_for_algorithm_not_updated_when_adding_new_non_default assert new_io != io assert not new_iot.is_default assert old_iot.is_default - - @pytest.mark.django_db - def test_is_default_is_updated_when_adding_an_already_existing_interface( - self, - ): - alg = AlgorithmFactory() - ci_1, ci_2 = ComponentInterfaceFactory.create_batch(2) - - io = AlgorithmInterfaceFactory() - io.inputs.set([ci_1]) - io.outputs.set([ci_2]) - - alg.interfaces.add(io, through_defaults={"is_default": True}) - old_iot = AlgorithmAlgorithmInterface.objects.get() - - form = AlgorithmInterfaceForm( - algorithm=alg, - data={ - "inputs": [ci_1.pk], - "outputs": [ci_2.pk], - "set_as_default": False, - }, - ) - form.is_valid() - new_io = form.save() - old_iot.refresh_from_db() - - assert new_io == io - assert not old_iot.is_default diff --git a/app/tests/algorithms_tests/test_models.py b/app/tests/algorithms_tests/test_models.py index b80a048e53..45274c8f43 100644 --- a/app/tests/algorithms_tests/test_models.py +++ b/app/tests/algorithms_tests/test_models.py @@ -818,6 +818,18 @@ def test_job_with_partially_overlapping_input( ComponentInterfaceValueFactory(), ] ) + j2 = AlgorithmJobFactory( + algorithm_image=alg.active_image, + time_limit=10, + algorithm_interface=alg.default_interface, + ) + j2.inputs.set( + [ + civs[0], + civs[1], + ComponentInterfaceValueFactory(), + ] + ) jobs = Job.objects.get_jobs_with_same_inputs( inputs=data, algorithm_image=alg.active_image, From 5de9ea1f79224994441a906ae510c8118b3e3deb Mon Sep 17 00:00:00 2001 From: Anne Mickan Date: Mon, 27 Jan 2025 15:24:31 +0100 Subject: [PATCH 09/18] Create job with interface through API (#3799) This adds input-to-interface matching to the JobPostSerializer so that algorithm jobs created through the API also have the interface configured. This also adds a serializer for algorithm interfaces. Part of https://github.com/DIAGNijmegen/rse-roadmap/issues/153 --- app/grandchallenge/algorithms/models.py | 27 ++-- app/grandchallenge/algorithms/serializers.py | 86 +++++----- app/tests/algorithms_tests/test_api.py | 75 +++++---- .../algorithms_tests/test_permissions.py | 26 +-- .../algorithms_tests/test_serializers.py | 149 ++++++++++++++++-- app/tests/algorithms_tests/test_views.py | 111 ++++++++++++- 6 files changed, 368 insertions(+), 106 deletions(-) diff --git a/app/grandchallenge/algorithms/models.py b/app/grandchallenge/algorithms/models.py index d28a12a93f..0392b369d8 100644 --- a/app/grandchallenge/algorithms/models.py +++ b/app/grandchallenge/algorithms/models.py @@ -96,6 +96,22 @@ def create( def delete(self): raise NotImplementedError("Bulk delete is not allowed.") + def with_input_output_counts(self, inputs=None, outputs=None): + return self.annotate( + input_count=Count("inputs", distinct=True), + output_count=Count("outputs", distinct=True), + relevant_input_count=Count( + "inputs", + filter=Q(inputs__in=inputs) if inputs is not None else Q(), + distinct=True, + ), + relevant_output_count=Count( + "outputs", + filter=Q(outputs__in=outputs) if outputs is not None else Q(), + distinct=True, + ), + ) + class AlgorithmInterface(UUIDModel): inputs = models.ManyToManyField( @@ -129,15 +145,8 @@ def get_existing_interface_for_inputs_and_outputs( *, inputs, outputs, model=AlgorithmInterface ): try: - return model.objects.annotate( - input_count=Count("inputs", distinct=True), - output_count=Count("outputs", distinct=True), - relevant_input_count=Count( - "inputs", filter=Q(inputs__in=inputs), distinct=True - ), - relevant_output_count=Count( - "outputs", filter=Q(outputs__in=outputs), distinct=True - ), + return model.objects.with_input_output_counts( + inputs=inputs, outputs=outputs ).get( relevant_input_count=len(inputs), relevant_output_count=len(outputs), diff --git a/app/grandchallenge/algorithms/serializers.py b/app/grandchallenge/algorithms/serializers.py index 28ece083a8..af9857b420 100644 --- a/app/grandchallenge/algorithms/serializers.py +++ b/app/grandchallenge/algorithms/serializers.py @@ -1,6 +1,6 @@ import logging -from django.core.exceptions import ValidationError +from django.core.exceptions import ObjectDoesNotExist, ValidationError from rest_framework import serializers from rest_framework.fields import ( CharField, @@ -16,13 +16,14 @@ from grandchallenge.algorithms.models import ( Algorithm, AlgorithmImage, + AlgorithmInterface, AlgorithmModel, Job, ) from grandchallenge.components.backends.exceptions import ( CIVNotEditableException, ) -from grandchallenge.components.models import CIVData, ComponentInterface +from grandchallenge.components.models import CIVData from grandchallenge.components.serializers import ( ComponentInterfaceSerializer, ComponentInterfaceValuePostSerializer, @@ -30,6 +31,7 @@ HyperlinkedComponentInterfaceValueSerializer, ) from grandchallenge.core.guardian import filter_by_permission +from grandchallenge.core.templatetags.remove_whitespace import oxford_comma from grandchallenge.hanging_protocols.serializers import ( HangingProtocolSerializer, ) @@ -37,12 +39,29 @@ logger = logging.getLogger(__name__) +class AlgorithmInterfaceSerializer(serializers.ModelSerializer): + """Serializer without hyperlinks for internal use""" + + inputs = ComponentInterfaceSerializer(many=True, read_only=True) + outputs = ComponentInterfaceSerializer(many=True, read_only=True) + + class Meta: + model = AlgorithmInterface + fields = [ + "inputs", + "outputs", + ] + + class AlgorithmSerializer(serializers.ModelSerializer): average_duration = SerializerMethodField() + # TODO: Still to be addressed for optional inputs pitch + # remove inputs and outputs inputs = ComponentInterfaceSerializer(many=True, read_only=True) outputs = ComponentInterfaceSerializer(many=True, read_only=True) logo = URLField(source="logo.x20.url", read_only=True) url = URLField(source="get_absolute_url", read_only=True) + interfaces = AlgorithmInterfaceSerializer(many=True, read_only=True) class Meta: model = Algorithm @@ -57,6 +76,7 @@ class Meta: "average_duration", "inputs", "outputs", + "interfaces", ] def get_average_duration(self, obj: Algorithm) -> float | None: @@ -158,7 +178,6 @@ class HyperlinkedJobSerializer(JobSerializer): view_name="api:algorithm-detail", read_only=True, ) - inputs = HyperlinkedComponentInterfaceValueSerializer(many=True) outputs = HyperlinkedComponentInterfaceValueSerializer(many=True) @@ -215,45 +234,10 @@ def validate(self, data): "You have run out of algorithm credits" ) - # validate that no inputs are provided that are not configured for the - # algorithm and that all interfaces without defaults are provided - algorithm_input_pks = {a.pk for a in self._algorithm.inputs.all()} - input_pks = {i["interface"].pk for i in data["inputs"]} - - # surplus inputs: provided but interfaces not configured for the algorithm - surplus = ComponentInterface.objects.filter( - id__in=list(input_pks - algorithm_input_pks) - ) - if surplus: - titles = ", ".join(ci.title for ci in surplus) - raise serializers.ValidationError( - f"Provided inputs(s) {titles} are not defined for this algorithm" - ) - - # missing inputs - missing = self._algorithm.inputs.filter( - id__in=list(algorithm_input_pks - input_pks), - default_value__isnull=True, - ) - if missing: - titles = ", ".join(ci.title for ci in missing) - raise serializers.ValidationError( - f"Interface(s) {titles} do not have a default value and should be provided." - ) - inputs = data.pop("inputs") - - default_inputs = self._algorithm.inputs.filter( - id__in=list(algorithm_input_pks - input_pks), - default_value__isnull=False, + data["algorithm_interface"] = ( + self.validate_inputs_and_return_matching_interface(inputs=inputs) ) - # Use default interface values if not present - for interface in default_inputs: - if interface.default_value: - inputs.append( - {"interface": interface, "value": interface.default_value} - ) - self.inputs = self.reformat_inputs(serialized_civs=inputs) if Job.objects.get_jobs_with_same_inputs( @@ -298,6 +282,28 @@ def create(self, validated_data): return job + def validate_inputs_and_return_matching_interface(self, *, inputs): + """ + Validates that the provided inputs match one of the configured interfaces of + the algorithm and returns that AlgorithmInterface + """ + provided_inputs = {i["interface"] for i in inputs} + try: + interface = self._algorithm.interfaces.with_input_output_counts( + inputs=provided_inputs + ).get( + relevant_input_count=len(provided_inputs), + input_count=len(provided_inputs), + ) + return interface + except ObjectDoesNotExist: + raise serializers.ValidationError( + f"The set of inputs provided does not match " + f"any of the algorithm's interfaces. This algorithm supports the " + f"following input combinations: " + f"{oxford_comma([f'Interface {n}: {oxford_comma(interface.inputs.all())}' for n, interface in enumerate(self._algorithm.interfaces.all(), start=1)])}" + ) + @staticmethod def reformat_inputs(*, serialized_civs): """Takes serialized CIV data and returns list of CIVData objects.""" diff --git a/app/tests/algorithms_tests/test_api.py b/app/tests/algorithms_tests/test_api.py index de7591df68..3b91399d83 100644 --- a/app/tests/algorithms_tests/test_api.py +++ b/app/tests/algorithms_tests/test_api.py @@ -18,6 +18,7 @@ from grandchallenge.uploads.models import UserUpload from tests.algorithms_tests.factories import ( AlgorithmImageFactory, + AlgorithmInterfaceFactory, AlgorithmJobFactory, AlgorithmModelFactory, ) @@ -93,7 +94,6 @@ def test_job_list_view_num_queries( assert len(response.json()["results"]) == num_jobs -@pytest.mark.xfail(reason="Still to be addressed for optional inputs pitch") @pytest.mark.django_db class TestJobCreationThroughAPI: @@ -151,15 +151,19 @@ def test_create_job_with_multiple_new_inputs( algorithm_with_multiple_inputs, ): # configure multiple inputs - algorithm_with_multiple_inputs.algorithm.inputs.set( - [ + interface = AlgorithmInterfaceFactory( + inputs=[ algorithm_with_multiple_inputs.ci_json_in_db_with_schema, algorithm_with_multiple_inputs.ci_existing_img, algorithm_with_multiple_inputs.ci_str, algorithm_with_multiple_inputs.ci_bool, algorithm_with_multiple_inputs.ci_json_file, algorithm_with_multiple_inputs.ci_img_upload, - ] + ], + outputs=[ComponentInterfaceFactory()], + ) + algorithm_with_multiple_inputs.algorithm.interfaces.add( + interface, through_defaults={"is_default": True} ) assert ComponentInterfaceValue.objects.count() == 0 @@ -217,12 +221,9 @@ def test_create_job_with_multiple_new_inputs( pk=algorithm_with_multiple_inputs.file_upload.pk ).exists() - assert sorted( - [ - int.pk - for int in algorithm_with_multiple_inputs.algorithm.inputs.all() - ] - ) == sorted([civ.interface.pk for civ in job.inputs.all()]) + assert sorted([int.pk for int in interface.inputs.all()]) == sorted( + [civ.interface.pk for civ in job.inputs.all()] + ) value_inputs = [civ.value for civ in job.inputs.all() if civ.value] assert "Foo" in value_inputs @@ -245,13 +246,17 @@ def test_create_job_with_existing_inputs( algorithm_with_multiple_inputs, ): # configure multiple inputs - algorithm_with_multiple_inputs.algorithm.inputs.set( - [ + interface = AlgorithmInterfaceFactory( + inputs=[ algorithm_with_multiple_inputs.ci_json_in_db_with_schema, algorithm_with_multiple_inputs.ci_existing_img, algorithm_with_multiple_inputs.ci_str, algorithm_with_multiple_inputs.ci_bool, - ] + ], + outputs=[ComponentInterfaceFactory()], + ) + algorithm_with_multiple_inputs.algorithm.interfaces.add( + interface, through_defaults={"is_default": True} ) civ1, civ2, civ3, civ4 = self.create_existing_civs( @@ -302,13 +307,17 @@ def test_create_job_is_idempotent( algorithm_with_multiple_inputs, ): # configure multiple inputs - algorithm_with_multiple_inputs.algorithm.inputs.set( - [ + interface = AlgorithmInterfaceFactory( + inputs=[ algorithm_with_multiple_inputs.ci_str, algorithm_with_multiple_inputs.ci_bool, algorithm_with_multiple_inputs.ci_existing_img, algorithm_with_multiple_inputs.ci_json_in_db_with_schema, - ] + ], + outputs=[ComponentInterfaceFactory()], + ) + algorithm_with_multiple_inputs.algorithm.interfaces.add( + interface, through_defaults={"is_default": True} ) civ1, civ2, civ3, civ4 = self.create_existing_civs( interface_data=algorithm_with_multiple_inputs @@ -366,8 +375,12 @@ def test_create_job_with_faulty_file_input( algorithm_with_multiple_inputs, ): # configure file input - algorithm_with_multiple_inputs.algorithm.inputs.set( - [algorithm_with_multiple_inputs.ci_json_file] + interface = AlgorithmInterfaceFactory( + inputs=[algorithm_with_multiple_inputs.ci_json_file], + outputs=[ComponentInterfaceFactory()], + ) + algorithm_with_multiple_inputs.algorithm.interfaces.add( + interface, through_defaults={"is_default": True} ) file_upload = UserUploadFactory( filename="file.json", creator=algorithm_with_multiple_inputs.editor @@ -414,8 +427,12 @@ def test_create_job_with_faulty_json_input( django_capture_on_commit_callbacks, algorithm_with_multiple_inputs, ): - algorithm_with_multiple_inputs.algorithm.inputs.set( - [algorithm_with_multiple_inputs.ci_json_in_db_with_schema] + interface = AlgorithmInterfaceFactory( + inputs=[algorithm_with_multiple_inputs.ci_json_in_db_with_schema], + outputs=[ComponentInterfaceFactory()], + ) + algorithm_with_multiple_inputs.algorithm.interfaces.add( + interface, through_defaults={"is_default": True} ) response = self.create_job( @@ -445,8 +462,12 @@ def test_create_job_with_faulty_image_input( django_capture_on_commit_callbacks, algorithm_with_multiple_inputs, ): - algorithm_with_multiple_inputs.algorithm.inputs.set( - [algorithm_with_multiple_inputs.ci_img_upload] + interface = AlgorithmInterfaceFactory( + inputs=[algorithm_with_multiple_inputs.ci_img_upload], + outputs=[ComponentInterfaceFactory()], + ) + algorithm_with_multiple_inputs.algorithm.interfaces.add( + interface, through_defaults={"is_default": True} ) user_upload = create_upload_from_file( creator=algorithm_with_multiple_inputs.editor, @@ -501,11 +522,11 @@ def test_create_job_with_multiple_faulty_existing_image_inputs( ] ci.save() - algorithm_with_multiple_inputs.algorithm.inputs.set( - [ - ci1, - ci2, - ] + interface = AlgorithmInterfaceFactory( + inputs=[ci1, ci2], outputs=[ComponentInterfaceFactory()] + ) + algorithm_with_multiple_inputs.algorithm.interfaces.add( + interface, through_defaults={"is_default": True} ) im = ImageFactory() diff --git a/app/tests/algorithms_tests/test_permissions.py b/app/tests/algorithms_tests/test_permissions.py index 3a0ff8951c..c96b0f03e5 100644 --- a/app/tests/algorithms_tests/test_permissions.py +++ b/app/tests/algorithms_tests/test_permissions.py @@ -345,9 +345,6 @@ def test_job_permissions_from_template(self, client): algorithm_image=algorithm_image, job=job, user=user ) - @pytest.mark.xfail( - reason="Still to be addressed for optional inputs pitch" - ) def test_job_permissions_from_api(self, rf): # setup user = UserFactory() @@ -356,18 +353,23 @@ def test_job_permissions_from_api(self, rf): is_in_registry=True, is_desired_version=True, ) - interfaces = { - ComponentInterfaceFactory( - kind=ComponentInterface.Kind.STRING, - title="TestInterface 1", - default_value="default", - ), - } - algorithm_image.algorithm.inputs.set(interfaces) + ci = ComponentInterfaceFactory( + kind=ComponentInterface.Kind.STRING, + title="TestInterface 1", + default_value="default", + ) + + interface = AlgorithmInterfaceFactory(inputs=[ci]) + algorithm_image.algorithm.interfaces.add( + interface, through_defaults={"is_default": True} + ) algorithm_image.algorithm.add_user(user) algorithm_image.algorithm.add_editor(UserFactory()) - job = {"algorithm": algorithm_image.algorithm.api_url, "inputs": []} + job = { + "algorithm": algorithm_image.algorithm.api_url, + "inputs": [{"interface": ci.slug, "value": "foo"}], + } # test request = rf.get("/foo") diff --git a/app/tests/algorithms_tests/test_serializers.py b/app/tests/algorithms_tests/test_serializers.py index eb5cd14412..4e9c90d7a2 100644 --- a/app/tests/algorithms_tests/test_serializers.py +++ b/app/tests/algorithms_tests/test_serializers.py @@ -12,6 +12,7 @@ from tests.algorithms_tests.factories import ( AlgorithmFactory, AlgorithmImageFactory, + AlgorithmInterfaceFactory, AlgorithmJobFactory, ) from tests.cases_tests.factories import RawImageUploadSessionFactory @@ -35,7 +36,6 @@ def test_algorithm_relations_on_job_serializer(rf): ) -@pytest.mark.xfail(reason="Still to be addressed for optional inputs pitch") @pytest.mark.django_db @pytest.mark.parametrize( "title, " @@ -79,7 +79,7 @@ def test_algorithm_relations_on_job_serializer(rf): True, ("TestInterface 1", "TestInterface 2"), ("testinterface-1",), - "Interface(s) TestInterface 2 do not have a default value and should be provided.", + "The set of inputs provided does not match any of the algorithm's interfaces.", False, ), ( @@ -88,7 +88,7 @@ def test_algorithm_relations_on_job_serializer(rf): True, ("TestInterface 1",), ("testinterface-1", "testinterface-2"), - "Provided inputs(s) TestInterface 2 are not defined for this algorithm", + "The set of inputs provided does not match any of the algorithm's interfaces.", False, ), ( @@ -137,8 +137,11 @@ def test_algorithm_job_post_serializer_validations( is_desired_version=image_ready, ) algorithm_image.algorithm.title = title - algorithm_image.algorithm.inputs.set( - [interfaces[title] for title in algorithm_interface_titles] + interface = AlgorithmInterfaceFactory( + inputs=[interfaces[title] for title in algorithm_interface_titles] + ) + algorithm_image.algorithm.interfaces.add( + interface, through_defaults={"is_default": True} ) if add_user: algorithm_image.algorithm.add_user(user) @@ -161,8 +164,7 @@ def test_algorithm_job_post_serializer_validations( job = { "algorithm": algorithm_image.algorithm.api_url, "inputs": [ - {"interface": interface, "value": "dummy"} - for interface in job_interface_slugs + {"interface": int, "value": "dummy"} for int in job_interface_slugs ], } @@ -184,7 +186,6 @@ def test_algorithm_job_post_serializer_validations( assert len(job.inputs.all()) == 2 -@pytest.mark.xfail(reason="Still to be addressed for optional inputs pitch") @pytest.mark.django_db def test_algorithm_job_post_serializer_create( rf, settings, django_capture_on_commit_callbacks @@ -212,7 +213,10 @@ def test_algorithm_job_post_serializer_create( ci_img1 = ComponentInterfaceFactory(kind=ComponentInterface.Kind.IMAGE) ci_img2 = ComponentInterfaceFactory(kind=ComponentInterface.Kind.IMAGE) - algorithm_image.algorithm.inputs.set([ci_string, ci_img2, ci_img1]) + interface = AlgorithmInterfaceFactory(inputs=[ci_string, ci_img2, ci_img1]) + algorithm_image.algorithm.interfaces.add( + interface, through_defaults={"is_default": True} + ) algorithm_image.algorithm.add_editor(user) job = { @@ -228,8 +232,22 @@ def test_algorithm_job_post_serializer_create( request.user = user serializer = JobPostSerializer(data=job, context={"request": request}) - # verify + # all inputs need to be provided, also those with default value + assert not serializer.is_valid() + + # add missing input + job = { + "algorithm": algorithm_image.algorithm.api_url, + "inputs": [ + {"interface": ci_img1.slug, "upload_session": upload.api_url}, + {"interface": ci_img2.slug, "image": image2.api_url}, + {"interface": ci_string.slug, "value": "foo"}, + ], + } + serializer = JobPostSerializer(data=job, context={"request": request}) + assert serializer.is_valid() + # fake successful upload upload.status = RawImageUploadSession.SUCCESS upload.save() @@ -240,9 +258,9 @@ def test_algorithm_job_post_serializer_create( job = Job.objects.first() assert job.creator == user assert len(job.inputs.all()) == 3 + assert job.algorithm_interface == interface -@pytest.mark.xfail(reason="Still to be addressed for optional inputs pitch") @pytest.mark.django_db class TestJobCreateLimits: def test_form_invalid_without_enough_credits(self, rf, settings): @@ -292,13 +310,20 @@ def test_form_valid_for_editor(self, rf, settings): user = UserFactory() algorithm_image.algorithm.add_editor(user=user) + ci = ComponentInterfaceFactory(kind=ComponentInterface.Kind.STRING) + interface = AlgorithmInterfaceFactory(inputs=[ci]) + algorithm_image.algorithm.interfaces.add( + interface, through_defaults={"is_default": True} + ) request = rf.get("/foo") request.user = user serializer = JobPostSerializer( data={ "algorithm": algorithm_image.algorithm.api_url, - "inputs": [], + "inputs": [ + {"interface": ci.slug, "value": "foo"}, + ], }, context={"request": request}, ) @@ -315,7 +340,9 @@ def test_form_valid_for_editor(self, rf, settings): serializer = JobPostSerializer( data={ "algorithm": algorithm_image.algorithm.api_url, - "inputs": [], + "inputs": [ + {"interface": ci.slug, "value": "foo"}, + ], }, context={"request": request}, ) @@ -333,13 +360,20 @@ def test_form_valid_with_credits(self, rf): user = UserFactory() algorithm_image.algorithm.add_user(user=user) + ci = ComponentInterfaceFactory(kind=ComponentInterface.Kind.STRING) + interface = AlgorithmInterfaceFactory(inputs=[ci]) + algorithm_image.algorithm.interfaces.add( + interface, through_defaults={"is_default": True} + ) request = rf.get("/foo") request.user = user serializer = JobPostSerializer( data={ "algorithm": algorithm_image.algorithm.api_url, - "inputs": [], + "inputs": [ + {"interface": ci.slug, "value": "foo"}, + ], }, context={"request": request}, ) @@ -383,7 +417,6 @@ def test_reformat_inputs(rf): ) -@pytest.mark.xfail(reason="Still to be addressed for optional inputs pitch") @pytest.mark.django_db def test_algorithm_post_serializer_image_and_time_limit_fixed(rf): request = rf.get("/foo") @@ -398,7 +431,9 @@ def test_algorithm_post_serializer_image_and_time_limit_fixed(rf): ) different_ai = AlgorithmImageFactory(algorithm=alg) ci = ComponentInterfaceFactory(kind=ComponentInterface.Kind.STRING) - alg.inputs.set([ci]) + interface = AlgorithmInterfaceFactory(inputs=[ci]) + alg.interfaces.add(interface, through_defaults={"is_default": True}) + serializer = JobPostSerializer( data={ "algorithm": alg.api_url, @@ -416,3 +451,85 @@ def test_algorithm_post_serializer_image_and_time_limit_fixed(rf): assert job.algorithm_image != different_ai assert not job.algorithm_model assert job.time_limit == 10 + + +@pytest.mark.parametrize( + "inputs, interface", + ( + ([1], 1), # matches interface 1 of algorithm + ([1, 2], 2), # matches interface 2 of algorithm + ([3, 4, 5], 3), # matches interface 3 of algorithm + ([4], None), # matches interface 4, but not configured for algorithm + ( + [1, 2, 3], + None, + ), # matches interface 5, but not configured for algorithm + ([2], None), # matches no interface (implements part of interface 2) + ( + [1, 3, 4], + None, + ), # matches no interface (implements interface 3 and an additional input) + ), +) +@pytest.mark.django_db +def test_validate_inputs_on_job_serializer(inputs, interface, rf): + user = UserFactory() + algorithm = AlgorithmFactory() + algorithm.add_editor(user) + AlgorithmImageFactory( + algorithm=algorithm, + is_desired_version=True, + is_manifest_valid=True, + is_in_registry=True, + ) + + io1, io2, io3, io4, io5 = AlgorithmInterfaceFactory.create_batch(5) + ci1, ci2, ci3, ci4, ci5, ci6 = ComponentInterfaceFactory.create_batch( + 6, kind=ComponentInterface.Kind.STRING + ) + + interfaces = [io1, io2, io3] + cis = [ci1, ci2, ci3, ci4, ci5, ci6] + + io1.inputs.set([ci1]) + io2.inputs.set([ci1, ci2]) + io3.inputs.set([ci3, ci4, ci5]) + io4.inputs.set([ci1, ci2, ci3]) + io5.inputs.set([ci4]) + io1.outputs.set([ci6]) + io2.outputs.set([ci3]) + io3.outputs.set([ci1]) + io4.outputs.set([ci1]) + io5.outputs.set([ci1]) + + algorithm.interfaces.add(io1, through_defaults={"is_default": True}) + algorithm.interfaces.add(io2) + algorithm.interfaces.add(io3) + + algorithm_interface = interfaces[interface - 1] if interface else None + inputs = [cis[i - 1] for i in inputs] + + job = { + "algorithm": algorithm.api_url, + "inputs": [ + {"interface": int.slug, "value": "dummy"} for int in inputs + ], + } + + request = rf.get("/foo") + request.user = user + serializer = JobPostSerializer(data=job, context={"request": request}) + + if interface: + assert serializer.is_valid() + assert ( + serializer.validated_data["algorithm_interface"] + == algorithm_interface + ) + else: + assert not serializer.is_valid() + assert ( + "The set of inputs provided does not match any of the algorithm's interfaces." + in str(serializer.errors) + ) + assert "algorithm_interface" not in serializer.validated_data diff --git a/app/tests/algorithms_tests/test_views.py b/app/tests/algorithms_tests/test_views.py index 51d7fee597..cedf9e818e 100644 --- a/app/tests/algorithms_tests/test_views.py +++ b/app/tests/algorithms_tests/test_views.py @@ -830,6 +830,110 @@ def list_algorithms(self, **__): "look_up_table": None, }, ], + "interfaces": [ + { + "inputs": [ + { + "title": "Coronal T2 Prostate MRI", + "description": "Coronal T2 MRI of the Prostate", + "slug": "coronal-t2-prostate-mri", + "kind": "Image", + "pk": 31, + "default_value": None, + "super_kind": "Image", + "relative_path": "images/coronal-t2-prostate-mri", + "overlay_segments": [], + "look_up_table": None, + }, + { + "title": "Transverse T2 Prostate MRI", + "description": "Transverse T2 MRI of the Prostate", + "slug": "transverse-t2-prostate-mri", + "kind": "Image", + "pk": 32, + "default_value": None, + "super_kind": "Image", + "relative_path": "images/transverse-t2-prostate-mri", + "overlay_segments": [], + "look_up_table": None, + }, + { + "title": "Sagittal T2 Prostate MRI", + "description": "Sagittal T2 MRI of the Prostate", + "slug": "sagittal-t2-prostate-mri", + "kind": "Image", + "pk": 33, + "default_value": None, + "super_kind": "Image", + "relative_path": "images/sagittal-t2-prostate-mri", + "overlay_segments": [], + "look_up_table": None, + }, + { + "title": "Transverse HBV Prostate MRI", + "description": "Transverse High B-Value Prostate MRI", + "slug": "transverse-hbv-prostate-mri", + "kind": "Image", + "pk": 47, + "default_value": None, + "super_kind": "Image", + "relative_path": "images/transverse-hbv-prostate-mri", + "overlay_segments": [], + "look_up_table": None, + }, + { + "title": "Transverse ADC Prostate MRI", + "description": "Transverse Apparent Diffusion Coefficient Prostate MRI", + "slug": "transverse-adc-prostate-mri", + "kind": "Image", + "pk": 48, + "default_value": None, + "super_kind": "Image", + "relative_path": "images/transverse-adc-prostate-mri", + "overlay_segments": [], + "look_up_table": None, + }, + { + "title": "Clinical Information Prostate MRI", + "description": "Clinical information to support clinically significant prostate cancer detection in prostate MRI. Provided information: patient age at time of examination (patient_age), PSA level in ng/mL as reported (PSA_report), PSA density in ng/mL^2 as reported (PSAD_report), prostate volume as reported (prostate_volume_report), prostate volume derived from automatic whole-gland segmentation (prostate_volume_automatic), scanner manufacturer (scanner_manufacturer), scanner model name (scanner_model_name), diffusion b-value of (calculated) high b-value diffusion map (diffusion_high_bvalue). Values acquired from radiology reports will be missing, if not reported.", + "slug": "clinical-information-prostate-mri", + "kind": "Anything", + "pk": 156, + "default_value": None, + "super_kind": "Value", + "relative_path": "clinical-information-prostate-mri.json", + "overlay_segments": [], + "look_up_table": None, + }, + ], + "outputs": [ + { + "title": "Case-level Cancer Likelihood Prostate MRI", + "description": "Case-level likelihood of harboring clinically significant prostate cancer, in range [0,1].", + "slug": "prostate-cancer-likelihood", + "kind": "Float", + "pk": 144, + "default_value": None, + "super_kind": "Value", + "relative_path": "cspca-case-level-likelihood.json", + "overlay_segments": [], + "look_up_table": None, + }, + { + "title": "Transverse Cancer Detection Map Prostate MRI", + "description": "Single-class, detection map of clinically significant prostate cancer lesions in 3D, where each voxel represents a floating point in range [0,1].", + "slug": "cspca-detection-map", + "kind": "Heat Map", + "pk": 151, + "default_value": None, + "super_kind": "Image", + "relative_path": "images/cspca-detection-map", + "overlay_segments": [], + "look_up_table": None, + }, + ], + } + ], } ], } @@ -1805,7 +1909,6 @@ def test_job_gpu_type_set(client, settings): assert algorithm.credits_per_job == 190 -@pytest.mark.xfail(reason="Still to be addressed for optional inputs pitch") @pytest.mark.django_db def test_job_gpu_type_set_with_api(client, settings): settings.COMPONENTS_DEFAULT_BACKEND = "grandchallenge.components.backends.amazon_sagemaker_training.AmazonSageMakerTrainingExecutor" @@ -1827,7 +1930,10 @@ def test_job_gpu_type_set_with_api(client, settings): ci = ComponentInterfaceFactory( kind=InterfaceKind.InterfaceKindChoices.ANY, store_in_database=True ) - algorithm.inputs.set([ci]) + interface = AlgorithmInterfaceFactory( + inputs=[ci], + ) + algorithm.interfaces.add(interface, through_defaults={"is_default": True}) response = get_view_for_user( viewname="api:algorithms-job-list", @@ -1851,6 +1957,7 @@ def test_job_gpu_type_set_with_api(client, settings): job = Job.objects.get() + assert job.algorithm_interface == interface assert job.algorithm_image == algorithm_image assert job.requires_gpu_type == GPUTypeChoices.A10G assert job.requires_memory_gb == 64 From 9a1bd62ce2bc8daeaf6ab069e7522cf261ebe7b2 Mon Sep 17 00:00:00 2001 From: Anne Mickan Date: Thu, 30 Jan 2025 15:33:22 +0100 Subject: [PATCH 10/18] Move input/output annotation from manager to separate method (#3804) I added an input/output count annotation to the `AlgorithmInterfaceManager` in my last PR, but had not tested the migrations again. In my migration check for the phase interfaces, I realized that the migrations break with that addition since they don't have access to custom manager methods. So I'm moving the annotation to a separate function here. The upside of this is that I can now use it in 2 other places where we're making the same annotation. So I'm calling this a win. Note that this does not add or change functionality, it just moves the code. So I didn't spend time on adding a test for the new function - it is indirectly covered by tests of all the places where it is used. --- app/grandchallenge/algorithms/forms.py | 58 +++++++++----------- app/grandchallenge/algorithms/models.py | 58 +++++++++----------- app/grandchallenge/algorithms/serializers.py | 8 ++- 3 files changed, 59 insertions(+), 65 deletions(-) diff --git a/app/grandchallenge/algorithms/forms.py b/app/grandchallenge/algorithms/forms.py index 4154a48c67..fd7a296751 100644 --- a/app/grandchallenge/algorithms/forms.py +++ b/app/grandchallenge/algorithms/forms.py @@ -56,6 +56,7 @@ AlgorithmModel, AlgorithmPermissionRequest, Job, + annotate_input_output_counts, ) from grandchallenge.algorithms.serializers import ( AlgorithmImageSerializer, @@ -462,37 +463,32 @@ def user_algorithms_for_phase(self): desired_model_subquery = AlgorithmModel.objects.filter( algorithm=OuterRef("pk"), is_desired_version=True ) - return ( - get_objects_for_user(self._user, "algorithms.change_algorithm") - .annotate( - total_input_count=Count("inputs", distinct=True), - total_output_count=Count("outputs", distinct=True), - relevant_input_count=Count( - "inputs", filter=Q(inputs__in=inputs), distinct=True - ), - relevant_output_count=Count( - "outputs", filter=Q(outputs__in=outputs), distinct=True - ), - has_active_image=Exists(desired_image_subquery), - active_image_pk=desired_image_subquery.values_list( - "pk", flat=True - ), - active_model_pk=desired_model_subquery.values_list( - "pk", flat=True - ), - active_image_comment=desired_image_subquery.values_list( - "comment", flat=True - ), - active_model_comment=desired_model_subquery.values_list( - "comment", flat=True - ), - ) - .filter( - total_input_count=len(inputs), - total_output_count=len(outputs), - relevant_input_count=len(inputs), - relevant_output_count=len(outputs), - ) + annotated_qs = annotate_input_output_counts( + queryset=get_objects_for_user( + self._user, "algorithms.change_algorithm" + ), + inputs=inputs, + outputs=outputs, + ) + return annotated_qs.annotate( + has_active_image=Exists(desired_image_subquery), + active_image_pk=desired_image_subquery.values_list( + "pk", flat=True + ), + active_model_pk=desired_model_subquery.values_list( + "pk", flat=True + ), + active_image_comment=desired_image_subquery.values_list( + "comment", flat=True + ), + active_model_comment=desired_model_subquery.values_list( + "comment", flat=True + ), + ).filter( + input_count=len(inputs), + output_count=len(outputs), + relevant_input_count=len(inputs), + relevant_output_count=len(outputs), ) @cached_property diff --git a/app/grandchallenge/algorithms/models.py b/app/grandchallenge/algorithms/models.py index 0392b369d8..2c2958a93b 100644 --- a/app/grandchallenge/algorithms/models.py +++ b/app/grandchallenge/algorithms/models.py @@ -69,6 +69,23 @@ JINJA_ENGINE = sandbox.ImmutableSandboxedEnvironment() +def annotate_input_output_counts(queryset, inputs=None, outputs=None): + return queryset.annotate( + input_count=Count("inputs", distinct=True), + output_count=Count("outputs", distinct=True), + relevant_input_count=Count( + "inputs", + filter=Q(inputs__in=inputs) if inputs is not None else Q(), + distinct=True, + ), + relevant_output_count=Count( + "outputs", + filter=Q(outputs__in=outputs) if outputs is not None else Q(), + distinct=True, + ), + ) + + class AlgorithmInterfaceManager(models.Manager): def create( @@ -96,22 +113,6 @@ def create( def delete(self): raise NotImplementedError("Bulk delete is not allowed.") - def with_input_output_counts(self, inputs=None, outputs=None): - return self.annotate( - input_count=Count("inputs", distinct=True), - output_count=Count("outputs", distinct=True), - relevant_input_count=Count( - "inputs", - filter=Q(inputs__in=inputs) if inputs is not None else Q(), - distinct=True, - ), - relevant_output_count=Count( - "outputs", - filter=Q(outputs__in=outputs) if outputs is not None else Q(), - distinct=True, - ), - ) - class AlgorithmInterface(UUIDModel): inputs = models.ManyToManyField( @@ -144,10 +145,11 @@ class AlgorithmInterfaceOutput(models.Model): def get_existing_interface_for_inputs_and_outputs( *, inputs, outputs, model=AlgorithmInterface ): + annotated_qs = annotate_input_output_counts( + model.objects.all(), inputs=inputs, outputs=outputs + ) try: - return model.objects.with_input_output_counts( - inputs=inputs, outputs=outputs - ).get( + return annotated_qs.get( relevant_input_count=len(inputs), relevant_output_count=len(outputs), input_count=len(inputs), @@ -998,18 +1000,12 @@ def get_jobs_with_same_inputs( # the existing civs and filter on both counts so as to not include jobs # with partially overlapping inputs # or jobs with more inputs than the existing civs - existing_jobs = ( - Job.objects.filter(**unique_kwargs) - .annotate( - input_count=Count("inputs", distinct=True), - input_match_count=Count( - "inputs", filter=Q(inputs__in=existing_civs), distinct=True - ), - ) - .filter( - input_count=input_interface_count, - input_match_count=input_interface_count, - ) + annotated_qs = annotate_input_output_counts( + queryset=Job.objects.filter(**unique_kwargs), inputs=existing_civs + ) + existing_jobs = annotated_qs.filter( + input_count=input_interface_count, + relevant_input_count=input_interface_count, ) return existing_jobs diff --git a/app/grandchallenge/algorithms/serializers.py b/app/grandchallenge/algorithms/serializers.py index af9857b420..ac930f13d9 100644 --- a/app/grandchallenge/algorithms/serializers.py +++ b/app/grandchallenge/algorithms/serializers.py @@ -19,6 +19,7 @@ AlgorithmInterface, AlgorithmModel, Job, + annotate_input_output_counts, ) from grandchallenge.components.backends.exceptions import ( CIVNotEditableException, @@ -288,10 +289,11 @@ def validate_inputs_and_return_matching_interface(self, *, inputs): the algorithm and returns that AlgorithmInterface """ provided_inputs = {i["interface"] for i in inputs} + annotated_qs = annotate_input_output_counts( + self._algorithm.interfaces, inputs=provided_inputs + ) try: - interface = self._algorithm.interfaces.with_input_output_counts( - inputs=provided_inputs - ).get( + interface = annotated_qs.get( relevant_input_count=len(provided_inputs), input_count=len(provided_inputs), ) From 66a21cbb37c52f98659fee284c6cbc25064f549a Mon Sep 17 00:00:00 2001 From: Anne Mickan Date: Fri, 31 Jan 2025 07:56:37 +0100 Subject: [PATCH 11/18] Remove is_default field for algorithm interfaces (#3811) Discussed with James that we don't need the option to mark an interface as default, so removing it here. --- app/grandchallenge/algorithms/admin.py | 4 +- app/grandchallenge/algorithms/forms.py | 40 +----- ...rface_algorithminterfaceoutput_and_more.py | 9 -- .../0065_create_algorithm_interfaces.py | 2 +- .../0066_create_interfaces_for_jobs.py | 5 +- app/grandchallenge/algorithms/models.py | 21 --- .../algorithmalgorithminterface_list.html | 4 +- app/grandchallenge/algorithms/views.py | 3 +- app/tests/algorithms_tests/test_api.py | 28 +--- app/tests/algorithms_tests/test_forms.py | 122 ++---------------- app/tests/algorithms_tests/test_models.py | 63 ++------- .../algorithms_tests/test_permissions.py | 10 +- .../algorithms_tests/test_serializers.py | 21 +-- app/tests/algorithms_tests/test_tasks.py | 12 +- app/tests/algorithms_tests/test_views.py | 112 +++++----------- app/tests/conftest.py | 2 +- 16 files changed, 85 insertions(+), 373 deletions(-) diff --git a/app/grandchallenge/algorithms/admin.py b/app/grandchallenge/algorithms/admin.py index 669d270ff7..b04a7499e8 100644 --- a/app/grandchallenge/algorithms/admin.py +++ b/app/grandchallenge/algorithms/admin.py @@ -53,7 +53,6 @@ class Meta: class AlgorithmAdmin(GuardedModelAdmin): readonly_fields = ( "algorithm_forge_json", - "default_interface", "inputs", "outputs", ) @@ -281,10 +280,9 @@ class AlgorithmAlgorithmInterfaceAdmin(GuardedModelAdmin): list_display = ( "pk", "interface", - "is_default", "algorithm", ) - list_filter = ("is_default", "algorithm") + list_filter = ("algorithm",) def has_add_permission(self, request, obj=None): # through table entries should only be created through the UI diff --git a/app/grandchallenge/algorithms/forms.py b/app/grandchallenge/algorithms/forms.py index fd7a296751..ff0f4c1cb2 100644 --- a/app/grandchallenge/algorithms/forms.py +++ b/app/grandchallenge/algorithms/forms.py @@ -26,7 +26,6 @@ from django.db.models import Count, Exists, Max, OuterRef, Q from django.db.transaction import on_commit from django.forms import ( - BooleanField, CharField, Form, HiddenInput, @@ -1372,31 +1371,18 @@ class AlgorithmInterfaceForm(SaveFormInitMixin, ModelForm): ), widget=Select2MultipleWidget, ) - set_as_default = BooleanField(required=False) class Meta: model = AlgorithmInterface fields = ( "inputs", "outputs", - "set_as_default", ) def __init__(self, *args, algorithm, **kwargs): super().__init__(*args, **kwargs) self._algorithm = algorithm - if not self._algorithm.default_interface: - self.fields["set_as_default"].initial = True - - def clean_set_as_default(self): - set_as_default = self.cleaned_data["set_as_default"] - - if not set_as_default and not self._algorithm.default_interface: - raise ValidationError("Your algorithm needs a default interface.") - - return set_as_default - def clean_inputs(self): inputs = self.cleaned_data.get("inputs", []) @@ -1459,29 +1445,7 @@ def save(self): inputs=self.cleaned_data["inputs"], outputs=self.cleaned_data["outputs"], ) - - if self.cleaned_data["set_as_default"]: - AlgorithmAlgorithmInterface.objects.filter( - algorithm=self._algorithm - ).update(is_default=False) - - matched_rows = AlgorithmAlgorithmInterface.objects.filter( - algorithm=self._algorithm, interface=interface - ).update(is_default=self.cleaned_data["set_as_default"]) - - if matched_rows == 0: - self._algorithm.interfaces.add( - interface, - through_defaults={ - "is_default": self.cleaned_data["set_as_default"] - }, - ) - elif matched_rows > 1: - raise RuntimeError( - "This algorithm and interface are associated " - "with each other more than once." - ) - + self._algorithm.interfaces.add(interface) return interface @@ -1502,7 +1466,7 @@ def __init__(self, *args, algorithm, **kwargs): self._algorithm.interfaces.all() ) self.fields["algorithm_interface"].initial = ( - self._algorithm.default_interface + self._algorithm.interfaces.first() ) self.fields["algorithm_interface"].widget.choices = { ( diff --git a/app/grandchallenge/algorithms/migrations/0064_algorithminterface_algorithminterfaceoutput_and_more.py b/app/grandchallenge/algorithms/migrations/0064_algorithminterface_algorithminterfaceoutput_and_more.py index 82c19b6d33..b79bff954a 100644 --- a/app/grandchallenge/algorithms/migrations/0064_algorithminterface_algorithminterfaceoutput_and_more.py +++ b/app/grandchallenge/algorithms/migrations/0064_algorithminterface_algorithminterfaceoutput_and_more.py @@ -121,7 +121,6 @@ class Migration(migrations.Migration): verbose_name="ID", ), ), - ("is_default", models.BooleanField(default=False)), ( "algorithm", models.ForeignKey( @@ -147,14 +146,6 @@ class Migration(migrations.Migration): to="algorithms.algorithminterface", ), ), - migrations.AddConstraint( - model_name="algorithmalgorithminterface", - constraint=models.UniqueConstraint( - condition=models.Q(("is_default", True)), - fields=("algorithm",), - name="unique_default_interface_per_algorithm", - ), - ), migrations.AddConstraint( model_name="algorithmalgorithminterface", constraint=models.UniqueConstraint( diff --git a/app/grandchallenge/algorithms/migrations/0065_create_algorithm_interfaces.py b/app/grandchallenge/algorithms/migrations/0065_create_algorithm_interfaces.py index 8e74cc9cd8..949f4eea93 100644 --- a/app/grandchallenge/algorithms/migrations/0065_create_algorithm_interfaces.py +++ b/app/grandchallenge/algorithms/migrations/0065_create_algorithm_interfaces.py @@ -25,7 +25,7 @@ def create_algorithm_interfaces(apps, _schema_editor): io.inputs.set(inputs) io.outputs.set(outputs) - algorithm.interfaces.add(io, through_defaults={"is_default": True}) + algorithm.interfaces.add(io) class Migration(migrations.Migration): diff --git a/app/grandchallenge/algorithms/migrations/0066_create_interfaces_for_jobs.py b/app/grandchallenge/algorithms/migrations/0066_create_interfaces_for_jobs.py index 550a4066b9..55f7c33332 100644 --- a/app/grandchallenge/algorithms/migrations/0066_create_interfaces_for_jobs.py +++ b/app/grandchallenge/algorithms/migrations/0066_create_interfaces_for_jobs.py @@ -6,11 +6,8 @@ def add_algorithm_interfaces_to_jobs(apps, _schema_editor): Job = apps.get_model("algorithms", "Job") # noqa: N806 for algorithm in Algorithm.objects.prefetch_related("interfaces").all(): - default_interface = algorithm.interfaces.get( - algorithmalgorithminterface__is_default=True - ) Job.objects.filter(algorithm_image__algorithm=algorithm).update( - algorithm_interface=default_interface + algorithm_interface=algorithm.interfaces.get() ) diff --git a/app/grandchallenge/algorithms/models.py b/app/grandchallenge/algorithms/models.py index 2c2958a93b..caa8be7b85 100644 --- a/app/grandchallenge/algorithms/models.py +++ b/app/grandchallenge/algorithms/models.py @@ -520,15 +520,6 @@ def default_workstation(self): return w - @cached_property - def default_interface(self): - try: - return self.interfaces.get( - algorithmalgorithminterface__is_default=True - ) - except ObjectDoesNotExist: - return None - def is_editor(self, user): return user.groups.filter(pk=self.editors_group.pk).exists() @@ -632,15 +623,9 @@ def form_field_label(self): class AlgorithmAlgorithmInterface(models.Model): algorithm = models.ForeignKey(Algorithm, on_delete=models.CASCADE) interface = models.ForeignKey(AlgorithmInterface, on_delete=models.CASCADE) - is_default = models.BooleanField(default=False) class Meta: constraints = [ - models.UniqueConstraint( - fields=["algorithm"], - condition=Q(is_default=True), - name="unique_default_interface_per_algorithm", - ), models.UniqueConstraint( fields=["algorithm", "interface"], name="unique_algorithm_interface_combination", @@ -650,12 +635,6 @@ class Meta: def __str__(self): return str(self.interface) - def clean(self): - super().clean() - - if not self.is_default and not self.algorithm.default_interface: - raise ValidationError("This algorithm needs a default interface.") - class AlgorithmUserObjectPermission(UserObjectPermissionBase): content_object = models.ForeignKey(Algorithm, on_delete=models.CASCADE) diff --git a/app/grandchallenge/algorithms/templates/algorithms/algorithmalgorithminterface_list.html b/app/grandchallenge/algorithms/templates/algorithms/algorithmalgorithminterface_list.html index 360db2f1fd..1efb894bb7 100644 --- a/app/grandchallenge/algorithms/templates/algorithms/algorithmalgorithminterface_list.html +++ b/app/grandchallenge/algorithms/templates/algorithms/algorithmalgorithminterface_list.html @@ -34,7 +34,6 @@

    Algorithm Interfaces for {{ algorithm }}

    Inputs Outputs - Default Delete @@ -42,9 +41,8 @@

    Algorithm Interfaces for {{ algorithm }}

    {{ object.interface.inputs.all|oxford_comma }} {{ object.interface.outputs.all|oxford_comma }} - {% if object.is_default %}{% else %}{% endif %} - diff --git a/app/grandchallenge/algorithms/views.py b/app/grandchallenge/algorithms/views.py index de391b0e82..83272f2984 100644 --- a/app/grandchallenge/algorithms/views.py +++ b/app/grandchallenge/algorithms/views.py @@ -511,7 +511,7 @@ class JobInterfaceSelect( def get(self, request, *args, **kwargs): if self.algorithm.interfaces.count() == 1: - self.selected_interface = self.algorithm.default_interface + self.selected_interface = self.algorithm.interfaces.get() return HttpResponseRedirect(self.get_success_url()) else: return super().get(request, *args, **kwargs) @@ -1229,7 +1229,6 @@ def algorithm_interface(self): klass=AlgorithmAlgorithmInterface, algorithm=self.algorithm, interface__pk=self.kwargs["interface_pk"], - is_default=False, ) def get_object(self, queryset=None): diff --git a/app/tests/algorithms_tests/test_api.py b/app/tests/algorithms_tests/test_api.py index 3b91399d83..78540a4f9e 100644 --- a/app/tests/algorithms_tests/test_api.py +++ b/app/tests/algorithms_tests/test_api.py @@ -162,9 +162,7 @@ def test_create_job_with_multiple_new_inputs( ], outputs=[ComponentInterfaceFactory()], ) - algorithm_with_multiple_inputs.algorithm.interfaces.add( - interface, through_defaults={"is_default": True} - ) + algorithm_with_multiple_inputs.algorithm.interfaces.add(interface) assert ComponentInterfaceValue.objects.count() == 0 @@ -255,9 +253,7 @@ def test_create_job_with_existing_inputs( ], outputs=[ComponentInterfaceFactory()], ) - algorithm_with_multiple_inputs.algorithm.interfaces.add( - interface, through_defaults={"is_default": True} - ) + algorithm_with_multiple_inputs.algorithm.interfaces.add(interface) civ1, civ2, civ3, civ4 = self.create_existing_civs( interface_data=algorithm_with_multiple_inputs @@ -316,9 +312,7 @@ def test_create_job_is_idempotent( ], outputs=[ComponentInterfaceFactory()], ) - algorithm_with_multiple_inputs.algorithm.interfaces.add( - interface, through_defaults={"is_default": True} - ) + algorithm_with_multiple_inputs.algorithm.interfaces.add(interface) civ1, civ2, civ3, civ4 = self.create_existing_civs( interface_data=algorithm_with_multiple_inputs ) @@ -379,9 +373,7 @@ def test_create_job_with_faulty_file_input( inputs=[algorithm_with_multiple_inputs.ci_json_file], outputs=[ComponentInterfaceFactory()], ) - algorithm_with_multiple_inputs.algorithm.interfaces.add( - interface, through_defaults={"is_default": True} - ) + algorithm_with_multiple_inputs.algorithm.interfaces.add(interface) file_upload = UserUploadFactory( filename="file.json", creator=algorithm_with_multiple_inputs.editor ) @@ -431,9 +423,7 @@ def test_create_job_with_faulty_json_input( inputs=[algorithm_with_multiple_inputs.ci_json_in_db_with_schema], outputs=[ComponentInterfaceFactory()], ) - algorithm_with_multiple_inputs.algorithm.interfaces.add( - interface, through_defaults={"is_default": True} - ) + algorithm_with_multiple_inputs.algorithm.interfaces.add(interface) response = self.create_job( client=client, @@ -466,9 +456,7 @@ def test_create_job_with_faulty_image_input( inputs=[algorithm_with_multiple_inputs.ci_img_upload], outputs=[ComponentInterfaceFactory()], ) - algorithm_with_multiple_inputs.algorithm.interfaces.add( - interface, through_defaults={"is_default": True} - ) + algorithm_with_multiple_inputs.algorithm.interfaces.add(interface) user_upload = create_upload_from_file( creator=algorithm_with_multiple_inputs.editor, file_path=RESOURCE_PATH / "corrupt.png", @@ -525,9 +513,7 @@ def test_create_job_with_multiple_faulty_existing_image_inputs( interface = AlgorithmInterfaceFactory( inputs=[ci1, ci2], outputs=[ComponentInterfaceFactory()] ) - algorithm_with_multiple_inputs.algorithm.interfaces.add( - interface, through_defaults={"is_default": True} - ) + algorithm_with_multiple_inputs.algorithm.interfaces.add(interface) im = ImageFactory() im.files.set([ImageFileFactoryWithMHDFile()]) diff --git a/app/tests/algorithms_tests/test_forms.py b/app/tests/algorithms_tests/test_forms.py index 52a111de12..7358a36cb8 100644 --- a/app/tests/algorithms_tests/test_forms.py +++ b/app/tests/algorithms_tests/test_forms.py @@ -18,7 +18,6 @@ ) from grandchallenge.algorithms.models import ( Algorithm, - AlgorithmAlgorithmInterface, AlgorithmPermissionRequest, Job, ) @@ -323,7 +322,7 @@ def test_create_job_input_fields( client=client, reverse_kwargs={ "slug": alg.slug, - "interface_pk": alg.default_interface.pk, + "interface_pk": alg.interfaces.first().pk, }, follow=True, user=creator, @@ -358,7 +357,7 @@ def test_create_job_json_input_field_validation( client=client, reverse_kwargs={ "slug": alg.slug, - "interface_pk": alg.default_interface.pk, + "interface_pk": alg.interfaces.first().pk, }, method=client.post, follow=True, @@ -392,7 +391,7 @@ def test_create_job_simple_input_field_validation( client=client, reverse_kwargs={ "slug": alg.slug, - "interface_pk": alg.default_interface.pk, + "interface_pk": alg.interfaces.first().pk, }, method=client.post, follow=True, @@ -413,7 +412,7 @@ def create_algorithm_with_input(slug): inputs=[ComponentInterface.objects.get(slug=slug)], outputs=[ComponentInterfaceFactory()], ) - alg.interfaces.add(interface, through_defaults={"is_default": True}) + alg.interfaces.add(interface) AlgorithmImageFactory( algorithm=alg, is_manifest_valid=True, @@ -741,7 +740,7 @@ def test_creator_queryset( form = JobCreateForm( algorithm=algorithm, user=editor, - interface=algorithm.default_interface, + interface=algorithm.interfaces.first(), data={}, ) assert list(form.fields["creator"].queryset.all()) == [editor] @@ -762,7 +761,7 @@ def test_algorithm_image_queryset( form = JobCreateForm( algorithm=algorithm, user=editor, - interface=algorithm.default_interface, + interface=algorithm.interfaces.first(), data={}, ) ai_qs = form.fields["algorithm_image"].queryset.all() @@ -783,14 +782,14 @@ def test_cannot_create_job_with_same_inputs_twice( algorithm_model=algorithm.active_model, status=Job.SUCCESS, time_limit=123, - algorithm_interface=algorithm.default_interface, + algorithm_interface=algorithm.interfaces.first(), ) job.inputs.set(civs) form = JobCreateForm( algorithm=algorithm, user=editor, - interface=algorithm.default_interface, + interface=algorithm.interfaces.first(), data={ "algorithm_image": algorithm.active_image, "algorithm_model": algorithm.active_model, @@ -819,9 +818,7 @@ def test_all_inputs_required_on_job_creation(algorithm_with_multiple_inputs): inputs=[ci_json_in_db_without_schema], outputs=[ComponentInterfaceFactory()], ) - algorithm_with_multiple_inputs.algorithm.interfaces.add( - interface, through_defaults={"is_default": True} - ) + algorithm_with_multiple_inputs.algorithm.interfaces.add(interface) form = JobCreateForm( algorithm=algorithm_with_multiple_inputs.algorithm, @@ -1409,36 +1406,12 @@ def test_algorithm_interface_disjoint_interfaces(): assert "The sets of Inputs and Outputs must be unique" in str(form.errors) -@pytest.mark.django_db -def test_algorithm_interface_default_interface_required(): - ci1, ci2 = ComponentInterfaceFactory.create_batch(2) - alg = AlgorithmFactory() - form = AlgorithmInterfaceForm( - algorithm=alg, - data={"inputs": [ci1], "outputs": [ci2], "set_as_default": False}, - ) - assert form.is_valid() is False - assert "Your algorithm needs a default interface" in str( - form.errors["set_as_default"] - ) - - alg.interfaces.add( - AlgorithmInterfaceFactory(), through_defaults={"is_default": True} - ) - del alg.default_interface - form = AlgorithmInterfaceForm( - algorithm=alg, - data={"inputs": [ci1], "outputs": [ci2], "set_as_default": False}, - ) - assert form.is_valid() - - @pytest.mark.django_db def test_algorithm_interface_unique_inputs_required(): ci1, ci2 = ComponentInterfaceFactory.create_batch(2) alg = AlgorithmFactory() interface = AlgorithmInterfaceFactory(inputs=[ci1]) - alg.interfaces.add(interface, through_defaults={"is_default": True}) + alg.interfaces.add(interface) form = AlgorithmInterfaceForm( algorithm=alg, data={"inputs": [ci1], "outputs": [ci2]} @@ -1478,26 +1451,6 @@ def test_algorithm_for_phase_form_memory(): class TestAlgorithmInterfaceForm: - @pytest.mark.django_db - def test_set_as_default_initial_value(self): - alg = AlgorithmFactory() - - form = AlgorithmInterfaceForm( - algorithm=alg, - ) - assert form.fields["set_as_default"].initial - - alg.interfaces.add( - AlgorithmInterfaceFactory(), through_defaults={"is_default": True} - ) - - del alg.default_interface - - form = AlgorithmInterfaceForm( - algorithm=alg, - ) - assert not form.fields["set_as_default"].initial - @pytest.mark.django_db def test_existing_io_is_reused(self): inp = ComponentInterfaceFactory() @@ -1513,64 +1466,9 @@ def test_existing_io_is_reused(self): data={ "inputs": [inp.pk], "outputs": [out.pk], - "set_as_default": True, }, ) assert form.is_valid() new_io = form.save() assert io == new_io - - @pytest.mark.django_db - def test_new_default_interface_updates_related_interfaces(self): - ci_1, ci_2 = ComponentInterfaceFactory.create_batch(2) - alg = AlgorithmFactory() - io = AlgorithmInterfaceFactory() - alg.interfaces.add(io, through_defaults={"is_default": True}) - - old_iot = AlgorithmAlgorithmInterface.objects.get() - - form = AlgorithmInterfaceForm( - algorithm=alg, - data={ - "inputs": [ci_1.pk], - "outputs": [ci_2.pk], - "set_as_default": True, - }, - ) - form.is_valid() - new_io = form.save() - new_iot = AlgorithmAlgorithmInterface.objects.get(interface=new_io) - old_iot.refresh_from_db() - - assert new_io != io - assert new_iot.is_default - assert not old_iot.is_default - - @pytest.mark.django_db - def test_default_interface_for_algorithm_not_updated_when_adding_new_non_default_interface( - self, - ): - alg = AlgorithmFactory() - io = AlgorithmInterfaceFactory() - alg.interfaces.add(io, through_defaults={"is_default": True}) - old_iot = AlgorithmAlgorithmInterface.objects.get() - - ci_1, ci_2 = ComponentInterfaceFactory.create_batch(2) - - form = AlgorithmInterfaceForm( - algorithm=alg, - data={ - "inputs": [ci_1.pk], - "outputs": [ci_2.pk], - "set_as_default": False, - }, - ) - form.is_valid() - new_io = form.save() - old_iot.refresh_from_db() - new_iot = AlgorithmAlgorithmInterface.objects.get(interface=new_io) - - assert new_io != io - assert not new_iot.is_default - assert old_iot.is_default diff --git a/app/tests/algorithms_tests/test_models.py b/app/tests/algorithms_tests/test_models.py index 45274c8f43..96af3ff593 100644 --- a/app/tests/algorithms_tests/test_models.py +++ b/app/tests/algorithms_tests/test_models.py @@ -77,14 +77,6 @@ def test_group_deletion_reverse(group): getattr(algorithm, group).delete() -@pytest.mark.django_db -def test_no_default_interfaces_created(): - a = AlgorithmFactory() - - assert {i.kind for i in a.inputs.all()} == set() - assert {o.kind for o in a.outputs.all()} == set() - - @pytest.mark.django_db def test_rendered_result_text(): def create_result(jb, result: dict): @@ -658,7 +650,7 @@ def test_job_with_same_image_different_model( j = AlgorithmJobFactory( algorithm_image=alg.active_image, time_limit=10, - algorithm_interface=alg.default_interface, + algorithm_interface=alg.interfaces.first(), ) j.inputs.set(civs) @@ -680,7 +672,7 @@ def test_job_with_same_model_different_image( algorithm_image=AlgorithmImageFactory(), algorithm_model=alg.active_model, time_limit=10, - algorithm_interface=alg.default_interface, + algorithm_interface=alg.interfaces.first(), ) j.inputs.set(civs) jobs = Job.objects.get_jobs_with_same_inputs( @@ -701,7 +693,7 @@ def test_job_with_same_model_and_image( algorithm_model=alg.active_model, algorithm_image=alg.active_image, time_limit=10, - algorithm_interface=alg.default_interface, + algorithm_interface=alg.interfaces.first(), ) j.inputs.set(civs) jobs = Job.objects.get_jobs_with_same_inputs( @@ -723,7 +715,7 @@ def test_job_with_different_image_and_model( algorithm_model=AlgorithmModelFactory(), algorithm_image=AlgorithmImageFactory(), time_limit=10, - algorithm_interface=alg.default_interface, + algorithm_interface=alg.interfaces.first(), ) j.inputs.set(civs) jobs = Job.objects.get_jobs_with_same_inputs( @@ -744,7 +736,7 @@ def test_job_with_same_image_no_model_provided( algorithm_model=alg.active_model, algorithm_image=alg.active_image, time_limit=10, - algorithm_interface=alg.default_interface, + algorithm_interface=alg.interfaces.first(), ) j.inputs.set(civs) jobs = Job.objects.get_jobs_with_same_inputs( @@ -764,7 +756,7 @@ def test_job_with_same_image_and_without_model( j = AlgorithmJobFactory( algorithm_image=alg.active_image, time_limit=10, - algorithm_interface=alg.default_interface, + algorithm_interface=alg.interfaces.first(), ) j.inputs.set(civs) jobs = Job.objects.get_jobs_with_same_inputs( @@ -785,7 +777,7 @@ def test_job_with_different_input( j = AlgorithmJobFactory( algorithm_image=alg.active_image, time_limit=10, - algorithm_interface=alg.default_interface, + algorithm_interface=alg.interfaces.first(), ) j.inputs.set( [ @@ -810,7 +802,7 @@ def test_job_with_partially_overlapping_input( j = AlgorithmJobFactory( algorithm_image=alg.active_image, time_limit=10, - algorithm_interface=alg.default_interface, + algorithm_interface=alg.interfaces.first(), ) j.inputs.set( [ @@ -821,7 +813,7 @@ def test_job_with_partially_overlapping_input( j2 = AlgorithmJobFactory( algorithm_image=alg.active_image, time_limit=10, - algorithm_interface=alg.default_interface, + algorithm_interface=alg.interfaces.first(), ) j2.inputs.set( [ @@ -1038,11 +1030,11 @@ def test_inputs_complete(): interface = AlgorithmInterfaceFactory( inputs=[ci1, ci2, ci3], outputs=[ComponentInterfaceFactory()] ) - alg.interfaces.add(interface, through_defaults={"is_default": True}) + alg.interfaces.add(interface) job = AlgorithmJobFactory( algorithm_image__algorithm=alg, time_limit=10, - algorithm_interface=alg.default_interface, + algorithm_interface=alg.interfaces.first(), ) civ_with_value_1 = ComponentInterfaceValueFactory( interface=ci1, value="Foo" @@ -1462,30 +1454,17 @@ def test_algorithmalgorithminterface_unique_constraints(): interface1, interface2 = AlgorithmInterfaceFactory.create_batch(2) algorithm = AlgorithmFactory() - io = AlgorithmAlgorithmInterface.objects.create( - interface=interface1, algorithm=algorithm, is_default=True + AlgorithmAlgorithmInterface.objects.create( + interface=interface1, algorithm=algorithm ) - # cannot add a second default interface to an algorithm - with pytest.raises(IntegrityError): - with transaction.atomic(): - AlgorithmAlgorithmInterface.objects.create( - interface=interface2, algorithm=algorithm, is_default=True - ) - # cannot add a second time the same interface for the same algorithm with pytest.raises(IntegrityError): with transaction.atomic(): AlgorithmAlgorithmInterface.objects.create( - interface=interface1, algorithm=algorithm, is_default=False + interface=interface1, algorithm=algorithm ) - # but you can update an existing entry from default to non-default - io.is_default = False - io.save() - assert AlgorithmAlgorithmInterface.objects.count() == 1 - assert not AlgorithmAlgorithmInterface.objects.get().is_default - @pytest.mark.parametrize( "inputs, outputs, expected_output", @@ -1557,17 +1536,3 @@ def test_algorithminterface_create(): io = AlgorithmInterface.objects.create(inputs=inputs, outputs=outputs) assert list(io.inputs.all()) == inputs assert list(io.outputs.all()) == outputs - - -@pytest.mark.django_db -def test_has_default_interface(): - alg1, alg2, alg3 = AlgorithmFactory.create_batch(3) - io1, io2 = AlgorithmInterfaceFactory.create_batch(2) - - alg1.interfaces.add(io1, through_defaults={"is_default": True}) - alg1.interfaces.add(io2) - alg2.interfaces.add(io1) - - assert alg1.default_interface == io1 - assert not alg2.default_interface - assert not alg3.default_interface diff --git a/app/tests/algorithms_tests/test_permissions.py b/app/tests/algorithms_tests/test_permissions.py index c96b0f03e5..081ffea444 100644 --- a/app/tests/algorithms_tests/test_permissions.py +++ b/app/tests/algorithms_tests/test_permissions.py @@ -317,9 +317,7 @@ def test_job_permissions_from_template(self, client): interface = AlgorithmInterfaceFactory( inputs=[ci], outputs=[ComponentInterfaceFactory()] ) - algorithm_image.algorithm.interfaces.add( - interface, through_defaults={"is_default": True} - ) + algorithm_image.algorithm.interfaces.add(interface) response = get_view_for_user( viewname="algorithms:job-create", @@ -327,7 +325,7 @@ def test_job_permissions_from_template(self, client): method=client.post, reverse_kwargs={ "slug": algorithm_image.algorithm.slug, - "interface_pk": algorithm_image.algorithm.default_interface.pk, + "interface_pk": algorithm_image.algorithm.interfaces.first().pk, }, user=user, follow=True, @@ -360,9 +358,7 @@ def test_job_permissions_from_api(self, rf): ) interface = AlgorithmInterfaceFactory(inputs=[ci]) - algorithm_image.algorithm.interfaces.add( - interface, through_defaults={"is_default": True} - ) + algorithm_image.algorithm.interfaces.add(interface) algorithm_image.algorithm.add_user(user) algorithm_image.algorithm.add_editor(UserFactory()) diff --git a/app/tests/algorithms_tests/test_serializers.py b/app/tests/algorithms_tests/test_serializers.py index 4e9c90d7a2..b176f78b3e 100644 --- a/app/tests/algorithms_tests/test_serializers.py +++ b/app/tests/algorithms_tests/test_serializers.py @@ -140,9 +140,8 @@ def test_algorithm_job_post_serializer_validations( interface = AlgorithmInterfaceFactory( inputs=[interfaces[title] for title in algorithm_interface_titles] ) - algorithm_image.algorithm.interfaces.add( - interface, through_defaults={"is_default": True} - ) + algorithm_image.algorithm.interfaces.add(interface) + if add_user: algorithm_image.algorithm.add_user(user) @@ -214,9 +213,7 @@ def test_algorithm_job_post_serializer_create( ci_img2 = ComponentInterfaceFactory(kind=ComponentInterface.Kind.IMAGE) interface = AlgorithmInterfaceFactory(inputs=[ci_string, ci_img2, ci_img1]) - algorithm_image.algorithm.interfaces.add( - interface, through_defaults={"is_default": True} - ) + algorithm_image.algorithm.interfaces.add(interface) algorithm_image.algorithm.add_editor(user) job = { @@ -312,9 +309,7 @@ def test_form_valid_for_editor(self, rf, settings): algorithm_image.algorithm.add_editor(user=user) ci = ComponentInterfaceFactory(kind=ComponentInterface.Kind.STRING) interface = AlgorithmInterfaceFactory(inputs=[ci]) - algorithm_image.algorithm.interfaces.add( - interface, through_defaults={"is_default": True} - ) + algorithm_image.algorithm.interfaces.add(interface) request = rf.get("/foo") request.user = user @@ -362,9 +357,7 @@ def test_form_valid_with_credits(self, rf): algorithm_image.algorithm.add_user(user=user) ci = ComponentInterfaceFactory(kind=ComponentInterface.Kind.STRING) interface = AlgorithmInterfaceFactory(inputs=[ci]) - algorithm_image.algorithm.interfaces.add( - interface, through_defaults={"is_default": True} - ) + algorithm_image.algorithm.interfaces.add(interface) request = rf.get("/foo") request.user = user @@ -432,7 +425,7 @@ def test_algorithm_post_serializer_image_and_time_limit_fixed(rf): different_ai = AlgorithmImageFactory(algorithm=alg) ci = ComponentInterfaceFactory(kind=ComponentInterface.Kind.STRING) interface = AlgorithmInterfaceFactory(inputs=[ci]) - alg.interfaces.add(interface, through_defaults={"is_default": True}) + alg.interfaces.add(interface) serializer = JobPostSerializer( data={ @@ -502,7 +495,7 @@ def test_validate_inputs_on_job_serializer(inputs, interface, rf): io4.outputs.set([ci1]) io5.outputs.set([ci1]) - algorithm.interfaces.add(io1, through_defaults={"is_default": True}) + algorithm.interfaces.add(io1) algorithm.interfaces.add(io2) algorithm.interfaces.add(io3) diff --git a/app/tests/algorithms_tests/test_tasks.py b/app/tests/algorithms_tests/test_tasks.py index 3f2baa705e..e4e75d2759 100644 --- a/app/tests/algorithms_tests/test_tasks.py +++ b/app/tests/algorithms_tests/test_tasks.py @@ -232,9 +232,7 @@ def test_algorithm( inputs=[input_interface], outputs=[json_result_interface, heatmap_interface], ) - ai.algorithm.interfaces.add( - interface, through_defaults={"is_default": True} - ) + ai.algorithm.interfaces.add(interface) with django_capture_on_commit_callbacks() as callbacks: get_view_for_user( @@ -385,9 +383,7 @@ def test_algorithm_with_invalid_output( interface = AlgorithmInterfaceFactory( inputs=[input_interface], outputs=[detection_interface] ) - ai.algorithm.interfaces.add( - interface, through_defaults={"is_default": True} - ) + ai.algorithm.interfaces.add(interface) image_file = ImageFileFactory( file__from_path=Path(__file__).parent / "resources" / "input_file.tif" @@ -475,9 +471,7 @@ def test_execute_algorithm_job_for_missing_inputs(settings): interface = AlgorithmInterfaceFactory( inputs=[ci], outputs=[ComponentInterfaceFactory()] ) - alg.algorithm.interfaces.add( - interface, through_defaults={"is_default": True} - ) + alg.algorithm.interfaces.add(interface) job = AlgorithmJobFactory( creator=creator, algorithm_image=alg, diff --git a/app/tests/algorithms_tests/test_views.py b/app/tests/algorithms_tests/test_views.py index cedf9e818e..bf057cb9bb 100644 --- a/app/tests/algorithms_tests/test_views.py +++ b/app/tests/algorithms_tests/test_views.py @@ -361,9 +361,7 @@ def test_permission_required_views(self, client): inputs=[ComponentInterfaceFactory()], outputs=[ComponentInterfaceFactory()], ) - ai.algorithm.interfaces.add( - interface, through_defaults={"is_default": True} - ) + ai.algorithm.interfaces.add(interface) VerificationFactory(user=u, is_verified=True) @@ -439,7 +437,7 @@ def test_permission_required_views(self, client): "job-create", { "slug": ai.algorithm.slug, - "interface_pk": ai.algorithm.default_interface.pk, + "interface_pk": ai.algorithm.interfaces.first().pk, }, "execute_algorithm", ai.algorithm, @@ -1072,9 +1070,7 @@ def test_create_job_with_json_file( interface = AlgorithmInterfaceFactory( inputs=[ci], ) - ai.algorithm.interfaces.add( - interface, through_defaults={"is_default": True} - ) + ai.algorithm.interfaces.add(interface) with tempfile.NamedTemporaryFile(mode="w+", suffix=".json") as file: json.dump('{"Foo": "bar"}', file) @@ -1131,9 +1127,7 @@ def test_algorithm_job_create_with_image_input( interface = AlgorithmInterfaceFactory( inputs=[ci], ) - ai.algorithm.interfaces.add( - interface, through_defaults={"is_default": True} - ) + ai.algorithm.interfaces.add(interface) image1, image2 = ImageFactory.create_batch(2) assign_perm("cases.view_image", editor, image1) @@ -1241,7 +1235,7 @@ def create_job( user=user, reverse_kwargs={ "slug": algorithm.slug, - "interface_pk": algorithm.default_interface.pk, + "interface_pk": algorithm.interfaces.first().pk, }, follow=True, data=inputs, @@ -1291,9 +1285,7 @@ def test_create_job_with_multiple_new_inputs( ], outputs=[ComponentInterfaceFactory()], ) - algorithm_with_multiple_inputs.algorithm.interfaces.add( - interface, through_defaults={"is_default": True} - ) + algorithm_with_multiple_inputs.algorithm.interfaces.add(interface) assert ComponentInterfaceValue.objects.count() == 0 @@ -1353,7 +1345,7 @@ def test_create_job_with_multiple_new_inputs( assert sorted( [ int.pk - for int in algorithm_with_multiple_inputs.algorithm.default_interface.inputs.all() + for int in algorithm_with_multiple_inputs.algorithm.interfaces.first().inputs.all() ] ) == sorted([civ.interface.pk for civ in job.inputs.all()]) @@ -1390,9 +1382,7 @@ def test_create_job_with_existing_inputs( ], outputs=[ComponentInterfaceFactory()], ) - algorithm_with_multiple_inputs.algorithm.interfaces.add( - interface, through_defaults={"is_default": True} - ) + algorithm_with_multiple_inputs.algorithm.interfaces.add(interface) civ1, civ2, civ3, civ4, civ5 = self.create_existing_civs( interface_data=algorithm_with_multiple_inputs @@ -1467,9 +1457,7 @@ def test_create_job_is_idempotent( ], outputs=[ComponentInterfaceFactory()], ) - algorithm_with_multiple_inputs.algorithm.interfaces.add( - interface, through_defaults={"is_default": True} - ) + algorithm_with_multiple_inputs.algorithm.interfaces.add(interface) civ1, civ2, civ3, civ4, civ5 = self.create_existing_civs( interface_data=algorithm_with_multiple_inputs ) @@ -1530,9 +1518,7 @@ def test_create_job_with_faulty_file_input( inputs=[algorithm_with_multiple_inputs.ci_json_file], outputs=[ComponentInterfaceFactory()], ) - algorithm_with_multiple_inputs.algorithm.interfaces.add( - interface, through_defaults={"is_default": True} - ) + algorithm_with_multiple_inputs.algorithm.interfaces.add(interface) file_upload = UserUploadFactory( filename="file.json", creator=algorithm_with_multiple_inputs.editor ) @@ -1581,9 +1567,7 @@ def test_create_job_with_faulty_json_input( inputs=[algorithm_with_multiple_inputs.ci_json_in_db_with_schema], outputs=[ComponentInterfaceFactory()], ) - algorithm_with_multiple_inputs.algorithm.interfaces.add( - interface, through_defaults={"is_default": True} - ) + algorithm_with_multiple_inputs.algorithm.interfaces.add(interface) response = self.create_job( client=client, django_capture_on_commit_callbacks=django_capture_on_commit_callbacks, @@ -1615,9 +1599,7 @@ def test_create_job_with_faulty_image_input( inputs=[algorithm_with_multiple_inputs.ci_img_upload], outputs=[ComponentInterfaceFactory()], ) - algorithm_with_multiple_inputs.algorithm.interfaces.add( - interface, through_defaults={"is_default": True} - ) + algorithm_with_multiple_inputs.algorithm.interfaces.add(interface) user_upload = create_upload_from_file( creator=algorithm_with_multiple_inputs.editor, file_path=RESOURCE_PATH / "corrupt.png", @@ -1671,9 +1653,7 @@ def test_create_job_with_multiple_faulty_existing_image_inputs( interface = AlgorithmInterfaceFactory( inputs=[ci1, ci2], outputs=[ComponentInterfaceFactory()] ) - algorithm_with_multiple_inputs.algorithm.interfaces.add( - interface, through_defaults={"is_default": True} - ) + algorithm_with_multiple_inputs.algorithm.interfaces.add(interface) assert ComponentInterfaceValue.objects.count() == 0 @@ -1828,7 +1808,7 @@ def test_job_time_limit(client): interface = AlgorithmInterfaceFactory( inputs=[ci], outputs=[ComponentInterfaceFactory()] ) - algorithm.interfaces.add(interface, through_defaults={"is_default": True}) + algorithm.interfaces.add(interface) response = get_view_for_user( viewname="algorithms:job-create", @@ -1836,7 +1816,7 @@ def test_job_time_limit(client): method=client.post, reverse_kwargs={ "slug": algorithm.slug, - "interface_pk": algorithm.default_interface.pk, + "interface_pk": algorithm.interfaces.first().pk, }, user=user, follow=True, @@ -1879,7 +1859,7 @@ def test_job_gpu_type_set(client, settings): interface = AlgorithmInterfaceFactory( inputs=[ci], outputs=[ComponentInterfaceFactory()] ) - algorithm.interfaces.add(interface, through_defaults={"is_default": True}) + algorithm.interfaces.add(interface) response = get_view_for_user( viewname="algorithms:job-create", @@ -1887,7 +1867,7 @@ def test_job_gpu_type_set(client, settings): method=client.post, reverse_kwargs={ "slug": algorithm.slug, - "interface_pk": algorithm.default_interface.pk, + "interface_pk": algorithm.interfaces.first().pk, }, user=user, follow=True, @@ -1933,7 +1913,7 @@ def test_job_gpu_type_set_with_api(client, settings): interface = AlgorithmInterfaceFactory( inputs=[ci], ) - algorithm.interfaces.add(interface, through_defaults={"is_default": True}) + algorithm.interfaces.add(interface) response = get_view_for_user( viewname="api:algorithms-job-list", @@ -1978,13 +1958,13 @@ def test_job_create_view_for_verified_users_only(client): inputs=[ComponentInterfaceFactory()], outputs=[ComponentInterfaceFactory()], ) - alg.interfaces.add(interface, through_defaults={"is_default": True}) + alg.interfaces.add(interface) response = get_view_for_user( viewname="algorithms:job-create", reverse_kwargs={ "slug": alg.slug, - "interface_pk": alg.default_interface.pk, + "interface_pk": alg.interfaces.first().pk, }, client=client, user=user, @@ -1995,7 +1975,7 @@ def test_job_create_view_for_verified_users_only(client): viewname="algorithms:job-create", reverse_kwargs={ "slug": alg.slug, - "interface_pk": alg.default_interface.pk, + "interface_pk": alg.interfaces.first().pk, }, client=client, user=editor, @@ -2097,7 +2077,7 @@ def test_job_create_denied_for_same_input_model_and_image(client): interface = AlgorithmInterfaceFactory( inputs=[ci], outputs=[ComponentInterfaceFactory()] ) - alg.interfaces.add(interface, through_defaults={"is_default": True}) + alg.interfaces.add(interface) ai = AlgorithmImageFactory( algorithm=alg, is_manifest_valid=True, @@ -2120,7 +2100,7 @@ def test_job_create_denied_for_same_input_model_and_image(client): method=client.post, reverse_kwargs={ "slug": alg.slug, - "interface_pk": alg.default_interface.pk, + "interface_pk": alg.interfaces.first().pk, }, user=creator, data={ @@ -2391,14 +2371,14 @@ def test_algorithm_interface_delete_permission(client): alg.add_editor(algorithm_editor_without_alg_add) int1, int2 = AlgorithmInterfaceFactory.create_batch(2) - alg.interfaces.add(int1, through_defaults={"is_default": True}) + alg.interfaces.add(int1) alg.interfaces.add(int2) - for user, status1, status2 in [ - [user_with_alg_add_perm, 403, 403], - [user_without_alg_add_perm, 403, 403], - [algorithm_editor_with_alg_add, 200, 404], - [algorithm_editor_without_alg_add, 403, 403], + for user, status in [ + [user_with_alg_add_perm, 403], + [user_without_alg_add_perm, 403], + [algorithm_editor_with_alg_add, 200], + [algorithm_editor_without_alg_add, 403], ]: response = get_view_for_user( viewname="algorithms:interface-delete", @@ -2409,19 +2389,7 @@ def test_algorithm_interface_delete_permission(client): }, user=user, ) - assert response.status_code == status1 - - # default interface cannot be deleted - response = get_view_for_user( - viewname="algorithms:interface-delete", - client=client, - reverse_kwargs={ - "slug": alg.slug, - "interface_pk": int1.pk, - }, - user=user, - ) - assert response.status_code == status2 + assert response.status_code == status @pytest.mark.django_db @@ -2442,7 +2410,6 @@ def test_algorithm_interface_create(client): data={ "inputs": [ci_1.pk], "outputs": [ci_2.pk], - "set_as_default": True, }, user=user, ) @@ -2457,7 +2424,6 @@ def test_algorithm_interface_create(client): io_through = AlgorithmAlgorithmInterface.objects.get() assert io_through.algorithm == alg assert io_through.interface == io - assert io_through.is_default @pytest.mark.django_db @@ -2497,21 +2463,9 @@ def test_algorithm_interface_delete(client): alg.add_editor(user) int1, int2 = AlgorithmInterfaceFactory.create_batch(2) - alg.interfaces.add(int1, through_defaults={"is_default": True}) + alg.interfaces.add(int1) alg.interfaces.add(int2) - response = get_view_for_user( - viewname="algorithms:interface-delete", - client=client, - method=client.post, - reverse_kwargs={ - "slug": alg.slug, - "interface_pk": int1.pk, - }, - user=user, - ) - assert response.status_code == 404 - response = get_view_for_user( viewname="algorithms:interface-delete", client=client, @@ -2547,7 +2501,7 @@ def test_interface_select_for_job_view_permission(client): inputs=[ComponentInterfaceFactory()], outputs=[ComponentInterfaceFactory()], ) - alg.interfaces.add(interface1, through_defaults={"is_default": True}) + alg.interfaces.add(interface1) alg.interfaces.add(interface2) response = get_view_for_user( @@ -2578,7 +2532,7 @@ def test_interface_select_automatic_redirect(client): inputs=[ComponentInterfaceFactory()], outputs=[ComponentInterfaceFactory()], ) - alg.interfaces.add(interface, through_defaults={"is_default": True}) + alg.interfaces.add(interface) # with just 1 interface, user gets redirected immediately response = get_view_for_user( diff --git a/app/tests/conftest.py b/app/tests/conftest.py index a46b213262..6fbdb3ed3a 100644 --- a/app/tests/conftest.py +++ b/app/tests/conftest.py @@ -484,7 +484,7 @@ def algorithm_with_image_and_model_and_two_inputs(): interface = AlgorithmInterfaceFactory( inputs=[ci1, ci2], outputs=[ComponentInterfaceFactory()] ) - alg.interfaces.add(interface, through_defaults={"is_default": True}) + alg.interfaces.add(interface) civs = [ ComponentInterfaceValueFactory(interface=ci1, value="foo"), ComponentInterfaceValueFactory(interface=ci2, value="bar"), From c75cd76617db0b5f19bf65a134b056ea707bb32a Mon Sep 17 00:00:00 2001 From: Anne Mickan Date: Fri, 31 Jan 2025 12:13:12 +0100 Subject: [PATCH 12/18] Add algorithm interfaces to phases and migrate (#3803) This adds a `algorithm_interfaces` M2M to the `Phase` model and adds a custom through model for it. It also removes the input / output configuration option from both the phase admin and the `ConfigureAlgorithmPhases` view. In anticipation of PR 2, it refactors the interface create form so that it can be easily used for both phases and algorithms alike. Views to create and manage interfaces for phases will be added in a separate PR. Part of https://github.com/DIAGNijmegen/rse-roadmap/issues/153 --- app/grandchallenge/algorithms/forms.py | 14 ++--- .../0065_create_algorithm_interfaces.py | 3 + app/grandchallenge/algorithms/models.py | 8 +++ app/grandchallenge/algorithms/views.py | 2 +- app/grandchallenge/evaluation/admin.py | 42 +++++++------ app/grandchallenge/evaluation/forms.py | 14 +---- ...ace_phase_algorithm_interfaces_and_more.py | 62 +++++++++++++++++++ .../0073_migrate_interfaces_for_phases.py | 49 +++++++++++++++ app/grandchallenge/evaluation/models.py | 31 +++++++++- .../configure_algorithm_phases_form.html | 3 + .../evaluation/views/__init__.py | 6 -- app/tests/algorithms_tests/test_forms.py | 6 +- app/tests/evaluation_tests/test_admin.py | 19 +----- app/tests/evaluation_tests/test_models.py | 3 +- app/tests/evaluation_tests/test_views.py | 4 -- 15 files changed, 189 insertions(+), 77 deletions(-) create mode 100644 app/grandchallenge/evaluation/migrations/0072_phasealgorithminterface_phase_algorithm_interfaces_and_more.py create mode 100644 app/grandchallenge/evaluation/migrations/0073_migrate_interfaces_for_phases.py diff --git a/app/grandchallenge/algorithms/forms.py b/app/grandchallenge/algorithms/forms.py index ff0f4c1cb2..7d988d7ed2 100644 --- a/app/grandchallenge/algorithms/forms.py +++ b/app/grandchallenge/algorithms/forms.py @@ -49,7 +49,6 @@ from grandchallenge.algorithms.models import ( Algorithm, - AlgorithmAlgorithmInterface, AlgorithmImage, AlgorithmInterface, AlgorithmModel, @@ -1379,9 +1378,9 @@ class Meta: "outputs", ) - def __init__(self, *args, algorithm, **kwargs): + def __init__(self, *args, base_obj, **kwargs): super().__init__(*args, **kwargs) - self._algorithm = algorithm + self._base_obj = base_obj def clean_inputs(self): inputs = self.cleaned_data.get("inputs", []) @@ -1390,10 +1389,7 @@ def clean_inputs(self): raise ValidationError("You must provide at least 1 input.") if ( - AlgorithmAlgorithmInterface.objects.filter( - algorithm=self._algorithm - ) - .annotate( + self._base_obj.algorithm_interface_through_model_manager.annotate( input_count=Count("interface__inputs", distinct=True), relevant_input_count=Count( "interface__inputs", @@ -1405,7 +1401,7 @@ def clean_inputs(self): .exists() ): raise ValidationError( - "An AlgorithmInterface for this algorithm with the " + f"An AlgorithmInterface for this {self._base_obj._meta.verbose_name} with the " "same inputs already exists. " "Algorithm interfaces need to have unique sets of inputs." ) @@ -1445,7 +1441,7 @@ def save(self): inputs=self.cleaned_data["inputs"], outputs=self.cleaned_data["outputs"], ) - self._algorithm.interfaces.add(interface) + self._base_obj.algorithm_interface_manager.add(interface) return interface diff --git a/app/grandchallenge/algorithms/migrations/0065_create_algorithm_interfaces.py b/app/grandchallenge/algorithms/migrations/0065_create_algorithm_interfaces.py index 949f4eea93..3caf518a7a 100644 --- a/app/grandchallenge/algorithms/migrations/0065_create_algorithm_interfaces.py +++ b/app/grandchallenge/algorithms/migrations/0065_create_algorithm_interfaces.py @@ -17,6 +17,9 @@ def create_algorithm_interfaces(apps, _schema_editor): inputs = algorithm.inputs.all() outputs = algorithm.outputs.all() + if not inputs or not outputs: + raise RuntimeError(f"{algorithm} is improperly configured.") + io = get_existing_interface_for_inputs_and_outputs( model=AlgorithmInterface, inputs=inputs, outputs=outputs ) diff --git a/app/grandchallenge/algorithms/models.py b/app/grandchallenge/algorithms/models.py index caa8be7b85..77666071e1 100644 --- a/app/grandchallenge/algorithms/models.py +++ b/app/grandchallenge/algorithms/models.py @@ -520,6 +520,14 @@ def default_workstation(self): return w + @property + def algorithm_interface_manager(self): + return self.interfaces + + @property + def algorithm_interface_through_model_manager(self): + return AlgorithmAlgorithmInterface.objects.filter(algorithm=self) + def is_editor(self, user): return user.groups.filter(pk=self.editors_group.pk).exists() diff --git a/app/grandchallenge/algorithms/views.py b/app/grandchallenge/algorithms/views.py index 83272f2984..0e550eac14 100644 --- a/app/grandchallenge/algorithms/views.py +++ b/app/grandchallenge/algorithms/views.py @@ -1195,7 +1195,7 @@ def get_context_data(self, *args, **kwargs): def get_form_kwargs(self): kwargs = super().get_form_kwargs() - kwargs.update({"algorithm": self.algorithm}) + kwargs.update({"base_obj": self.algorithm}) return kwargs diff --git a/app/grandchallenge/evaluation/admin.py b/app/grandchallenge/evaluation/admin.py index aad7d674c0..d922ec8a66 100644 --- a/app/grandchallenge/evaluation/admin.py +++ b/app/grandchallenge/evaluation/admin.py @@ -1,9 +1,9 @@ import json from django.contrib import admin -from django.core.exceptions import ValidationError from django.forms import ModelForm from django.utils.html import format_html +from guardian.admin import GuardedModelAdmin from grandchallenge.components.admin import ( ComponentImageAdmin, @@ -15,7 +15,6 @@ GroupObjectPermissionAdmin, UserObjectPermissionAdmin, ) -from grandchallenge.core.templatetags.remove_whitespace import oxford_comma from grandchallenge.core.utils.grand_challenge_forge import ( get_forge_challenge_pack_context, ) @@ -29,6 +28,7 @@ MethodGroupObjectPermission, MethodUserObjectPermission, Phase, + PhaseAlgorithmInterface, PhaseGroupObjectPermission, PhaseUserObjectPermission, Submission, @@ -52,21 +52,6 @@ def __init__(self, *args, **kwargs): ) in self.instance.read_only_fields_for_dependent_phases: self.fields[field_name].disabled = True - def clean(self): - cleaned_data = super().clean() - - duplicate_interfaces = { - *cleaned_data.get("algorithm_inputs", []) - }.intersection({*cleaned_data.get("algorithm_outputs", [])}) - - if duplicate_interfaces: - raise ValidationError( - f"The sets of Algorithm Inputs and Algorithm Outputs must be unique: " - f"{oxford_comma(duplicate_interfaces)} present in both" - ) - - return cleaned_data - @admin.register(Phase) class PhaseAdmin(admin.ModelAdmin): @@ -97,13 +82,14 @@ class PhaseAdmin(admin.ModelAdmin): autocomplete_fields = ( "inputs", "outputs", - "algorithm_inputs", - "algorithm_outputs", "archive", ) readonly_fields = ( "give_algorithm_editors_job_view_permissions", "challenge_forge_json", + "algorithm_interfaces", + "algorithm_inputs", + "algorithm_outputs", ) form = PhaseAdminForm @@ -226,6 +212,24 @@ class EvaluationGroundTruthAdmin(admin.ModelAdmin): readonly_fields = ("creator", "phase", "sha256", "size_in_storage") +@admin.register(PhaseAlgorithmInterface) +class PhaseAlgorithmInterfaceAdmin(GuardedModelAdmin): + list_display = ( + "pk", + "interface", + "phase", + ) + list_filter = ("phase",) + + def has_add_permission(self, request, obj=None): + # through table entries should only be created through the UI + return False + + def has_change_permission(self, request, obj=None): + # through table entries should only be updated through the UI + return False + + admin.site.register(PhaseUserObjectPermission, UserObjectPermissionAdmin) admin.site.register(PhaseGroupObjectPermission, GroupObjectPermissionAdmin) admin.site.register(Method, ComponentImageAdmin) diff --git a/app/grandchallenge/evaluation/forms.py b/app/grandchallenge/evaluation/forms.py index 1bef795ad3..b2f5d768e3 100644 --- a/app/grandchallenge/evaluation/forms.py +++ b/app/grandchallenge/evaluation/forms.py @@ -21,16 +21,12 @@ ) from django.utils.html import format_html from django.utils.text import format_lazy -from django_select2.forms import Select2MultipleWidget from grandchallenge.algorithms.forms import UserAlgorithmsForPhaseMixin from grandchallenge.algorithms.models import Job from grandchallenge.challenges.models import Challenge, ChallengeRequest from grandchallenge.components.forms import ContainerImageForm -from grandchallenge.components.models import ( - ComponentInterface, - ImportStatusChoices, -) +from grandchallenge.components.models import ImportStatusChoices from grandchallenge.components.schemas import GPUTypeChoices from grandchallenge.components.tasks import assign_tarball_from_upload from grandchallenge.core.forms import ( @@ -742,14 +738,6 @@ class ConfigureAlgorithmPhasesForm(SaveFormInitMixin, Form): queryset=None, widget=CheckboxSelectMultiple, ) - algorithm_inputs = ModelMultipleChoiceField( - queryset=ComponentInterface.objects.all(), - widget=Select2MultipleWidget, - ) - algorithm_outputs = ModelMultipleChoiceField( - queryset=ComponentInterface.objects.all(), - widget=Select2MultipleWidget, - ) algorithm_time_limit = IntegerField( widget=forms.HiddenInput(), disabled=True, diff --git a/app/grandchallenge/evaluation/migrations/0072_phasealgorithminterface_phase_algorithm_interfaces_and_more.py b/app/grandchallenge/evaluation/migrations/0072_phasealgorithminterface_phase_algorithm_interfaces_and_more.py new file mode 100644 index 0000000000..6e3e24e173 --- /dev/null +++ b/app/grandchallenge/evaluation/migrations/0072_phasealgorithminterface_phase_algorithm_interfaces_and_more.py @@ -0,0 +1,62 @@ +# Generated by Django 4.2.18 on 2025-01-29 10:23 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("algorithms", "0066_create_interfaces_for_jobs"), + ( + "evaluation", + "0071_alter_combinedleaderboardphase_unique_together_and_more", + ), + ] + + operations = [ + migrations.CreateModel( + name="PhaseAlgorithmInterface", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "interface", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="algorithms.algorithminterface", + ), + ), + ( + "phase", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="evaluation.phase", + ), + ), + ], + ), + migrations.AddField( + model_name="phase", + name="algorithm_interfaces", + field=models.ManyToManyField( + blank=True, + help_text="The interfaces that an algorithm for this phase must implement.", + through="evaluation.PhaseAlgorithmInterface", + to="algorithms.algorithminterface", + ), + ), + migrations.AddConstraint( + model_name="phasealgorithminterface", + constraint=models.UniqueConstraint( + fields=("phase", "interface"), + name="unique_phase_interface_combination", + ), + ), + ] diff --git a/app/grandchallenge/evaluation/migrations/0073_migrate_interfaces_for_phases.py b/app/grandchallenge/evaluation/migrations/0073_migrate_interfaces_for_phases.py new file mode 100644 index 0000000000..56a0e90bb7 --- /dev/null +++ b/app/grandchallenge/evaluation/migrations/0073_migrate_interfaces_for_phases.py @@ -0,0 +1,49 @@ +from django.db import migrations + +from grandchallenge.algorithms.models import ( + get_existing_interface_for_inputs_and_outputs, +) +from grandchallenge.evaluation.utils import SubmissionKindChoices + + +def add_algorithm_interfaces_for_phases(apps, _schema_editor): + Phase = apps.get_model("evaluation", "Phase") # noqa: N806 + AlgorithmInterface = apps.get_model( # noqa: N806 + "algorithms", "AlgorithmInterface" + ) + + for phase in ( + Phase.objects.filter(submission_kind=SubmissionKindChoices.ALGORITHM) + .prefetch_related("algorithm_inputs", "algorithm_outputs") + .all() + ): + inputs = phase.algorithm_inputs.all() + outputs = phase.algorithm_outputs.all() + + if not inputs or not outputs: + raise RuntimeError(f"{phase} is improperly configured.") + + io = get_existing_interface_for_inputs_and_outputs( + model=AlgorithmInterface, inputs=inputs, outputs=outputs + ) + if not io: + io = AlgorithmInterface.objects.create() + io.inputs.set(inputs) + io.outputs.set(outputs) + + phase.algorithm_interfaces.add(io) + + +class Migration(migrations.Migration): + dependencies = [ + ( + "evaluation", + "0072_phasealgorithminterface_phase_algorithm_interfaces_and_more", + ), + ] + + operations = [ + migrations.RunPython( + add_algorithm_interfaces_for_phases, elidable=True + ), + ] diff --git a/app/grandchallenge/evaluation/models.py b/app/grandchallenge/evaluation/models.py index f57bfe926b..9459947be9 100644 --- a/app/grandchallenge/evaluation/models.py +++ b/app/grandchallenge/evaluation/models.py @@ -24,6 +24,7 @@ from grandchallenge.algorithms.models import ( AlgorithmImage, + AlgorithmInterface, AlgorithmModel, Job, ) @@ -460,7 +461,12 @@ class Phase(FieldChangeMixin, HangingProtocolMixin, UUIDModel): "metrics.json over the API." ), ) - + algorithm_interfaces = models.ManyToManyField( + to=AlgorithmInterface, + through="evaluation.PhaseAlgorithmInterface", + blank=True, + help_text="The interfaces that an algorithm for this phase must implement.", + ) inputs = models.ManyToManyField( to=ComponentInterface, related_name="evaluation_inputs" ) @@ -843,7 +849,7 @@ def valid_metrics(self): def read_only_fields_for_dependent_phases(self): common_fields = ["submission_kind"] if self.submission_kind == SubmissionKindChoices.ALGORITHM: - common_fields += ["algorithm_inputs", "algorithm_outputs"] + common_fields += ["algorithm_interfaces"] return common_fields def _clean_parent_phase(self): @@ -1198,6 +1204,14 @@ def parent_phase_choices(self): ) ) + @property + def algorithm_interface_manager(self): + return self.algorithm_interfaces + + @property + def algorithm_interface_through_model_manager(self): + return PhaseAlgorithmInterface.objects.filter(phase=self) + class PhaseUserObjectPermission(UserObjectPermissionBase): content_object = models.ForeignKey(Phase, on_delete=models.CASCADE) @@ -1207,6 +1221,19 @@ class PhaseGroupObjectPermission(GroupObjectPermissionBase): content_object = models.ForeignKey(Phase, on_delete=models.CASCADE) +class PhaseAlgorithmInterface(models.Model): + phase = models.ForeignKey(Phase, on_delete=models.CASCADE) + interface = models.ForeignKey(AlgorithmInterface, on_delete=models.CASCADE) + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["phase", "interface"], + name="unique_phase_interface_combination", + ), + ] + + class Method(UUIDModel, ComponentImage): """Store the methods for performing an evaluation.""" diff --git a/app/grandchallenge/evaluation/templates/evaluation/configure_algorithm_phases_form.html b/app/grandchallenge/evaluation/templates/evaluation/configure_algorithm_phases_form.html index 612cb945d4..7a62485ac5 100644 --- a/app/grandchallenge/evaluation/templates/evaluation/configure_algorithm_phases_form.html +++ b/app/grandchallenge/evaluation/templates/evaluation/configure_algorithm_phases_form.html @@ -20,6 +20,9 @@ {% block content %}

    Configure algorithm submission phases

    +

    Select the phases you would like to turn into algorithm submission phases.

    +

    This will create an archive for each selected phase, and populate the inference time limit, gpu type choices and memory requirements based on the corresponding challenge request.

    +

    To configure algorithm interfaces for each phase, please visit the phase settings after this step.

    {% crispy form %} {% endblock %} diff --git a/app/grandchallenge/evaluation/views/__init__.py b/app/grandchallenge/evaluation/views/__init__.py index dbb94d17c8..8edbcd4ce3 100644 --- a/app/grandchallenge/evaluation/views/__init__.py +++ b/app/grandchallenge/evaluation/views/__init__.py @@ -1037,8 +1037,6 @@ def form_valid(self, form): for phase in form.cleaned_data["phases"]: self.turn_phase_into_algorithm_phase( phase=phase, - inputs=form.cleaned_data["algorithm_inputs"], - outputs=form.cleaned_data["algorithm_outputs"], algorithm_time_limit=form.cleaned_data["algorithm_time_limit"], algorithm_selectable_gpu_type_choices=form.cleaned_data[ "algorithm_selectable_gpu_type_choices" @@ -1059,8 +1057,6 @@ def turn_phase_into_algorithm_phase( self, *, phase, - inputs, - outputs, algorithm_time_limit, algorithm_selectable_gpu_type_choices, algorithm_maximum_settable_memory_gb, @@ -1097,8 +1093,6 @@ def turn_phase_into_algorithm_phase( phase.submission_kind = phase.SubmissionKindChoices.ALGORITHM phase.creator_must_be_verified = True phase.save() - phase.algorithm_outputs.add(*outputs) - phase.algorithm_inputs.add(*inputs) class EvaluationGroundTruthCreate( diff --git a/app/tests/algorithms_tests/test_forms.py b/app/tests/algorithms_tests/test_forms.py index 7358a36cb8..3e2499fd05 100644 --- a/app/tests/algorithms_tests/test_forms.py +++ b/app/tests/algorithms_tests/test_forms.py @@ -1400,7 +1400,7 @@ def test_algorithm_for_phase_form_memory_limited(): def test_algorithm_interface_disjoint_interfaces(): ci = ComponentInterfaceFactory() form = AlgorithmInterfaceForm( - algorithm=AlgorithmFactory(), data={"inputs": [ci], "outputs": [ci]} + base_obj=AlgorithmFactory(), data={"inputs": [ci], "outputs": [ci]} ) assert form.is_valid() is False assert "The sets of Inputs and Outputs must be unique" in str(form.errors) @@ -1414,7 +1414,7 @@ def test_algorithm_interface_unique_inputs_required(): alg.interfaces.add(interface) form = AlgorithmInterfaceForm( - algorithm=alg, data={"inputs": [ci1], "outputs": [ci2]} + base_obj=alg, data={"inputs": [ci1], "outputs": [ci2]} ) assert form.is_valid() is False assert ( @@ -1462,7 +1462,7 @@ def test_existing_io_is_reused(self): alg = AlgorithmFactory() form = AlgorithmInterfaceForm( - algorithm=alg, + base_obj=alg, data={ "inputs": [inp.pk], "outputs": [out.pk], diff --git a/app/tests/evaluation_tests/test_admin.py b/app/tests/evaluation_tests/test_admin.py index 9037e67a3d..9779ea94ad 100644 --- a/app/tests/evaluation_tests/test_admin.py +++ b/app/tests/evaluation_tests/test_admin.py @@ -2,26 +2,10 @@ from grandchallenge.evaluation.admin import PhaseAdmin from grandchallenge.evaluation.utils import SubmissionKindChoices -from tests.components_tests.factories import ComponentInterfaceFactory from tests.evaluation_tests.factories import PhaseFactory from tests.factories import ChallengeFactory -@pytest.mark.django_db -def test_disjoint_interfaces(): - i = ComponentInterfaceFactory() - p = PhaseFactory(challenge=ChallengeFactory()) - form = PhaseAdmin.form( - instance=p, - data={"algorithm_inputs": [i.pk], "algorithm_outputs": [i.pk]}, - ) - assert form.is_valid() is False - assert ( - "The sets of Algorithm Inputs and Algorithm Outputs must be unique" - in str(form.errors) - ) - - @pytest.mark.django_db def test_read_only_fields_disabled(): p1, p2 = PhaseFactory.create_batch( @@ -34,8 +18,7 @@ def test_read_only_fields_disabled(): form = PhaseAdmin.form( instance=p1, ) - assert form.fields["algorithm_inputs"].disabled - assert form.fields["algorithm_outputs"].disabled + assert form.fields["algorithm_interfaces"].disabled assert form.fields["submission_kind"].disabled p3, p4 = PhaseFactory.create_batch( diff --git a/app/tests/evaluation_tests/test_models.py b/app/tests/evaluation_tests/test_models.py index 96ea6f9d45..2e8360efd5 100644 --- a/app/tests/evaluation_tests/test_models.py +++ b/app/tests/evaluation_tests/test_models.py @@ -1056,8 +1056,7 @@ def test_read_only_fields_for_dependent_phases(): ) assert p1.read_only_fields_for_dependent_phases == [ "submission_kind", - "algorithm_inputs", - "algorithm_outputs", + "algorithm_interfaces", ] assert p2.read_only_fields_for_dependent_phases == ["submission_kind"] diff --git a/app/tests/evaluation_tests/test_views.py b/app/tests/evaluation_tests/test_views.py index 5759c17528..91120b7fdc 100644 --- a/app/tests/evaluation_tests/test_views.py +++ b/app/tests/evaluation_tests/test_views.py @@ -1373,8 +1373,6 @@ def test_configure_algorithm_phases_view(client): }, data={ "phases": [phase.pk], - "algorithm_inputs": [ci1.pk], - "algorithm_outputs": [ci2.pk], }, ) assert response.status_code == 302 @@ -1385,8 +1383,6 @@ def test_configure_algorithm_phases_view(client): phase.archive.title == f"{phase.challenge.short_name} {phase.title} dataset" ) - assert list(phase.algorithm_inputs.all()) == [ci1] - assert list(phase.algorithm_outputs.all()) == [ci2] assert ( phase.algorithm_time_limit == challenge_request.inference_time_limit_in_minutes * 60 From d840750a99a2a96b9074051de493de53729b142c Mon Sep 17 00:00:00 2001 From: Anne Mickan Date: Wed, 5 Feb 2025 10:35:55 +0100 Subject: [PATCH 13/18] Managements views for algorithm interfaces for phases (#3812) This adds the management views for algorithm interfaces for phases. The views are only accessible to staff users, and only for algorithm submission phases that are not external. The logic behind the views is the same as for the algorithm interfaces, but it only made sense to refactor the create view logic, and parts of the templates. ![image](https://github.com/user-attachments/assets/b9c3d86b-fd2e-4ee4-a558-7bef40f15ab5) Part of https://github.com/DIAGNijmegen/rse-roadmap/issues/153 --- app/config/urls/challenge_subdomain.py | 4 + app/grandchallenge/algorithms/models.py | 14 ++ .../algorithms/algorithm_detail.html | 3 + ...ithmalgorithminterface_confirm_delete.html | 21 +- .../algorithmalgorithminterface_list.html | 35 +--- .../algorithms/algorithminterface_form.html | 25 +-- .../algorithminterface_confirm_delete.html | 20 ++ .../partials/algorithminterface_form.html | 15 ++ .../partials/algorithminterface_list.html | 36 ++++ app/grandchallenge/algorithms/views.py | 31 ++- app/grandchallenge/evaluation/models.py | 18 ++ .../algorithminterface_for_phase_form.html | 26 +++ ...hasealgorithminterface_confirm_delete.html | 26 +++ .../phasealgorithminterface_list.html | 23 +++ app/grandchallenge/evaluation/urls.py | 70 ++++--- .../evaluation/views/__init__.py | 102 ++++++++- .../templates/pages/phase_menu_sidebar.html | 14 +- app/tests/evaluation_tests/test_views.py | 194 +++++++++++++++++- 18 files changed, 564 insertions(+), 113 deletions(-) create mode 100644 app/grandchallenge/algorithms/templates/algorithms/partials/algorithminterface_confirm_delete.html create mode 100644 app/grandchallenge/algorithms/templates/algorithms/partials/algorithminterface_form.html create mode 100644 app/grandchallenge/algorithms/templates/algorithms/partials/algorithminterface_list.html create mode 100644 app/grandchallenge/evaluation/templates/evaluation/algorithminterface_for_phase_form.html create mode 100644 app/grandchallenge/evaluation/templates/evaluation/phasealgorithminterface_confirm_delete.html create mode 100644 app/grandchallenge/evaluation/templates/evaluation/phasealgorithminterface_list.html diff --git a/app/config/urls/challenge_subdomain.py b/app/config/urls/challenge_subdomain.py index 6c0b84b24f..07fff57d58 100644 --- a/app/config/urls/challenge_subdomain.py +++ b/app/config/urls/challenge_subdomain.py @@ -15,6 +15,10 @@ ), name="subdomain_robots_txt", ), + path( + "components/", + include("grandchallenge.components.urls", namespace="components"), + ), path( "evaluation/", include("grandchallenge.evaluation.urls", namespace="evaluation"), diff --git a/app/grandchallenge/algorithms/models.py b/app/grandchallenge/algorithms/models.py index 77666071e1..4345deb9cd 100644 --- a/app/grandchallenge/algorithms/models.py +++ b/app/grandchallenge/algorithms/models.py @@ -528,6 +528,20 @@ def algorithm_interface_manager(self): def algorithm_interface_through_model_manager(self): return AlgorithmAlgorithmInterface.objects.filter(algorithm=self) + @property + def algorithm_interface_create_url(self): + return reverse( + "algorithms:interface-create", kwargs={"slug": self.slug} + ) + + @property + def algorithm_interface_delete_viewname(self): + return "algorithms:interface-delete" + + @property + def algorithm_interface_list_url(self): + return reverse("algorithms:interface-list", kwargs={"slug": self.slug}) + def is_editor(self, user): return user.groups.filter(pk=self.editors_group.pk).exists() diff --git a/app/grandchallenge/algorithms/templates/algorithms/algorithm_detail.html b/app/grandchallenge/algorithms/templates/algorithms/algorithm_detail.html index ded523c980..380a29a4f4 100644 --- a/app/grandchallenge/algorithms/templates/algorithms/algorithm_detail.html +++ b/app/grandchallenge/algorithms/templates/algorithms/algorithm_detail.html @@ -39,6 +39,9 @@  Interfaces + {% if not object.interfaces.all %}  + + {% endif %} Confirm Algorithm Interface Deletion -
    - {% csrf_token %} -

    Are you sure that you want to delete the following algorithm interface from your algorithm?

    -
      -
    • Inputs: {{ object.interface.inputs.all|oxford_comma }}
      • -
    • Outputs: {{ object.interface.outputs.all|oxford_comma }}
      • -
    -

    - WARNING: You are not able to undo this action. Once the interface is deleted, it is deleted forever. -

    -     Cancel - -
    - + {% include 'algorithms/partials/algorithminterface_confirm_delete.html' with base_obj=algorithm %} {% endblock %} diff --git a/app/grandchallenge/algorithms/templates/algorithms/algorithmalgorithminterface_list.html b/app/grandchallenge/algorithms/templates/algorithms/algorithmalgorithminterface_list.html index 1efb894bb7..883c635a44 100644 --- a/app/grandchallenge/algorithms/templates/algorithms/algorithmalgorithminterface_list.html +++ b/app/grandchallenge/algorithms/templates/algorithms/algorithmalgorithminterface_list.html @@ -1,6 +1,4 @@ {% extends "base.html" %} -{% load crispy_forms_tags %} -{% load remove_whitespace %} {% block title %} Interfaces - {{ algorithm.title }} - {{ block.super }} @@ -22,36 +20,5 @@ {% endblock %} {% block content %} -

    Algorithm Interfaces for {{ algorithm }}

    - -

    - The following interfaces are configured for your algorithm: -

    -

    Add new interface

    - -
    - - - - - - - - {% for object in object_list %} - - - - - - {% empty %} - - {% endfor %} - -
    InputsOutputsDelete
    {{ object.interface.inputs.all|oxford_comma }}{{ object.interface.outputs.all|oxford_comma }} - - - -
    This algorithm does not have any interfaces defined yet.
    -
    + {% include 'algorithms/partials/algorithminterface_list.html' with base_obj=algorithm %} {% endblock %} diff --git a/app/grandchallenge/algorithms/templates/algorithms/algorithminterface_form.html b/app/grandchallenge/algorithms/templates/algorithms/algorithminterface_form.html index 356412ffa0..02614e610f 100644 --- a/app/grandchallenge/algorithms/templates/algorithms/algorithminterface_form.html +++ b/app/grandchallenge/algorithms/templates/algorithms/algorithminterface_form.html @@ -1,8 +1,7 @@ {% extends "base.html" %} -{% load crispy_forms_tags %} {% block title %} - Create Interface - {{ algorithm.title }} - {{ block.super }} + Create Interface - {{ base_obj.title }} - {{ block.super }} {% endblock %} {% block breadcrumbs %} @@ -10,10 +9,10 @@
  • Algorithms
  • {{ algorithm.title }} + href="{{ algorithm.get_absolute_url }}">{{ base_obj.title }}
  • Interfaces + href="{% url 'algorithms:interface-list' slug=base_obj.slug %}">Interfaces
  • Create Interface @@ -22,21 +21,5 @@ {% endblock %} {% block content %} - -

    Create An Algorithm Interface

    -
    -

    - Create an interface for your algorithm: define any combination of inputs and outputs, and optionally mark the interface as default for the algorithm. -

    -

    - Please see the list of input options and the - list of output options - for more information and examples. -

    -

    - If you cannot find suitable inputs or outputs, please contact support@grand-challenge.org. -

    - - {% crispy form %} - + {% include 'algorithms/partials/algorithminterface_form.html' %} {% endblock %} diff --git a/app/grandchallenge/algorithms/templates/algorithms/partials/algorithminterface_confirm_delete.html b/app/grandchallenge/algorithms/templates/algorithms/partials/algorithminterface_confirm_delete.html new file mode 100644 index 0000000000..2f166af3b7 --- /dev/null +++ b/app/grandchallenge/algorithms/templates/algorithms/partials/algorithminterface_confirm_delete.html @@ -0,0 +1,20 @@ +{% load remove_whitespace %} + +

    Confirm Algorithm Interface Deletion

    +
    + {% csrf_token %} +

    Are you sure that you want to delete the following algorithm interface from this {{ base_obj.verbose_name }}?

    +
      +
    • Inputs: {{ object.interface.inputs.all|oxford_comma }}
      • +
    • Outputs: {{ object.interface.outputs.all|oxford_comma }}
      • +
    +

    + WARNING: You are not able to undo this action. Once the interface is deleted, it is deleted forever. +

    + Cancel + +
    diff --git a/app/grandchallenge/algorithms/templates/algorithms/partials/algorithminterface_form.html b/app/grandchallenge/algorithms/templates/algorithms/partials/algorithminterface_form.html new file mode 100644 index 0000000000..40ed27f74b --- /dev/null +++ b/app/grandchallenge/algorithms/templates/algorithms/partials/algorithminterface_form.html @@ -0,0 +1,15 @@ +{% load crispy_forms_tags %} +{% load meta_attr %} + +

    Create An Algorithm Interface

    +

    + Create an interface: define any combination of inputs and outputs, and optionally mark the interface as default for the {{ base_obj|verbose_name }}.

    +

    + Please see the list of input options and the + list of output options for more information and examples. +

    +

    + If you cannot find suitable inputs or outputs, please contact support@grand-challenge.org. +

    + +{% crispy form %} diff --git a/app/grandchallenge/algorithms/templates/algorithms/partials/algorithminterface_list.html b/app/grandchallenge/algorithms/templates/algorithms/partials/algorithminterface_list.html new file mode 100644 index 0000000000..fd1aca7033 --- /dev/null +++ b/app/grandchallenge/algorithms/templates/algorithms/partials/algorithminterface_list.html @@ -0,0 +1,36 @@ +{% load crispy_forms_tags %} +{% load remove_whitespace %} +{% load meta_attr %} + +

    Algorithm Interfaces for {{ base_obj }}

    + +

    The following interfaces are configured for your {{ base_obj|verbose_name }}:

    +

    + Add new interface +

    + +
    + + + + + + + + {% for object in object_list %} + + + + + + {% empty %} + + {% endfor %} + +
    InputsOutputsDelete
    {{ object.interface.inputs.all|oxford_comma }}{{ object.interface.outputs.all|oxford_comma }} + + + +
    This {{ base_obj|verbose_name }} does not have any interfaces defined yet.
    +
    diff --git a/app/grandchallenge/algorithms/views.py b/app/grandchallenge/algorithms/views.py index 0e550eac14..ac238d71d2 100644 --- a/app/grandchallenge/algorithms/views.py +++ b/app/grandchallenge/algorithms/views.py @@ -1175,30 +1175,43 @@ def dispatch(self, request, *args, **kwargs): return self.handle_no_permission() -class AlgorithmInterfaceForAlgorithmCreate( - AlgorithmInterfacePermissionMixin, CreateView -): +class AlgorithmInterfaceCreateBase(CreateView): model = AlgorithmInterface form_class = AlgorithmInterfaceForm success_message = "Algorithm interface successfully added" def get_success_url(self): - return reverse( - "algorithms:interface-list", - kwargs={"slug": self.algorithm.slug}, - ) + return NotImplementedError + + @property + def base_obj(self): + return NotImplementedError def get_context_data(self, *args, **kwargs): context = super().get_context_data(*args, **kwargs) - context.update({"algorithm": self.algorithm}) + context.update({"base_obj": self.base_obj}) return context def get_form_kwargs(self): kwargs = super().get_form_kwargs() - kwargs.update({"base_obj": self.algorithm}) + kwargs.update({"base_obj": self.base_obj}) return kwargs +class AlgorithmInterfaceForAlgorithmCreate( + AlgorithmInterfacePermissionMixin, AlgorithmInterfaceCreateBase +): + @property + def base_obj(self): + return self.algorithm + + def get_success_url(self): + return reverse( + "algorithms:interface-list", + kwargs={"slug": self.algorithm.slug}, + ) + + class AlgorithmInterfacesForAlgorithmList( AlgorithmInterfacePermissionMixin, ListView ): diff --git a/app/grandchallenge/evaluation/models.py b/app/grandchallenge/evaluation/models.py index 9459947be9..e04dfde56b 100644 --- a/app/grandchallenge/evaluation/models.py +++ b/app/grandchallenge/evaluation/models.py @@ -1212,6 +1212,24 @@ def algorithm_interface_manager(self): def algorithm_interface_through_model_manager(self): return PhaseAlgorithmInterface.objects.filter(phase=self) + @property + def algorithm_interface_create_url(self): + return reverse( + "evaluation:interface-create", + kwargs={"challenge_short_name": self.challenge, "slug": self.slug}, + ) + + @property + def algorithm_interface_delete_viewname(self): + return "evaluation:interface-delete" + + @property + def algorithm_interface_list_url(self): + return reverse( + "evaluation:interface-list", + kwargs={"challenge_short_name": self.challenge, "slug": self.slug}, + ) + class PhaseUserObjectPermission(UserObjectPermissionBase): content_object = models.ForeignKey(Phase, on_delete=models.CASCADE) diff --git a/app/grandchallenge/evaluation/templates/evaluation/algorithminterface_for_phase_form.html b/app/grandchallenge/evaluation/templates/evaluation/algorithminterface_for_phase_form.html new file mode 100644 index 0000000000..85aa24ef3e --- /dev/null +++ b/app/grandchallenge/evaluation/templates/evaluation/algorithminterface_for_phase_form.html @@ -0,0 +1,26 @@ +{% extends "pages/challenge_settings_base.html" %} +{% load url %} +{% load static %} + +{% block title %} + Create Interface - {{ base_obj.title }} - {% firstof challenge.title challenge.short_name %} - {{ block.super }} +{% endblock %} + +{% block breadcrumbs %} +
      +
    1. Challenges +
      2. +
    2. {% firstof challenge.title challenge.short_name %}
      3. +
    3. Interfaces +
      4. +
    4. Create interface
      5. +
    +{% endblock %} + +{% block content %} + {% include 'algorithms/partials/algorithminterface_form.html' %} +{% endblock %} diff --git a/app/grandchallenge/evaluation/templates/evaluation/phasealgorithminterface_confirm_delete.html b/app/grandchallenge/evaluation/templates/evaluation/phasealgorithminterface_confirm_delete.html new file mode 100644 index 0000000000..5ac02e4a66 --- /dev/null +++ b/app/grandchallenge/evaluation/templates/evaluation/phasealgorithminterface_confirm_delete.html @@ -0,0 +1,26 @@ +{% extends "pages/challenge_settings_base.html" %} +{% load url %} +{% load static %} + +{% block title %} + Delete interface - {{ phase.title }} - {% firstof challenge.title challenge.short_name %} - {{ block.super }} +{% endblock %} + +{% block breadcrumbs %} +
      +
    1. Challenges +
      2. +
    2. {% firstof challenge.title challenge.short_name %}
      3. +
    3. Interfaces +
      4. +
    4. Delete interface
      5. +
    +{% endblock %} + +{% block content %} + {% include 'algorithms/partials/algorithminterface_confirm_delete.html' with base_obj=phase %} +{% endblock %} diff --git a/app/grandchallenge/evaluation/templates/evaluation/phasealgorithminterface_list.html b/app/grandchallenge/evaluation/templates/evaluation/phasealgorithminterface_list.html new file mode 100644 index 0000000000..793b5b8f02 --- /dev/null +++ b/app/grandchallenge/evaluation/templates/evaluation/phasealgorithminterface_list.html @@ -0,0 +1,23 @@ +{% extends "pages/challenge_settings_base.html" %} +{% load url %} +{% load static %} + +{% block title %} + Algorithm interfaces for {{ phase.title }} - {% firstof challenge.title challenge.short_name %} - {{ block.super }} +{% endblock %} + +{% block breadcrumbs %} +
      +
    1. Challenges +
      2. +
    2. {% firstof challenge.title challenge.short_name %}
      3. +
    3. Algorithm Interfaces for {{ phase.title }}
      4. +
    +{% endblock %} + +{% block content %} + {% include 'algorithms/partials/algorithminterface_list.html' with base_obj=phase %} +{% endblock %} diff --git a/app/grandchallenge/evaluation/urls.py b/app/grandchallenge/evaluation/urls.py index 43fbdfbf55..7cfff19363 100644 --- a/app/grandchallenge/evaluation/urls.py +++ b/app/grandchallenge/evaluation/urls.py @@ -1,6 +1,9 @@ from django.urls import path from grandchallenge.evaluation.views import ( + AlgorithmInterfaceForPhaseCreate, + AlgorithmInterfaceForPhaseDelete, + AlgorithmInterfacesForPhaseList, CombinedLeaderboardCreate, CombinedLeaderboardDelete, CombinedLeaderboardDetail, @@ -64,109 +67,128 @@ name="combined-leaderboard-create", ), path( - "combined-leaderboards//", + "combined-leaderboards//", CombinedLeaderboardDetail.as_view(), name="combined-leaderboard-detail", ), path( - "combined-leaderboards//update/", + "combined-leaderboards//update/", CombinedLeaderboardUpdate.as_view(), name="combined-leaderboard-update", ), path( - "combined-leaderboards//delete/", + "combined-leaderboards//delete/", CombinedLeaderboardDelete.as_view(), name="combined-leaderboard-delete", ), - path("/", EvaluationList.as_view(), name="list"), + path("/", EvaluationList.as_view(), name="list"), path( - "/admin/", + "/admin/", EvaluationAdminList.as_view(), name="evaluation-admin-list", ), path( - "/algorithms/create/", + "/algorithms/create/", PhaseAlgorithmCreate.as_view(), name="phase-algorithm-create", ), path( - "/linked-archive/", + "/interfaces/", + AlgorithmInterfacesForPhaseList.as_view(), + name="interface-list", + ), + path( + "/interfaces/create/", + AlgorithmInterfaceForPhaseCreate.as_view(), + name="interface-create", + ), + path( + "/interfaces//delete/", + AlgorithmInterfaceForPhaseDelete.as_view(), + name="interface-delete", + ), + path( + "/linked-archive/", PhaseArchiveInfo.as_view(), name="phase-archive-info", ), path( - "/ground-truths/", + "/ground-truths/", EvaluationGroundTruthList.as_view(), name="ground-truth-list", ), path( - "/ground-truths/activate/", + "/ground-truths/activate/", EvaluationGroundTruthVersionManagement.as_view(activate=True), name="ground-truth-activate", ), path( - "/ground-truths/create/", + "/ground-truths/create/", EvaluationGroundTruthCreate.as_view(), name="ground-truth-create", ), path( - "/ground-truths/deactivate/", + "/ground-truths/deactivate/", EvaluationGroundTruthVersionManagement.as_view(activate=False), name="ground-truth-deactivate", ), path( - "/ground-truths//", + "/ground-truths//", EvaluationGroundTruthDetail.as_view(), name="ground-truth-detail", ), path( - "/ground-truths//import-status/", + "/ground-truths//import-status/", EvaluationGroundTruthImportStatusDetail.as_view(), name="ground-truth-import-status-detail", ), path( - "/ground-truths//update/", + "/ground-truths//update/", EvaluationGroundTruthUpdate.as_view(), name="ground-truth-update", ), path( - "/leaderboard/", LeaderboardDetail.as_view(), name="leaderboard" + "/leaderboard/", + LeaderboardDetail.as_view(), + name="leaderboard", ), - path("/methods/", MethodList.as_view(), name="method-list"), + path("/methods/", MethodList.as_view(), name="method-list"), path( - "/methods/create/", MethodCreate.as_view(), name="method-create" + "/methods/create/", + MethodCreate.as_view(), + name="method-create", ), path( - "/methods//", + "/methods//", MethodDetail.as_view(), name="method-detail", ), path( - "/methods//import-status/", + "/methods//import-status/", MethodImportStatusDetail.as_view(), name="method-import-status-detail", ), path( - "/methods//update/", + "/methods//update/", MethodUpdate.as_view(), name="method-update", ), path( - "/submissions/create/", + "/submissions/create/", SubmissionCreate.as_view(), name="submission-create", ), path( - "/submissions//", + "/submissions//", SubmissionDetail.as_view(), name="submission-detail", ), path( - "/submissions//evaluations/create/", + "/submissions//evaluations/create/", EvaluationCreate.as_view(), name="evaluation-create", ), - path("/update/", PhaseUpdate.as_view(), name="phase-update"), + path("/update/", PhaseUpdate.as_view(), name="phase-update"), path("results/", LeaderboardRedirect.as_view()), path("leaderboard/", LeaderboardRedirect.as_view()), ] diff --git a/app/grandchallenge/evaluation/views/__init__.py b/app/grandchallenge/evaluation/views/__init__.py index 8edbcd4ce3..92c4ca1b80 100644 --- a/app/grandchallenge/evaluation/views/__init__.py +++ b/app/grandchallenge/evaluation/views/__init__.py @@ -29,6 +29,7 @@ from grandchallenge.algorithms.forms import AlgorithmForPhaseForm from grandchallenge.algorithms.models import Algorithm, Job +from grandchallenge.algorithms.views import AlgorithmInterfaceCreateBase from grandchallenge.archives.models import Archive from grandchallenge.challenges.views import ActiveChallengeRequiredMixin from grandchallenge.components.models import ImportStatusChoices @@ -60,6 +61,7 @@ EvaluationGroundTruth, Method, Phase, + PhaseAlgorithmInterface, Submission, ) from grandchallenge.evaluation.tasks import create_evaluation @@ -1022,9 +1024,14 @@ def get_permission_object(self): return self.get_object().challenge -class ConfigureAlgorithmPhasesView(PermissionRequiredMixin, FormView): - form_class = ConfigureAlgorithmPhasesForm +class ConfigureAlgorithmPhasesPermissionMixin(PermissionRequiredMixin): permission_required = "evaluation.configure_algorithm_phase" + + +class ConfigureAlgorithmPhasesView( + ConfigureAlgorithmPhasesPermissionMixin, FormView +): + form_class = ConfigureAlgorithmPhasesForm template_name = "evaluation/configure_algorithm_phases_form.html" raise_exception = True @@ -1252,3 +1259,94 @@ def get_object(self, queryset=None): challenge=self.request.challenge, slug=self.kwargs["slug"], ) + + +class AlgorithmInterfaceForPhaseMixin: + + @property + def phase(self): + return get_object_or_404( + Phase, + challenge=self.request.challenge, + challenge__phase__submission_kind=SubmissionKindChoices.ALGORITHM, + slug=self.kwargs["slug"], + ) + + +class AlgorithmInterfaceForPhaseCreate( + ConfigureAlgorithmPhasesPermissionMixin, + AlgorithmInterfaceForPhaseMixin, + AlgorithmInterfaceCreateBase, +): + template_name = "evaluation/algorithminterface_for_phase_form.html" + + @property + def base_obj(self): + return self.phase + + def get_success_url(self): + return reverse( + "evaluation:interface-list", + kwargs={ + "slug": self.phase.slug, + "challenge_short_name": self.request.challenge.short_name, + }, + ) + + +class AlgorithmInterfacesForPhaseList( + ConfigureAlgorithmPhasesPermissionMixin, + AlgorithmInterfaceForPhaseMixin, + ListView, +): + model = PhaseAlgorithmInterface + + def get_queryset(self): + qs = super().get_queryset() + return qs.filter(phase=self.phase) + + def get_context_data(self, *args, **kwargs): + context = super().get_context_data(*args, **kwargs) + context.update( + { + "phase": self.phase, + } + ) + return context + + +class AlgorithmInterfaceForPhaseDelete( + ConfigureAlgorithmPhasesPermissionMixin, + AlgorithmInterfaceForPhaseMixin, + DeleteView, +): + model = PhaseAlgorithmInterface + + @property + def algorithm_interface(self): + return get_object_or_404( + klass=PhaseAlgorithmInterface, + phase=self.phase, + interface__pk=self.kwargs["interface_pk"], + ) + + def get_object(self, queryset=None): + return self.algorithm_interface + + def get_success_url(self): + return reverse( + "evaluation:interface-list", + kwargs={ + "slug": self.phase.slug, + "challenge_short_name": self.request.challenge.short_name, + }, + ) + + def get_context_data(self, *args, **kwargs): + context = super().get_context_data(*args, **kwargs) + context.update( + { + "phase": self.phase, + } + ) + return context diff --git a/app/grandchallenge/pages/templates/pages/phase_menu_sidebar.html b/app/grandchallenge/pages/templates/pages/phase_menu_sidebar.html index c7f0b0fb4e..55e71a9e2e 100644 --- a/app/grandchallenge/pages/templates/pages/phase_menu_sidebar.html +++ b/app/grandchallenge/pages/templates/pages/phase_menu_sidebar.html @@ -1,15 +1,17 @@ +{% load guardian_tags %} +
  • {% if not phase.public %}{% endif %} {{ phase.title }} -
    {% endif %} -
    -
    Inputs:
    -
    -
      - {% for input in object.inputs.all %} -
    • {{ input.title }} {% if input.description %} - ({{ input.description }}){% endif %}
      • - {% endfor %} -
    -
    -
    +

    Interfaces

    +

    This algorithm implements all of the following input-output combinations:

    + {% include 'algorithms/partials/algorithminterface_table.html' with base_obj=object interfaces=object.interfaces.all delete_option=False %} -
    -
    Outputs:
    -
    -
      - {% for output in object.outputs.all %} -
    • {{ output.title }} {% if output.description %} - ({{ output.description }}){% endif %}
      • - {% endfor %} -
    -
    -
    {% if best_evaluation_per_phase %}

    Challenge Performance

    diff --git a/app/grandchallenge/algorithms/templates/algorithms/algorithmimage_form.html b/app/grandchallenge/algorithms/templates/algorithms/algorithmimage_form.html index 5343922a9b..8f1f18e61f 100644 --- a/app/grandchallenge/algorithms/templates/algorithms/algorithmimage_form.html +++ b/app/grandchallenge/algorithms/templates/algorithms/algorithmimage_form.html @@ -24,53 +24,20 @@

    Create An Algorithm Container Image

    - Upload a container image that implements the selected inputs and outputs. - In order to work this container will need to read the following input: + Upload a container image that implements all of the configured interfaces (i.e. input-output combinations):

    -
      - {% for interface in algorithm.inputs.all %} -
    • - {{ interface }} at - {% if interface.is_image_kind %} - /input/{{ interface.relative_path }}/<uuid>.mha or - /input/{{ interface.relative_path }}/<uuid>.tif - {% else %} - /input/{{ interface.relative_path }} - {% endif %} -
      • - {% endfor %} -
    - -

    - Please see the - list of input options - more information and examples. -

    - -

    - The container will also need to write the following output: -

    - -
      - {% for interface in algorithm.outputs.all %} -
    • - {{ interface }} to - {% if interface.is_image_kind %} - /output/{{ interface.relative_path }}/<uuid>.mha or - /output/{{ interface.relative_path }}/<uuid>.tif - {% else %} - /output/{{ interface.relative_path }} - {% endif %} -
      • - {% endfor %} -
    - -

    - Please see the - list of output options - more information and examples. -

    + {% include 'algorithms/partials/algorithminterface_table.html' with base_obj=algorithm interfaces=algorithm.interfaces.all delete_option=False read_write_paths=True %} + + {% if perms.algorithms.add_algorithm %} +

    + To add or update interfaces for your algorithm, go here. +

    + {% else %} +

    + To add or update interfaces for your algorithm, please contact support@grand-challenge.org. +

    + {% endif %}

    Container Image Options

    diff --git a/app/grandchallenge/algorithms/templates/algorithms/partials/algorithminterface_list.html b/app/grandchallenge/algorithms/templates/algorithms/partials/algorithminterface_list.html index fd1aca7033..e3ce744b95 100644 --- a/app/grandchallenge/algorithms/templates/algorithms/partials/algorithminterface_list.html +++ b/app/grandchallenge/algorithms/templates/algorithms/partials/algorithminterface_list.html @@ -4,33 +4,9 @@

    Algorithm Interfaces for {{ base_obj }}

    -

    The following interfaces are configured for your {{ base_obj|verbose_name }}:

    +

    The following interfaces (i.e. input-output combinations) are configured for your {{ base_obj|verbose_name }}:

    Add new interface

    -
    - - - - - - - - {% for object in object_list %} - - - - - - {% empty %} - - {% endfor %} - -
    InputsOutputsDelete
    {{ object.interface.inputs.all|oxford_comma }}{{ object.interface.outputs.all|oxford_comma }} - - - -
    This {{ base_obj|verbose_name }} does not have any interfaces defined yet.
    -
    +{% include 'algorithms/partials/algorithminterface_table.html' with base_obj=base_obj interfaces=interfaces delete_option=True %} diff --git a/app/grandchallenge/algorithms/templates/algorithms/partials/algorithminterface_table.html b/app/grandchallenge/algorithms/templates/algorithms/partials/algorithminterface_table.html new file mode 100644 index 0000000000..54671a19df --- /dev/null +++ b/app/grandchallenge/algorithms/templates/algorithms/partials/algorithminterface_table.html @@ -0,0 +1,61 @@ +{% load meta_attr %} + +
    + + + + + + {% if delete_option %} + + {% endif %} + + + {% for interface in interfaces %} + + + + + {% if delete_option %} + + {% endif %} + + {% empty %} + + {% endfor %} + +
    InputsOutputsDelete
    {{ forloop.counter }} + {% for input in interface.inputs.all %} +
  • {{ input }} + {% if read_write_paths %} + at + {% if input.is_image_kind %} + /input{% if input.relative_path %}/{{ input.relative_path }}{% endif %}/<uuid>.mha or + /input{% if input.relative_path %}/{{ input.relative_path }}{% endif %}/<uuid>.tif + {% else %} + /input/{{ input.relative_path }} + {% endif %} + {% endif %} +
    • + {% endfor %} +     		+ {% for output in interface.outputs.all %} +
  • {{ output }} + {% if read_write_paths %} + at + {% if output.is_image_kind %} + /output{% if output.relative_path %}/{{ output.relative_path }}{% endif %}/<uuid>.mha or + /output{% if output.relative_path %}/{{ output.relative_path }}{% endif %}/<uuid>.tif + {% else %} + /output/{{ output.relative_path }} + {% endif %} + {% endif %} +
    • + {% endfor %} +     		+ + + +
    This {{ base_obj|verbose_name }} does not have any interfaces defined yet.
    +
    diff --git a/app/grandchallenge/algorithms/views.py b/app/grandchallenge/algorithms/views.py index ac238d71d2..b3971cda28 100644 --- a/app/grandchallenge/algorithms/views.py +++ b/app/grandchallenge/algorithms/views.py @@ -1226,6 +1226,7 @@ def get_context_data(self, *args, **kwargs): context.update( { "algorithm": self.algorithm, + "interfaces": [obj.interface for obj in self.object_list], } ) return context diff --git a/app/grandchallenge/evaluation/forms.py b/app/grandchallenge/evaluation/forms.py index b2f5d768e3..c641ca4563 100644 --- a/app/grandchallenge/evaluation/forms.py +++ b/app/grandchallenge/evaluation/forms.py @@ -37,7 +37,6 @@ filter_by_permission, get_objects_for_user, ) -from grandchallenge.core.templatetags.remove_whitespace import oxford_comma from grandchallenge.core.widgets import ( JSONEditorWidget, MarkdownEditorInlineWidget, @@ -304,7 +303,10 @@ class SubmissionForm( label="Predictions File", queryset=None, ) - algorithm = AlgorithmChoiceField(queryset=None) + algorithm = AlgorithmChoiceField( + queryset=None, + help_text="Select one of your algorithms to submit as a solution to this phase. See above for information regarding the necessary configuration of the algorithm.", + ) confirm_submission = forms.BooleanField( required=True, label="I understand that by submitting my algorithm image and model " @@ -423,24 +425,6 @@ def __init__(self, *args, user, phase: Phase, **kwargs): # noqa: C901 self.fields["algorithm_image"].required = False self.fields["algorithm_model"].widget = HiddenInput() - self._algorithm_inputs = self._phase.algorithm_inputs.all() - self._algorithm_outputs = self._phase.algorithm_outputs.all() - self.fields["algorithm"].help_text = format_lazy( - "Select one of your algorithms to submit as a solution to this " - "challenge. The algorithms need to work with the following inputs: {} " - "and the following outputs: {}. If you have not created your " - "algorithm yet you can " - "do so on this page.", - oxford_comma(self._algorithm_inputs), - oxford_comma(self._algorithm_outputs), - reverse( - "evaluation:phase-algorithm-create", - kwargs={ - "slug": phase.slug, - "challenge_short_name": phase.challenge.short_name, - }, - ), - ) if ( not self._phase.active_image and not self._phase.external_evaluation diff --git a/app/grandchallenge/evaluation/templates/evaluation/partials/algorithm_interfaces_information.html b/app/grandchallenge/evaluation/templates/evaluation/partials/algorithm_interfaces_information.html new file mode 100644 index 0000000000..795d8f5c7c --- /dev/null +++ b/app/grandchallenge/evaluation/templates/evaluation/partials/algorithm_interfaces_information.html @@ -0,0 +1,21 @@ +{% load url %} + +
    +
    The algorithms for this phase need to implement all of the following input-output combinations:
    +
    + + Manage your algorithms + +
    +
    +{% include 'algorithms/partials/algorithminterface_table.html' with base_obj=phase interfaces=phase.algorithm_interfaces.all delete_option=False %} + +{% if phase.submission_kind == phase.SubmissionKindChoices.ALGORITHM and not form.fields.algorithm.queryset.exists %} +

    + Currently, you either do not have an algorithm that is compatible with this phase, + or none of your compatible algorithms have an active algorithm image. + Either upload a new algorithm image to one of your compatible algorithms, or create a new algorithm (see 'Manage your algorithms' button above). + As soon as your new algorithm image is active you can return to this page to submit it to this phase. +

    +{% endif %} diff --git a/app/grandchallenge/evaluation/templates/evaluation/submission_form.html b/app/grandchallenge/evaluation/templates/evaluation/submission_form.html index e7de89174a..4490bc6754 100644 --- a/app/grandchallenge/evaluation/templates/evaluation/submission_form.html +++ b/app/grandchallenge/evaluation/templates/evaluation/submission_form.html @@ -81,6 +81,8 @@

    Create a new submission

    + {% include 'evaluation/partials/algorithm_interfaces_information.html' with phase=phase form=form %} + {% crispy form %} {% else %} @@ -112,19 +114,6 @@

    Create a new submission

    {% endif %} - {% elif phase.submission_kind == phase.SubmissionKindChoices.ALGORITHM and not form.fields.algorithm_image.queryset.exists %} - -

    - Currently, you do not have any Algorithm Image that can be submitted to this phase. - Either upload a new Algorithm Image to one of your existing Algorithms, or create a new Algorithm. - As soon as your new Algorithm Image is active you can return to this page to submit it to this phase. -

    - - - Add a new Algorithm - - {% else %} {% endif %} + {% include 'evaluation/partials/algorithm_interfaces_information.html' with phase=phase form=form %} + {% crispy form %} {% endif %} diff --git a/app/grandchallenge/evaluation/views/__init__.py b/app/grandchallenge/evaluation/views/__init__.py index 92c4ca1b80..efc377ca91 100644 --- a/app/grandchallenge/evaluation/views/__init__.py +++ b/app/grandchallenge/evaluation/views/__init__.py @@ -1310,6 +1310,7 @@ def get_context_data(self, *args, **kwargs): context.update( { "phase": self.phase, + "interfaces": [obj.interface for obj in self.object_list], } ) return context From 028a605843196d59e51901cfaf743bcbc10e4fc6 Mon Sep 17 00:00:00 2001 From: Anne Mickan Date: Thu, 13 Feb 2025 13:36:20 +0100 Subject: [PATCH 15/18] Enable submission with optional inputs (#3820) Part of https://github.com/DIAGNijmegen/rse-roadmap/issues/153 --- app/grandchallenge/algorithms/forms.py | 107 +- app/grandchallenge/algorithms/models.py | 5 +- app/grandchallenge/algorithms/tasks.py | 161 +-- .../core/utils/grand_challenge_forge.py | 30 +- app/grandchallenge/evaluation/forms.py | 2 +- app/grandchallenge/evaluation/models.py | 225 ++-- app/grandchallenge/evaluation/tasks.py | 23 +- .../evaluation/phase_archive_info.html | 35 +- .../templates/evaluation/submission_form.html | 6 +- .../evaluation/views/__init__.py | 6 +- app/tests/algorithms_tests/test_admin.py | 24 +- app/tests/algorithms_tests/test_forms.py | 96 +- .../algorithms_tests/test_permissions.py | 31 +- .../algorithms_tests/test_serializers.py | 3 - app/tests/algorithms_tests/test_tasks.py | 362 ++++-- app/tests/archives_tests/test_forms.py | 4 +- app/tests/conftest.py | 258 ++++- .../core_tests/test_grand_challenge_forge.py | 68 +- app/tests/evaluation_tests/test_api.py | 8 +- app/tests/evaluation_tests/test_forms.py | 174 ++- app/tests/evaluation_tests/test_models.py | 1018 +++++++++++++---- app/tests/evaluation_tests/test_tasks.py | 436 ++++--- app/tests/evaluation_tests/test_views.py | 65 +- 23 files changed, 2144 insertions(+), 1003 deletions(-) diff --git a/app/grandchallenge/algorithms/forms.py b/app/grandchallenge/algorithms/forms.py index 7d988d7ed2..7e580e8244 100644 --- a/app/grandchallenge/algorithms/forms.py +++ b/app/grandchallenge/algorithms/forms.py @@ -54,7 +54,6 @@ AlgorithmModel, AlgorithmPermissionRequest, Job, - annotate_input_output_counts, ) from grandchallenge.algorithms.serializers import ( AlgorithmImageSerializer, @@ -357,33 +356,22 @@ def job_requirement_properties_from_phases(self): qs = get_objects_for_user( self._user, "evaluation.create_phase_submission" ) - inputs = self.instance.inputs.all() - outputs = self.instance.outputs.all() + interfaces = self.instance.interfaces.all() return ( qs.annotate( - total_algorithm_input_count=Count( - "algorithm_inputs", distinct=True + total_algorithm_interface_count=Count( + "algorithm_interfaces", distinct=True ), - total_algorithm_output_count=Count( - "algorithm_outputs", distinct=True - ), - relevant_algorithm_input_count=Count( - "algorithm_inputs", - filter=Q(algorithm_inputs__in=inputs), - distinct=True, - ), - relevant_algorithm_output_count=Count( - "algorithm_outputs", - filter=Q(algorithm_outputs__in=outputs), + relevant_algorithm_interface_count=Count( + "algorithm_interfaces", + filter=Q(algorithm_interfaces__in=interfaces), distinct=True, ), ) .filter( submission_kind=SubmissionKindChoices.ALGORITHM, - total_algorithm_input_count=len(inputs), - total_algorithm_output_count=len(outputs), - relevant_algorithm_input_count=len(inputs), - relevant_algorithm_output_count=len(outputs), + total_algorithm_interface_count=len(interfaces), + relevant_algorithm_interface_count=len(interfaces), ) .aggregate( max_memory=Max("algorithm_maximum_settable_memory_gb"), @@ -446,47 +434,45 @@ def __init__(self, *args, user, phase, **kwargs): self._user = user self._phase = phase - def get_phase_algorithm_inputs_outputs(self): - return ( - self._phase.algorithm_inputs.all(), - self._phase.algorithm_outputs.all(), - ) - @cached_property def user_algorithms_for_phase(self): - inputs, outputs = self.get_phase_algorithm_inputs_outputs() + interfaces = self._phase.algorithm_interfaces.all() desired_image_subquery = AlgorithmImage.objects.filter( algorithm=OuterRef("pk"), is_desired_version=True ) desired_model_subquery = AlgorithmModel.objects.filter( algorithm=OuterRef("pk"), is_desired_version=True ) - annotated_qs = annotate_input_output_counts( - queryset=get_objects_for_user( - self._user, "algorithms.change_algorithm" - ), - inputs=inputs, - outputs=outputs, - ) - return annotated_qs.annotate( - has_active_image=Exists(desired_image_subquery), - active_image_pk=desired_image_subquery.values_list( - "pk", flat=True - ), - active_model_pk=desired_model_subquery.values_list( - "pk", flat=True - ), - active_image_comment=desired_image_subquery.values_list( - "comment", flat=True - ), - active_model_comment=desired_model_subquery.values_list( - "comment", flat=True - ), - ).filter( - input_count=len(inputs), - output_count=len(outputs), - relevant_input_count=len(inputs), - relevant_output_count=len(outputs), + + return ( + get_objects_for_user(self._user, "algorithms.change_algorithm") + .annotate( + interface_count=Count("interfaces", distinct=True), + relevant_interfaces_count=Count( + "interfaces", + filter=Q(interfaces__in=interfaces), + distinct=True, + ), + ) + .filter( + interface_count=len(interfaces), + relevant_interfaces_count=len(interfaces), + ) + .annotate( + has_active_image=Exists(desired_image_subquery), + active_image_pk=desired_image_subquery.values_list( + "pk", flat=True + ), + active_model_pk=desired_model_subquery.values_list( + "pk", flat=True + ), + active_image_comment=desired_image_subquery.values_list( + "comment", flat=True + ), + active_model_comment=desired_model_subquery.values_list( + "comment", flat=True + ), + ) ) @cached_property @@ -504,8 +490,7 @@ class Meta: "description", "modalities", "structures", - "inputs", - "outputs", + "interfaces", "workstation", "workstation_config", "hanging_protocol", @@ -527,8 +512,7 @@ class Meta: "display_editors": HiddenInput(), "contact_email": HiddenInput(), "workstation": HiddenInput(), - "inputs": MultipleHiddenInput(), - "outputs": MultipleHiddenInput(), + "interfaces": MultipleHiddenInput(), "modalities": MultipleHiddenInput(), "structures": MultipleHiddenInput(), "logo": HiddenInput(), @@ -551,8 +535,7 @@ def __init__( display_editors, contact_email, workstation, - inputs, - outputs, + interfaces, structures, modalities, logo, @@ -583,10 +566,8 @@ def __init__( ) ) self.fields["workstation"].disabled = True - self.fields["inputs"].initial = inputs - self.fields["inputs"].disabled = True - self.fields["outputs"].initial = outputs - self.fields["outputs"].disabled = True + self.fields["interfaces"].initial = interfaces + self.fields["interfaces"].disabled = True self.fields["modalities"].initial = modalities self.fields["modalities"].disabled = True self.fields["structures"].initial = structures diff --git a/app/grandchallenge/algorithms/models.py b/app/grandchallenge/algorithms/models.py index 4345deb9cd..e563138fde 100644 --- a/app/grandchallenge/algorithms/models.py +++ b/app/grandchallenge/algorithms/models.py @@ -1151,8 +1151,11 @@ def output_interfaces(self): @cached_property def inputs_complete(self): # check if all inputs are present and if they all have a value + # interfaces that do not require a value will be considered complete regardless return { - civ.interface for civ in self.inputs.all() if civ.has_value + civ.interface + for civ in self.inputs.all() + if civ.has_value or not civ.interface.value_required } == {*self.algorithm_interface.inputs.all()} @cached_property diff --git a/app/grandchallenge/algorithms/tasks.py b/app/grandchallenge/algorithms/tasks.py index 9622c27caf..15db440c4e 100644 --- a/app/grandchallenge/algorithms/tasks.py +++ b/app/grandchallenge/algorithms/tasks.py @@ -8,7 +8,6 @@ from django.core.cache import cache from django.core.files.base import File from django.db import transaction -from django.db.models import Count, Q from django.db.transaction import on_commit from django.utils._os import safe_join @@ -97,12 +96,9 @@ def create_algorithm_jobs_for_archive( create_algorithm_jobs( algorithm_image=algorithm.active_image, algorithm_model=algorithm.active_model, - civ_sets=[ - {*ai.values.all()} - for ai in archive_items.prefetch_related( - "values__interface" - ) - ], + archive_items=archive_items.prefetch_related( + "values__interface" + ), extra_viewer_groups=archive_groups, # NOTE: no emails in case the logs leak data # to the algorithm editors @@ -116,7 +112,7 @@ def create_algorithm_jobs_for_archive( def create_algorithm_jobs( *, algorithm_image, - civ_sets, + archive_items, time_limit, requires_gpu_type, requires_memory_gb, @@ -134,8 +130,8 @@ def create_algorithm_jobs( ---------- algorithm_image The algorithm image to use - civ_sets - The sets of component interface values that will be used as input + archive_items + Archive items whose values will be used as input for the algorithm image time_limit The time limit for the Job @@ -163,54 +159,61 @@ def create_algorithm_jobs( if not algorithm_image: raise RuntimeError("Algorithm image required to create jobs.") - civ_sets = filter_civs_for_algorithm( - civ_sets=civ_sets, + valid_job_inputs = filter_archive_items_for_algorithm( + archive_items=archive_items, algorithm_image=algorithm_image, algorithm_model=algorithm_model, ) - if max_jobs is not None: - civ_sets = civ_sets[:max_jobs] - if time_limit is None: time_limit = settings.ALGORITHMS_JOB_DEFAULT_TIME_LIMIT_SECONDS jobs = [] - for civ_set in civ_sets: - - if len(jobs) >= settings.ALGORITHMS_JOB_BATCH_LIMIT: - raise TooManyJobsScheduled - - with transaction.atomic(): - job = Job.objects.create( - creator=None, # System jobs, so no creator - algorithm_image=algorithm_image, - algorithm_model=algorithm_model, - task_on_success=task_on_success, - task_on_failure=task_on_failure, - time_limit=time_limit, - requires_gpu_type=requires_gpu_type, - requires_memory_gb=requires_memory_gb, - extra_viewer_groups=extra_viewer_groups, - extra_logs_viewer_groups=extra_logs_viewer_groups, - input_civ_set=civ_set, - ) - on_commit(job.execute) + for interface, archive_items in valid_job_inputs.items(): + for ai in archive_items: + if max_jobs is not None and len(jobs) >= max_jobs: + # only schedule max_jobs amount of jobs + # the rest will be scheduled only after these have succeeded + # we do not want to retry the task here, so just stop the loop + break + + if len(jobs) >= settings.ALGORITHMS_JOB_BATCH_LIMIT: + # raise exception so that we retry the task + raise TooManyJobsScheduled + + with transaction.atomic(): + job = Job.objects.create( + creator=None, # System jobs, so no creator + algorithm_image=algorithm_image, + algorithm_model=algorithm_model, + algorithm_interface=interface, + task_on_success=task_on_success, + task_on_failure=task_on_failure, + time_limit=time_limit, + requires_gpu_type=requires_gpu_type, + requires_memory_gb=requires_memory_gb, + extra_viewer_groups=extra_viewer_groups, + extra_logs_viewer_groups=extra_logs_viewer_groups, + input_civ_set=ai.values.all(), + ) + on_commit(job.execute) - jobs.append(job) + jobs.append(job) return jobs -def filter_civs_for_algorithm(*, civ_sets, algorithm_image, algorithm_model): +def filter_archive_items_for_algorithm( + *, archive_items, algorithm_image, algorithm_model=None +): """ - Removes sets of civs that are invalid for new jobs + Removes archive items that are invalid for new jobs. + The archive items need to contain values for all inputs of one of the algorithm's interfaces. Parameters ---------- - civ_sets - Iterable of sets of ComponentInterfaceValues that are candidate for - new Jobs + archive_items + Archive items whose values are candidates for new jobs' inputs algorithm_image The algorithm image to use for new job algorithm_model @@ -218,50 +221,48 @@ def filter_civs_for_algorithm(*, civ_sets, algorithm_image, algorithm_model): Returns ------- - Filtered set of ComponentInterfaceValues + Dictionary of valid ArchiveItems for new jobs, grouped by AlgorithmInterface """ - from grandchallenge.algorithms.models import Job - - input_interfaces = {*algorithm_image.algorithm.inputs.all()} + from grandchallenge.evaluation.models import ( + get_archive_items_for_interfaces, + get_valid_jobs_for_interfaces_and_archive_items, + ) - existing_jobs = { - frozenset(j.inputs.all()) - for j in Job.objects.filter( - algorithm_image=algorithm_image, algorithm_model=algorithm_model - ) - .annotate( - inputs_match_count=Count( - "inputs", - filter=Q( - inputs__in={civ for civ_set in civ_sets for civ in civ_set} - ), - ) - ) - .filter(inputs_match_count=len(input_interfaces), creator=None) - .prefetch_related("inputs") - } - - valid_job_inputs = [] - - for civ_set in civ_sets: - # Check interfaces are complete - civ_interfaces = {civ.interface for civ in civ_set} - if input_interfaces.issubset(civ_interfaces): - # If the algorithm works with a subset of the interfaces - # present in the set then only feed these through to the algorithm - valid_input = { - civ for civ in civ_set if civ.interface in input_interfaces - } - else: - continue + algorithm_interfaces = ( + algorithm_image.algorithm.interfaces.prefetch_related("inputs").all() + ) - # Check job has not been run - if frozenset(valid_input) in existing_jobs: - continue + # First, sort archive items by algorithm interface: + # An archive item is only valid for an interface if it has values + # for all inputs of the interface + valid_job_inputs = get_archive_items_for_interfaces( + algorithm_interfaces=algorithm_interfaces, archive_items=archive_items + ) - valid_job_inputs.append(valid_input) + # Next, group all system jobs that have been run for the provided archive items + # with the same model and image by interface + existing_jobs_for_interfaces = ( + get_valid_jobs_for_interfaces_and_archive_items( + algorithm_image=algorithm_image, + algorithm_model=algorithm_model, + algorithm_interfaces=algorithm_interfaces, + valid_archive_items_per_interface=valid_job_inputs, + ) + ) + # Finally, exclude archive items for which there already is a job + filtered_valid_job_inputs = {} + for interface, archive_items in valid_job_inputs.items(): + job_input_sets_for_interface = { + frozenset(j.inputs.all()) + for j in existing_jobs_for_interfaces[interface] + } + filtered_valid_job_inputs[interface] = [ + ai + for ai in archive_items + if not frozenset(ai.values.all()) in job_input_sets_for_interface + ] - return valid_job_inputs + return filtered_valid_job_inputs @acks_late_micro_short_task diff --git a/app/grandchallenge/core/utils/grand_challenge_forge.py b/app/grandchallenge/core/utils/grand_challenge_forge.py index e300caeda5..25223d9248 100644 --- a/app/grandchallenge/core/utils/grand_challenge_forge.py +++ b/app/grandchallenge/core/utils/grand_challenge_forge.py @@ -25,7 +25,7 @@ def get_forge_challenge_pack_context(challenge, phase_pks=None): phases = challenge.phase_set.filter( archive__isnull=False, submission_kind=SubmissionKindChoices.ALGORITHM, - ).prefetch_related("archive", "algorithm_inputs", "algorithm_outputs") + ).prefetch_related("archive", "algorithm_interfaces") if phase_pks is not None: phases = phases.filter(pk__in=phase_pks) @@ -39,16 +39,21 @@ def process_archive(archive): } def process_phase(phase): + interfaces = phase.algorithm_interfaces.all() + inputs = { + ci for interface in interfaces for ci in interface.inputs.all() + } + outputs = { + ci for interface in interfaces for ci in interface.outputs.all() + } return { "slug": phase.slug, "archive": process_archive(phase.archive), "algorithm_inputs": [ - _process_component_interface(ci) - for ci in phase.algorithm_inputs.all() + _process_component_interface(ci) for ci in inputs ], "algorithm_outputs": [ - _process_component_interface(ci) - for ci in phase.algorithm_outputs.all() + _process_component_interface(ci) for ci in outputs ], } @@ -63,18 +68,17 @@ def process_phase(phase): def get_forge_algorithm_template_context(algorithm): + interfaces = algorithm.interfaces.all() + inputs = {ci for interface in interfaces for ci in interface.inputs.all()} + outputs = { + ci for interface in interfaces for ci in interface.outputs.all() + } return { "algorithm": { "title": algorithm.title, "slug": algorithm.slug, "url": algorithm.get_absolute_url(), - "inputs": [ - _process_component_interface(ci) - for ci in algorithm.inputs.all() - ], - "outputs": [ - _process_component_interface(ci) - for ci in algorithm.outputs.all() - ], + "inputs": [_process_component_interface(ci) for ci in inputs], + "outputs": [_process_component_interface(ci) for ci in outputs], } } diff --git a/app/grandchallenge/evaluation/forms.py b/app/grandchallenge/evaluation/forms.py index c641ca4563..f21094713d 100644 --- a/app/grandchallenge/evaluation/forms.py +++ b/app/grandchallenge/evaluation/forms.py @@ -466,7 +466,7 @@ def clean_phase(self): if ( phase.submission_kind == SubmissionKindChoices.ALGORITHM and not phase.external_evaluation - and phase.count_valid_archive_items == 0 + and phase.jobs_to_schedule_per_submission == 0 ): self.add_error( None, diff --git a/app/grandchallenge/evaluation/models.py b/app/grandchallenge/evaluation/models.py index e04dfde56b..56fee2edf0 100644 --- a/app/grandchallenge/evaluation/models.py +++ b/app/grandchallenge/evaluation/models.py @@ -28,7 +28,7 @@ AlgorithmModel, Job, ) -from grandchallenge.archives.models import Archive, ArchiveItem +from grandchallenge.archives.models import Archive from grandchallenge.challenges.models import Challenge from grandchallenge.components.models import ( ComponentImage, @@ -140,6 +140,79 @@ } +def get_archive_items_for_interfaces(*, algorithm_interfaces, archive_items): + valid_archive_items_per_interface = {} + for interface in algorithm_interfaces: + inputs = interface.inputs.all() + valid_archive_items_per_interface[interface] = ( + archive_items.annotate( + input_count=Count("values", distinct=True), + relevant_input_count=Count( + "values", + filter=Q(values__interface__in=inputs), + distinct=True, + ), + ) + .filter(input_count=len(inputs), relevant_input_count=len(inputs)) + .prefetch_related("values") + ) + return valid_archive_items_per_interface + + +def get_valid_jobs_for_interfaces_and_archive_items( + *, + algorithm_image, + algorithm_interfaces, + valid_archive_items_per_interface, + algorithm_model=None, + successful_jobs_only=False, +): + if algorithm_model: + extra_filter = {"algorithm_model": algorithm_model} + else: + extra_filter = {"algorithm_model__isnull": True} + + if successful_jobs_only: + extra_filter["status"] = Job.SUCCESS + + jobs = Job.objects.filter( + algorithm_image=algorithm_image, + creator=None, + **extra_filter, + ) + + jobs_per_interface = {} + for interface in algorithm_interfaces: + jobs_per_interface[interface] = [] + jobs_for_interface = ( + jobs.filter( + algorithm_interface=interface, + inputs__archive_items__in=valid_archive_items_per_interface[ + interface + ], + ) + .distinct() + .prefetch_related("inputs") + .select_related("algorithm_image__algorithm") + ) + + archive_item_value_sets = { + frozenset(value.pk for value in item.values.all()) + for item in valid_archive_items_per_interface[interface] + } + + for job in jobs_for_interface: + # subset to jobs whose input set exactly matches + # one of the valid archive items' value sets + if ( + frozenset(inpt.pk for inpt in job.inputs.all()) + in archive_item_value_sets + ): + jobs_per_interface[interface].append(job) + + return jobs_per_interface + + class PhaseManager(models.Manager): def get_queryset(self): return ( @@ -725,15 +798,11 @@ def _clean_algorithm_submission_settings(self): if ( self.submissions_limit_per_user_per_period > 0 and not self.external_evaluation - and ( - not self.archive - or not self.algorithm_inputs - or not self.algorithm_outputs - ) + and (not self.archive or not self.algorithm_interfaces) ): raise ValidationError( "To change the submission limit to above 0, you need to first link an archive containing the secret " - "test data to this phase and define the inputs and outputs that the submitted algorithms need to " + "test data to this phase and define the interfaces (input-output combinations) that the submitted algorithms need to " "read/write. To configure these settings, please get in touch with support@grand-challenge.org." ) if ( @@ -864,7 +933,7 @@ def _clean_parent_phase(self): f"the current phase's children set as its parent." ) - if self.parent.count_valid_archive_items < 1: + if self.parent.jobs_to_schedule_per_submission < 1: raise ValidationError( "The parent phase needs to have at least 1 valid archive item." ) @@ -1098,23 +1167,33 @@ def ground_truth_upload_in_progress(self): ).exists() @cached_property - def valid_archive_items(self): - """Returns the archive items that are valid for this phase""" - if self.archive and self.algorithm_inputs: - return self.archive.items.annotate( - interface_match_count=Count( - "values", - filter=Q( - values__interface__in={*self.algorithm_inputs.all()} - ), - ) - ).filter(interface_match_count=len(self.algorithm_inputs.all())) + def valid_archive_items_per_interface(self): + """ + Returns the archive items that are valid for + each interface configured for this phase + """ + if self.archive and self.algorithm_interfaces: + return get_archive_items_for_interfaces( + algorithm_interfaces=self.algorithm_interfaces.prefetch_related( + "inputs" + ), + archive_items=self.archive.items.prefetch_related( + "values__interface" + ), + ) else: - return ArchiveItem.objects.none() + return {} @cached_property - def count_valid_archive_items(self): - return self.valid_archive_items.count() + def valid_archive_item_count_per_interface(self): + return { + interface: len(valid_archive_items) + for interface, valid_archive_items in self.valid_archive_items_per_interface.items() + } + + @cached_property + def jobs_to_schedule_per_submission(self): + return sum(self.valid_archive_item_count_per_interface.values()) def send_give_algorithm_editors_job_view_permissions_changed_email(self): message = format_html( @@ -1165,29 +1244,20 @@ def parent_phase_choices(self): extra_filters = {} extra_annotations = {} if self.submission_kind == SubmissionKindChoices.ALGORITHM: - algorithm_inputs = self.algorithm_inputs.all() - algorithm_outputs = self.algorithm_outputs.all() + algorithm_interfaces = self.algorithm_interfaces.all() extra_annotations = { - "total_input_count": Count("algorithm_inputs", distinct=True), - "total_output_count": Count( - "algorithm_outputs", distinct=True - ), - "relevant_input_count": Count( - "algorithm_inputs", - filter=Q(algorithm_inputs__in=algorithm_inputs), - distinct=True, + "total_interface_count": Count( + "algorithm_interfaces", distinct=True ), - "relevant_output_count": Count( - "algorithm_outputs", - filter=Q(algorithm_outputs__in=algorithm_outputs), + "relevant_interface_count": Count( + "algorithm_interfaces", + filter=Q(algorithm_interfaces__in=algorithm_interfaces), distinct=True, ), } extra_filters = { - "total_input_count": len(algorithm_inputs), - "total_output_count": len(algorithm_outputs), - "relevant_input_count": len(algorithm_inputs), - "relevant_output_count": len(algorithm_outputs), + "total_interface_count": len(algorithm_interfaces), + "relevant_interface_count": len(algorithm_interfaces), } return ( Phase.objects.annotate(**extra_annotations) @@ -1671,59 +1741,50 @@ def algorithm_inputs(self): return self.submission.phase.algorithm_inputs.all() @cached_property - def valid_archive_item_values(self): - return { - i.values.all() - for i in self.submission.phase.archive.items.annotate( - interface_match_count=Count( - "values", - filter=Q(values__interface__in=self.algorithm_inputs), - ) + def successful_jobs_per_interface(self): + algorithm_interfaces = ( + self.submission.phase.algorithm_interfaces.prefetch_related( + "inputs" ) - .filter(interface_match_count=len(self.algorithm_inputs)) - .prefetch_related("values") + ) + + successful_jobs_per_interface = get_valid_jobs_for_interfaces_and_archive_items( + successful_jobs_only=True, + algorithm_image=self.submission.algorithm_image, + algorithm_model=self.submission.algorithm_model, + algorithm_interfaces=algorithm_interfaces, + valid_archive_items_per_interface=self.submission.phase.valid_archive_items_per_interface, + ) + + return successful_jobs_per_interface + + @cached_property + def successful_job_count_per_interface(self): + return { + interface: len(successful_jobs) + for interface, successful_jobs in self.successful_jobs_per_interface.items() } @cached_property - def successful_jobs(self): - if self.submission.algorithm_model: - extra_filter = {"algorithm_model": self.submission.algorithm_model} - else: - extra_filter = {"algorithm_model__isnull": True} + def total_successful_jobs(self): + return sum(self.successful_job_count_per_interface.values()) - successful_jobs = ( - Job.objects.filter( - algorithm_image=self.submission.algorithm_image, - status=Job.SUCCESS, - **extra_filter, - ) - .annotate( - inputs_match_count=Count( - "inputs", - filter=Q( - inputs__in={ - civ - for civ_set in self.valid_archive_item_values - for civ in civ_set - } - ), - ), - ) - .filter( - inputs_match_count=self.algorithm_inputs.count(), - creator=None, - ) - .distinct() - .prefetch_related("outputs__interface", "inputs__interface") - .select_related("algorithm_image__algorithm") + @cached_property + def successful_jobs(self): + return Job.objects.filter( + pk__in=[ + j.pk + for sublist in self.successful_jobs_per_interface.values() + for j in sublist + ] ) - return successful_jobs @cached_property def inputs_complete(self): if self.submission.algorithm_image: - return self.successful_jobs.count() == len( - self.valid_archive_item_values + return ( + self.total_successful_jobs + == self.submission.phase.jobs_to_schedule_per_submission ) elif self.submission.predictions_file: return True diff --git a/app/grandchallenge/evaluation/tasks.py b/app/grandchallenge/evaluation/tasks.py index de8e678375..ef6078488f 100644 --- a/app/grandchallenge/evaluation/tasks.py +++ b/app/grandchallenge/evaluation/tasks.py @@ -212,20 +212,17 @@ def create_algorithm_jobs_for_evaluation(*, evaluation_pk, max_jobs=1): jobs = create_algorithm_jobs( algorithm_image=evaluation.submission.algorithm_image, algorithm_model=evaluation.submission.algorithm_model, - civ_sets=[ - {*ai.values.all()} - for ai in evaluation.submission.phase.archive.items.prefetch_related( - "values__interface" - ) - .annotate( - has_title=Case( - When(title="", then=Value(1)), - default=Value(0), - output_field=IntegerField(), - ) + archive_items=evaluation.submission.phase.archive.items.prefetch_related( + "values__interface" + ) + .annotate( + has_title=Case( + When(title="", then=Value(1)), + default=Value(0), + output_field=IntegerField(), ) - .order_by("has_title", "title", "created") - ], + ) + .order_by("has_title", "title", "created"), extra_viewer_groups=viewer_groups, extra_logs_viewer_groups=viewer_groups, task_on_success=task_on_success, diff --git a/app/grandchallenge/evaluation/templates/evaluation/phase_archive_info.html b/app/grandchallenge/evaluation/templates/evaluation/phase_archive_info.html index 5335adc6d8..70bc120f1d 100644 --- a/app/grandchallenge/evaluation/templates/evaluation/phase_archive_info.html +++ b/app/grandchallenge/evaluation/templates/evaluation/phase_archive_info.html @@ -23,24 +23,35 @@

    Linked archive for {{ object.title }}

    The (hidden) input data for this phase needs to be uploaded to an archive on Grand Challenge.

    - {% if object.archive and object.submission_kind == object.SubmissionKindChoices.ALGORITHM and not object.external_evaluation %} + + {% if object.archive and object.submission_kind == object.SubmissionKindChoices.ALGORITHM %}

    This phase is linked to archive {{ object.archive }}.

    - {% if object.algorithm_inputs.all %} -

    Each item in the archive needs to contain data for the following interfaces, which are defined as algorithm inputs:

    -
      - {% for input in object.algorithm_inputs.all %} -
    • {{ input }}
      • - {% endfor %} -
    + {% if object.algorithm_interfaces.all %} +

    Each item in the archive needs to contain data for one of the configured interfaces (i.e. input combinations):

    + {% for interface in object.algorithm_interfaces.all %} +
      +
    • Interface {{ forloop.counter }}
      • +
        + {% for input in interface.inputs.all %} +
      • {{ input }}
        • + {% endfor %} +
      +
    + {% endfor %}

    - For each submission, {{ object.count_valid_archive_items }} - algorithm job{{ object.count_valid_archive_items|pluralize }} will be created - from the {{ object.count_valid_archive_items }} valid item{{ object.count_valid_archive_items|pluralize }}. + Currently, for each submission, {{ object.jobs_to_schedule_per_submission }} + algorithm job{{ object.jobs_to_schedule_per_submission|pluralize }} will be created from the {{ object.jobs_to_schedule_per_submission }} valid archive item{{ object.jobs_to_schedule_per_submission|pluralize }}.

    +

    The jobs are spread across the configured algorithm interfaces as follows: +

      + {% for interface, count in object.valid_archive_item_count_per_interface.items %} +
    • Interface {{ forloop.counter }}: {{ count }} job{{ count|pluralize }}
      • + {% endfor %} +

    Upload data to {{ object.archive }} {% else %} -

    Before you can upload data to your archive, you need to define the algorithm inputs and outputs. Contact Grand Challenge Support for help with getting this set up.

    +

    Before you can upload data to your archive, you need to define the interfaces (i.e. possible input and output combinations) for algorithms submitted to this phase. Contact Grand Challenge Support for help with getting this set up.

    {% endif %} {% elif object.submission_kind == object.SubmissionKindChoices.ALGORITHM and not object.external_evaluation and not object.archive %} diff --git a/app/grandchallenge/evaluation/templates/evaluation/submission_form.html b/app/grandchallenge/evaluation/templates/evaluation/submission_form.html index 4490bc6754..752d82cd1b 100644 --- a/app/grandchallenge/evaluation/templates/evaluation/submission_form.html +++ b/app/grandchallenge/evaluation/templates/evaluation/submission_form.html @@ -69,9 +69,9 @@

    Create a new submission

    {% if phase.submission_kind == phase.SubmissionKindChoices.ALGORITHM and not phase.external_evaluation %}

    - For each submission, {{ phase.count_valid_archive_items }} - algorithm job{{ phase.count_valid_archive_items|pluralize }} will be created - from the {{ phase.count_valid_archive_items }} valid item{{ phase.count_valid_archive_items|pluralize }} + For each submission, {{ phase.jobs_to_schedule_per_submission }} + algorithm job{{ phase.jobs_to_schedule_per_submission|pluralize }} will be created + from the {{ phase.jobs_to_schedule_per_submission }} valid item{{ phase.jobs_to_schedule_per_submission|pluralize }} in {{ phase.archive }}. Each individual algorithm job will have a time limit of {{ phase.algorithm_time_limit|naturaldelta }}.

    diff --git a/app/grandchallenge/evaluation/views/__init__.py b/app/grandchallenge/evaluation/views/__init__.py index efc377ca91..e5eee85830 100644 --- a/app/grandchallenge/evaluation/views/__init__.py +++ b/app/grandchallenge/evaluation/views/__init__.py @@ -139,8 +139,7 @@ def dispatch(self, request, *args, **kwargs): return self.handle_no_permission() elif ( not self.phase.submission_kind == SubmissionKindChoices.ALGORITHM - or not self.phase.algorithm_inputs - or not self.phase.algorithm_outputs + or not self.phase.algorithm_interfaces or not self.phase.archive ): error_message = ( @@ -891,8 +890,7 @@ def get_form_kwargs(self): "display_editors": True, "contact_email": self.request.user.email, "workstation": self.phase.workstation, - "inputs": self.phase.algorithm_inputs.all(), - "outputs": self.phase.algorithm_outputs.all(), + "interfaces": self.phase.algorithm_interfaces.all(), "modalities": self.phase.challenge.modalities.all(), "structures": self.phase.challenge.structures.all(), "logo": self.phase.challenge.logo, diff --git a/app/tests/algorithms_tests/test_admin.py b/app/tests/algorithms_tests/test_admin.py index 8c90afbdfb..73031993a9 100644 --- a/app/tests/algorithms_tests/test_admin.py +++ b/app/tests/algorithms_tests/test_admin.py @@ -5,9 +5,14 @@ from grandchallenge.algorithms.models import Job from grandchallenge.algorithms.tasks import create_algorithm_jobs +from grandchallenge.archives.models import ArchiveItem from grandchallenge.components.admin import requeue_jobs from grandchallenge.components.models import ComponentInterface -from tests.algorithms_tests.factories import AlgorithmImageFactory +from tests.algorithms_tests.factories import ( + AlgorithmImageFactory, + AlgorithmInterfaceFactory, +) +from tests.archives_tests.factories import ArchiveItemFactory from tests.components_tests.factories import ( ComponentInterfaceFactory, ComponentInterfaceValueFactory, @@ -16,7 +21,6 @@ from tests.utils import recurse_callbacks -@pytest.mark.xfail(reason="Still to be addressed for optional inputs pitch") @pytest.mark.django_db def test_job_updated_start_and_complete_times_after_admin_requeue( algorithm_image, settings, django_capture_on_commit_callbacks @@ -37,30 +41,32 @@ def test_job_updated_start_and_complete_times_after_admin_requeue( ai.refresh_from_db() # Make sure the job fails when trying to upload an invalid file - input_interface = ComponentInterface.objects.get( - slug="generic-medical-image" - ) + ci = ComponentInterface.objects.get(slug="generic-medical-image") detection_interface = ComponentInterfaceFactory( store_in_database=False, relative_path="some_text.txt", slug="detection-json-file", kind=ComponentInterface.Kind.ANY, ) - ai.algorithm.inputs.add(input_interface) - ai.algorithm.outputs.add(detection_interface) + interface = AlgorithmInterfaceFactory( + inputs=[ci], outputs=[detection_interface] + ) + ai.algorithm.interfaces.add(interface) image_file = ImageFileFactory( file__from_path=Path(__file__).parent / "resources" / "input_file.tif" ) civ = ComponentInterfaceValueFactory( - image=image_file.image, interface=input_interface, file=None + image=image_file.image, interface=ci, file=None ) + item = ArchiveItemFactory() + item.values.add(civ) with django_capture_on_commit_callbacks() as callbacks: create_algorithm_jobs( algorithm_image=ai, - civ_sets=[{civ}], + archive_items=ArchiveItem.objects.all(), time_limit=ai.algorithm.time_limit, requires_gpu_type=ai.algorithm.job_requires_gpu_type, requires_memory_gb=ai.algorithm.job_requires_memory_gb, diff --git a/app/tests/algorithms_tests/test_forms.py b/app/tests/algorithms_tests/test_forms.py index 3e2499fd05..47dad06b8c 100644 --- a/app/tests/algorithms_tests/test_forms.py +++ b/app/tests/algorithms_tests/test_forms.py @@ -853,8 +853,8 @@ def test_algorithm_form_gpu_choices_from_phases(): ci1, ci2, ci3, ci4, ci5, ci6 = ComponentInterfaceFactory.create_batch(6) inputs = [ci1, ci2] outputs = [ci3, ci4] - algorithm.inputs.set(inputs) - algorithm.outputs.set(outputs) + interface = AlgorithmInterfaceFactory(inputs=inputs, outputs=outputs) + algorithm.interfaces.set([interface]) def assert_gpu_type_choices(expected_choices): form = AlgorithmForm(instance=algorithm, user=user) @@ -885,8 +885,7 @@ def assert_gpu_type_choices(expected_choices): GPUTypeChoices.T4, ], ) - phase.algorithm_inputs.set(inputs) - phase.algorithm_outputs.set(outputs) + phase.algorithm_interfaces.set([interface]) phase.challenge.add_participant(user) phases.append(phase) @@ -936,7 +935,19 @@ def assert_gpu_type_choices(expected_choices): ] ) - phases[0].algorithm_inputs.set([ci1, ci5]) + interface2 = AlgorithmInterfaceFactory(inputs=[ci1, ci5], outputs=outputs) + # add additional interface + phases[0].algorithm_interfaces.add(interface2) + + assert_gpu_type_choices( + [ + (GPUTypeChoices.NO_GPU, "No GPU"), + (GPUTypeChoices.T4, "NVIDIA T4 Tensor Core GPU"), + ] + ) + + # replace with different interface + phases[0].algorithm_interfaces.set([interface2]) assert_gpu_type_choices( [ @@ -954,12 +965,21 @@ def assert_gpu_type_choices(expected_choices): (GPUTypeChoices.T4, "NVIDIA T4 Tensor Core GPU"), ] ) + interface3 = AlgorithmInterfaceFactory(inputs=inputs, outputs=[ci4, ci6]) + phases[3].algorithm_interfaces.add(interface3) - phases[3].algorithm_outputs.set([ci4, ci6]) + assert_gpu_type_choices( + [ + (GPUTypeChoices.NO_GPU, "No GPU"), + (GPUTypeChoices.T4, "NVIDIA T4 Tensor Core GPU"), + ] + ) + algorithm.interfaces.add(interface3) assert_gpu_type_choices( [ (GPUTypeChoices.NO_GPU, "No GPU"), + (GPUTypeChoices.K80, "NVIDIA K80 GPU"), (GPUTypeChoices.T4, "NVIDIA T4 Tensor Core GPU"), ] ) @@ -1026,8 +1046,9 @@ def test_algorithm_form_gpu_choices_from_organizations_and_phases(): ci1, ci2, ci3, ci4, ci5, ci6 = ComponentInterfaceFactory.create_batch(6) inputs = [ci1, ci2] outputs = [ci3, ci4] - algorithm.inputs.set(inputs) - algorithm.outputs.set(outputs) + interface = AlgorithmInterfaceFactory(inputs=inputs, outputs=outputs) + algorithm.interfaces.set([interface]) + org1 = OrganizationFactory( algorithm_selectable_gpu_type_choices=[ GPUTypeChoices.NO_GPU, @@ -1070,8 +1091,7 @@ def assert_gpu_type_choices(expected_choices): GPUTypeChoices.T4, ], ) - phase.algorithm_inputs.set(inputs) - phase.algorithm_outputs.set(outputs) + phase.algorithm_interfaces.set([interface]) phase.challenge.add_participant(user) phases.append(phase) @@ -1108,8 +1128,7 @@ def test_algorithm_for_phase_form_gpu_limited_choices(): display_editors=True, contact_email="test@test.com", workstation=WorkstationFactory.build(), - inputs=[ComponentInterfaceFactory.build()], - outputs=[ComponentInterfaceFactory.build()], + interfaces=[AlgorithmInterfaceFactory.build()], structures=[], modalities=[], logo=ImageField(filename="test.jpeg"), @@ -1136,8 +1155,7 @@ def test_algorithm_for_phase_form_gpu_additional_choices(): display_editors=True, contact_email="test@test.com", workstation=WorkstationFactory.build(), - inputs=[ComponentInterfaceFactory.build()], - outputs=[ComponentInterfaceFactory.build()], + interfaces=[AlgorithmInterfaceFactory.build()], structures=[], modalities=[], logo=ImageField(filename="test.jpeg"), @@ -1188,16 +1206,16 @@ def test_algorithm_form_max_memory_from_phases_for_admins(): ci1, ci2, ci3, ci4, ci5, ci6 = ComponentInterfaceFactory.create_batch(6) inputs = [ci1, ci2] outputs = [ci3, ci4] - algorithm.inputs.set(inputs) - algorithm.outputs.set(outputs) + interface = AlgorithmInterfaceFactory(inputs=inputs, outputs=outputs) + algorithm.interfaces.set([interface]) + phases = [] for max_memory in [42, 100, 200, 300, 400]: phase = PhaseFactory( submission_kind=SubmissionKindChoices.ALGORITHM, algorithm_maximum_settable_memory_gb=max_memory, ) - phase.algorithm_inputs.set(inputs) - phase.algorithm_outputs.set(outputs) + phase.algorithm_interfaces.set([interface]) phase.challenge.add_admin(user) phases.append(phase) @@ -1228,11 +1246,13 @@ def assert_max_value_validator(value): assert_max_value_validator(200) - phases[2].algorithm_inputs.set([ci1, ci5]) + interface2 = AlgorithmInterfaceFactory(inputs=[ci1, ci5], outputs=outputs) + phases[2].algorithm_interfaces.set([interface2]) assert_max_value_validator(100) - phases[1].algorithm_outputs.set([ci4, ci6]) + interface3 = AlgorithmInterfaceFactory(inputs=inputs, outputs=[ci4, ci6]) + phases[1].algorithm_interfaces.set([interface3]) assert_max_value_validator(42) @@ -1244,16 +1264,16 @@ def test_algorithm_form_max_memory_from_phases(): ci1, ci2, ci3, ci4, ci5, ci6 = ComponentInterfaceFactory.create_batch(6) inputs = [ci1, ci2] outputs = [ci3, ci4] - algorithm.inputs.set(inputs) - algorithm.outputs.set(outputs) + interface = AlgorithmInterfaceFactory(inputs=inputs, outputs=outputs) + algorithm.interfaces.set([interface]) + phases = [] for max_memory in [42, 100, 200, 300, 400, 500]: phase = PhaseFactory( submission_kind=SubmissionKindChoices.ALGORITHM, algorithm_maximum_settable_memory_gb=max_memory, ) - phase.algorithm_inputs.set(inputs) - phase.algorithm_outputs.set(outputs) + phase.algorithm_interfaces.set([interface]) phase.challenge.add_participant(user) phases.append(phase) @@ -1284,14 +1304,23 @@ def assert_max_value_validator(value): assert_max_value_validator(200) - phases[2].algorithm_inputs.set([ci1, ci5]) - + interface2 = AlgorithmInterfaceFactory(inputs=[ci1, ci5], outputs=outputs) + # adding an interface + phases[2].algorithm_interfaces.add(interface2) + assert_max_value_validator(100) + # replacing the interface + phases[2].algorithm_interfaces.set([interface2]) assert_max_value_validator(100) - phases[1].algorithm_outputs.set([ci4, ci6]) + interface3 = AlgorithmInterfaceFactory(inputs=inputs, outputs=[ci4, ci6]) + phases[1].algorithm_interfaces.set([interface3]) assert_max_value_validator(42) + # updating the algorithm's interface + algorithm.interfaces.set([interface3]) + assert_max_value_validator(100) + @pytest.mark.django_db def test_algorithm_form_max_memory_from_organizations(): @@ -1328,8 +1357,8 @@ def test_algorithm_form_max_memory_from_organizations_and_phases(): ci1, ci2, ci3, ci4, ci5, ci6 = ComponentInterfaceFactory.create_batch(6) inputs = [ci1, ci2] outputs = [ci3, ci4] - algorithm.inputs.set(inputs) - algorithm.outputs.set(outputs) + interface = AlgorithmInterfaceFactory(inputs=inputs, outputs=outputs) + algorithm.interfaces.set([interface]) org1 = OrganizationFactory(algorithm_maximum_settable_memory_gb=42) org2 = OrganizationFactory(algorithm_maximum_settable_memory_gb=1337) @@ -1351,8 +1380,7 @@ def assert_max_value_validator(value): submission_kind=SubmissionKindChoices.ALGORITHM, algorithm_maximum_settable_memory_gb=max_memory, ) - phase.algorithm_inputs.set(inputs) - phase.algorithm_outputs.set(outputs) + phase.algorithm_interfaces.set([interface]) phase.challenge.add_participant(user) assert_max_value_validator(42) @@ -1372,8 +1400,7 @@ def test_algorithm_for_phase_form_memory_limited(): display_editors=True, contact_email="test@test.com", workstation=WorkstationFactory.build(), - inputs=[ComponentInterfaceFactory.build()], - outputs=[ComponentInterfaceFactory.build()], + interfaces=[AlgorithmInterfaceFactory.build()], structures=[], modalities=[], logo=ImageField(filename="test.jpeg"), @@ -1432,8 +1459,7 @@ def test_algorithm_for_phase_form_memory(): display_editors=True, contact_email="test@test.com", workstation=WorkstationFactory.build(), - inputs=[ComponentInterfaceFactory.build()], - outputs=[ComponentInterfaceFactory.build()], + interfaces=[AlgorithmInterfaceFactory.build()], structures=[], modalities=[], logo=ImageField(filename="test.jpeg"), diff --git a/app/tests/algorithms_tests/test_permissions.py b/app/tests/algorithms_tests/test_permissions.py index 081ffea444..94bf5fc02b 100644 --- a/app/tests/algorithms_tests/test_permissions.py +++ b/app/tests/algorithms_tests/test_permissions.py @@ -399,12 +399,10 @@ def test_job_permissions_for_archive( im = ImageFactory() s.image_set.set([im]) - input_interface = ComponentInterface.objects.get( - slug="generic-medical-image" - ) - civ = ComponentInterfaceValueFactory( - image=im, interface=input_interface - ) + ci = ComponentInterface.objects.get(slug="generic-medical-image") + interface = AlgorithmInterfaceFactory(inputs=[ci]) + ai.algorithm.interfaces.add(interface) + civ = ComponentInterfaceValueFactory(image=im, interface=ci) archive_item = ArchiveItemFactory(archive=archive) with django_capture_on_commit_callbacks(execute=True): @@ -459,12 +457,10 @@ def test_job_permissions_for_normal_phase( im = ImageFactory() s.image_set.set([im]) - input_interface = ComponentInterface.objects.get( - slug="generic-medical-image" - ) - civ = ComponentInterfaceValueFactory( - image=im, interface=input_interface - ) + ci = ComponentInterface.objects.get(slug="generic-medical-image") + interface = AlgorithmInterfaceFactory(inputs=[ci]) + ai.algorithm.interfaces.add(interface) + civ = ComponentInterfaceValueFactory(image=im, interface=ci) archive_item = ArchiveItemFactory(archive=archive) with django_capture_on_commit_callbacks(execute=True): @@ -517,16 +513,15 @@ def test_job_permissions_for_debug_phase( im = ImageFactory() s.image_set.set([im]) - input_interface = ComponentInterface.objects.get( - slug="generic-medical-image" - ) - civ = ComponentInterfaceValueFactory( - image=im, interface=input_interface - ) + ci = ComponentInterface.objects.get(slug="generic-medical-image") + civ = ComponentInterfaceValueFactory(image=im, interface=ci) archive_item = ArchiveItemFactory(archive=archive) with django_capture_on_commit_callbacks(execute=True): archive_item.values.add(civ) + interface = AlgorithmInterfaceFactory(inputs=[ci]) + ai.algorithm.interfaces.add(interface) + create_algorithm_jobs_for_evaluation(evaluation_pk=evaluation.pk) job = Job.objects.get() diff --git a/app/tests/algorithms_tests/test_serializers.py b/app/tests/algorithms_tests/test_serializers.py index b176f78b3e..523bdc9023 100644 --- a/app/tests/algorithms_tests/test_serializers.py +++ b/app/tests/algorithms_tests/test_serializers.py @@ -269,7 +269,6 @@ def test_form_invalid_without_enough_credits(self, rf, settings): settings.ALGORITHMS_GENERAL_CREDITS_PER_MONTH_PER_USER + 1 ), ) - algorithm_image.algorithm.inputs.clear() user = UserFactory() algorithm_image.algorithm.add_user(user=user) @@ -303,7 +302,6 @@ def test_form_valid_for_editor(self, rf, settings): settings.ALGORITHMS_GENERAL_CREDITS_PER_MONTH_PER_USER + 1 ), ) - algorithm_image.algorithm.inputs.clear() user = UserFactory() algorithm_image.algorithm.add_editor(user=user) @@ -351,7 +349,6 @@ def test_form_valid_with_credits(self, rf): is_desired_version=True, algorithm__minimum_credits_per_job=1, ) - algorithm_image.algorithm.inputs.clear() user = UserFactory() algorithm_image.algorithm.add_user(user=user) diff --git a/app/tests/algorithms_tests/test_tasks.py b/app/tests/algorithms_tests/test_tasks.py index e4e75d2759..ecb2ffb25c 100644 --- a/app/tests/algorithms_tests/test_tasks.py +++ b/app/tests/algorithms_tests/test_tasks.py @@ -12,12 +12,14 @@ create_algorithm_jobs, create_algorithm_jobs_for_archive, execute_algorithm_job_for_inputs, - filter_civs_for_algorithm, + filter_archive_items_for_algorithm, send_failed_job_notification, ) +from grandchallenge.archives.models import ArchiveItem from grandchallenge.components.models import ( ComponentInterface, ComponentInterfaceValue, + InterfaceKindChoices, ) from grandchallenge.components.schemas import GPUTypeChoices from grandchallenge.components.tasks import ( @@ -58,9 +60,12 @@ def default_input_interface(self): def test_no_images_does_nothing(self): ai = AlgorithmImageFactory() + interface = AlgorithmInterfaceFactory() + ai.algorithm.interfaces.set([interface]) + create_algorithm_jobs( algorithm_image=ai, - civ_sets=[], + archive_items=ArchiveItem.objects.none(), time_limit=ai.algorithm.time_limit, requires_gpu_type=GPUTypeChoices.NO_GPU, requires_memory_gb=4, @@ -71,7 +76,7 @@ def test_no_algorithm_image_errors_out(self): with pytest.raises(RuntimeError): create_algorithm_jobs( algorithm_image=None, - civ_sets=[], + archive_items=ArchiveItem.objects.none(), time_limit=60, requires_gpu_type=GPUTypeChoices.NO_GPU, requires_memory_gb=4, @@ -80,15 +85,18 @@ def test_no_algorithm_image_errors_out(self): def test_creates_job_correctly(self): ai = AlgorithmImageFactory() image = ImageFactory() - interface = ComponentInterface.objects.get( - slug="generic-medical-image" - ) - ai.algorithm.inputs.set([interface]) - civ = ComponentInterfaceValueFactory(image=image, interface=interface) + ci = ComponentInterface.objects.get(slug="generic-medical-image") + interface = AlgorithmInterfaceFactory(inputs=[ci]) + ai.algorithm.interfaces.set([interface]) + + civ = ComponentInterfaceValueFactory(image=image, interface=ci) + item = ArchiveItemFactory() + item.values.add(civ) + assert Job.objects.count() == 0 jobs = create_algorithm_jobs( algorithm_image=ai, - civ_sets=[{civ}], + archive_items=ArchiveItem.objects.all(), time_limit=ai.algorithm.time_limit, requires_gpu_type=ai.algorithm.job_requires_gpu_type, requires_memory_gb=ai.algorithm.job_requires_memory_gb, @@ -97,25 +105,94 @@ def test_creates_job_correctly(self): j = Job.objects.first() assert j.algorithm_image == ai assert j.creator is None + assert j.algorithm_interface == interface assert ( j.inputs.get(interface__slug="generic-medical-image").image == image ) assert j.pk == jobs[0].pk - def test_is_idempotent(self): + def test_creates_job_for_multiple_interfaces_correctly(self): ai = AlgorithmImageFactory() image = ImageFactory() - interface = ComponentInterface.objects.get( - slug="generic-medical-image" + ci1 = ComponentInterfaceFactory(kind=InterfaceKindChoices.BOOL) + ci2 = ComponentInterfaceFactory(kind=InterfaceKindChoices.IMAGE) + ci3 = ComponentInterfaceFactory(kind=InterfaceKindChoices.STRING) + + interface1 = AlgorithmInterfaceFactory(inputs=[ci1]) + interface2 = AlgorithmInterfaceFactory(inputs=[ci2]) + interface3 = AlgorithmInterfaceFactory(inputs=[ci3]) + interface4 = AlgorithmInterfaceFactory(inputs=[ci1, ci3]) + interface5 = AlgorithmInterfaceFactory(inputs=[ci1, ci2, ci3]) + ai.algorithm.interfaces.set( + [interface1, interface2, interface3, interface4, interface5] + ) + + civ1 = ComponentInterfaceValueFactory(value=False, interface=ci1) + civ2 = ComponentInterfaceValueFactory(image=image, interface=ci2) + civ3 = ComponentInterfaceValueFactory(value="foo", interface=ci3) + civ4 = ComponentInterfaceValueFactory() + + archive = ArchiveFactory() + item1, item2, item3, item4, item5, item6 = ( + ArchiveItemFactory.create_batch(6, archive=archive) + ) + item1.values.add(civ1) # item for interface 1 only + item2.values.add(civ2) # item for interface 2 only + item3.values.add(civ3) # item for interface 3 only + item4.values.set([civ1, civ3]) # item for interface 4 only + item5.values.set([civ4]) # not a match for any interface + item6.values.set([civ2, civ3]) # not a match for any interface + + assert Job.objects.count() == 0 + create_algorithm_jobs( + algorithm_image=ai, + archive_items=ArchiveItem.objects.all(), + time_limit=ai.algorithm.time_limit, + requires_gpu_type=ai.algorithm.job_requires_gpu_type, + requires_memory_gb=ai.algorithm.job_requires_memory_gb, + ) + assert Job.objects.count() == 4 + + for j in Job.objects.all(): + assert j.algorithm_image == ai + assert j.creator is None + + assert not ( + Job.objects.get(algorithm_interface=interface1).inputs.get().value + ) + assert ( + Job.objects.get(algorithm_interface=interface2).inputs.get().image + == image + ) + assert ( + Job.objects.get(algorithm_interface=interface3).inputs.get().value + == "foo" + ) + assert ( + Job.objects.get(algorithm_interface=interface4).inputs.count() == 2 + ) + assert [False, "foo"] == list( + Job.objects.get(algorithm_interface=interface4).inputs.values_list( + "value", flat=True + ) ) - civ = ComponentInterfaceValueFactory(image=image, interface=interface) + + def test_is_idempotent(self): + ai = AlgorithmImageFactory() + image = ImageFactory() + ci = ComponentInterface.objects.get(slug="generic-medical-image") + interface = AlgorithmInterfaceFactory(inputs=[ci]) + ai.algorithm.interfaces.set([interface]) + civ = ComponentInterfaceValueFactory(image=image, interface=ci) + item = ArchiveItemFactory() + item.values.add(civ) assert Job.objects.count() == 0 create_algorithm_jobs( algorithm_image=ai, - civ_sets=[{civ}], + archive_items=ArchiveItem.objects.all(), time_limit=ai.algorithm.time_limit, requires_gpu_type=ai.algorithm.job_requires_gpu_type, requires_memory_gb=ai.algorithm.job_requires_memory_gb, @@ -125,7 +202,7 @@ def test_is_idempotent(self): jobs = create_algorithm_jobs( algorithm_image=ai, - civ_sets=[{civ}], + archive_items=ArchiveItem.objects.all(), time_limit=ai.algorithm.time_limit, requires_gpu_type=ai.algorithm.job_requires_gpu_type, requires_memory_gb=ai.algorithm.job_requires_memory_gb, @@ -136,14 +213,16 @@ def test_is_idempotent(self): def test_extra_viewer_groups(self): ai = AlgorithmImageFactory() - interface = ComponentInterface.objects.get( - slug="generic-medical-image" - ) - civ = ComponentInterfaceValueFactory(interface=interface) + ci = ComponentInterface.objects.get(slug="generic-medical-image") + interface = AlgorithmInterfaceFactory(inputs=[ci]) + ai.algorithm.interfaces.set([interface]) + civ = ComponentInterfaceValueFactory(interface=ci) + item = ArchiveItemFactory() + item.values.add(civ) groups = (GroupFactory(), GroupFactory(), GroupFactory()) jobs = create_algorithm_jobs( algorithm_image=ai, - civ_sets=[{civ}], + archive_items=ArchiveItem.objects.all(), extra_viewer_groups=groups, time_limit=ai.algorithm.time_limit, requires_gpu_type=ai.algorithm.job_requires_gpu_type, @@ -159,7 +238,7 @@ def test_no_jobs_workflow(django_capture_on_commit_callbacks): with django_capture_on_commit_callbacks() as callbacks: create_algorithm_jobs( algorithm_image=ai, - civ_sets=[], + archive_items=ArchiveItem.objects.none(), time_limit=ai.algorithm.time_limit, requires_gpu_type=ai.algorithm.job_requires_gpu_type, requires_memory_gb=ai.algorithm.job_requires_memory_gb, @@ -171,15 +250,19 @@ def test_no_jobs_workflow(django_capture_on_commit_callbacks): def test_jobs_workflow(django_capture_on_commit_callbacks): ai = AlgorithmImageFactory() images = [ImageFactory(), ImageFactory()] - interface = ComponentInterface.objects.get(slug="generic-medical-image") - civ_sets = [ - {ComponentInterfaceValueFactory(image=im, interface=interface)} - for im in images - ] + ci = ComponentInterface.objects.get(slug="generic-medical-image") + archive = ArchiveFactory() + for im in images: + item = ArchiveItemFactory(archive=archive) + item.values.add(ComponentInterfaceValueFactory(image=im, interface=ci)) + + interface = AlgorithmInterfaceFactory(inputs=[ci]) + ai.algorithm.interfaces.set([interface]) + with django_capture_on_commit_callbacks() as callbacks: create_algorithm_jobs( algorithm_image=ai, - civ_sets=civ_sets, + archive_items=ArchiveItem.objects.all(), time_limit=ai.algorithm.time_limit, requires_gpu_type=ai.algorithm.job_requires_gpu_type, requires_memory_gb=ai.algorithm.job_requires_memory_gb, @@ -488,120 +571,154 @@ def test_execute_algorithm_job_for_missing_inputs(settings): @pytest.mark.django_db class TestJobCreation: - def test_unmatched_interface_filter(self): - ai = AlgorithmImageFactory() - cis = ComponentInterfaceFactory.create_batch(2) - ai.algorithm.inputs.set(cis) - - civ_sets = [ - {}, # No interfaces - { - ComponentInterfaceValueFactory(interface=cis[0]) - }, # Missing interface - { - # OK - ComponentInterfaceValueFactory(interface=cis[0]), - ComponentInterfaceValueFactory(interface=cis[1]), - }, - { - # Unmatched interface - ComponentInterfaceValueFactory(interface=cis[0]), + def test_interface_matching(self): + ai1, ai2, ai3, ai4 = AlgorithmImageFactory.create_batch(4) + ci1, ci2, ci3, ci4 = ComponentInterfaceFactory.create_batch(4) + interface1 = AlgorithmInterfaceFactory(inputs=[ci1]) + interface2 = AlgorithmInterfaceFactory(inputs=[ci1, ci2]) + interface3 = AlgorithmInterfaceFactory(inputs=[ci2, ci3, ci4]) + interface4 = AlgorithmInterfaceFactory(inputs=[ci2]) + + ai1.algorithm.interfaces.set([interface1]) + ai2.algorithm.interfaces.set([interface1, interface2]) + ai3.algorithm.interfaces.set([interface1, interface3, interface4]) + ai4.algorithm.interfaces.set([interface4]) + + archive = ArchiveFactory() + i1, i2, i3, i4 = ArchiveItemFactory.create_batch(4, archive=archive) + i1.values.add( + ComponentInterfaceValueFactory(interface=ci1) + ) # Valid for interface 1 + i2.values.set( + [ + ComponentInterfaceValueFactory(interface=ci1), + ComponentInterfaceValueFactory(interface=ci2), + ] + ) # valid for interface 2 + i3.values.set( + [ + ComponentInterfaceValueFactory(interface=ci1), ComponentInterfaceValueFactory( interface=ComponentInterfaceFactory() ), - }, - ] - - filtered_civ_sets = filter_civs_for_algorithm( - civ_sets=civ_sets, algorithm_image=ai, algorithm_model=None - ) - - assert filtered_civ_sets == [civ_sets[2]] - - def test_unmatched_interface_filter_subset(self): - ai = AlgorithmImageFactory() - cis = ComponentInterfaceFactory.create_batch(2) - ai.algorithm.inputs.set(cis) - - civ_sets = [ - { - # Extra interface - ComponentInterfaceValueFactory(interface=cis[0]), - ComponentInterfaceValueFactory(interface=cis[1]), + ] + ) # valid for no interface, because of additional / mismatching interface + i4.values.set( + [ + ComponentInterfaceValueFactory(interface=ci2), ComponentInterfaceValueFactory( interface=ComponentInterfaceFactory() ), - } - ] - - filtered_civ_sets = filter_civs_for_algorithm( - civ_sets=civ_sets, algorithm_image=ai, algorithm_model=None + ] + ) # valid for no interface, because of additional / mismatching interface + + # filtered archive items depend on defined interfaces and archive item values + filtered_archive_items = filter_archive_items_for_algorithm( + archive_items=ArchiveItem.objects.all(), + algorithm_image=ai1, + algorithm_model=None, ) + assert filtered_archive_items.keys() == {interface1} + assert filtered_archive_items[interface1] == [i1] - assert len(filtered_civ_sets) == 1 - assert {civ.interface for civ in filtered_civ_sets[0]} == {*cis} + filtered_archive_items = filter_archive_items_for_algorithm( + archive_items=ArchiveItem.objects.all(), + algorithm_image=ai2, + algorithm_model=None, + ) + assert filtered_archive_items.keys() == {interface1, interface2} + assert filtered_archive_items[interface1] == [i1] + assert filtered_archive_items[interface2] == [i2] + + filtered_archive_items = filter_archive_items_for_algorithm( + archive_items=ArchiveItem.objects.all(), + algorithm_image=ai3, + algorithm_model=None, + ) + assert filtered_archive_items.keys() == { + interface1, + interface3, + interface4, + } + assert filtered_archive_items[interface1] == [i1] + assert filtered_archive_items[interface3] == [] + assert filtered_archive_items[interface4] == [] + + filtered_archive_items = filter_archive_items_for_algorithm( + archive_items=ArchiveItem.objects.all(), + algorithm_image=ai4, + algorithm_model=None, + ) + assert filtered_archive_items.keys() == {interface4} + assert filtered_archive_items[interface4] == [] - def test_existing_jobs(self): + def test_jobs_with_creator_ignored(self): alg = AlgorithmFactory() ai = AlgorithmImageFactory(algorithm=alg) - am = AlgorithmModelFactory(algorithm=alg) cis = ComponentInterfaceFactory.create_batch(2) - ai.algorithm.inputs.set(cis) + interface = AlgorithmInterfaceFactory(inputs=cis) + ai.algorithm.interfaces.set([interface]) civs1 = [ComponentInterfaceValueFactory(interface=c) for c in cis] civs2 = [ComponentInterfaceValueFactory(interface=c) for c in cis] - civs3 = [ComponentInterfaceValueFactory(interface=c) for c in cis] j1 = AlgorithmJobFactory( creator=None, algorithm_image=ai, + algorithm_interface=interface, time_limit=ai.algorithm.time_limit, ) j1.inputs.set(civs1) + # non-system job j2 = AlgorithmJobFactory( - algorithm_image=ai, time_limit=ai.algorithm.time_limit - ) - j2.inputs.set(civs2) - j3 = AlgorithmJobFactory( - creator=None, + creator=UserFactory(), algorithm_image=ai, - algorithm_model=am, + algorithm_interface=interface, time_limit=ai.algorithm.time_limit, ) - j3.inputs.set(civs3) - - civ_sets = [ - {civ for civ in civs1}, # Job already exists (system job) - { - civ for civ in civs2 - }, # Job already exists but with a creator set and hence should be ignored - { - civ for civ in civs3 - }, # Job exists but with an algorithm model set and should be ignored - { - # New values - ComponentInterfaceValueFactory(interface=cis[0]), - ComponentInterfaceValueFactory(interface=cis[1]), - }, - { - # Changed values - civs1[0], - ComponentInterfaceValueFactory(interface=cis[1]), - }, - ] + j2.inputs.set(civs2) + + archive = ArchiveFactory() + item1, item2 = ArchiveItemFactory.create_batch(2, archive=archive) + item1.values.set(civs1) # non-system job already exists + item2.values.set(civs2) # non-system job should be ignored - filtered_civ_sets = filter_civs_for_algorithm( - civ_sets=civ_sets, algorithm_image=ai, algorithm_model=None + filtered_civ_sets = filter_archive_items_for_algorithm( + archive_items=ArchiveItem.objects.all(), + algorithm_image=ai, ) - assert sorted(filtered_civ_sets) == sorted(civ_sets[1:]) + assert filtered_civ_sets.keys() == {interface} + assert filtered_civ_sets[interface] == [item2] + + def test_existing_jobs(self, archive_items_and_jobs_for_interfaces): + # image used for all jobs + image = archive_items_and_jobs_for_interfaces.jobs_for_interface1[ + 0 + ].algorithm_image - def test_existing_jobs_with_algorithm_model(self): + filtered_civ_sets = filter_archive_items_for_algorithm( + archive_items=ArchiveItem.objects.all(), + algorithm_image=image, + ) + # this should return 1 archive item per interface + # for which there is no job yet + assert filtered_civ_sets == { + archive_items_and_jobs_for_interfaces.interface1: [ + archive_items_and_jobs_for_interfaces.items_for_interface1[1] + ], + archive_items_and_jobs_for_interfaces.interface2: [ + archive_items_and_jobs_for_interfaces.items_for_interface2[1] + ], + } + + def test_model_filter_for_jobs_works(self): alg = AlgorithmFactory() ai = AlgorithmImageFactory(algorithm=alg) am = AlgorithmModelFactory(algorithm=alg) cis = ComponentInterfaceFactory.create_batch(2) - ai.algorithm.inputs.set(cis) + interface = AlgorithmInterfaceFactory(inputs=cis) + ai.algorithm.interfaces.set([interface]) civs1 = [ComponentInterfaceValueFactory(interface=c) for c in cis] civs2 = [ComponentInterfaceValueFactory(interface=c) for c in cis] @@ -610,29 +727,31 @@ def test_existing_jobs_with_algorithm_model(self): creator=None, algorithm_image=ai, algorithm_model=am, + algorithm_interface=interface, time_limit=ai.algorithm.time_limit, ) j1.inputs.set(civs1) j2 = AlgorithmJobFactory( creator=None, algorithm_image=ai, + algorithm_interface=interface, time_limit=ai.algorithm.time_limit, ) j2.inputs.set(civs2) - civ_sets = [ - {civ for civ in civs1}, # Job already exists with image and model - { - civ - for civ in civs2 # Job exists but only with image, so should be ignored - }, - ] + archive = ArchiveFactory() + item1, item2 = ArchiveItemFactory.create_batch(2, archive=archive) + item1.values.set(civs1) # Job already exists with image and model + item2.values.set(civs2) # Job exists but only with image - filtered_civ_sets = filter_civs_for_algorithm( - civ_sets=civ_sets, algorithm_image=ai, algorithm_model=am + filtered_civ_sets = filter_archive_items_for_algorithm( + archive_items=ArchiveItem.objects.all(), + algorithm_image=ai, + algorithm_model=am, ) - assert filtered_civ_sets == sorted(civ_sets[1:]) + assert filtered_civ_sets.keys() == {interface} + assert filtered_civ_sets[interface] == [item2] @pytest.mark.django_db @@ -765,10 +884,11 @@ def test_archive_job_gets_gpu_and_memory_set( im = ImageFactory() session.image_set.set([im]) - input_interface = ComponentInterface.objects.get( - slug="generic-medical-image" - ) - civ = ComponentInterfaceValueFactory(image=im, interface=input_interface) + ci = ComponentInterface.objects.get(slug="generic-medical-image") + interface = AlgorithmInterfaceFactory(inputs=[ci]) + algorithm_image.algorithm.interfaces.set([interface]) + + civ = ComponentInterfaceValueFactory(image=im, interface=ci) archive_item = ArchiveItemFactory(archive=archive) with django_capture_on_commit_callbacks(execute=True): diff --git a/app/tests/archives_tests/test_forms.py b/app/tests/archives_tests/test_forms.py index 91cae12332..ed812df2b1 100644 --- a/app/tests/archives_tests/test_forms.py +++ b/app/tests/archives_tests/test_forms.py @@ -20,6 +20,7 @@ from tests.algorithms_tests.factories import ( AlgorithmFactory, AlgorithmImageFactory, + AlgorithmInterfaceFactory, ) from tests.archives_tests.factories import ( ArchiveFactory, @@ -336,13 +337,14 @@ def test_archive_item_update_triggers_algorithm_jobs( assert Job.objects.count() == 0 alg = AlgorithmFactory() + interface = AlgorithmInterfaceFactory(inputs=[ci]) + alg.interfaces.add(interface) AlgorithmImageFactory( algorithm=alg, is_manifest_valid=True, is_in_registry=True, is_desired_version=True, ) - alg.inputs.set([ci]) with django_capture_on_commit_callbacks(execute=True): archive.algorithms.add(alg) diff --git a/app/tests/conftest.py b/app/tests/conftest.py index fb70ddf7a6..1ce27bd9bf 100644 --- a/app/tests/conftest.py +++ b/app/tests/conftest.py @@ -12,6 +12,7 @@ from guardian.shortcuts import assign_perm from requests import put +from grandchallenge.algorithms.models import Job from grandchallenge.cases.widgets import ImageWidgetChoices from grandchallenge.components.backends import docker_client from grandchallenge.components.form_fields import INTERFACE_FORM_FIELD_PREFIX @@ -22,8 +23,10 @@ AlgorithmFactory, AlgorithmImageFactory, AlgorithmInterfaceFactory, + AlgorithmJobFactory, AlgorithmModelFactory, ) +from tests.archives_tests.factories import ArchiveFactory, ArchiveItemFactory from tests.cases_tests import RESOURCE_PATH from tests.cases_tests.factories import ( ImageFileFactoryWithMHDFile, @@ -33,7 +36,11 @@ ComponentInterfaceFactory, ComponentInterfaceValueFactory, ) -from tests.evaluation_tests.factories import MethodFactory, PhaseFactory +from tests.evaluation_tests.factories import ( + MethodFactory, + PhaseFactory, + SubmissionFactory, +) from tests.factories import ChallengeFactory, ImageFactory, UserFactory from tests.reader_studies_tests.factories import ( AnswerFactory, @@ -623,3 +630,252 @@ def get_interface_form_data(*, interface_slug, data, existing_data=False): ] = "uuid" return form_data + + +class InterfacesAndJobs(NamedTuple): + archive: ArchiveFactory + algorithm_image: AlgorithmImageFactory + interface1: AlgorithmInterfaceFactory + interface2: AlgorithmInterfaceFactory + interface3: AlgorithmInterfaceFactory + jobs_for_interface1: list[AlgorithmJobFactory] + jobs_for_interface2: list[AlgorithmJobFactory] + jobs_for_interface3: list[AlgorithmJobFactory] + items_for_interface1: list[ArchiveItemFactory] + items_for_interface2: list[ArchiveItemFactory] + items_for_interface3: list[ArchiveItemFactory] + civs_for_interface1: list[ComponentInterfaceValueFactory] + civs_for_interface2: list[ComponentInterfaceValueFactory] + civs_for_interface3: list[ComponentInterfaceValueFactory] + output_civs: list[ComponentInterfaceValueFactory] + + +@pytest.fixture +def archive_items_and_jobs_for_interfaces(): + ci1, ci2, ci3 = ComponentInterfaceFactory.create_batch(3) + + interface1 = AlgorithmInterfaceFactory(inputs=[ci1], outputs=[ci3]) + interface2 = AlgorithmInterfaceFactory(inputs=[ci1, ci2], outputs=[ci3]) + interface3 = AlgorithmInterfaceFactory(inputs=[ci1, ci3], outputs=[ci3]) + + archive = ArchiveFactory() + ai1, ai2, ai3, ai4, ai5, ai6 = ArchiveItemFactory.create_batch( + 6, archive=archive + ) + + civ_int_1a = ComponentInterfaceValueFactory(interface=ci1) + civ_int_1b = ComponentInterfaceValueFactory(interface=ci1) + civ_int_1c = ComponentInterfaceValueFactory(interface=ci1) + civ_int_2a = ComponentInterfaceValueFactory(interface=ci2) + civ_int_2b = ComponentInterfaceValueFactory(interface=ci2) + civ_int_3a = ComponentInterfaceValueFactory(interface=ci3) + + civs_out = ComponentInterfaceValueFactory.create_batch(6, interface=ci3) + + ai1.values.set([civ_int_1a]) # valid for interface 1 + ai2.values.set([civ_int_1b]) # valid for interface 1 + ai3.values.set([civ_int_1c, civ_int_2a]) # valid for interface 2 + ai4.values.set([civ_int_1b, civ_int_2b]) # valid for interface 2 + ai5.values.set([civ_int_1a, civ_int_3a]) # valid for interface 3 + ai6.values.set([civ_int_3a]) # not valid for any interface + + algorithm_image = AlgorithmImageFactory() + algorithm_image.algorithm.interfaces.set([interface1, interface2]) + + # create jobs for interface 1 + j1, j2, j3 = AlgorithmJobFactory.create_batch( + 3, + algorithm_image=algorithm_image, + algorithm_interface=interface1, + time_limit=algorithm_image.algorithm.time_limit, + creator=None, + ) + # outputs don't matter + j1.inputs.set([civ_int_1a]) # corresponds to item ai1 + j2.inputs.set( + [civ_int_1c] + ) # matches interface1, but does not match an item (uses value from an item for interface2) + j3.inputs.set( + [ComponentInterfaceValueFactory(interface=ci1)] + ) # matches interface1 but does not correspond to an item (new value) + + # create jobs for interface 2 + j4, j5, j6 = AlgorithmJobFactory.create_batch( + 3, + algorithm_image=algorithm_image, + algorithm_interface=interface2, + time_limit=algorithm_image.algorithm.time_limit, + creator=None, + ) + j4.inputs.set([civ_int_1c, civ_int_2a]) # corresponds to item ai3 + j5.inputs.set( + [civ_int_1a, civ_int_2a] + ) # valid for interface 2 but does not correspond to an item (mixes values from different items) + j6.inputs.set( + [ + ComponentInterfaceValueFactory(interface=ci1), + ComponentInterfaceValueFactory(interface=ci2), + ] + ) # valid for interface 2 but does not correspond to an item (new values) + + # create jobs for interface 3 (which is not configured for the algorithm) + ( + j7, + j8, + ) = AlgorithmJobFactory.create_batch( + 2, + algorithm_image=algorithm_image, + algorithm_interface=interface3, + time_limit=algorithm_image.algorithm.time_limit, + creator=None, + ) + j7.inputs.set( + [civ_int_1a, civ_int_3a] + ) # valid for interface3, corresponds to item ai5 + j8.inputs.set( + [civ_int_1b, civ_int_3a] + ) # valid for interface3, but does not match item + + return InterfacesAndJobs( + archive=archive, + algorithm_image=algorithm_image, + interface1=interface1, + interface2=interface2, + interface3=interface3, + jobs_for_interface1=[j1, j2, j3], + jobs_for_interface2=[j4, j5, j6], + jobs_for_interface3=[j7, j8], + items_for_interface1=[ai1, ai2], + items_for_interface2=[ai3, ai4], + items_for_interface3=[ai5], + civs_for_interface1=[civ_int_1a, civ_int_1b, civ_int_1c], + civs_for_interface2=[ + [civ_int_1c, civ_int_2a], + [civ_int_1b, civ_int_2b], + ], + civs_for_interface3=[civ_int_3a], + output_civs=civs_out, + ) + + +@pytest.fixture +def jobs_for_optional_inputs(archive_items_and_jobs_for_interfaces): + # delete existing jobs + Job.objects.all().delete() + + # create 2 successful jobs per interface, for each of the archive items + j1, j2 = AlgorithmJobFactory.create_batch( + 2, + status=Job.SUCCESS, + creator=None, + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_interface=archive_items_and_jobs_for_interfaces.interface1, + time_limit=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm.time_limit, + ) + j1.inputs.set( + [archive_items_and_jobs_for_interfaces.civs_for_interface1[0]] + ) + j1.outputs.set([archive_items_and_jobs_for_interfaces.output_civs[0]]) + j2.inputs.set( + [archive_items_and_jobs_for_interfaces.civs_for_interface1[1]] + ) + j2.outputs.set([archive_items_and_jobs_for_interfaces.output_civs[1]]) + + # create 2 jobs per interface, for each of the archive items + j3, j4 = AlgorithmJobFactory.create_batch( + 2, + status=Job.SUCCESS, + creator=None, + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_interface=archive_items_and_jobs_for_interfaces.interface2, + time_limit=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm.time_limit, + ) + j3.inputs.set(archive_items_and_jobs_for_interfaces.civs_for_interface2[0]) + j3.outputs.set([archive_items_and_jobs_for_interfaces.output_civs[2]]) + j4.inputs.set(archive_items_and_jobs_for_interfaces.civs_for_interface2[1]) + j4.outputs.set([archive_items_and_jobs_for_interfaces.output_civs[3]]) + + return InterfacesAndJobs( + archive=archive_items_and_jobs_for_interfaces.archive, + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + interface1=archive_items_and_jobs_for_interfaces.interface1, + interface2=archive_items_and_jobs_for_interfaces.interface2, + interface3=archive_items_and_jobs_for_interfaces.interface3, + jobs_for_interface1=[j1, j2], + jobs_for_interface2=[j3, j4], + jobs_for_interface3=[], + items_for_interface1=archive_items_and_jobs_for_interfaces.items_for_interface1, + items_for_interface2=archive_items_and_jobs_for_interfaces.items_for_interface2, + items_for_interface3=archive_items_and_jobs_for_interfaces.items_for_interface3, + civs_for_interface1=archive_items_and_jobs_for_interfaces.civs_for_interface1, + civs_for_interface2=archive_items_and_jobs_for_interfaces.civs_for_interface2, + civs_for_interface3=archive_items_and_jobs_for_interfaces.civs_for_interface3, + output_civs=archive_items_and_jobs_for_interfaces.output_civs[:4], + ) + + +class SubmissionWithJobs(NamedTuple): + submission: SubmissionFactory + jobs: list[AlgorithmJobFactory] + interface1: AlgorithmInterfaceFactory + interface2: AlgorithmInterfaceFactory + civs_for_interface1: list[ComponentInterfaceValueFactory] + civs_for_interface2: list[ComponentInterfaceValueFactory] + output_civs: list[ComponentInterfaceValueFactory] + + +@pytest.fixture +def submission_without_model_for_optional_inputs(jobs_for_optional_inputs): + submission = SubmissionFactory( + algorithm_image=jobs_for_optional_inputs.algorithm_image + ) + submission.phase.archive = jobs_for_optional_inputs.archive + submission.phase.save() + submission.phase.algorithm_interfaces.set( + [ + jobs_for_optional_inputs.interface1, + jobs_for_optional_inputs.interface2, + ] + ) + return SubmissionWithJobs( + submission=submission, + jobs=[ + *jobs_for_optional_inputs.jobs_for_interface1, + *jobs_for_optional_inputs.jobs_for_interface2, + ], + interface1=jobs_for_optional_inputs.interface1, + interface2=jobs_for_optional_inputs.interface2, + civs_for_interface1=jobs_for_optional_inputs.civs_for_interface1, + civs_for_interface2=jobs_for_optional_inputs.civs_for_interface2, + output_civs=jobs_for_optional_inputs.output_civs, + ) + + +@pytest.fixture +def submission_with_model_for_optional_inputs(jobs_for_optional_inputs): + submission_with_model = SubmissionFactory( + algorithm_image=jobs_for_optional_inputs.algorithm_image, + algorithm_model=AlgorithmModelFactory( + algorithm=jobs_for_optional_inputs.algorithm_image.algorithm + ), + ) + submission_with_model.phase.archive = jobs_for_optional_inputs.archive + submission_with_model.phase.save() + submission_with_model.phase.algorithm_interfaces.set( + [ + jobs_for_optional_inputs.interface1, + jobs_for_optional_inputs.interface2, + ] + ) + return SubmissionWithJobs( + submission=submission_with_model, + jobs=[ + *jobs_for_optional_inputs.jobs_for_interface1, + *jobs_for_optional_inputs.jobs_for_interface2, + ], + interface1=jobs_for_optional_inputs.interface1, + interface2=jobs_for_optional_inputs.interface2, + civs_for_interface1=jobs_for_optional_inputs.civs_for_interface1, + civs_for_interface2=jobs_for_optional_inputs.civs_for_interface2, + output_civs=jobs_for_optional_inputs.output_civs, + ) diff --git a/app/tests/core_tests/test_grand_challenge_forge.py b/app/tests/core_tests/test_grand_challenge_forge.py index 03803e72d9..e7b7d9df93 100644 --- a/app/tests/core_tests/test_grand_challenge_forge.py +++ b/app/tests/core_tests/test_grand_challenge_forge.py @@ -6,7 +6,10 @@ get_forge_challenge_pack_context, ) from grandchallenge.evaluation.utils import SubmissionKindChoices -from tests.algorithms_tests.factories import AlgorithmFactory +from tests.algorithms_tests.factories import ( + AlgorithmFactory, + AlgorithmInterfaceFactory, +) from tests.archives_tests.factories import ArchiveFactory from tests.components_tests.factories import ( ComponentInterfaceExampleValueFactory, @@ -26,7 +29,14 @@ def test_get_challenge_pack_context(): outputs = [ ComponentInterfaceFactory(), ComponentInterfaceFactory(), + ComponentInterfaceFactory(), ] + interface1 = AlgorithmInterfaceFactory( + inputs=[inputs[0]], outputs=outputs[:2] + ) + interface2 = AlgorithmInterfaceFactory( + inputs=[inputs[1]], outputs=[outputs[2]] + ) # Add an example ComponentInterfaceExampleValueFactory( @@ -46,8 +56,7 @@ def test_get_challenge_pack_context(): submission_kind=SubmissionKindChoices.ALGORITHM, ) for phase in phase_1, phase_2: - phase.algorithm_inputs.set(inputs) - phase.algorithm_outputs.set(outputs) + phase.algorithm_interfaces.set([interface1, interface2]) # Setup phases that should not pass the filters phase_3 = PhaseFactory( @@ -89,12 +98,6 @@ def test_get_challenge_pack_context(): ]: assert ci_key in component_interface - # Quick check on CI input and outputs - for index, ci in enumerate( - context["challenge"]["phases"][0]["algorithm_inputs"] - ): - assert inputs[index].slug == ci["slug"] - # Test assigned example value assert ( context["challenge"]["phases"][0]["algorithm_inputs"][0][ @@ -103,10 +106,23 @@ def test_get_challenge_pack_context(): == 87 ) - for index, ci in enumerate( - context["challenge"]["phases"][0]["algorithm_outputs"] - ): - assert outputs[index].slug == ci["slug"] + # Quick check on CI input and outputs + input_slugs = [ + input["slug"] + for input in context["challenge"]["phases"][0]["algorithm_inputs"] + ] + assert len(input_slugs) == len(inputs) + assert inputs[0].slug in input_slugs + assert inputs[1].slug in input_slugs + + output_slugs = [ + output["slug"] + for output in context["challenge"]["phases"][0]["algorithm_outputs"] + ] + assert len(output_slugs) == len(outputs) + assert outputs[0].slug in output_slugs + assert outputs[1].slug in output_slugs + assert outputs[2].slug in output_slugs context = get_forge_challenge_pack_context( challenge, phase_pks=[phase_1.pk] @@ -125,25 +141,37 @@ def test_get_algorithm_template_context(): inputs = [ ComponentInterfaceFactory(kind=ComponentInterface.Kind.INTEGER), + ComponentInterfaceFactory(kind=ComponentInterface.Kind.STRING), ] - algorithm.inputs.set(inputs) - outputs = [ ComponentInterfaceFactory(), ComponentInterfaceFactory(), + ComponentInterfaceFactory(), ] - algorithm.outputs.set(outputs) + interface1 = AlgorithmInterfaceFactory( + inputs=[inputs[0]], outputs=outputs[:2] + ) + interface2 = AlgorithmInterfaceFactory( + inputs=[inputs[1]], outputs=[outputs[2]] + ) + algorithm.interfaces.set([interface1, interface2]) context = get_forge_algorithm_template_context(algorithm=algorithm) for key in ["title", "slug", "url", "inputs", "outputs"]: assert key in context["algorithm"] - for index, ci in enumerate(context["algorithm"]["inputs"]): - assert inputs[index].slug == ci["slug"] + input_slugs = [input["slug"] for input in context["algorithm"]["inputs"]] + assert len(input_slugs) == len(inputs) + assert inputs[0].slug in input_slugs + assert inputs[1].slug in input_slugs - for index, ci in enumerate(context["algorithm"]["outputs"]): - assert outputs[index].slug == ci["slug"] + output_slugs = [ + output["slug"] for output in context["algorithm"]["outputs"] + ] + assert len(output_slugs) == len(outputs) + assert outputs[0].slug in output_slugs + assert outputs[1].slug in output_slugs # Test adding default examples assert context["algorithm"]["inputs"][0]["example_value"] == 42 diff --git a/app/tests/evaluation_tests/test_api.py b/app/tests/evaluation_tests/test_api.py index 51c219cd42..787694304a 100644 --- a/app/tests/evaluation_tests/test_api.py +++ b/app/tests/evaluation_tests/test_api.py @@ -14,6 +14,7 @@ from tests.algorithms_tests.factories import ( AlgorithmFactory, AlgorithmImageFactory, + AlgorithmInterfaceFactory, ) from tests.components_tests.factories import ( ComponentInterfaceFactory, @@ -42,9 +43,9 @@ def generate_claimable_evaluation(): 2, challenge=challenge, submission_kind=SubmissionKindChoices.ALGORITHM ) ci1, ci2 = ComponentInterfaceFactory.create_batch(2) + interface = AlgorithmInterfaceFactory(inputs=[ci1], outputs=[ci2]) for phase in [p1, p2]: - phase.algorithm_outputs.set([ci1]) - phase.algorithm_inputs.set([ci2]) + phase.algorithm_interfaces.set([interface]) p2.external_evaluation = True p2.parent = p1 p2.save() @@ -54,8 +55,7 @@ def generate_claimable_evaluation(): is_in_registry=True, is_desired_version=True, ) - ai.algorithm.inputs.set([ci1]) - ai.algorithm.inputs.set([ci2]) + ai.algorithm.interfaces.set([interface]) eval = EvaluationFactory( submission__algorithm_image=ai, diff --git a/app/tests/evaluation_tests/test_forms.py b/app/tests/evaluation_tests/test_forms.py index 50ae58c906..e1f3b3d4bc 100644 --- a/app/tests/evaluation_tests/test_forms.py +++ b/app/tests/evaluation_tests/test_forms.py @@ -26,6 +26,7 @@ from tests.algorithms_tests.factories import ( AlgorithmFactory, AlgorithmImageFactory, + AlgorithmInterfaceFactory, AlgorithmJobFactory, AlgorithmModelFactory, ) @@ -101,12 +102,12 @@ def test_algorithm_queryset(self): alg2.add_editor(editor) alg4.add_editor(editor) ci1, ci2, ci3, ci4 = ComponentInterfaceFactory.create_batch(4) - alg1.inputs.set([ci1, ci2]) - alg1.outputs.set([ci3, ci4]) - alg3.inputs.set([ci1, ci2]) - alg3.outputs.set([ci3, ci4]) - alg4.inputs.set([ci1, ci2]) - alg4.outputs.set([ci3, ci4]) + interface = AlgorithmInterfaceFactory( + inputs=[ci1, ci2], outputs=[ci3, ci4] + ) + alg1.interfaces.set([interface]) + alg3.interfaces.set([interface]) + alg4.interfaces.set([interface]) for alg in [alg1, alg2, alg3]: AlgorithmImageFactory( algorithm=alg, @@ -116,8 +117,7 @@ def test_algorithm_queryset(self): ) AlgorithmImageFactory(algorithm=alg4) p = PhaseFactory(submission_kind=SubmissionKindChoices.ALGORITHM) - p.algorithm_inputs.set([ci1, ci2]) - p.algorithm_outputs.set([ci3, ci4]) + p.algorithm_interfaces.set([interface]) form = SubmissionForm( user=editor, phase=p, @@ -134,6 +134,9 @@ def test_algorithm_queryset_if_parent_phase_exists(self): AlgorithmFactory.create_batch(10) ) ci1, ci2, ci3, ci4 = ComponentInterfaceFactory.create_batch(4) + interface = AlgorithmInterfaceFactory( + inputs=[ci1, ci2], outputs=[ci3, ci4] + ) for alg in [ alg1, alg2, @@ -147,8 +150,7 @@ def test_algorithm_queryset_if_parent_phase_exists(self): alg10, ]: alg.add_editor(editor) - alg.inputs.set([ci1, ci2]) - alg.outputs.set([ci3, ci4]) + alg.interfaces.set([interface]) AlgorithmImageFactory( algorithm=alg, is_in_registry=True, @@ -164,6 +166,7 @@ def test_algorithm_queryset_if_parent_phase_exists(self): AlgorithmJobFactory( algorithm_image=alg.active_image, algorithm_model=alg.active_model, + algorithm_interface=interface, status=Job.SUCCESS, time_limit=alg.time_limit, ) @@ -174,8 +177,7 @@ def test_algorithm_queryset_if_parent_phase_exists(self): challenge=ChallengeFactory(), ) for p in [p_parent, p_child]: - p.algorithm_inputs.set([ci1, ci2]) - p.algorithm_outputs.set([ci3, ci4]) + p.algorithm_interfaces.set([interface]) p_child.parent = p_parent p_child.save() @@ -234,6 +236,7 @@ def test_algorithm_queryset_if_parent_phase_exists(self): AlgorithmJobFactory( algorithm_image=alg.active_image, algorithm_model=alg.active_model, + algorithm_interface=interface, status=Job.FAILURE, time_limit=alg.time_limit, ) @@ -248,6 +251,7 @@ def test_algorithm_queryset_if_parent_phase_exists(self): AlgorithmJobFactory( algorithm_image=AlgorithmImageFactory(algorithm=alg8), algorithm_model=alg.active_model, + algorithm_interface=interface, status=Job.SUCCESS, time_limit=alg.time_limit, ) @@ -262,6 +266,7 @@ def test_algorithm_queryset_if_parent_phase_exists(self): AlgorithmJobFactory( algorithm_image=alg9.active_image, algorithm_model=AlgorithmModelFactory(algorithm=alg9), + algorithm_interface=interface, status=Job.SUCCESS, time_limit=alg9.time_limit, ) @@ -388,16 +393,15 @@ def test_algorithm_with_permission(self): alg.add_editor(user=user) ci1 = ComponentInterfaceFactory() ci2 = ComponentInterfaceFactory() - alg.inputs.set([ci1]) - alg.outputs.set([ci2]) + interface = AlgorithmInterfaceFactory(inputs=[ci1], outputs=[ci2]) + alg.interfaces.set([interface]) archive = ArchiveFactory() p = PhaseFactory( submission_kind=SubmissionKindChoices.ALGORITHM, submissions_limit_per_user_per_period=10, archive=archive, ) - p.algorithm_inputs.set([ci1]) - p.algorithm_outputs.set([ci2]) + p.algorithm_interfaces.set([interface]) civ = ComponentInterfaceValueFactory(interface=ci1) i = ArchiveItemFactory(archive=p.archive) i.values.add(civ) @@ -419,7 +423,10 @@ def test_algorithm_with_permission(self): algorithm=alg, ) AlgorithmJobFactory( - algorithm_image=ai, status=4, time_limit=ai.algorithm.time_limit + algorithm_image=ai, + algorithm_interface=interface, + status=4, + time_limit=ai.algorithm.time_limit, ) MethodFactory( is_manifest_valid=True, @@ -444,16 +451,15 @@ def test_algorithm_image_and_model_set(self): alg.add_editor(user=user) ci1 = ComponentInterfaceFactory() ci2 = ComponentInterfaceFactory() - alg.inputs.set([ci1]) - alg.outputs.set([ci2]) + interface = AlgorithmInterfaceFactory(inputs=[ci1], outputs=[ci2]) + alg.interfaces.set([interface]) archive = ArchiveFactory() p = PhaseFactory( submission_kind=SubmissionKindChoices.ALGORITHM, submissions_limit_per_user_per_period=10, archive=archive, ) - p.algorithm_inputs.set([ci1]) - p.algorithm_outputs.set([ci2]) + p.algorithm_interfaces.set([interface]) civ = ComponentInterfaceValueFactory(interface=ci1) i = ArchiveItemFactory(archive=p.archive) i.values.add(civ) @@ -477,6 +483,7 @@ def test_algorithm_image_and_model_set(self): am = AlgorithmModelFactory(algorithm=alg, is_desired_version=True) AlgorithmJobFactory( algorithm_image=ai, + algorithm_interface=interface, status=Job.SUCCESS, time_limit=ai.algorithm.time_limit, ) @@ -549,16 +556,15 @@ def test_no_valid_archive_items(self): alg.add_editor(user=user) ci1 = ComponentInterfaceFactory() ci2 = ComponentInterfaceFactory() - alg.inputs.set([ci1]) - alg.outputs.set([ci2]) + interface = AlgorithmInterfaceFactory(inputs=[ci1], outputs=[ci2]) + alg.interfaces.set([interface]) archive = ArchiveFactory() p_alg = PhaseFactory( submission_kind=SubmissionKindChoices.ALGORITHM, submissions_limit_per_user_per_period=10, archive=archive, ) - p_alg.algorithm_inputs.set([ci1]) - p_alg.algorithm_outputs.set([ci2]) + p_alg.algorithm_interfaces.set([interface]) MethodFactory( phase=p_alg, is_manifest_valid=True, @@ -590,7 +596,10 @@ def test_no_valid_archive_items(self): algorithm=alg, ) AlgorithmJobFactory( - algorithm_image=ai, status=4, time_limit=ai.algorithm.time_limit + algorithm_image=ai, + algorithm_interface=interface, + status=4, + time_limit=ai.algorithm.time_limit, ) upload = UserUploadFactory(creator=user) @@ -652,16 +661,15 @@ def test_submission_or_eval_exists_for_image(self): alg.add_editor(user=user) ci1 = ComponentInterfaceFactory() ci2 = ComponentInterfaceFactory() - alg.inputs.set([ci1]) - alg.outputs.set([ci2]) + interface = AlgorithmInterfaceFactory(inputs=[ci1], outputs=[ci2]) + alg.interfaces.set([interface]) archive = ArchiveFactory() p = PhaseFactory( submission_kind=SubmissionKindChoices.ALGORITHM, submissions_limit_per_user_per_period=10, archive=archive, ) - p.algorithm_inputs.set([ci1]) - p.algorithm_outputs.set([ci2]) + p.algorithm_interfaces.set([interface]) civ = ComponentInterfaceValueFactory(interface=ci1) i = ArchiveItemFactory(archive=p.archive) i.values.add(civ) @@ -683,7 +691,10 @@ def test_submission_or_eval_exists_for_image(self): algorithm=alg, ) AlgorithmJobFactory( - algorithm_image=ai, status=4, time_limit=ai.algorithm.time_limit + algorithm_image=ai, + algorithm_interface=interface, + status=4, + time_limit=ai.algorithm.time_limit, ) SubmissionFactory( phase=p, @@ -744,16 +755,15 @@ def test_eval_exists_for_image_and_model( alg.add_editor(user=user) ci1 = ComponentInterfaceFactory() ci2 = ComponentInterfaceFactory() - alg.inputs.set([ci1]) - alg.outputs.set([ci2]) + interface = AlgorithmInterfaceFactory(inputs=[ci1], outputs=[ci2]) + alg.interfaces.set([interface]) archive = ArchiveFactory() p = PhaseFactory( submission_kind=SubmissionKindChoices.ALGORITHM, submissions_limit_per_user_per_period=10, archive=archive, ) - p.algorithm_inputs.set([ci1]) - p.algorithm_outputs.set([ci2]) + p.algorithm_interfaces.set([interface]) civ = ComponentInterfaceValueFactory(interface=ci1) i = ArchiveItemFactory(archive=p.archive) i.values.add(civ) @@ -894,8 +904,7 @@ def test_algorithm_for_phase_form(): display_editors=True, contact_email="test@test.com", workstation=WorkstationFactory.build(), - inputs=[ComponentInterfaceFactory.build()], - outputs=[ComponentInterfaceFactory.build()], + interfaces=[AlgorithmInterfaceFactory.build()], structures=[], modalities=[], logo=ImageField(filename="test.jpeg"), @@ -903,8 +912,7 @@ def test_algorithm_for_phase_form(): user=UserFactory.build(), ) - assert form.fields["inputs"].disabled - assert form.fields["outputs"].disabled + assert form.fields["interfaces"].disabled assert form.fields["workstation_config"].disabled assert form.fields["hanging_protocol"].disabled assert form.fields["optional_hanging_protocols"].disabled @@ -922,8 +930,7 @@ def test_algorithm_for_phase_form(): assert not form.fields["job_requires_memory_gb"].disabled assert { - form.fields["inputs"], - form.fields["outputs"], + form.fields["interfaces"], form.fields["workstation_config"], form.fields["hanging_protocol"], form.fields["optional_hanging_protocols"], @@ -951,12 +958,15 @@ def test_algorithm_for_phase_form_validation(): phase = PhaseFactory() alg1, alg2, alg3 = AlgorithmFactory.create_batch(3) ci1, ci2, ci3, ci4 = ComponentInterfaceFactory.create_batch(4) - phase.algorithm_inputs.set([ci1, ci2]) - phase.algorithm_outputs.set([ci3, ci4]) + + interface = AlgorithmInterfaceFactory( + inputs=[ci1, ci2], outputs=[ci3, ci4] + ) + + phase.algorithm_interfaces.set([interface]) for alg in [alg1, alg2]: alg.add_editor(user) - alg.inputs.set([ci1, ci2]) - alg.outputs.set([ci3, ci4]) + alg.interfaces.set([interface]) form = AlgorithmForPhaseForm( workstation_config=WorkstationConfigFactory(), @@ -966,8 +976,7 @@ def test_algorithm_for_phase_form_validation(): display_editors=True, contact_email="test@test.com", workstation=WorkstationFactory(), - inputs=[ci1, ci2], - outputs=[ci3, ci4], + interfaces=[interface], structures=[], modalities=[], logo=ImageField(filename="test.jpeg"), @@ -984,8 +993,7 @@ def test_algorithm_for_phase_form_validation(): ) alg3.add_editor(user) - alg3.inputs.set([ci1, ci2]) - alg3.outputs.set([ci3, ci4]) + alg3.interfaces.set([interface]) form = AlgorithmForPhaseForm( workstation_config=WorkstationConfigFactory(), @@ -995,8 +1003,7 @@ def test_algorithm_for_phase_form_validation(): display_editors=True, contact_email="test@test.com", workstation=WorkstationFactory(), - inputs=[ci1, ci2], - outputs=[ci3, ci4], + interfaces=[interface], structures=[], modalities=[], logo=ImageField(filename="test.jpeg"), @@ -1013,6 +1020,68 @@ def test_algorithm_for_phase_form_validation(): ) +@pytest.mark.django_db +def test_user_algorithms_for_phase(): + + def populate_form(interfaces): + return AlgorithmForPhaseForm( + workstation_config=WorkstationConfigFactory(), + hanging_protocol=HangingProtocolFactory(), + optional_hanging_protocols=[HangingProtocolFactory()], + view_content=None, + display_editors=True, + contact_email="test@test.com", + workstation=WorkstationFactory(), + interfaces=interfaces, + structures=[], + modalities=[], + logo=ImageField(filename="test.jpeg"), + phase=phase, + user=user, + data={ + "title": "foo", + }, + ) + + user = UserFactory() + phase = PhaseFactory() + alg1, alg2, alg3, alg4, alg5 = AlgorithmFactory.create_batch(5) + ci1, ci2, ci3, ci4 = ComponentInterfaceFactory.create_batch(4) + + interface1 = AlgorithmInterfaceFactory( + inputs=[ci1, ci2], outputs=[ci3, ci4] + ) + interface2 = AlgorithmInterfaceFactory(inputs=[ci1], outputs=[ci3]) + interface3 = AlgorithmInterfaceFactory(inputs=[ci3], outputs=[ci1, ci4]) + + for alg in [alg1, alg2, alg3, alg4, alg5]: + alg.add_editor(user) + + # phase has 1 interface + phase.algorithm_interfaces.set([interface1]) + # only algorithms that have this interface set only should match + # partial matches are not valid + alg1.interfaces.set([interface1]) # exact match + alg2.interfaces.set( + [interface3] + ) # same number of interfaces, but different interface + alg3.interfaces.set([interface1, interface3]) # additional interface + alg4.interfaces.set([interface2, interface3]) # diff num, diff interfaces + + form = populate_form(interfaces=[interface1]) + assert list(form.user_algorithms_for_phase) == [alg1] + + # phase with 2 interfaces + phase.algorithm_interfaces.set([interface1, interface3]) + form = populate_form(interfaces=[interface1, interface3]) + assert list(form.user_algorithms_for_phase) == [alg3] + + # user needs to be owner of algorithm + alg3.remove_editor(user) + form = populate_form(interfaces=[interface1, interface3]) + assert list(form.user_algorithms_for_phase) == [] + + @pytest.mark.django_db def test_configure_algorithm_phases_form(): ch = ChallengeFactory() @@ -1023,7 +1092,6 @@ def test_configure_algorithm_phases_form(): SubmissionFactory(phase=p1) MethodFactory(phase=p2) PhaseFactory(submission_kind=SubmissionKindChoices.ALGORITHM) - ci1, ci2 = ComponentInterfaceFactory.create_batch(2) form = ConfigureAlgorithmPhasesForm(challenge=ch) assert list(form.fields["phases"].queryset) == [p3] @@ -1032,8 +1100,6 @@ def test_configure_algorithm_phases_form(): challenge=ch, data={ "phases": [p3], - "algorithm_inputs": [ci1], - "algorithm_outputs": [ci2], }, ) assert form3.is_valid() diff --git a/app/tests/evaluation_tests/test_models.py b/app/tests/evaluation_tests/test_models.py index 2e8360efd5..b09e237dd0 100644 --- a/app/tests/evaluation_tests/test_models.py +++ b/app/tests/evaluation_tests/test_models.py @@ -5,11 +5,11 @@ import pytest from django.core import mail from django.core.exceptions import ValidationError -from django.test import TestCase from django.utils import timezone from django.utils.timezone import now from grandchallenge.algorithms.models import Job +from grandchallenge.archives.models import ArchiveItem from grandchallenge.components.models import ComponentInterface from grandchallenge.components.schemas import GPUTypeChoices from grandchallenge.evaluation.models import ( @@ -17,6 +17,8 @@ CombinedLeaderboard, Evaluation, Phase, + get_archive_items_for_interfaces, + get_valid_jobs_for_interfaces_and_archive_items, ) from grandchallenge.evaluation.tasks import ( calculate_ranks, @@ -28,6 +30,7 @@ from grandchallenge.invoices.models import PaymentStatusChoices from tests.algorithms_tests.factories import ( AlgorithmImageFactory, + AlgorithmInterfaceFactory, AlgorithmJobFactory, AlgorithmModelFactory, ) @@ -63,13 +66,14 @@ def algorithm_submission(): ) algorithm_image = AlgorithmImageFactory() - interface = ComponentInterfaceFactory() - algorithm_image.algorithm.inputs.set([interface]) + ci = ComponentInterfaceFactory() + interface = AlgorithmInterfaceFactory(inputs=[ci]) + algorithm_image.algorithm.interfaces.set([interface]) images = ImageFactory.create_batch(3) for image in images[:2]: - civ = ComponentInterfaceValueFactory(image=image, interface=interface) + civ = ComponentInterfaceValueFactory(image=image, interface=ci) ai = ArchiveItemFactory(archive=method.phase.archive) ai.values.add(civ) @@ -160,8 +164,8 @@ def test_create_algorithm_jobs_for_evaluation_sets_gpu_and_memory(): is_in_registry=True, is_desired_version=True, ) - algorithm_image.algorithm.inputs.set(inputs) - algorithm_image.algorithm.outputs.set(outputs) + interface = AlgorithmInterfaceFactory(inputs=inputs, outputs=outputs) + algorithm_image.algorithm.interfaces.set([interface]) archive = ArchiveFactory() archive_item = ArchiveItemFactory(archive=archive) @@ -177,8 +181,7 @@ def test_create_algorithm_jobs_for_evaluation_sets_gpu_and_memory(): submission_kind=SubmissionKindChoices.ALGORITHM, submissions_limit_per_user_per_period=1, ) - phase.algorithm_inputs.set(inputs) - phase.algorithm_outputs.set(outputs) + phase.algorithm_interfaces.set([interface]) method = MethodFactory( phase=phase, @@ -852,19 +855,19 @@ def test_evaluation_invalid_metrics( @pytest.mark.django_db -def test_count_valid_archive_items(): +def test_valid_archive_items_per_interface(): archive = ArchiveFactory() phase = PhaseFactory(archive=archive) i1, i2, i3 = ComponentInterfaceFactory.create_batch(3) - - phase.algorithm_inputs.set([i1, i2]) + interface = AlgorithmInterfaceFactory(inputs=[i1, i2]) + phase.algorithm_interfaces.set([interface]) # Valid archive item ai1 = ArchiveItemFactory(archive=archive) ai1.values.add(ComponentInterfaceValueFactory(interface=i1)) ai1.values.add(ComponentInterfaceValueFactory(interface=i2)) - # Valid, but with extra value + # Invalid, because it has extra value ai2 = ArchiveItemFactory(archive=archive) ai2.values.add(ComponentInterfaceValueFactory(interface=i1)) ai2.values.add(ComponentInterfaceValueFactory(interface=i2)) @@ -882,16 +885,26 @@ def test_count_valid_archive_items(): ai5.values.add(ComponentInterfaceValueFactory(interface=i1)) ai5.values.add(ComponentInterfaceValueFactory(interface=i3)) - cciv1 = ComponentInterfaceValueFactory(interface=i1) - cciv2 = ComponentInterfaceValueFactory(interface=i2) + civ1 = ComponentInterfaceValueFactory(interface=i1) + civ2 = ComponentInterfaceValueFactory(interface=i2) # Valid, reusing interfaces ai6 = ArchiveItemFactory(archive=archive) - ai6.values.set([cciv1, cciv2]) + ai6.values.set([civ1, civ2]) ai7 = ArchiveItemFactory(archive=archive) - ai7.values.set([cciv1, cciv2]) + ai7.values.set([civ1, civ2]) - assert {*phase.valid_archive_items} == {ai1, ai2, ai6, ai7} + assert phase.valid_archive_items_per_interface.keys() == {interface} + assert [ + item.pk + for qs in phase.valid_archive_items_per_interface.values() + for item in qs.order_by("pk") + ] == [ + item.pk + for item in ArchiveItem.objects.filter( + pk__in=[ai1.pk, ai6.pk, ai7.pk] + ).order_by("pk") + ] @pytest.mark.django_db @@ -963,14 +976,14 @@ def test_parent_phase_choices(): p5.submission_kind = SubmissionKindChoices.CSV ci1, ci2, ci3, ci4 = ComponentInterfaceFactory.create_batch(4) - - for phase in [p1, p2, p3, p4, p5, p6]: - phase.algorithm_inputs.set([ci1]) + interface1 = AlgorithmInterfaceFactory(inputs=[ci1], outputs=[ci2, ci3]) + interface2 = AlgorithmInterfaceFactory(inputs=[ci1], outputs=[ci2, ci4]) + interface3 = AlgorithmInterfaceFactory(inputs=[ci1], outputs=[ci2]) for phase in [p1, p4, p5]: - phase.algorithm_outputs.set([ci2, ci3]) - p2.algorithm_outputs.set([ci2, ci4]) - p3.algorithm_outputs.set([ci2]) + phase.algorithm_interfaces.set([interface1]) + p2.algorithm_interfaces.set([interface2]) + p3.algorithm_interfaces.set([interface3]) for phase in [p1, p2, p3, p4, p5, p6]: phase.save() @@ -986,10 +999,10 @@ def test_parent_phase_choices_no_circular_dependency(): submission_kind=SubmissionKindChoices.ALGORITHM, ) ci1, ci2 = ComponentInterfaceFactory.create_batch(2) + interface = AlgorithmInterfaceFactory(inputs=[ci1], outputs=[ci2]) for phase in [p1, p2, p3, p4]: - phase.algorithm_inputs.set([ci1]) - phase.algorithm_outputs.set([ci2]) + phase.algorithm_interfaces.set([interface]) p1.parent = p2 p2.parent = p3 @@ -1010,11 +1023,11 @@ def test_clean_parent_phase(): 4, challenge=ChallengeFactory(), creator_must_be_verified=True ) ci1, ci2 = ComponentInterfaceFactory.create_batch(2) + interface = AlgorithmInterfaceFactory(inputs=[ci1], outputs=[ci2]) for phase in [p1, p2, p3, p4]: phase.submission_kind = SubmissionKindChoices.ALGORITHM - phase.algorithm_inputs.set([ci1]) - phase.algorithm_outputs.set([ci2]) + phase.algorithm_interfaces.set([interface]) phase.save() ai = ArchiveItemFactory() @@ -1143,288 +1156,671 @@ def test_is_evaluated_with_active_image_and_ground_truth(): @pytest.mark.django_db -class TestInputsComplete(TestCase): +class TestInputsComplete: + def test_inputs_complete_for_prediction_submission(self): + eval_pred = EvaluationFactory( + submission__predictions_file=None, time_limit=10 + ) + assert not eval_pred.inputs_complete + + eval_pred2 = EvaluationFactory(time_limit=10) + assert eval_pred2.inputs_complete - def setUp(self): - self.interface = ComponentInterface.objects.get( - slug="generic-medical-image" + def test_non_successful_jobs_ignored( + self, archive_items_and_jobs_for_interfaces + ): + submission = SubmissionFactory( + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image + ) + submission.phase.archive = ( + archive_items_and_jobs_for_interfaces.archive + ) + submission.phase.save() + submission.phase.algorithm_interfaces.set( + [ + archive_items_and_jobs_for_interfaces.interface1, + archive_items_and_jobs_for_interfaces.interface2, + ] ) - archive = ArchiveFactory() - self.archive_items = ArchiveItemFactory.create_batch(2) - archive.items.set(self.archive_items) + eval_alg = EvaluationFactory(submission=submission, time_limit=10) + assert not eval_alg.inputs_complete - input_civs = ComponentInterfaceValueFactory.create_batch( - 2, interface=self.interface + # create 2 jobs per interface, for each of the archive items + j1, j2 = AlgorithmJobFactory.create_batch( + 2, + creator=None, + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_interface=archive_items_and_jobs_for_interfaces.interface1, + time_limit=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm.time_limit, + ) + j1.inputs.set( + [archive_items_and_jobs_for_interfaces.civs_for_interface1[0]] ) - output_civs = ComponentInterfaceValueFactory.create_batch( - 2, interface=self.interface + j2.inputs.set( + [archive_items_and_jobs_for_interfaces.civs_for_interface1[1]] ) - for ai, civ in zip(self.archive_items, input_civs, strict=True): - ai.values.set([civ]) + # create 2 jobs per interface, for each of the archive items + j3, j4 = AlgorithmJobFactory.create_batch( + 2, + creator=None, + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_interface=archive_items_and_jobs_for_interfaces.interface2, + time_limit=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm.time_limit, + ) + j3.inputs.set( + archive_items_and_jobs_for_interfaces.civs_for_interface2[0] + ) + j4.inputs.set( + archive_items_and_jobs_for_interfaces.civs_for_interface2[1] + ) - self.algorithm_image = AlgorithmImageFactory() - self.algorithm_model = AlgorithmModelFactory() - submission = SubmissionFactory(algorithm_image=self.algorithm_image) - submission.phase.archive = archive - submission.phase.save() - submission.phase.algorithm_inputs.set([self.interface]) + del eval_alg.successful_jobs_per_interface + del eval_alg.successful_job_count_per_interface + del eval_alg.total_successful_jobs + del eval_alg.inputs_complete + assert not eval_alg.inputs_complete - submission_with_model = SubmissionFactory( - algorithm_image=self.algorithm_image, - algorithm_model=self.algorithm_model, + def test_inputs_complete_for_algorithm_submission_without_model( + self, archive_items_and_jobs_for_interfaces + ): + submission = SubmissionFactory( + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image + ) + submission.phase.archive = ( + archive_items_and_jobs_for_interfaces.archive + ) + submission.phase.save() + submission.phase.algorithm_interfaces.set( + [ + archive_items_and_jobs_for_interfaces.interface1, + archive_items_and_jobs_for_interfaces.interface2, + ] ) - submission_with_model.phase.archive = archive - submission_with_model.phase.save() - submission_with_model.phase.algorithm_inputs.set([self.interface]) - self.submission = submission - self.submission_with_model = submission_with_model - self.input_civs = input_civs - self.output_civs = output_civs + eval_alg = EvaluationFactory(submission=submission, time_limit=10) + assert not eval_alg.inputs_complete - def test_inputs_complete_for_prediction_submission(self): - eval_pred = EvaluationFactory( - submission__predictions_file=None, time_limit=10 + # create 2 jobs per interface, for each of the archive items + j1, j2 = AlgorithmJobFactory.create_batch( + 2, + status=Job.SUCCESS, + creator=None, + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_interface=archive_items_and_jobs_for_interfaces.interface1, + time_limit=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm.time_limit, + ) + j1.inputs.set( + [archive_items_and_jobs_for_interfaces.civs_for_interface1[0]] + ) + j2.inputs.set( + [archive_items_and_jobs_for_interfaces.civs_for_interface1[1]] ) - assert not eval_pred.inputs_complete - eval_pred2 = EvaluationFactory(time_limit=10) - assert eval_pred2.inputs_complete + # create 2 jobs per interface, for each of the archive items + j3, j4 = AlgorithmJobFactory.create_batch( + 2, + status=Job.SUCCESS, + creator=None, + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_interface=archive_items_and_jobs_for_interfaces.interface2, + time_limit=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm.time_limit, + ) + j3.inputs.set( + archive_items_and_jobs_for_interfaces.civs_for_interface2[0] + ) + j4.inputs.set( + archive_items_and_jobs_for_interfaces.civs_for_interface2[1] + ) - def test_inputs_complete_for_algorithm_submission_without_model(self): - eval_alg = EvaluationFactory(submission=self.submission, time_limit=10) - assert not eval_alg.inputs_complete + # no need to set outputs, we assume that only a job with valid outputs has a + # status of SUCCESS - for inpt, output in zip( - self.input_civs, self.output_civs, strict=True - ): - j = AlgorithmJobFactory( - status=Job.SUCCESS, - algorithm_image=self.algorithm_image, - time_limit=self.algorithm_image.algorithm.time_limit, - ) - j.inputs.set([inpt]) - j.outputs.set([output]) - j.creator = None - j.save() - - del eval_alg.algorithm_inputs - del eval_alg.successful_jobs + del eval_alg.successful_jobs_per_interface + del eval_alg.successful_job_count_per_interface + del eval_alg.total_successful_jobs del eval_alg.inputs_complete assert eval_alg.inputs_complete - def test_inputs_complete_for_algorithm_submission_with_model(self): - eval_alg = EvaluationFactory( - submission=self.submission_with_model, time_limit=10 + def test_inputs_complete_for_algorithm_submission_with_model( + self, archive_items_and_jobs_for_interfaces + ): + algorithm_model = AlgorithmModelFactory( + algorithm=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm + ) + submission = SubmissionFactory( + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_model=algorithm_model, + ) + submission.phase.archive = ( + archive_items_and_jobs_for_interfaces.archive ) + submission.phase.save() + submission.phase.algorithm_interfaces.set( + [ + archive_items_and_jobs_for_interfaces.interface1, + archive_items_and_jobs_for_interfaces.interface2, + ] + ) + + eval_alg = EvaluationFactory(submission=submission, time_limit=10) assert not eval_alg.inputs_complete - for inpt, output in zip( - self.input_civs, self.output_civs, strict=True - ): - j = AlgorithmJobFactory( - status=Job.SUCCESS, - algorithm_image=self.algorithm_image, - algorithm_model=self.algorithm_model, - time_limit=self.algorithm_image.algorithm.time_limit, - ) - j.inputs.set([inpt]) - j.outputs.set([output]) - j.creator = None - j.save() - del eval_alg.algorithm_inputs - del eval_alg.successful_jobs + # create 2 jobs per interface, for each of the archive items + j1, j2 = AlgorithmJobFactory.create_batch( + 2, + status=Job.SUCCESS, + creator=None, + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_model=algorithm_model, + algorithm_interface=archive_items_and_jobs_for_interfaces.interface1, + time_limit=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm.time_limit, + ) + j1.inputs.set( + [archive_items_and_jobs_for_interfaces.civs_for_interface1[0]] + ) + j2.inputs.set( + [archive_items_and_jobs_for_interfaces.civs_for_interface1[1]] + ) + + # create 2 jobs per interface, for each of the archive items + j3, j4 = AlgorithmJobFactory.create_batch( + 2, + status=Job.SUCCESS, + creator=None, + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_model=algorithm_model, + algorithm_interface=archive_items_and_jobs_for_interfaces.interface2, + time_limit=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm.time_limit, + ) + j3.inputs.set( + archive_items_and_jobs_for_interfaces.civs_for_interface2[0] + ) + j4.inputs.set( + archive_items_and_jobs_for_interfaces.civs_for_interface2[1] + ) + + del eval_alg.successful_jobs_per_interface + del eval_alg.successful_job_count_per_interface + del eval_alg.total_successful_jobs del eval_alg.inputs_complete assert eval_alg.inputs_complete - def test_jobs_with_creator_ignored(self): - eval_alg = EvaluationFactory(submission=self.submission, time_limit=10) - assert not eval_alg.inputs_complete + def test_jobs_with_creator_ignored( + self, archive_items_and_jobs_for_interfaces + ): + submission = SubmissionFactory( + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image + ) + submission.phase.archive = ( + archive_items_and_jobs_for_interfaces.archive + ) + submission.phase.save() + submission.phase.algorithm_interfaces.set( + [ + archive_items_and_jobs_for_interfaces.interface1, + archive_items_and_jobs_for_interfaces.interface2, + ] + ) - for inpt, output in zip( - self.input_civs, self.output_civs, strict=True - ): - j_irrelevant = AlgorithmJobFactory( - status=Job.SUCCESS, - algorithm_image=self.algorithm_image, - creator=self.algorithm_image.creator, - time_limit=self.algorithm_image.algorithm.time_limit, - ) - j_irrelevant.inputs.set([inpt]) - j_irrelevant.outputs.set([output]) - del eval_alg.algorithm_inputs - del eval_alg.successful_jobs - del eval_alg.inputs_complete + eval_alg = EvaluationFactory(submission=submission, time_limit=10) assert not eval_alg.inputs_complete - def test_failed_jobs_ignored(self): - eval_alg = EvaluationFactory(submission=self.submission, time_limit=10) - assert not eval_alg.inputs_complete + # create 2 jobs per interface, for each of the archive items + j1, j2 = AlgorithmJobFactory.create_batch( + 2, + status=Job.SUCCESS, + creator=archive_items_and_jobs_for_interfaces.algorithm_image.creator, + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_interface=archive_items_and_jobs_for_interfaces.interface1, + time_limit=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm.time_limit, + ) + j1.inputs.set( + [archive_items_and_jobs_for_interfaces.civs_for_interface1[0]] + ) + j2.inputs.set( + [archive_items_and_jobs_for_interfaces.civs_for_interface1[1]] + ) + + # create 2 jobs per interface, for each of the archive items + j3, j4 = AlgorithmJobFactory.create_batch( + 2, + status=Job.SUCCESS, + creator=archive_items_and_jobs_for_interfaces.algorithm_image.creator, + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_interface=archive_items_and_jobs_for_interfaces.interface2, + time_limit=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm.time_limit, + ) + j3.inputs.set( + archive_items_and_jobs_for_interfaces.civs_for_interface2[0] + ) + j4.inputs.set( + archive_items_and_jobs_for_interfaces.civs_for_interface2[1] + ) - for inpt, output in zip( - self.input_civs, self.output_civs, strict=True - ): - j_irrelevant = AlgorithmJobFactory( - status=Job.FAILURE, - algorithm_image=self.algorithm_image, - time_limit=self.algorithm_image.algorithm.time_limit, - ) - j_irrelevant.inputs.set([inpt]) - j_irrelevant.outputs.set([output]) - del eval_alg.algorithm_inputs - del eval_alg.successful_jobs + del eval_alg.successful_jobs_per_interface + del eval_alg.successful_job_count_per_interface + del eval_alg.total_successful_jobs del eval_alg.inputs_complete assert not eval_alg.inputs_complete - def test_jobs_with_other_inputs_ignored(self): - eval_alg = EvaluationFactory(submission=self.submission, time_limit=10) + def test_jobs_with_other_inputs_ignored( + self, archive_items_and_jobs_for_interfaces + ): + submission = SubmissionFactory( + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image + ) + submission.phase.archive = ( + archive_items_and_jobs_for_interfaces.archive + ) + submission.phase.save() + submission.phase.algorithm_interfaces.set( + [ + archive_items_and_jobs_for_interfaces.interface1, + archive_items_and_jobs_for_interfaces.interface2, + ] + ) + + eval_alg = EvaluationFactory(submission=submission, time_limit=10) assert not eval_alg.inputs_complete - other_input_civs = ComponentInterfaceValueFactory.create_batch( - 2, interface=self.interface - ) - other_output_civs = ComponentInterfaceValueFactory.create_batch( - 2, interface=self.interface - ) - for inpt, output in zip( - other_input_civs, other_output_civs, strict=True - ): - j_irrelevant = AlgorithmJobFactory( - status=Job.SUCCESS, - algorithm_image=self.algorithm_image, - time_limit=self.algorithm_image.algorithm.time_limit, - ) - j_irrelevant.inputs.set([inpt]) - j_irrelevant.outputs.set([output]) - del eval_alg.algorithm_inputs - del eval_alg.successful_jobs + # create 2 jobs per interface, for each of the archive items + j1, j2 = AlgorithmJobFactory.create_batch( + 2, + status=Job.SUCCESS, + creator=None, + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_interface=archive_items_and_jobs_for_interfaces.interface1, + time_limit=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm.time_limit, + ) + j1.inputs.set( + [archive_items_and_jobs_for_interfaces.civs_for_interface1[0]] + ) + j2.inputs.set( + [archive_items_and_jobs_for_interfaces.civs_for_interface1[1]] + ) + + # create 2 jobs per interface, for each of the archive items + j3, j4 = AlgorithmJobFactory.create_batch( + 2, + status=Job.SUCCESS, + creator=None, + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_interface=archive_items_and_jobs_for_interfaces.interface2, + time_limit=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm.time_limit, + ) + j3.inputs.set( + archive_items_and_jobs_for_interfaces.civs_for_interface2[0] + ) + j4.inputs.set( + archive_items_and_jobs_for_interfaces.civs_for_interface2[1] + ) + + # now also create jobs with other inputs, those should be ignored + j5, j6 = AlgorithmJobFactory.create_batch( + 2, + status=Job.SUCCESS, + creator=None, + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_interface=archive_items_and_jobs_for_interfaces.interface1, + time_limit=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm.time_limit, + ) + j5.inputs.set( + [ + ComponentInterfaceValueFactory( + interface=archive_items_and_jobs_for_interfaces.interface1.inputs.get() + ) + ] + ) + j6.inputs.set( + [ + ComponentInterfaceValueFactory( + interface=archive_items_and_jobs_for_interfaces.interface1.inputs.get() + ) + ] + ) + + # create 2 jobs per interface, for each of the archive items + j7, j8 = AlgorithmJobFactory.create_batch( + 2, + status=Job.SUCCESS, + creator=None, + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_interface=archive_items_and_jobs_for_interfaces.interface2, + time_limit=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm.time_limit, + ) + j7.inputs.set( + [ + ComponentInterfaceValueFactory( + interface=archive_items_and_jobs_for_interfaces.interface2.inputs.first() + ), + ComponentInterfaceValueFactory( + interface=archive_items_and_jobs_for_interfaces.interface2.inputs.last() + ), + ] + ) + j8.inputs.set( + [ + ComponentInterfaceValueFactory( + interface=archive_items_and_jobs_for_interfaces.interface2.inputs.first() + ), + ComponentInterfaceValueFactory( + interface=archive_items_and_jobs_for_interfaces.interface2.inputs.last() + ), + ] + ) + + del eval_alg.successful_jobs_per_interface + del eval_alg.successful_job_count_per_interface + del eval_alg.total_successful_jobs del eval_alg.inputs_complete - assert not eval_alg.inputs_complete + assert eval_alg.inputs_complete - def test_jobs_with_partial_inputs_ignored(self): - eval_alg = EvaluationFactory(submission=self.submission, time_limit=10) + def test_jobs_with_partial_inputs_ignored( + self, archive_items_and_jobs_for_interfaces + ): + submission = SubmissionFactory( + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image + ) + submission.phase.archive = ( + archive_items_and_jobs_for_interfaces.archive + ) + submission.phase.save() + submission.phase.algorithm_interfaces.set( + [ + archive_items_and_jobs_for_interfaces.interface1, + archive_items_and_jobs_for_interfaces.interface2, + ] + ) + + eval_alg = EvaluationFactory(submission=submission, time_limit=10) assert not eval_alg.inputs_complete - # add values to archive items - new_interface = ComponentInterface.objects.get(slug="generic-overlay") - self.submission.phase.algorithm_inputs.add(new_interface) - self.submission.phase.save() - - new_input_civs = ComponentInterfaceValueFactory.create_batch( - 2, interface=new_interface - ) - for ai, civ in zip(self.archive_items, new_input_civs, strict=True): - ai.values.add(civ) - - for inpt, output in zip( - self.input_civs, self.output_civs, strict=True - ): - j = AlgorithmJobFactory( - status=Job.SUCCESS, - algorithm_image=self.algorithm_image, - time_limit=self.algorithm_image.algorithm.time_limit, - ) - j.inputs.set([inpt]) - j.outputs.set([output]) - j.creator = None - j.save() - - del eval_alg.algorithm_inputs - del eval_alg.successful_jobs + # create 2 complete jobs for interface1 + j1, j2 = AlgorithmJobFactory.create_batch( + 2, + status=Job.SUCCESS, + creator=None, + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_interface=archive_items_and_jobs_for_interfaces.interface1, + time_limit=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm.time_limit, + ) + j1.inputs.set( + [archive_items_and_jobs_for_interfaces.civs_for_interface1[0]] + ) + j2.inputs.set( + [archive_items_and_jobs_for_interfaces.civs_for_interface1[1]] + ) + + # create jobs for interface 2 with only part of the required inputs, + # those should not count as complete jobs + j3, j4 = AlgorithmJobFactory.create_batch( + 2, + status=Job.SUCCESS, + creator=None, + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_interface=archive_items_and_jobs_for_interfaces.interface2, + time_limit=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm.time_limit, + ) + j3.inputs.set( + [archive_items_and_jobs_for_interfaces.civs_for_interface2[0][0]] + ) + j4.inputs.set( + [archive_items_and_jobs_for_interfaces.civs_for_interface2[1][1]] + ) + + del eval_alg.successful_jobs_per_interface + del eval_alg.successful_job_count_per_interface + del eval_alg.total_successful_jobs del eval_alg.inputs_complete assert not eval_alg.inputs_complete def test_jobs_with_different_image_ignored_for_submission_without_model( - self, + self, archive_items_and_jobs_for_interfaces ): - eval_alg = EvaluationFactory(submission=self.submission, time_limit=10) + submission = SubmissionFactory( + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image + ) + submission.phase.archive = ( + archive_items_and_jobs_for_interfaces.archive + ) + submission.phase.save() + submission.phase.algorithm_interfaces.set( + [ + archive_items_and_jobs_for_interfaces.interface1, + archive_items_and_jobs_for_interfaces.interface2, + ] + ) + + eval_alg = EvaluationFactory(submission=submission, time_limit=10) assert not eval_alg.inputs_complete - for inpt, output in zip( - self.input_civs, self.output_civs, strict=True - ): - j_irrelevant = AlgorithmJobFactory( - status=Job.SUCCESS, - algorithm_image=AlgorithmImageFactory(), - time_limit=10, - ) - j_irrelevant.inputs.set([inpt]) - j_irrelevant.outputs.set([output]) - del eval_alg.algorithm_inputs - del eval_alg.successful_jobs + # create 2 jobs per interface, for each of the archive items + j1, j2 = AlgorithmJobFactory.create_batch( + 2, + status=Job.SUCCESS, + creator=None, + algorithm_image=AlgorithmImageFactory(), + algorithm_interface=archive_items_and_jobs_for_interfaces.interface1, + time_limit=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm.time_limit, + ) + j1.inputs.set( + [archive_items_and_jobs_for_interfaces.civs_for_interface1[0]] + ) + j2.inputs.set( + [archive_items_and_jobs_for_interfaces.civs_for_interface1[1]] + ) + + # create 2 jobs per interface, for each of the archive items + j3, j4 = AlgorithmJobFactory.create_batch( + 2, + status=Job.SUCCESS, + creator=None, + algorithm_image=AlgorithmImageFactory(), + algorithm_interface=archive_items_and_jobs_for_interfaces.interface2, + time_limit=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm.time_limit, + ) + j3.inputs.set( + archive_items_and_jobs_for_interfaces.civs_for_interface2[0] + ) + j4.inputs.set( + archive_items_and_jobs_for_interfaces.civs_for_interface2[1] + ) + + del eval_alg.successful_jobs_per_interface + del eval_alg.successful_job_count_per_interface + del eval_alg.total_successful_jobs del eval_alg.inputs_complete assert not eval_alg.inputs_complete def test_successful_job_with_model_ignored_for_submission_without_model( - self, + self, archive_items_and_jobs_for_interfaces ): - eval_alg = EvaluationFactory(submission=self.submission, time_limit=10) + submission = SubmissionFactory( + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image + ) + submission.phase.archive = ( + archive_items_and_jobs_for_interfaces.archive + ) + submission.phase.save() + submission.phase.algorithm_interfaces.set( + [ + archive_items_and_jobs_for_interfaces.interface1, + archive_items_and_jobs_for_interfaces.interface2, + ] + ) + + eval_alg = EvaluationFactory(submission=submission, time_limit=10) assert not eval_alg.inputs_complete - for inpt, output in zip( - self.input_civs, self.output_civs, strict=True - ): - j_irrelevant = AlgorithmJobFactory( - status=Job.SUCCESS, - algorithm_image=self.algorithm_image, - algorithm_model=AlgorithmModelFactory(), - time_limit=10, - ) - j_irrelevant.inputs.set([inpt]) - j_irrelevant.outputs.set([output]) - del eval_alg.algorithm_inputs - del eval_alg.successful_jobs + # create 2 jobs per interface, for each of the archive items + j1, j2 = AlgorithmJobFactory.create_batch( + 2, + status=Job.SUCCESS, + creator=None, + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_model=AlgorithmModelFactory(), + algorithm_interface=archive_items_and_jobs_for_interfaces.interface1, + time_limit=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm.time_limit, + ) + j1.inputs.set( + [archive_items_and_jobs_for_interfaces.civs_for_interface1[0]] + ) + j2.inputs.set( + [archive_items_and_jobs_for_interfaces.civs_for_interface1[1]] + ) + + # create 2 jobs per interface, for each of the archive items + j3, j4 = AlgorithmJobFactory.create_batch( + 2, + status=Job.SUCCESS, + creator=None, + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_model=AlgorithmModelFactory(), + algorithm_interface=archive_items_and_jobs_for_interfaces.interface2, + time_limit=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm.time_limit, + ) + j3.inputs.set( + archive_items_and_jobs_for_interfaces.civs_for_interface2[0] + ) + j4.inputs.set( + archive_items_and_jobs_for_interfaces.civs_for_interface2[1] + ) + + del eval_alg.successful_jobs_per_interface + del eval_alg.successful_job_count_per_interface + del eval_alg.total_successful_jobs del eval_alg.inputs_complete assert not eval_alg.inputs_complete - def test_jobs_with_different_model_ignored_for_submission_with_model(self): - eval_alg = EvaluationFactory( - submission=self.submission_with_model, time_limit=10 + def test_jobs_with_different_model_ignored_for_submission_with_model( + self, archive_items_and_jobs_for_interfaces + ): + algorithm_model = AlgorithmModelFactory( + algorithm=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm + ) + submission = SubmissionFactory( + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_model=algorithm_model, + ) + submission.phase.archive = ( + archive_items_and_jobs_for_interfaces.archive + ) + submission.phase.save() + submission.phase.algorithm_interfaces.set( + [ + archive_items_and_jobs_for_interfaces.interface1, + archive_items_and_jobs_for_interfaces.interface2, + ] ) + + eval_alg = EvaluationFactory(submission=submission, time_limit=10) assert not eval_alg.inputs_complete - for inpt, output in zip( - self.input_civs, self.output_civs, strict=True - ): - j_with_model = AlgorithmJobFactory( - status=Job.SUCCESS, - algorithm_image=eval_alg.submission.algorithm_image, - algorithm_model=AlgorithmModelFactory( - algorithm=eval_alg.submission.algorithm_image.algorithm - ), - time_limit=10, - ) - j_with_model.inputs.set([inpt]) - j_with_model.outputs.set([output]) - j_with_model.creator = None - j_with_model.save() - del eval_alg.algorithm_inputs - del eval_alg.successful_jobs + # create 2 jobs per interface, for each of the archive items + j1, j2 = AlgorithmJobFactory.create_batch( + 2, + status=Job.SUCCESS, + creator=None, + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_model=AlgorithmModelFactory( + algorithm=eval_alg.submission.algorithm_image.algorithm + ), + algorithm_interface=archive_items_and_jobs_for_interfaces.interface1, + time_limit=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm.time_limit, + ) + j1.inputs.set( + [archive_items_and_jobs_for_interfaces.civs_for_interface1[0]] + ) + j2.inputs.set( + [archive_items_and_jobs_for_interfaces.civs_for_interface1[1]] + ) + + # create 2 jobs per interface, for each of the archive items + j3, j4 = AlgorithmJobFactory.create_batch( + 2, + status=Job.SUCCESS, + creator=None, + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_model=AlgorithmModelFactory( + algorithm=eval_alg.submission.algorithm_image.algorithm + ), + algorithm_interface=archive_items_and_jobs_for_interfaces.interface2, + time_limit=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm.time_limit, + ) + j3.inputs.set( + archive_items_and_jobs_for_interfaces.civs_for_interface2[0] + ) + j4.inputs.set( + archive_items_and_jobs_for_interfaces.civs_for_interface2[1] + ) + + del eval_alg.successful_jobs_per_interface + del eval_alg.successful_job_count_per_interface + del eval_alg.total_successful_jobs del eval_alg.inputs_complete assert not eval_alg.inputs_complete - def test_jobs_without_model_ignored_for_submission_with_model(self): - eval_alg = EvaluationFactory( - submission=self.submission_with_model, time_limit=10 + def test_jobs_without_model_ignored_for_submission_with_model( + self, archive_items_and_jobs_for_interfaces + ): + algorithm_model = AlgorithmModelFactory( + algorithm=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm + ) + submission = SubmissionFactory( + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_model=algorithm_model, + ) + submission.phase.archive = ( + archive_items_and_jobs_for_interfaces.archive ) + submission.phase.save() + submission.phase.algorithm_interfaces.set( + [ + archive_items_and_jobs_for_interfaces.interface1, + archive_items_and_jobs_for_interfaces.interface2, + ] + ) + + eval_alg = EvaluationFactory(submission=submission, time_limit=10) assert not eval_alg.inputs_complete - for inpt, output in zip( - self.input_civs, self.output_civs, strict=True - ): - j_with_model = AlgorithmJobFactory( - status=Job.SUCCESS, - algorithm_image=eval_alg.submission.algorithm_image, - time_limit=10, - ) - j_with_model.inputs.set([inpt]) - j_with_model.outputs.set([output]) - j_with_model.creator = None - j_with_model.save() - del eval_alg.algorithm_inputs - del eval_alg.successful_jobs + # create 2 jobs per interface, for each of the archive items + j1, j2 = AlgorithmJobFactory.create_batch( + 2, + status=Job.SUCCESS, + creator=None, + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_interface=archive_items_and_jobs_for_interfaces.interface1, + time_limit=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm.time_limit, + ) + j1.inputs.set( + [archive_items_and_jobs_for_interfaces.civs_for_interface1[0]] + ) + j2.inputs.set( + [archive_items_and_jobs_for_interfaces.civs_for_interface1[1]] + ) + + # create 2 jobs per interface, for each of the archive items + j3, j4 = AlgorithmJobFactory.create_batch( + 2, + status=Job.SUCCESS, + creator=None, + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_interface=archive_items_and_jobs_for_interfaces.interface2, + time_limit=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm.time_limit, + ) + j3.inputs.set( + archive_items_and_jobs_for_interfaces.civs_for_interface2[0] + ) + j4.inputs.set( + archive_items_and_jobs_for_interfaces.civs_for_interface2[1] + ) + + del eval_alg.successful_jobs_per_interface + del eval_alg.successful_job_count_per_interface + del eval_alg.total_successful_jobs del eval_alg.inputs_complete assert not eval_alg.inputs_complete @@ -1451,3 +1847,135 @@ def test_algorithm_requires_memory_unchangable(): submission.save() assert "algorithm_requires_memory_gb cannot be changed" in str(error) + + +@pytest.mark.django_db +def test_archive_item_matching_to_interfaces(): + phase = PhaseFactory(submission_kind=SubmissionKindChoices.ALGORITHM) + + archive = ArchiveFactory() + phase.archive = archive + phase.save() + + i1, i2, i3, i4 = ArchiveItemFactory.create_batch(4, archive=archive) + + ci1, ci2, ci3, ci4 = ComponentInterfaceFactory.create_batch(4) + interface1 = AlgorithmInterfaceFactory(inputs=[ci1]) + interface2 = AlgorithmInterfaceFactory(inputs=[ci1, ci2]) + interface3 = AlgorithmInterfaceFactory(inputs=[ci2, ci3, ci4]) + + i1.values.add( + ComponentInterfaceValueFactory(interface=ci1) + ) # Valid for interface 1 + i2.values.set( + [ + ComponentInterfaceValueFactory(interface=ci1), + ComponentInterfaceValueFactory(interface=ci2), + ] + ) # valid for interface 2 + i3.values.set( + [ + ComponentInterfaceValueFactory(interface=ci1), + ComponentInterfaceValueFactory( + interface=ComponentInterfaceFactory() + ), + ] + ) # valid for no interface, because of additional / mismatching interface + i4.values.set( + [ + ComponentInterfaceValueFactory(interface=ci2), + ComponentInterfaceValueFactory( + interface=ComponentInterfaceFactory() + ), + ] + ) # valid for no interface, because of additional / mismatching interface + + phase.algorithm_interfaces.set([interface1]) + assert phase.valid_archive_items_per_interface.keys() == {interface1} + assert phase.valid_archive_items_per_interface[interface1].get() == i1 + assert phase.valid_archive_item_count_per_interface == {interface1: 1} + assert phase.jobs_to_schedule_per_submission == 1 + + del phase.valid_archive_items_per_interface + del phase.valid_archive_item_count_per_interface + del phase.jobs_to_schedule_per_submission + phase.algorithm_interfaces.set([interface2]) + assert phase.valid_archive_items_per_interface.keys() == {interface2} + assert phase.valid_archive_items_per_interface[interface2].get() == i2 + assert phase.valid_archive_item_count_per_interface == {interface2: 1} + assert phase.jobs_to_schedule_per_submission == 1 + + del phase.valid_archive_items_per_interface + del phase.valid_archive_item_count_per_interface + del phase.jobs_to_schedule_per_submission + phase.algorithm_interfaces.set([interface3]) + assert phase.valid_archive_items_per_interface.keys() == {interface3} + assert not phase.valid_archive_items_per_interface[interface3].exists() + assert phase.valid_archive_item_count_per_interface == {interface3: 0} + assert phase.jobs_to_schedule_per_submission == 0 + + del phase.valid_archive_items_per_interface + del phase.valid_archive_item_count_per_interface + del phase.jobs_to_schedule_per_submission + phase.algorithm_interfaces.set([interface1, interface3]) + assert phase.valid_archive_items_per_interface.keys() == { + interface1, + interface3, + } + assert phase.valid_archive_items_per_interface[interface1].get() == i1 + assert not phase.valid_archive_items_per_interface[interface3].exists() + assert phase.valid_archive_item_count_per_interface == { + interface1: 1, + interface3: 0, + } + assert phase.jobs_to_schedule_per_submission == 1 + + del phase.valid_archive_items_per_interface + del phase.valid_archive_item_count_per_interface + del phase.jobs_to_schedule_per_submission + phase.algorithm_interfaces.set([interface1, interface2, interface3]) + assert phase.valid_archive_items_per_interface.keys() == { + interface1, + interface2, + interface3, + } + assert phase.valid_archive_items_per_interface[interface1].get() == i1 + assert phase.valid_archive_items_per_interface[interface2].get() == i2 + assert not phase.valid_archive_items_per_interface[interface3].exists() + assert phase.valid_archive_item_count_per_interface == { + interface1: 1, + interface2: 1, + interface3: 0, + } + assert phase.jobs_to_schedule_per_submission == 2 + + +@pytest.mark.django_db +def test_get_valid_jobs_for_interfaces_and_archive_items( + archive_items_and_jobs_for_interfaces, +): + + valid_job_inputs = get_archive_items_for_interfaces( + algorithm_interfaces=[ + archive_items_and_jobs_for_interfaces.interface1, + archive_items_and_jobs_for_interfaces.interface2, + ], + archive_items=ArchiveItem.objects.all(), + ) + + jobs_per_interface = get_valid_jobs_for_interfaces_and_archive_items( + algorithm_interfaces=[ + archive_items_and_jobs_for_interfaces.interface1, + archive_items_and_jobs_for_interfaces.interface2, + ], + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + valid_archive_items_per_interface=valid_job_inputs, + ) + assert jobs_per_interface == { + archive_items_and_jobs_for_interfaces.interface1: [ + archive_items_and_jobs_for_interfaces.jobs_for_interface1[0] + ], + archive_items_and_jobs_for_interfaces.interface2: [ + archive_items_and_jobs_for_interfaces.jobs_for_interface2[0] + ], + } diff --git a/app/tests/evaluation_tests/test_tasks.py b/app/tests/evaluation_tests/test_tasks.py index bbbf233ab8..3493fe6e93 100644 --- a/app/tests/evaluation_tests/test_tasks.py +++ b/app/tests/evaluation_tests/test_tasks.py @@ -6,12 +6,11 @@ from actstream.actions import unfollow from django.conf import settings from django.core.cache import cache -from django.test import TestCase from django.utils.html import format_html from redis.exceptions import LockError from grandchallenge.algorithms.models import Job -from grandchallenge.components.models import ComponentInterface, InterfaceKind +from grandchallenge.components.models import InterfaceKind from grandchallenge.components.tasks import ( push_container_image, validate_docker_image, @@ -26,6 +25,7 @@ from grandchallenge.profiles.templatetags.profiles import user_profile_link from tests.algorithms_tests.factories import ( AlgorithmImageFactory, + AlgorithmInterfaceFactory, AlgorithmJobFactory, AlgorithmModelFactory, ) @@ -222,222 +222,287 @@ def test_method_validation_not_a_docker_tar(submission_file): assert method.status == "Could not decompress the container image file." -class TestSetEvaluationInputs(TestCase): - def setUp(self): - interface = ComponentInterface.objects.get( - slug="generic-medical-image" - ) - - archive = ArchiveFactory() - archive_items = ArchiveItemFactory.create_batch(2) - archive.items.set(archive_items) - - input_civs = ComponentInterfaceValueFactory.create_batch( - 2, interface=interface - ) - output_civs = ComponentInterfaceValueFactory.create_batch( - 2, interface=interface - ) - - for ai, civ in zip(archive_items, input_civs, strict=True): - ai.values.set([civ]) - - algorithm_image = AlgorithmImageFactory() - algorithm_model = AlgorithmModelFactory() - submission = SubmissionFactory(algorithm_image=algorithm_image) - submission.phase.archive = archive - submission.phase.save() - submission.phase.algorithm_inputs.set([interface]) - - submission_with_model = SubmissionFactory( - algorithm_image=algorithm_image, algorithm_model=algorithm_model - ) - submission_with_model.phase.archive = archive - submission_with_model.phase.save() - submission_with_model.phase.algorithm_inputs.set([interface]) - - jobs = [] - for inpt, output in zip(input_civs, output_civs, strict=True): - j = AlgorithmJobFactory( - status=Job.SUCCESS, - algorithm_image=algorithm_image, - time_limit=algorithm_image.algorithm.time_limit, - ) - j.inputs.set([inpt]) - j.outputs.set([output]) - j.creator = None - j.save() - jobs.append(j) - - # also create a job with the same input but a creator set, this job - # should be ignored - j_irrelevant = AlgorithmJobFactory( - status=Job.SUCCESS, - algorithm_image=algorithm_image, - creator=algorithm_image.creator, - time_limit=algorithm_image.algorithm.time_limit, - ) - j_irrelevant.inputs.set([input_civs[0]]) - j_irrelevant.outputs.set([output_civs[0]]) - - self.evaluation = EvaluationFactory( - submission=submission, - status=Evaluation.EXECUTING_PREREQUISITES, - time_limit=submission.phase.evaluation_time_limit, - ) - self.evaluation_with_model = EvaluationFactory( - submission=submission_with_model, +@pytest.mark.django_db +class TestSetEvaluationInputs: + def test_set_evaluation_inputs( + self, submission_without_model_for_optional_inputs + ): + eval = EvaluationFactory( + submission=submission_without_model_for_optional_inputs.submission, status=Evaluation.EXECUTING_PREREQUISITES, - time_limit=submission.phase.evaluation_time_limit, + time_limit=submission_without_model_for_optional_inputs.submission.phase.evaluation_time_limit, ) - self.input_civs = input_civs - self.output_civs = output_civs - self.jobs = jobs - self.output_civs = output_civs - - def test_set_evaluation_inputs(self): - set_evaluation_inputs(evaluation_pk=self.evaluation.pk) + set_evaluation_inputs(evaluation_pk=eval.pk) - self.evaluation.refresh_from_db() - assert self.evaluation.status == self.evaluation.PENDING - assert self.evaluation.error_message == "" - assert self.evaluation.inputs.count() == 3 - assert self.evaluation.input_prefixes == { + eval.refresh_from_db() + assert eval.status == eval.PENDING + assert eval.error_message == "" + assert eval.inputs.count() == 5 + assert eval.input_prefixes == { str(civ.pk): f"{alg.pk}/output/" - for alg, civ in zip(self.jobs, self.output_civs, strict=True) + for alg, civ in zip( + submission_without_model_for_optional_inputs.jobs, + submission_without_model_for_optional_inputs.output_civs, + strict=True, + ) } - def test_has_pending_jobs(self): + def test_has_pending_jobs( + self, submission_without_model_for_optional_inputs + ): + eval = EvaluationFactory( + submission=submission_without_model_for_optional_inputs.submission, + status=Evaluation.EXECUTING_PREREQUISITES, + time_limit=submission_without_model_for_optional_inputs.submission.phase.evaluation_time_limit, + ) AlgorithmJobFactory( status=Job.PENDING, creator=None, - algorithm_image=self.evaluation.submission.algorithm_image, - time_limit=self.evaluation.submission.algorithm_image.algorithm.time_limit, + algorithm_image=eval.submission.algorithm_image, + algorithm_interface=submission_without_model_for_optional_inputs.interface1, + time_limit=eval.submission.algorithm_image.algorithm.time_limit, ) # nothing happens because there are pending jobs - set_evaluation_inputs(evaluation_pk=self.evaluation.pk) - self.evaluation.refresh_from_db() - assert ( - self.evaluation.status == self.evaluation.EXECUTING_PREREQUISITES + set_evaluation_inputs(evaluation_pk=eval.pk) + eval.refresh_from_db() + assert eval.status == eval.EXECUTING_PREREQUISITES + assert eval.inputs.count() == 0 + assert eval.input_prefixes == {} + + def test_has_pending_jobs_with_image_and_model( + self, submission_with_model_for_optional_inputs + ): + evaluation_with_model = EvaluationFactory( + submission=submission_with_model_for_optional_inputs.submission, + status=Evaluation.EXECUTING_PREREQUISITES, + time_limit=submission_with_model_for_optional_inputs.submission.phase.evaluation_time_limit, ) - assert self.evaluation.inputs.count() == 0 - assert self.evaluation.input_prefixes == {} - - def test_has_pending_jobs_with_image_and_model(self): AlgorithmJobFactory( status=Job.PENDING, creator=None, - algorithm_image=self.evaluation_with_model.submission.algorithm_image, - algorithm_model=self.evaluation_with_model.submission.algorithm_model, - time_limit=self.evaluation_with_model.submission.algorithm_image.algorithm.time_limit, + algorithm_image=evaluation_with_model.submission.algorithm_image, + algorithm_model=evaluation_with_model.submission.algorithm_model, + algorithm_interface=submission_with_model_for_optional_inputs.interface1, + time_limit=evaluation_with_model.submission.algorithm_image.algorithm.time_limit, ) # nothing happens - set_evaluation_inputs(evaluation_pk=self.evaluation_with_model.pk) - self.evaluation_with_model.refresh_from_db() + set_evaluation_inputs(evaluation_pk=evaluation_with_model.pk) + evaluation_with_model.refresh_from_db() assert ( - self.evaluation_with_model.status - == self.evaluation_with_model.EXECUTING_PREREQUISITES + evaluation_with_model.status + == evaluation_with_model.EXECUTING_PREREQUISITES + ) + assert evaluation_with_model.inputs.count() == 0 + assert evaluation_with_model.input_prefixes == {} + + def test_has_pending_jobs_with_image_but_without_model( + self, submission_with_model_for_optional_inputs + ): + evaluation_with_model = EvaluationFactory( + submission=submission_with_model_for_optional_inputs.submission, + status=Evaluation.EXECUTING_PREREQUISITES, + time_limit=submission_with_model_for_optional_inputs.submission.phase.evaluation_time_limit, ) - assert self.evaluation_with_model.inputs.count() == 0 - assert self.evaluation_with_model.input_prefixes == {} - - def test_has_pending_jobs_with_image_but_without_model(self): # the pending job with only the image will be ignored, but task will still # fail because there are no successful jobs with both the image and model yet AlgorithmJobFactory( status=Job.PENDING, creator=None, - algorithm_image=self.evaluation_with_model.submission.algorithm_image, - time_limit=self.evaluation_with_model.submission.algorithm_image.algorithm.time_limit, + algorithm_image=evaluation_with_model.submission.algorithm_image, + algorithm_interface=submission_with_model_for_optional_inputs.interface1, + time_limit=evaluation_with_model.submission.algorithm_image.algorithm.time_limit, ) - set_evaluation_inputs(evaluation_pk=self.evaluation_with_model.pk) - self.evaluation_with_model.refresh_from_db() + set_evaluation_inputs(evaluation_pk=evaluation_with_model.pk) + evaluation_with_model.refresh_from_db() assert ( - self.evaluation_with_model.status - == self.evaluation_with_model.EXECUTING_PREREQUISITES - ) - assert self.evaluation_with_model.inputs.count() == 0 - assert self.evaluation_with_model.input_prefixes == {} - - # add jobs - jobs = [] - for inpt, output in zip( - self.input_civs, self.output_civs, strict=True - ): - j_with_model = AlgorithmJobFactory( - status=Job.SUCCESS, - algorithm_image=self.evaluation_with_model.submission.algorithm_image, - algorithm_model=self.evaluation_with_model.submission.algorithm_model, - time_limit=self.evaluation_with_model.submission.algorithm_image.algorithm.time_limit, - ) - j_with_model.inputs.set([inpt]) - j_with_model.outputs.set([output]) - j_with_model.creator = None - j_with_model.save() - jobs.append(j_with_model) - - set_evaluation_inputs(evaluation_pk=self.evaluation_with_model.pk) - self.evaluation_with_model.refresh_from_db() - assert ( - self.evaluation_with_model.status - == self.evaluation_with_model.PENDING + evaluation_with_model.status + == evaluation_with_model.EXECUTING_PREREQUISITES + ) + assert evaluation_with_model.inputs.count() == 0 + assert evaluation_with_model.input_prefixes == {} + + # add jobs, 2 for each interface with a model + j_with_model_1 = AlgorithmJobFactory( + status=Job.SUCCESS, + creator=None, + algorithm_image=evaluation_with_model.submission.algorithm_image, + algorithm_model=evaluation_with_model.submission.algorithm_model, + algorithm_interface=submission_with_model_for_optional_inputs.interface1, + time_limit=evaluation_with_model.submission.algorithm_image.algorithm.time_limit, + ) + j_with_model_2 = AlgorithmJobFactory( + status=Job.SUCCESS, + creator=None, + algorithm_image=evaluation_with_model.submission.algorithm_image, + algorithm_model=evaluation_with_model.submission.algorithm_model, + algorithm_interface=submission_with_model_for_optional_inputs.interface1, + time_limit=evaluation_with_model.submission.algorithm_image.algorithm.time_limit, + ) + j_with_model_1.inputs.set( + submission_with_model_for_optional_inputs.jobs[0].inputs.all() + ) + j_with_model_1.outputs.set( + submission_with_model_for_optional_inputs.jobs[0].outputs.all() + ) + j_with_model_2.inputs.set( + submission_with_model_for_optional_inputs.jobs[1].inputs.all() + ) + j_with_model_2.outputs.set( + submission_with_model_for_optional_inputs.jobs[1].outputs.all() + ) + + j_with_model_3 = AlgorithmJobFactory( + status=Job.SUCCESS, + creator=None, + algorithm_image=evaluation_with_model.submission.algorithm_image, + algorithm_model=evaluation_with_model.submission.algorithm_model, + algorithm_interface=submission_with_model_for_optional_inputs.interface2, + time_limit=evaluation_with_model.submission.algorithm_image.algorithm.time_limit, + ) + j_with_model_4 = AlgorithmJobFactory( + status=Job.SUCCESS, + creator=None, + algorithm_image=evaluation_with_model.submission.algorithm_image, + algorithm_model=evaluation_with_model.submission.algorithm_model, + algorithm_interface=submission_with_model_for_optional_inputs.interface2, + time_limit=evaluation_with_model.submission.algorithm_image.algorithm.time_limit, + ) + j_with_model_3.inputs.set( + submission_with_model_for_optional_inputs.jobs[2].inputs.all() ) - assert self.evaluation_with_model.inputs.count() == 3 - assert self.evaluation_with_model.input_prefixes == { + j_with_model_3.outputs.set( + submission_with_model_for_optional_inputs.jobs[2].outputs.all() + ) + j_with_model_4.inputs.set( + submission_with_model_for_optional_inputs.jobs[3].inputs.all() + ) + j_with_model_4.outputs.set( + submission_with_model_for_optional_inputs.jobs[3].outputs.all() + ) + + jobs = [j_with_model_1, j_with_model_2, j_with_model_3, j_with_model_4] + + set_evaluation_inputs(evaluation_pk=evaluation_with_model.pk) + evaluation_with_model.refresh_from_db() + assert evaluation_with_model.status == evaluation_with_model.PENDING + assert evaluation_with_model.inputs.count() == 5 + assert evaluation_with_model.input_prefixes == { str(civ.pk): f"{alg.pk}/output/" - for alg, civ in zip(jobs, self.output_civs, strict=True) + for alg, civ in zip( + jobs, + submission_with_model_for_optional_inputs.output_civs, + strict=True, + ) } - def test_has_pending_jobs_without_active_model(self): + def test_has_pending_jobs_without_active_model( + self, submission_without_model_for_optional_inputs + ): + eval = EvaluationFactory( + submission=submission_without_model_for_optional_inputs.submission, + status=Evaluation.EXECUTING_PREREQUISITES, + time_limit=submission_without_model_for_optional_inputs.submission.phase.evaluation_time_limit, + ) AlgorithmJobFactory( status=Job.PENDING, creator=None, - algorithm_image=self.evaluation.submission.algorithm_image, + algorithm_image=eval.submission.algorithm_image, algorithm_model=AlgorithmModelFactory( - algorithm=self.evaluation.submission.algorithm_image.algorithm + algorithm=eval.submission.algorithm_image.algorithm ), - time_limit=self.evaluation.submission.algorithm_image.algorithm.time_limit, + algorithm_interface=submission_without_model_for_optional_inputs.interface1, + time_limit=eval.submission.algorithm_image.algorithm.time_limit, ) # pending job for image with model should be ignored, # since active_model is None for the evaluation - set_evaluation_inputs(evaluation_pk=self.evaluation.pk) - self.evaluation.refresh_from_db() - assert self.evaluation.status == self.evaluation.PENDING - assert self.evaluation.inputs.count() == 3 - assert self.evaluation.input_prefixes == { + set_evaluation_inputs(evaluation_pk=eval.pk) + eval.refresh_from_db() + assert eval.status == eval.PENDING + assert eval.inputs.count() == 5 + assert eval.input_prefixes == { str(civ.pk): f"{alg.pk}/output/" - for alg, civ in zip(self.jobs, self.output_civs, strict=True) + for alg, civ in zip( + submission_without_model_for_optional_inputs.jobs, + submission_without_model_for_optional_inputs.output_civs, + strict=True, + ) } - def test_successful_jobs_without_model(self): + def test_successful_jobs_without_model( + self, submission_without_model_for_optional_inputs + ): # delete jobs created in setup(), # create new ones with a model, these should not count # towards successful jobs, and eval inputs should not be set Job.objects.all().delete() - jobs = [] - for inpt, output in zip( - self.input_civs, self.output_civs, strict=True - ): - j_with_model = AlgorithmJobFactory( - status=Job.SUCCESS, - algorithm_image=self.evaluation.submission.algorithm_image, - algorithm_model=AlgorithmModelFactory( - algorithm=self.evaluation.submission.algorithm_image.algorithm - ), - time_limit=self.evaluation.submission.algorithm_image.algorithm.time_limit, - ) - j_with_model.inputs.set([inpt]) - j_with_model.outputs.set([output]) - j_with_model.creator = None - j_with_model.save() - jobs.append(j_with_model) - set_evaluation_inputs(evaluation_pk=self.evaluation.pk) - self.evaluation.refresh_from_db() - assert ( - self.evaluation.status == self.evaluation.EXECUTING_PREREQUISITES + + eval = EvaluationFactory( + submission=submission_without_model_for_optional_inputs.submission, + status=Evaluation.EXECUTING_PREREQUISITES, + time_limit=submission_without_model_for_optional_inputs.submission.phase.evaluation_time_limit, + ) + + # create 2 jobs per interface, for each of the archive items + j1, j2 = AlgorithmJobFactory.create_batch( + 2, + creator=None, + algorithm_image=submission_without_model_for_optional_inputs.submission.algorithm_image, + algorithm_interface=submission_without_model_for_optional_inputs.interface1, + algorithm_model=AlgorithmModelFactory( + algorithm=eval.submission.algorithm_image.algorithm + ), + time_limit=submission_without_model_for_optional_inputs.submission.algorithm_image.algorithm.time_limit, + ) + j1.inputs.set( + [ + submission_without_model_for_optional_inputs.civs_for_interface1[ + 0 + ] + ] + ) + j1.outputs.set( + [submission_without_model_for_optional_inputs.output_civs[0]] + ) + j2.inputs.set( + [ + submission_without_model_for_optional_inputs.civs_for_interface1[ + 1 + ] + ] + ) + j2.outputs.set( + [submission_without_model_for_optional_inputs.output_civs[1]] ) - assert self.evaluation.inputs.count() == 0 - assert self.evaluation.input_prefixes == {} + + # create 2 jobs per interface, for each of the archive items + j3, j4 = AlgorithmJobFactory.create_batch( + 2, + creator=None, + algorithm_image=submission_without_model_for_optional_inputs.submission.algorithm_image, + algorithm_model=AlgorithmModelFactory( + algorithm=submission_without_model_for_optional_inputs.submission.algorithm_image.algorithm + ), + algorithm_interface=submission_without_model_for_optional_inputs.interface2, + time_limit=submission_without_model_for_optional_inputs.submission.algorithm_image.algorithm.time_limit, + ) + j3.inputs.set( + submission_without_model_for_optional_inputs.civs_for_interface2[0] + ) + j3.outputs.set( + [submission_without_model_for_optional_inputs.output_civs[2]] + ) + j4.inputs.set( + submission_without_model_for_optional_inputs.civs_for_interface2[1] + ) + j4.outputs.set( + [submission_without_model_for_optional_inputs.output_civs[3]] + ) + + set_evaluation_inputs(evaluation_pk=eval.pk) + eval.refresh_from_db() + assert eval.status == eval.EXECUTING_PREREQUISITES + assert eval.inputs.count() == 0 + assert eval.input_prefixes == {} @pytest.mark.django_db @@ -726,22 +791,18 @@ def test_evaluation_order_with_title(): time_limit=ai.algorithm.time_limit, ) - input_interface = ComponentInterfaceFactory( + input_ci = ComponentInterfaceFactory( kind=InterfaceKind.InterfaceKindChoices.BOOL ) - - evaluation.submission.phase.algorithm_inputs.set([input_interface]) - ai.algorithm.inputs.set([input_interface]) + interface = AlgorithmInterfaceFactory(inputs=[input_ci]) + ai.algorithm.interfaces.set([interface]) + evaluation.submission.phase.algorithm_interfaces.set([interface]) # Priority should be given to archive items with titles archive_item = ArchiveItemFactory(archive=archive) - archive_item.values.add( - ComponentInterfaceValueFactory(interface=input_interface) - ) + archive_item.values.add(ComponentInterfaceValueFactory(interface=input_ci)) - civs = ComponentInterfaceValueFactory.create_batch( - 5, interface=input_interface - ) + civs = ComponentInterfaceValueFactory.create_batch(5, interface=input_ci) for idx, civ in enumerate(civs): archive_item = ArchiveItemFactory(archive=archive, title=f"{5 - idx}") @@ -767,16 +828,15 @@ def test_evaluation_order_without_title(): time_limit=ai.algorithm.time_limit, ) - input_interface = ComponentInterfaceFactory( + input_ci = ComponentInterfaceFactory( kind=InterfaceKind.InterfaceKindChoices.BOOL ) + interface = AlgorithmInterfaceFactory(inputs=[input_ci]) + ai.algorithm.interfaces.set([interface]) - evaluation.submission.phase.algorithm_inputs.set([input_interface]) - ai.algorithm.inputs.set([input_interface]) + evaluation.submission.phase.algorithm_interfaces.set([interface]) - civs = ComponentInterfaceValueFactory.create_batch( - 5, interface=input_interface - ) + civs = ComponentInterfaceValueFactory.create_batch(5, interface=input_ci) for civ in civs: archive_item = ArchiveItemFactory(archive=archive) diff --git a/app/tests/evaluation_tests/test_views.py b/app/tests/evaluation_tests/test_views.py index 7a8ed10777..fa7cc84d0e 100644 --- a/app/tests/evaluation_tests/test_views.py +++ b/app/tests/evaluation_tests/test_views.py @@ -708,8 +708,8 @@ def test_create_algorithm_for_phase_permission(client, uploaded_image): phase.submission_kind = SubmissionKindChoices.ALGORITHM phase.creator_must_be_verified = True phase.archive = ArchiveFactory() - phase.algorithm_inputs.set([ComponentInterfaceFactory()]) - phase.algorithm_outputs.set([ComponentInterfaceFactory()]) + interface = AlgorithmInterfaceFactory() + phase.algorithm_interfaces.set([interface]) phase.save() response = get_view_for_user( @@ -814,8 +814,9 @@ def test_create_algorithm_for_phase_presets(client): ci1 = ComponentInterfaceFactory(kind=InterfaceKindChoices.STRING) ci2 = ComponentInterfaceFactory(kind=InterfaceKindChoices.STRING) optional_protocol = HangingProtocolFactory() - phase.algorithm_inputs.set([ci1]) - phase.algorithm_outputs.set([ci2]) + + interface1, interface2 = AlgorithmInterfaceFactory.create_batch(2) + phase.algorithm_interfaces.set([interface1, interface2]) phase.hanging_protocol = HangingProtocolFactory( json=[{"viewport_name": "main"}] ) @@ -834,8 +835,10 @@ def test_create_algorithm_for_phase_presets(client): client=client, user=admin, ) - assert response.context_data["form"]["inputs"].initial.get() == ci1 - assert response.context_data["form"]["outputs"].initial.get() == ci2 + assert list(response.context_data["form"]["interfaces"].initial.all()) == [ + interface1, + interface2, + ] assert response.context_data["form"][ "workstation" ].initial == Workstation.objects.get( @@ -881,11 +884,11 @@ def test_create_algorithm_for_phase_presets(client): data={ "title": "Test algorithm", "job_requires_memory_gb": 8, - "inputs": [ - response.context_data["form"]["inputs"].initial.get().pk - ], - "outputs": [ - response.context_data["form"]["outputs"].initial.get().pk + "interfaces": [ + interface.pk + for interface in response.context_data["form"][ + "interfaces" + ].initial.all() ], "workstation": response.context_data["form"][ "workstation" @@ -911,8 +914,7 @@ def test_create_algorithm_for_phase_presets(client): assert response.status_code == 302 assert Algorithm.objects.count() == 1 algorithm = Algorithm.objects.get() - assert algorithm.inputs.get() == ci1 - assert algorithm.outputs.get() == ci2 + assert list(algorithm.interfaces.all()) == [interface1, interface2] assert algorithm.hanging_protocol == phase.hanging_protocol assert algorithm.optional_hanging_protocols.get() == optional_protocol assert algorithm.workstation_config == phase.workstation_config @@ -957,15 +959,12 @@ def test_create_algorithm_for_phase_presets(client): # created algorithm has the initial values set, not the modified ones alg2 = Algorithm.objects.last() - assert alg2.inputs.get() == ci1 - assert alg2.outputs.get() == ci2 + assert list(alg2.interfaces.all()) == [interface1, interface2] assert alg2.hanging_protocol == phase.hanging_protocol assert alg2.optional_hanging_protocols.get() == optional_protocol assert alg2.workstation_config == phase.workstation_config assert alg2.view_content == phase.view_content assert alg2.workstation.slug == settings.DEFAULT_WORKSTATION_SLUG - assert alg2.inputs.get() != ci3 - assert alg2.outputs.get() != ci4 assert alg2.hanging_protocol != hp assert alg2.workstation_config != wsc assert alg2.view_content != "{}" @@ -982,8 +981,10 @@ def test_create_algorithm_for_phase_limits(client): phase.archive = ArchiveFactory() ci1 = ComponentInterfaceFactory() ci2 = ComponentInterfaceFactory() - phase.algorithm_inputs.set([ci1]) - phase.algorithm_outputs.set([ci2]) + + interface = AlgorithmInterfaceFactory(inputs=[ci1], outputs=[ci2]) + phase.algorithm_interfaces.set([interface]) + phase.submissions_limit_per_user_per_period = 10 phase.save() @@ -1005,14 +1006,16 @@ def test_create_algorithm_for_phase_limits(client): alg3.add_editor(u1) alg4.add_editor(u2) alg5.add_editor(u1) - alg6.add_editor(u1) - for alg in [alg1, alg2, alg3, alg4, alg5]: - alg.inputs.set([ci1]) - alg.outputs.set([ci2]) + alg6.add_editor(u2) + for alg in [alg1, alg2, alg3, alg4]: + alg.interfaces.set([interface]) ci3 = ComponentInterfaceFactory() - alg5.inputs.add(ci3) - alg6.inputs.set([ci3]) - alg6.outputs.set([ci2]) + + interface2 = AlgorithmInterfaceFactory(inputs=[ci1, ci3], outputs=[ci2]) + alg5.interfaces.set([interface2]) + + interface3 = AlgorithmInterfaceFactory(inputs=[ci3], outputs=[ci2]) + alg6.interfaces.set([interface3]) response = get_view_for_user( viewname="evaluation:phase-algorithm-create", @@ -1581,8 +1584,8 @@ def test_evaluation_details_zero_rank_message(client): @pytest.mark.django_db def test_submission_create_sets_limits_correctly_with_algorithm(client): - inputs = [ComponentInterfaceFactory()] - outputs = [ComponentInterfaceFactory()] + inputs = ComponentInterfaceFactory.create_batch(2) + interface = AlgorithmInterfaceFactory(inputs=inputs) algorithm_image = AlgorithmImageFactory( is_manifest_valid=True, @@ -1591,8 +1594,7 @@ def test_submission_create_sets_limits_correctly_with_algorithm(client): algorithm__job_requires_gpu_type=GPUTypeChoices.V100, algorithm__job_requires_memory_gb=1337, ) - algorithm_image.algorithm.inputs.set(inputs) - algorithm_image.algorithm.outputs.set(outputs) + algorithm_image.algorithm.interfaces.set([interface]) archive = ArchiveFactory() archive_item = ArchiveItemFactory(archive=archive) @@ -1610,8 +1612,7 @@ def test_submission_create_sets_limits_correctly_with_algorithm(client): algorithm_selectable_gpu_type_choices=[GPUTypeChoices.V100], algorithm_maximum_settable_memory_gb=1337, ) - phase.algorithm_inputs.set(inputs) - phase.algorithm_outputs.set(outputs) + phase.algorithm_interfaces.set([interface]) InvoiceFactory( challenge=phase.challenge, From eaa667bc3148b3fc4e8b06194fb9f93af25444c8 Mon Sep 17 00:00:00 2001 From: Anne Mickan Date: Wed, 19 Feb 2025 14:50:37 +0100 Subject: [PATCH 16/18] Remove inputs and outputs fields from Phase and Algorithm models (#3832) Final PR for https://github.com/DIAGNijmegen/rse-roadmap/issues/153 --- app/grandchallenge/algorithms/admin.py | 6 +- app/grandchallenge/algorithms/forms.py | 108 ++++++++++-------- ...lgorithm_inputs_alter_algorithm_outputs.py | 31 +++++ app/grandchallenge/algorithms/models.py | 21 +++- app/grandchallenge/algorithms/serializers.py | 6 - .../templates/algorithms/job_detail.html | 3 +- app/grandchallenge/algorithms/views.py | 5 +- app/grandchallenge/evaluation/admin.py | 2 - ...4_alter_phase_algorithm_inputs_and_more.py | 35 ++++++ app/grandchallenge/evaluation/models.py | 40 ++++--- app/grandchallenge/hanging_protocols/forms.py | 11 +- app/tests/algorithms_tests/test_forms.py | 22 +++- app/tests/algorithms_tests/test_views.py | 107 +---------------- .../core_tests/test_grand_challenge_forge.py | 12 +- app/tests/evaluation_tests/test_views.py | 4 +- .../hanging_protocols_tests/test_forms.py | 48 +++++--- app/tests/serving_tests/test_views.py | 8 +- .../test_session_control.py | 1 + scripts/algorithm_evaluation_fixtures.py | 19 ++- scripts/cost_fixtures.py | 21 +++- scripts/development_fixtures.py | 17 ++- .../external_algorithm_evaluation_fixtures.py | 8 +- scripts/pentest_fixtures.py | 20 +++- 23 files changed, 306 insertions(+), 249 deletions(-) create mode 100644 app/grandchallenge/algorithms/migrations/0067_alter_algorithm_inputs_alter_algorithm_outputs.py create mode 100644 app/grandchallenge/evaluation/migrations/0074_alter_phase_algorithm_inputs_and_more.py diff --git a/app/grandchallenge/algorithms/admin.py b/app/grandchallenge/algorithms/admin.py index b04a7499e8..b504f3e0d8 100644 --- a/app/grandchallenge/algorithms/admin.py +++ b/app/grandchallenge/algorithms/admin.py @@ -51,11 +51,7 @@ class Meta: @admin.register(Algorithm) class AlgorithmAdmin(GuardedModelAdmin): - readonly_fields = ( - "algorithm_forge_json", - "inputs", - "outputs", - ) + readonly_fields = ("algorithm_forge_json",) list_display = ( "title", "created", diff --git a/app/grandchallenge/algorithms/forms.py b/app/grandchallenge/algorithms/forms.py index 7e580e8244..957ca64999 100644 --- a/app/grandchallenge/algorithms/forms.py +++ b/app/grandchallenge/algorithms/forms.py @@ -178,7 +178,7 @@ def __init__(self, *args, algorithm, user, interface, **kwargs): instance=inp, initial=initial if initial else inp.default_value, user=self._user, - required=True, + required=inp.value_required, help_text=clean(inp.description) if inp.description else "", form_data=self.data, ).field @@ -952,7 +952,8 @@ def __init__(self, *args, user, **kwargs): self.algorithm_serializer = None self.algorithm_image_serializer = None self.algorithm = None - self.new_interfaces = None + self.algorithm_interfaces = [] + self.new_component_interfaces = [] @property def remote_instance_client(self): @@ -1058,27 +1059,16 @@ def _build_algorithm_image(self, headers, netloc): self.algorithm_image_serializer = algorithm_image_serializer def _build_interfaces(self): - remote_interfaces = { - interface["slug"]: interface - for interface in chain( - self.algorithm_serializer.initial_data["inputs"], - self.algorithm_serializer.initial_data["outputs"], + for remote_interface in self.algorithm_serializer.initial_data[ + "interfaces" + ]: + self._validate_interface_inputs_and_outputs( + remote_interface=remote_interface ) - } - - self.new_interfaces = [] - for slug, remote_interface in remote_interfaces.items(): - try: - self._validate_existing_interface( - slug=slug, remote_interface=remote_interface - ) - except ObjectDoesNotExist: - # The remote interface does not exist locally, create it - self._create_new_interface( - slug=slug, remote_interface=remote_interface - ) - def _validate_existing_interface(self, *, remote_interface, slug): + def _validate_existing_component_interface( + self, *, remote_component_interface, slug + ): serialized_local_interface = ComponentInterfaceSerializer( instance=ComponentInterface.objects.get(slug=slug) ) @@ -1087,27 +1077,72 @@ def _validate_existing_interface(self, *, remote_interface, slug): # Check all the values match, some are allowed to differ if ( key not in {"pk", "description"} - and value != remote_interface[key] + and value != remote_component_interface[key] ): raise ValidationError( f"Interface {key} does not match for `{slug}`" ) - def _create_new_interface(self, *, remote_interface, slug): - new_interface = ComponentInterfaceSerializer(data=remote_interface) + def _create_new_component_interface( + self, *, remote_component_interface, slug + ): + new_interface = ComponentInterfaceSerializer( + data=remote_component_interface + ) if not new_interface.is_valid(): raise ValidationError(f"New interface {slug!r} is invalid") - self.new_interfaces.append(new_interface) + self.new_component_interfaces.append(new_interface) + + def _validate_interface_inputs_and_outputs(self, *, remote_interface): + for input in remote_interface["inputs"]: + try: + self._validate_existing_component_interface( + remote_component_interface=input, slug=input["slug"] + ) + except ObjectDoesNotExist: + self._create_new_component_interface( + remote_component_interface=input, slug=input["slug"] + ) + + for output in remote_interface["outputs"]: + try: + self._validate_existing_component_interface( + remote_component_interface=output, slug=output["slug"] + ) + except ObjectDoesNotExist: + self._create_new_component_interface( + remote_component_interface=output, slug=output["slug"] + ) def save(self): + # first save new ComponentInterfaces + self._save_new_component_interfaces() + # then get or create algorithm interfaces self._save_new_interfaces() self._save_new_algorithm() self._save_new_algorithm_image() def _save_new_interfaces(self): - for interface in self.new_interfaces: + for interface in self.algorithm_serializer.initial_data["interfaces"]: + inputs = [ + ComponentInterface.objects.get(slug=input["slug"]) + for input in interface["inputs"] + ] + outputs = [ + ComponentInterface.objects.get(slug=output["slug"]) + for output in interface["outputs"] + ] + # either get or create an AlgorithmInterface + # with the given inputs / outputs using the custom model manager + interface = AlgorithmInterface.objects.create( + inputs=inputs, outputs=outputs + ) + self.algorithm_interfaces.append(interface) + + def _save_new_component_interfaces(self): + for interface in self.new_component_interfaces: interface.save( # The interface kind is a read only display value, this could # be better solved with a custom DRF Field but deadlines... @@ -1140,26 +1175,7 @@ def _save_new_algorithm(self): self.algorithm.add_editor(user=self.user) - self.algorithm.inputs.set( - ComponentInterface.objects.filter( - slug__in={ - interface["slug"] - for interface in self.algorithm_serializer.initial_data[ - "inputs" - ] - } - ) - ) - self.algorithm.outputs.set( - ComponentInterface.objects.filter( - slug__in={ - interface["slug"] - for interface in self.algorithm_serializer.initial_data[ - "outputs" - ] - } - ) - ) + self.algorithm.interfaces.set(self.algorithm_interfaces) if logo_url := self.algorithm_serializer.initial_data["logo"]: response = requests.get( diff --git a/app/grandchallenge/algorithms/migrations/0067_alter_algorithm_inputs_alter_algorithm_outputs.py b/app/grandchallenge/algorithms/migrations/0067_alter_algorithm_inputs_alter_algorithm_outputs.py new file mode 100644 index 0000000000..d1326ae490 --- /dev/null +++ b/app/grandchallenge/algorithms/migrations/0067_alter_algorithm_inputs_alter_algorithm_outputs.py @@ -0,0 +1,31 @@ +# Generated by Django 4.2.19 on 2025-02-17 09:48 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("components", "0024_alter_componentinterface_kind_and_more"), + ("algorithms", "0066_create_interfaces_for_jobs"), + ] + + operations = [ + migrations.AlterField( + model_name="algorithm", + name="inputs", + field=models.ManyToManyField( + null=True, + related_name="algorithm_inputs", + to="components.componentinterface", + ), + ), + migrations.AlterField( + model_name="algorithm", + name="outputs", + field=models.ManyToManyField( + null=True, + related_name="algorithm_outputs", + to="components.componentinterface", + ), + ), + ] diff --git a/app/grandchallenge/algorithms/models.py b/app/grandchallenge/algorithms/models.py index e563138fde..2b3e187b8c 100644 --- a/app/grandchallenge/algorithms/models.py +++ b/app/grandchallenge/algorithms/models.py @@ -20,6 +20,7 @@ from django.utils.functional import cached_property from django.utils.text import get_valid_filename from django.utils.timezone import now +from django_deprecate_fields import deprecate_field from django_extensions.db.models import TitleSlugDescriptionModel from guardian.models import GroupObjectPermissionBase, UserObjectPermissionBase from guardian.shortcuts import assign_perm, remove_perm @@ -243,11 +244,17 @@ class Algorithm(UUIDModel, TitleSlugDescriptionModel, HangingProtocolMixin): related_name="algorithm_interfaces", through="algorithms.AlgorithmAlgorithmInterface", ) - inputs = models.ManyToManyField( - to=ComponentInterface, related_name="algorithm_inputs", blank=False + inputs = deprecate_field( + models.ManyToManyField( + to=ComponentInterface, related_name="algorithm_inputs", blank=False + ) ) - outputs = models.ManyToManyField( - to=ComponentInterface, related_name="algorithm_outputs", blank=False + outputs = deprecate_field( + models.ManyToManyField( + to=ComponentInterface, + related_name="algorithm_outputs", + blank=False, + ) ) publications = models.ManyToManyField( Publication, @@ -562,7 +569,11 @@ def remove_user(self, user): @cached_property def linked_component_interfaces(self): - return (self.inputs.all() | self.outputs.all()).distinct() + return { + ci + for interface in self.interfaces.all() + for ci in (interface.inputs.all() | interface.outputs.all()) + } @cached_property def user_statistics(self): diff --git a/app/grandchallenge/algorithms/serializers.py b/app/grandchallenge/algorithms/serializers.py index ac930f13d9..13f81ded78 100644 --- a/app/grandchallenge/algorithms/serializers.py +++ b/app/grandchallenge/algorithms/serializers.py @@ -56,10 +56,6 @@ class Meta: class AlgorithmSerializer(serializers.ModelSerializer): average_duration = SerializerMethodField() - # TODO: Still to be addressed for optional inputs pitch - # remove inputs and outputs - inputs = ComponentInterfaceSerializer(many=True, read_only=True) - outputs = ComponentInterfaceSerializer(many=True, read_only=True) logo = URLField(source="logo.x20.url", read_only=True) url = URLField(source="get_absolute_url", read_only=True) interfaces = AlgorithmInterfaceSerializer(many=True, read_only=True) @@ -75,8 +71,6 @@ class Meta: "logo", "slug", "average_duration", - "inputs", - "outputs", "interfaces", ] diff --git a/app/grandchallenge/algorithms/templates/algorithms/job_detail.html b/app/grandchallenge/algorithms/templates/algorithms/job_detail.html index f96b87de04..bce180ec4f 100644 --- a/app/grandchallenge/algorithms/templates/algorithms/job_detail.html +++ b/app/grandchallenge/algorithms/templates/algorithms/job_detail.html @@ -52,10 +52,9 @@ aria-selected="false"> Logs {% endif %} - {% if object.status == object.SUCCESS and perms.reader_studies.add_readerstudy %} + href="{% url "algorithms:display-set-from-job-create" slug=object.algorithm_image.algorithm.slug pk=object.pk %}">  Edit in Reader Study {% endif %} diff --git a/app/grandchallenge/algorithms/views.py b/app/grandchallenge/algorithms/views.py index b3971cda28..84f24e8011 100644 --- a/app/grandchallenge/algorithms/views.py +++ b/app/grandchallenge/algorithms/views.py @@ -797,7 +797,7 @@ def form_valid(self, form): class AlgorithmViewSet(ReadOnlyModelViewSet): - queryset = Algorithm.objects.all().prefetch_related("outputs", "inputs") + queryset = Algorithm.objects.all().prefetch_related("interfaces") serializer_class = AlgorithmSerializer permission_classes = [DjangoObjectPermissions] filter_backends = [DjangoFilterBackend, ObjectPermissionsFilter] @@ -1132,8 +1132,7 @@ class AlgorithmImageTemplate(ObjectPermissionRequiredMixin, DetailView): permission_required = "algorithms.change_algorithm" raise_exception = True queryset = Algorithm.objects.prefetch_related( - "inputs", - "outputs", + "interfaces", ) def get(self, *_, **__): diff --git a/app/grandchallenge/evaluation/admin.py b/app/grandchallenge/evaluation/admin.py index d922ec8a66..94ab1effdc 100644 --- a/app/grandchallenge/evaluation/admin.py +++ b/app/grandchallenge/evaluation/admin.py @@ -88,8 +88,6 @@ class PhaseAdmin(admin.ModelAdmin): "give_algorithm_editors_job_view_permissions", "challenge_forge_json", "algorithm_interfaces", - "algorithm_inputs", - "algorithm_outputs", ) form = PhaseAdminForm diff --git a/app/grandchallenge/evaluation/migrations/0074_alter_phase_algorithm_inputs_and_more.py b/app/grandchallenge/evaluation/migrations/0074_alter_phase_algorithm_inputs_and_more.py new file mode 100644 index 0000000000..644f8e782b --- /dev/null +++ b/app/grandchallenge/evaluation/migrations/0074_alter_phase_algorithm_inputs_and_more.py @@ -0,0 +1,35 @@ +# Generated by Django 4.2.19 on 2025-02-17 09:48 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("components", "0024_alter_componentinterface_kind_and_more"), + ("evaluation", "0073_migrate_interfaces_for_phases"), + ] + + operations = [ + migrations.AlterField( + model_name="phase", + name="algorithm_inputs", + field=models.ManyToManyField( + blank=True, + help_text="The input interfaces that the algorithms for this phase must use", + null=True, + related_name="+", + to="components.componentinterface", + ), + ), + migrations.AlterField( + model_name="phase", + name="algorithm_outputs", + field=models.ManyToManyField( + blank=True, + help_text="The output interfaces that the algorithms for this phase must use", + null=True, + related_name="+", + to="components.componentinterface", + ), + ), + ] diff --git a/app/grandchallenge/evaluation/models.py b/app/grandchallenge/evaluation/models.py index 56fee2edf0..f17d26625d 100644 --- a/app/grandchallenge/evaluation/models.py +++ b/app/grandchallenge/evaluation/models.py @@ -18,6 +18,7 @@ from django.utils.html import format_html from django.utils.text import get_valid_filename from django.utils.timezone import localtime +from django_deprecate_fields import deprecate_field from django_extensions.db.fields import AutoSlugField from guardian.models import GroupObjectPermissionBase, UserObjectPermissionBase from guardian.shortcuts import assign_perm, remove_perm @@ -546,17 +547,21 @@ class Phase(FieldChangeMixin, HangingProtocolMixin, UUIDModel): outputs = models.ManyToManyField( to=ComponentInterface, related_name="evaluation_outputs" ) - algorithm_inputs = models.ManyToManyField( - to=ComponentInterface, - related_name="+", - blank=True, - help_text="The input interfaces that the algorithms for this phase must use", + algorithm_inputs = deprecate_field( + models.ManyToManyField( + to=ComponentInterface, + related_name="+", + blank=True, + help_text="The input interfaces that the algorithms for this phase must use", + ) ) - algorithm_outputs = models.ManyToManyField( - to=ComponentInterface, - related_name="+", - blank=True, - help_text="The output interfaces that the algorithms for this phase must use", + algorithm_outputs = deprecate_field( + models.ManyToManyField( + to=ComponentInterface, + related_name="+", + blank=True, + help_text="The output interfaces that the algorithms for this phase must use", + ) ) algorithm_selectable_gpu_type_choices = models.JSONField( default=get_default_gpu_type_choices, @@ -955,9 +960,14 @@ def set_default_interfaces(self): @cached_property def linked_component_interfaces(self): - return ( - self.algorithm_inputs.all() | self.algorithm_outputs.all() - ).distinct() + return { + ci + for interface in self.algorithm_interfaces.all() + for ci in ( + interface.inputs.order_by("pk") + | interface.outputs.order_by("pk") + ) + } def assign_permissions(self): assign_perm("view_phase", self.challenge.admins_group, self) @@ -1736,10 +1746,6 @@ def container(self): def output_interfaces(self): return self.submission.phase.outputs - @cached_property - def algorithm_inputs(self): - return self.submission.phase.algorithm_inputs.all() - @cached_property def successful_jobs_per_interface(self): algorithm_interfaces = ( diff --git a/app/grandchallenge/hanging_protocols/forms.py b/app/grandchallenge/hanging_protocols/forms.py index ad032fd3c9..c9846ffea3 100644 --- a/app/grandchallenge/hanging_protocols/forms.py +++ b/app/grandchallenge/hanging_protocols/forms.py @@ -183,20 +183,24 @@ def __init__(self, *args, **kwargs): ) def _get_interface_lists(self): + sorted_interfaces = sorted( + list(self.instance.linked_component_interfaces), + key=lambda x: x.slug, + ) images = [ interface.slug - for interface in self.instance.linked_component_interfaces + for interface in sorted_interfaces if interface.kind == InterfaceKindChoices.IMAGE ] mandatory_isolation_interfaces = [ interface.slug - for interface in self.instance.linked_component_interfaces + for interface in sorted_interfaces if interface.kind in InterfaceKind.interface_type_mandatory_isolation() ] overlays = [ interface.slug - for interface in self.instance.linked_component_interfaces + for interface in sorted_interfaces if interface.kind not in ( *InterfaceKind.interface_type_undisplayable(), @@ -266,7 +270,6 @@ def _build_view_content( def generate_view_content_example(self): interface_lists = self._get_interface_lists() - if ( not interface_lists.images and not interface_lists.mandatory_isolation_interfaces diff --git a/app/tests/algorithms_tests/test_forms.py b/app/tests/algorithms_tests/test_forms.py index 47dad06b8c..4b18130cff 100644 --- a/app/tests/algorithms_tests/test_forms.py +++ b/app/tests/algorithms_tests/test_forms.py @@ -809,16 +809,23 @@ def test_cannot_create_job_with_same_inputs_twice( @pytest.mark.django_db -def test_all_inputs_required_on_job_creation(algorithm_with_multiple_inputs): +def test_inputs_required_on_job_creation(algorithm_with_multiple_inputs): ci_json_in_db_without_schema = ComponentInterfaceFactory( kind=InterfaceKind.InterfaceKindChoices.ANY, store_in_database=True, ) interface = AlgorithmInterfaceFactory( - inputs=[ci_json_in_db_without_schema], + inputs=[ + ci_json_in_db_without_schema, + algorithm_with_multiple_inputs.ci_bool, + algorithm_with_multiple_inputs.ci_str, + algorithm_with_multiple_inputs.ci_json_in_db_with_schema, + algorithm_with_multiple_inputs.ci_existing_img, + algorithm_with_multiple_inputs.ci_json_file, + ], outputs=[ComponentInterfaceFactory()], ) - algorithm_with_multiple_inputs.algorithm.interfaces.add(interface) + algorithm_with_multiple_inputs.algorithm.interfaces.set([interface]) form = JobCreateForm( algorithm=algorithm_with_multiple_inputs.algorithm, @@ -828,7 +835,14 @@ def test_all_inputs_required_on_job_creation(algorithm_with_multiple_inputs): ) for name, field in form.fields.items(): - if name not in ["algorithm_model", "creator"]: + # boolean and json inputs that allow None should not be required, + # all other inputs should be + if name not in [ + "algorithm_model", + "creator", + f"{INTERFACE_FORM_FIELD_PREFIX}{algorithm_with_multiple_inputs.ci_bool.slug}", + f"{INTERFACE_FORM_FIELD_PREFIX}{ci_json_in_db_without_schema.slug}", + ]: assert field.required diff --git a/app/tests/algorithms_tests/test_views.py b/app/tests/algorithms_tests/test_views.py index bf057cb9bb..1067f9aca7 100644 --- a/app/tests/algorithms_tests/test_views.py +++ b/app/tests/algorithms_tests/test_views.py @@ -728,106 +728,6 @@ def list_algorithms(self, **__): "logo": "https://rumc-gcorg-p-public.s3.amazonaws.com/logos/algorithm/0d11fc7b-c63f-4fd7-b80b-51d2e21492c0/square_logo.x20.jpeg", "slug": "the-pi-cai-challenge-baseline-nndetection", "average_duration": 363.50596, - "inputs": [ - { - "title": "Coronal T2 Prostate MRI", - "description": "Coronal T2 MRI of the Prostate", - "slug": "coronal-t2-prostate-mri", - "kind": "Image", - "pk": 31, - "default_value": None, - "super_kind": "Image", - "relative_path": "images/coronal-t2-prostate-mri", - "overlay_segments": [], - "look_up_table": None, - }, - { - "title": "Transverse T2 Prostate MRI", - "description": "Transverse T2 MRI of the Prostate", - "slug": "transverse-t2-prostate-mri", - "kind": "Image", - "pk": 32, - "default_value": None, - "super_kind": "Image", - "relative_path": "images/transverse-t2-prostate-mri", - "overlay_segments": [], - "look_up_table": None, - }, - { - "title": "Sagittal T2 Prostate MRI", - "description": "Sagittal T2 MRI of the Prostate", - "slug": "sagittal-t2-prostate-mri", - "kind": "Image", - "pk": 33, - "default_value": None, - "super_kind": "Image", - "relative_path": "images/sagittal-t2-prostate-mri", - "overlay_segments": [], - "look_up_table": None, - }, - { - "title": "Transverse HBV Prostate MRI", - "description": "Transverse High B-Value Prostate MRI", - "slug": "transverse-hbv-prostate-mri", - "kind": "Image", - "pk": 47, - "default_value": None, - "super_kind": "Image", - "relative_path": "images/transverse-hbv-prostate-mri", - "overlay_segments": [], - "look_up_table": None, - }, - { - "title": "Transverse ADC Prostate MRI", - "description": "Transverse Apparent Diffusion Coefficient Prostate MRI", - "slug": "transverse-adc-prostate-mri", - "kind": "Image", - "pk": 48, - "default_value": None, - "super_kind": "Image", - "relative_path": "images/transverse-adc-prostate-mri", - "overlay_segments": [], - "look_up_table": None, - }, - { - "title": "Clinical Information Prostate MRI", - "description": "Clinical information to support clinically significant prostate cancer detection in prostate MRI. Provided information: patient age at time of examination (patient_age), PSA level in ng/mL as reported (PSA_report), PSA density in ng/mL^2 as reported (PSAD_report), prostate volume as reported (prostate_volume_report), prostate volume derived from automatic whole-gland segmentation (prostate_volume_automatic), scanner manufacturer (scanner_manufacturer), scanner model name (scanner_model_name), diffusion b-value of (calculated) high b-value diffusion map (diffusion_high_bvalue). Values acquired from radiology reports will be missing, if not reported.", - "slug": "clinical-information-prostate-mri", - "kind": "Anything", - "pk": 156, - "default_value": None, - "super_kind": "Value", - "relative_path": "clinical-information-prostate-mri.json", - "overlay_segments": [], - "look_up_table": None, - }, - ], - "outputs": [ - { - "title": "Case-level Cancer Likelihood Prostate MRI", - "description": "Case-level likelihood of harboring clinically significant prostate cancer, in range [0,1].", - "slug": "prostate-cancer-likelihood", - "kind": "Float", - "pk": 144, - "default_value": None, - "super_kind": "Value", - "relative_path": "cspca-case-level-likelihood.json", - "overlay_segments": [], - "look_up_table": None, - }, - { - "title": "Transverse Cancer Detection Map Prostate MRI", - "description": "Single-class, detection map of clinically significant prostate cancer lesions in 3D, where each voxel represents a floating point in range [0,1].", - "slug": "cspca-detection-map", - "kind": "Heat Map", - "pk": 151, - "default_value": None, - "super_kind": "Image", - "relative_path": "images/cspca-detection-map", - "overlay_segments": [], - "look_up_table": None, - }, - ], "interfaces": [ { "inputs": [ @@ -1007,7 +907,10 @@ def list_algorithm_images(self, **__): "logos/algorithm/0d11fc7b-c63f-4fd7-b80b-51d2e21492c0/square_logo" ) assert "Imported from [grand-challenge.org]" in algorithm.summary - assert {i.slug for i in algorithm.inputs.all()} == { + + assert algorithm.interfaces.count() == 1 + interface = algorithm.interfaces.get() + assert {i.slug for i in interface.inputs.all()} == { "clinical-information-prostate-mri", "coronal-t2-prostate-mri", "sagittal-t2-prostate-mri", @@ -1015,7 +918,7 @@ def list_algorithm_images(self, **__): "transverse-hbv-prostate-mri", "transverse-t2-prostate-mri", } - assert {i.slug for i in algorithm.outputs.all()} == { + assert {i.slug for i in interface.outputs.all()} == { "cspca-detection-map", "prostate-cancer-likelihood", } diff --git a/app/tests/core_tests/test_grand_challenge_forge.py b/app/tests/core_tests/test_grand_challenge_forge.py index e7b7d9df93..873d4c6e12 100644 --- a/app/tests/core_tests/test_grand_challenge_forge.py +++ b/app/tests/core_tests/test_grand_challenge_forge.py @@ -99,12 +99,12 @@ def test_get_challenge_pack_context(): assert ci_key in component_interface # Test assigned example value - assert ( - context["challenge"]["phases"][0]["algorithm_inputs"][0][ - "example_value" - ] - == 87 - ) + example_values = { + input["example_value"] + for input in context["challenge"]["phases"][0]["algorithm_inputs"] + if input["example_value"] + } + assert example_values == {87} # Quick check on CI input and outputs input_slugs = [ diff --git a/app/tests/evaluation_tests/test_views.py b/app/tests/evaluation_tests/test_views.py index fa7cc84d0e..f043a4e7bc 100644 --- a/app/tests/evaluation_tests/test_views.py +++ b/app/tests/evaluation_tests/test_views.py @@ -812,7 +812,6 @@ def test_create_algorithm_for_phase_presets(client): phase.creator_must_be_verified = True phase.archive = ArchiveFactory() ci1 = ComponentInterfaceFactory(kind=InterfaceKindChoices.STRING) - ci2 = ComponentInterfaceFactory(kind=InterfaceKindChoices.STRING) optional_protocol = HangingProtocolFactory() interface1, interface2 = AlgorithmInterfaceFactory.create_batch(2) @@ -947,8 +946,7 @@ def test_create_algorithm_for_phase_presets(client): data={ "title": "Test algorithm", "job_requires_memory_gb": 8, - "inputs": [ci3.pk], - "outputs": [ci2.pk], + "interfaces": [interface1.pk], "workstation": ws.pk, "hanging_protocol": hp.pk, "optional_hanging_protocols": [oph.pk], diff --git a/app/tests/hanging_protocols_tests/test_forms.py b/app/tests/hanging_protocols_tests/test_forms.py index 81fa7d73c2..589e7ee427 100644 --- a/app/tests/hanging_protocols_tests/test_forms.py +++ b/app/tests/hanging_protocols_tests/test_forms.py @@ -18,7 +18,10 @@ from grandchallenge.hanging_protocols.forms import HangingProtocolForm from grandchallenge.hanging_protocols.models import HangingProtocol from grandchallenge.reader_studies.forms import ReaderStudyUpdateForm -from tests.algorithms_tests.factories import AlgorithmFactory +from tests.algorithms_tests.factories import ( + AlgorithmFactory, + AlgorithmInterfaceFactory, +) from tests.archives_tests.factories import ArchiveItemFactory from tests.components_tests.factories import ( ComponentInterfaceFactory, @@ -529,7 +532,6 @@ def test_archive_and_reader_study_forms_view_content_help_text( 0, 1, ( - "The following interfaces are used in your algorithm: test-ci-overlay-0 and test-ci-undisplayable-0. " "No interfaces of type image, chart, pdf, mp4, thumbnail_jpg or thumbnail_png are used. At least one interface of those types is needed to configure the viewer. " 'Refer to the documentation for more information' ), @@ -540,7 +542,6 @@ def test_archive_and_reader_study_forms_view_content_help_text( 1, 1, ( - "The following interfaces are used in your algorithm: test-ci-isolated-0, test-ci-image-0, test-ci-overlay-0, and test-ci-undisplayable-0. " 'Example usage: {"main": ["test-ci-isolated-0"], "secondary": ["test-ci-image-0", "test-ci-overlay-0"]}. ' 'Refer to the documentation for more information' ), @@ -562,11 +563,18 @@ def test_algorithm_form_view_content_help_text( number_of_undisplayable_interfaces=number_of_undisplayable_interfaces, ) algorithm = AlgorithmFactory() - algorithm.inputs.set(ci_list) + if ci_list: + interface = AlgorithmInterfaceFactory( + inputs=ci_list[:-1], outputs=[ci_list[-1]] + ) + algorithm.interfaces.set([interface]) form = AlgorithmForm(user=UserFactory(), instance=algorithm) - assert form.fields["view_content"].help_text == expected_help_text + assert sorted( + [interface.pk for interface in algorithm.linked_component_interfaces] + ) == [interface.pk for interface in ci_list] + assert expected_help_text in form.fields["view_content"].help_text @pytest.mark.parametrize( @@ -588,7 +596,6 @@ def test_algorithm_form_view_content_help_text( 0, 1, ( - "The following interfaces are used in your phase: test-ci-overlay-0 and test-ci-undisplayable-0. " "No interfaces of type image, chart, pdf, mp4, thumbnail_jpg or thumbnail_png are used. At least one interface of those types is needed to configure the viewer. " 'Refer to the documentation for more information' ), @@ -599,7 +606,6 @@ def test_algorithm_form_view_content_help_text( 1, 1, ( - "The following interfaces are used in your phase: test-ci-isolated-0, test-ci-image-0, test-ci-overlay-0, and test-ci-undisplayable-0. " 'Example usage: {"main": ["test-ci-isolated-0"], "secondary": ["test-ci-image-0", "test-ci-overlay-0"]}. ' 'Refer to the documentation for more information' ), @@ -621,14 +627,20 @@ def test_phase_update_form_view_content_help_text( number_of_undisplayable_interfaces=number_of_undisplayable_interfaces, ) phase = PhaseFactory() - phase.algorithm_inputs.set(ci_list) - phase.algorithm_outputs.set([]) + if ci_list: + interface = AlgorithmInterfaceFactory( + inputs=ci_list[:-1], outputs=[ci_list[-1]] + ) + phase.algorithm_interfaces.set([interface]) form = PhaseUpdateForm( user=UserFactory(), instance=phase, **{"challenge": phase.challenge} ) - assert form.fields["view_content"].help_text == expected_help_text + assert sorted( + [interface.pk for interface in phase.linked_component_interfaces] + ) == [interface.pk for interface in ci_list] + assert expected_help_text in form.fields["view_content"].help_text @pytest.mark.django_db @@ -642,14 +654,14 @@ def test_phase_update_form_view_content_help_text( None, ), ( - 1, + 2, 0, 0, - {"main": ["test-ci-image-0"]}, + {"main": ["test-ci-image-0"], "secondary": ["test-ci-image-1"]}, ), ( 0, - 1, + 2, 0, None, ), @@ -741,9 +753,13 @@ def test_generate_view_content_example( number_of_isolated_interfaces=number_of_isolated_interfaces, number_of_undisplayable_interfaces=0, ) - base_obj = AlgorithmFactory() - base_obj.inputs.set(ci_list) - form = AlgorithmForm(user=UserFactory(), instance=base_obj) + alg = AlgorithmFactory() + if ci_list: + interface = AlgorithmInterfaceFactory( + inputs=ci_list[:-1], outputs=[ci_list[-1]] + ) + alg.interfaces.set([interface]) + form = AlgorithmForm(user=UserFactory(), instance=alg) view_content_example = form.generate_view_content_example() view_content_example_json = ( diff --git a/app/tests/serving_tests/test_views.py b/app/tests/serving_tests/test_views.py index 60a522c2ee..5402f5ccb6 100644 --- a/app/tests/serving_tests/test_views.py +++ b/app/tests/serving_tests/test_views.py @@ -10,7 +10,10 @@ ComponentInterface, ComponentInterfaceValue, ) -from tests.algorithms_tests.factories import AlgorithmJobFactory +from tests.algorithms_tests.factories import ( + AlgorithmInterfaceFactory, + AlgorithmJobFactory, +) from tests.archives_tests.factories import ArchiveFactory, ArchiveItemFactory from tests.cases_tests import RESOURCE_PATH from tests.evaluation_tests.factories import ( @@ -170,7 +173,8 @@ def has_correct_access(user_allowed, user_denied, url): # test algorithm job = AlgorithmJobFactory(creator=user1, time_limit=60) - job.algorithm_image.algorithm.outputs.add(detection_interface) + interface = AlgorithmInterfaceFactory(outputs=[detection_interface]) + job.algorithm_image.algorithm.interfaces.add(interface) job.outputs.add(output_civ) has_correct_access(user1, user2, job.outputs.first().file.url) diff --git a/app/tests/workstations_tests/test_session_control.py b/app/tests/workstations_tests/test_session_control.py index 3e7a7dc79d..8aecfe9501 100644 --- a/app/tests/workstations_tests/test_session_control.py +++ b/app/tests/workstations_tests/test_session_control.py @@ -17,6 +17,7 @@ class SessionCreationView(TemplateView): template_name = "new_session.html" +@pytest.mark.xfail(reason="Still to be addressed for optional inputs pitch") @pytest.mark.playwright def test_viewer_session_control(live_server, page, settings): settings.WORKSTATIONS_EXTRA_BROADCAST_DOMAINS = [live_server.url] diff --git a/scripts/algorithm_evaluation_fixtures.py b/scripts/algorithm_evaluation_fixtures.py index 84b58b22fc..add970d87d 100644 --- a/scripts/algorithm_evaluation_fixtures.py +++ b/scripts/algorithm_evaluation_fixtures.py @@ -9,7 +9,11 @@ from django.contrib.auth import get_user_model from django.core.files.base import ContentFile -from grandchallenge.algorithms.models import Algorithm, AlgorithmImage +from grandchallenge.algorithms.models import ( + Algorithm, + AlgorithmImage, + AlgorithmInterface, +) from grandchallenge.archives.models import Archive, ArchiveItem from grandchallenge.cases.models import Image, ImageFile from grandchallenge.challenges.models import Challenge @@ -122,9 +126,10 @@ def _create_challenge( p = Phase.objects.create( challenge=c, title="Phase 1", algorithm_time_limit=300 ) - - p.algorithm_inputs.set(inputs) - p.algorithm_outputs.set(outputs) + interface = AlgorithmInterface.objects.create( + inputs=inputs, outputs=outputs + ) + p.algorithm_interfaces.set([interface]) p.title = "Algorithm Evaluation" p.submission_kind = SubmissionKindChoices.ALGORITHM @@ -146,8 +151,10 @@ def _create_algorithm(*, creator, inputs, outputs, suffix): title=f"Test Algorithm Evaluation {suffix}", logo=create_uploaded_image(), ) - algorithm.inputs.set(inputs) - algorithm.outputs.set(outputs) + interface = AlgorithmInterface.objects.create( + inputs=inputs, outputs=outputs + ) + algorithm.interfaces.set([interface]) algorithm.add_editor(creator) algorithm_image = AlgorithmImage(creator=creator, algorithm=algorithm) diff --git a/scripts/cost_fixtures.py b/scripts/cost_fixtures.py index feffc67024..9786f3f45e 100644 --- a/scripts/cost_fixtures.py +++ b/scripts/cost_fixtures.py @@ -5,7 +5,12 @@ from django.contrib.auth import get_user_model from django.utils.timezone import now -from grandchallenge.algorithms.models import Algorithm, AlgorithmImage, Job +from grandchallenge.algorithms.models import ( + Algorithm, + AlgorithmImage, + AlgorithmInterface, + Job, +) from grandchallenge.archives.models import Archive, ArchiveItem from grandchallenge.cases.models import Image, ImageFile from grandchallenge.challenges.models import Challenge @@ -134,9 +139,10 @@ def _create_challenge( c.add_participant(participant) p = Phase.objects.create(challenge=c, title="Phase 1") - - p.algorithm_inputs.set(inputs) - p.algorithm_outputs.set(outputs) + interface = AlgorithmInterface.objects.create( + inputs=inputs, outputs=outputs + ) + p.algorithm_interfaces.set([interface]) p.title = f"Type 2 {suffix}" p.submission_kind = SubmissionKindChoices.ALGORITHM @@ -158,8 +164,10 @@ def _create_algorithm(*, creator, inputs, outputs, suffix): title=f"Test Algorithm Type 2 {suffix}", logo=create_uploaded_image(), ) - algorithm.inputs.set(inputs) - algorithm.outputs.set(outputs) + interface = AlgorithmInterface.objects.create( + inputs=inputs, outputs=outputs + ) + algorithm.interfaces.set([interface]) algorithm.add_editor(creator) algorithm_image = AlgorithmImage(creator=creator, algorithm=algorithm) @@ -176,6 +184,7 @@ def _create_submission(algorithm, challenge, archive_items): for _ in range(archive_items): job = Job.objects.create( algorithm_image=ai, + algorithm_interface=algorithm.interfaces.first(), started_at=now() - timedelta(minutes=random.randint(5, 120)), completed_at=now(), status=Job.SUCCESS, diff --git a/scripts/development_fixtures.py b/scripts/development_fixtures.py index ddc7066749..24f8f3faf9 100644 --- a/scripts/development_fixtures.py +++ b/scripts/development_fixtures.py @@ -17,7 +17,12 @@ from knox.settings import CONSTANTS from machina.apps.forum.models import Forum -from grandchallenge.algorithms.models import Algorithm, AlgorithmImage, Job +from grandchallenge.algorithms.models import ( + Algorithm, + AlgorithmImage, + AlgorithmInterface, + Job, +) from grandchallenge.anatomy.models import BodyRegion, BodyStructure from grandchallenge.archives.models import Archive, ArchiveItem from grandchallenge.challenges.models import Challenge, ChallengeSeries @@ -356,12 +361,11 @@ def _create_algorithm_demo(users): ) algorithm.editors_group.user_set.add(users["algorithm"], users["demo"]) algorithm.users_group.user_set.add(users["algorithmuser"]) - algorithm.inputs.set( - [ComponentInterface.objects.get(slug="generic-medical-image")] - ) - algorithm.outputs.set( - [ComponentInterface.objects.get(slug="results-json-file")] + interface = AlgorithmInterface.objects.create( + inputs=[ComponentInterface.objects.get(slug="generic-medical-image")], + outputs=[ComponentInterface.objects.get(slug="results-json-file")], ) + algorithm.interfaces.set([interface]) algorithm_image = AlgorithmImage( creator=users["algorithm"], algorithm=algorithm @@ -380,6 +384,7 @@ def _create_algorithm_demo(users): algorithms_job = Job.objects.create( creator=users["algorithm"], algorithm_image=algorithm_image, + algorithm_interface=interface, status=Evaluation.SUCCESS, time_limit=60, requires_gpu_type=algorithm_image.algorithm.job_requires_gpu_type, diff --git a/scripts/external_algorithm_evaluation_fixtures.py b/scripts/external_algorithm_evaluation_fixtures.py index d554df0884..88dd483927 100644 --- a/scripts/external_algorithm_evaluation_fixtures.py +++ b/scripts/external_algorithm_evaluation_fixtures.py @@ -4,6 +4,7 @@ from knox.models import AuthToken, hash_token from knox.settings import CONSTANTS +from grandchallenge.algorithms.models import AlgorithmInterface from grandchallenge.challenges.models import Challenge from grandchallenge.evaluation.models import Phase from grandchallenge.evaluation.utils import SubmissionKindChoices @@ -63,9 +64,10 @@ def _create_external_evaluation_phase( existing_phase = challenge.phase_set.get() p = Phase.objects.create(challenge=challenge, title="Phase 2") - - p.algorithm_inputs.set(inputs) - p.algorithm_outputs.set(outputs) + interface = AlgorithmInterface.objects.create( + inputs=inputs, outputs=outputs + ) + p.algorithm_interfaces.set([interface]) p.title = "External Algorithm Evaluation" p.submission_kind = SubmissionKindChoices.ALGORITHM diff --git a/scripts/pentest_fixtures.py b/scripts/pentest_fixtures.py index 3fa757a372..f3ccb626a2 100644 --- a/scripts/pentest_fixtures.py +++ b/scripts/pentest_fixtures.py @@ -25,7 +25,12 @@ from PIL import Image as PILImage from PIL import ImageDraw, ImageFont -from grandchallenge.algorithms.models import Algorithm, AlgorithmImage, Job +from grandchallenge.algorithms.models import ( + Algorithm, + AlgorithmImage, + AlgorithmInterface, + Job, +) from grandchallenge.archives.models import Archive, ArchiveItem from grandchallenge.cases.models import Image, ImageFile from grandchallenge.challenges.models import Challenge @@ -202,8 +207,10 @@ def create_challenge(*, challenge_num, container, public): give_algorithm_editors_job_view_permissions=give_algorithm_editors_job_view_permissions, ) - p.algorithm_inputs.set(_get_inputs()) - p.algorithm_outputs.set(_get_outputs()) + interface = AlgorithmInterface.objects.create( + inputs=_get_inputs(), outputs=_get_outputs() + ) + p.algorithm_interfaces.set([interface]) m = Method(creator=admin, phase=p) @@ -426,8 +433,10 @@ def create_algorithm( public=public, ) - algorithm.inputs.set(inputs) - algorithm.outputs.set(outputs) + interface = AlgorithmInterface.objects.create( + inputs=inputs, outputs=outputs + ) + algorithm.interfaces.set([interface]) if editor is None: editor = _create_user(f"algorithm-{algorithm_num}", "editor") @@ -465,6 +474,7 @@ def create_algorithm( job = Job.objects.create( algorithm_image=algorithm_image, + algorithm_interface=interface, creator=user, public=public, comment=job_title, From 3d9941a3631f6a1199ab0167ee72be9e44ba0231 Mon Sep 17 00:00:00 2001 From: amickan Date: Thu, 20 Feb 2025 12:21:24 +0100 Subject: [PATCH 17/18] Fix gc-forge test --- app/tests/core_tests/test_grand_challenge_forge.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/tests/core_tests/test_grand_challenge_forge.py b/app/tests/core_tests/test_grand_challenge_forge.py index 873d4c6e12..c48571d60d 100644 --- a/app/tests/core_tests/test_grand_challenge_forge.py +++ b/app/tests/core_tests/test_grand_challenge_forge.py @@ -24,7 +24,7 @@ def test_get_challenge_pack_context(): challenge = ChallengeFactory() inputs = [ ComponentInterfaceFactory(kind=ComponentInterface.Kind.INTEGER), - ComponentInterfaceFactory(), + ComponentInterfaceFactory(kind=ComponentInterface.Kind.IMAGE), ] outputs = [ ComponentInterfaceFactory(), @@ -99,12 +99,12 @@ def test_get_challenge_pack_context(): assert ci_key in component_interface # Test assigned example value - example_values = { + example_values = [ input["example_value"] for input in context["challenge"]["phases"][0]["algorithm_inputs"] if input["example_value"] - } - assert example_values == {87} + ] + assert example_values == [87] # Quick check on CI input and outputs input_slugs = [ From 22ff7474fad966b46b4f68e14f5ab7362a0cbf30 Mon Sep 17 00:00:00 2001 From: Anne Mickan Date: Mon, 24 Feb 2025 10:39:08 +0100 Subject: [PATCH 18/18] Rename interface to socket on user-facing views (#3846) This renames `interface` to `socket` on user-facing views and forms. I did not change the urls that include the word `interface`. That seems less important to me for now. We will need to add permanent redirects when we do change them. I based this on the feature branch for optional inputs since a lot of the places that require changes are touched in this branch. The users can wait 1 week for the renaming I think. The only places where the user is still confronted with the word `interface` now (as far as I can tell) is the API. Updating that is more complex. --- .../0025_algorithm_hanging_protocol.py | 2 +- app/grandchallenge/archives/forms.py | 14 +++++----- .../0011_archive_hanging_protocol.py | 2 +- app/grandchallenge/cases/models.py | 2 +- app/grandchallenge/components/forms.py | 10 +++---- .../templates/components/civ_set_form.html | 2 +- .../componentinterface_io_switch.html | 27 ++++++++++--------- .../components/componentinterface_list.html | 10 +++---- app/grandchallenge/core/error_handlers.py | 8 +++--- .../migrations/0027_auto_20220707_0933.py | 2 +- app/grandchallenge/hanging_protocols/forms.py | 4 +-- .../hanging_protocols/models.py | 16 +++++------ app/grandchallenge/reader_studies/forms.py | 3 ++- .../0019_readerstudy_hanging_protocol.py | 2 +- app/grandchallenge/reader_studies/models.py | 2 +- .../readerstudy_add_object.html | 2 +- app/tests/cases_tests/test_forms.py | 2 +- app/tests/components_tests/test_tasks.py | 6 ++--- .../hanging_protocols_tests/test_forms.py | 20 +++++++------- .../hanging_protocols_tests/test_models.py | 10 +++---- app/tests/reader_studies_tests/test_models.py | 2 +- 21 files changed, 74 insertions(+), 74 deletions(-) diff --git a/app/grandchallenge/algorithms/migrations/0025_algorithm_hanging_protocol.py b/app/grandchallenge/algorithms/migrations/0025_algorithm_hanging_protocol.py index b872664deb..d7d50e23a7 100644 --- a/app/grandchallenge/algorithms/migrations/0025_algorithm_hanging_protocol.py +++ b/app/grandchallenge/algorithms/migrations/0025_algorithm_hanging_protocol.py @@ -17,7 +17,7 @@ class Migration(migrations.Migration): name="hanging_protocol", field=models.ForeignKey( blank=True, - help_text='Indicate which Component Interfaces need to be displayed in which image port. E.g. {"main": ["interface1"]}. The first item in the list of interfaces will be the main image in the image port. The first overlay type interface thereafter will be rendered as an overlay. For now, any other items will be ignored by the viewer.', + help_text='Indicate which sockets need to be displayed in which image port. E.g. {"main": ["socket1"]}. The first item in the list of sockets will be the main image in the image port. The first overlay type socket thereafter will be rendered as an overlay. For now, any other items will be ignored by the viewer.', null=True, on_delete=django.db.models.deletion.SET_NULL, to="hanging_protocols.hangingprotocol", diff --git a/app/grandchallenge/archives/forms.py b/app/grandchallenge/archives/forms.py index 59212637a8..29e33d6cc5 100644 --- a/app/grandchallenge/archives/forms.py +++ b/app/grandchallenge/archives/forms.py @@ -206,14 +206,14 @@ def __init__(self, *args, user, archive, **kwargs): class AddCasesForm(UploadRawImagesForm): - interface = ModelChoiceField( + socket = ModelChoiceField( queryset=ComponentInterface.objects.filter( kind__in=InterfaceKind.interface_type_image() ).order_by("title"), widget=autocomplete.ModelSelect2( url="components:component-interface-autocomplete", attrs={ - "data-placeholder": "Search for an interface ...", + "data-placeholder": "Search for a socket ...", "data-minimum-input-length": 3, "data-theme": settings.CRISPY_TEMPLATE_PACK, "data-html": True, @@ -223,18 +223,18 @@ class AddCasesForm(UploadRawImagesForm): def __init__(self, *args, interface_viewname, **kwargs): super().__init__(*args, **kwargs) - self.fields["interface"].help_text = format_lazy( + self.fields["socket"].help_text = format_lazy( ( - 'See the list of interfaces for more ' - "information about each interface. " - "Please contact support if your desired interface is missing." + 'See the list of sockets for more ' + "information about each socket. " + "Please contact support if your desired socket is missing." ), reverse_lazy(interface_viewname), ) def save(self, *args, **kwargs): self._linked_task.kwargs.update( - {"interface_pk": self.cleaned_data["interface"].pk} + {"interface_pk": self.cleaned_data["socket"].pk} ) return super().save(*args, **kwargs) diff --git a/app/grandchallenge/archives/migrations/0011_archive_hanging_protocol.py b/app/grandchallenge/archives/migrations/0011_archive_hanging_protocol.py index d9d243a7b8..1349a209cc 100644 --- a/app/grandchallenge/archives/migrations/0011_archive_hanging_protocol.py +++ b/app/grandchallenge/archives/migrations/0011_archive_hanging_protocol.py @@ -17,7 +17,7 @@ class Migration(migrations.Migration): name="hanging_protocol", field=models.ForeignKey( blank=True, - help_text='Indicate which Component Interfaces need to be displayed in which image port. E.g. {"main": ["interface1"]}. The first item in the list of interfaces will be the main image in the image port. The first overlay type interface thereafter will be rendered as an overlay. For now, any other items will be ignored by the viewer.', + help_text='Indicate which sockets need to be displayed in which image port. E.g. {"main": ["socket1"]}. The first item in the list of sockets will be the main image in the image port. The first overlay type socket thereafter will be rendered as an overlay. For now, any other items will be ignored by the viewer.', null=True, on_delete=django.db.models.deletion.SET_NULL, to="hanging_protocols.hangingprotocol", diff --git a/app/grandchallenge/cases/models.py b/app/grandchallenge/cases/models.py index 6b6f31f9f1..8eee6fe8e6 100644 --- a/app/grandchallenge/cases/models.py +++ b/app/grandchallenge/cases/models.py @@ -153,7 +153,7 @@ def update_status( if detailed_error_message: notification_description = oxford_comma( [ - f"Image validation for interface {key} failed with error: {val}. " + f"Image validation for socket {key} failed with error: {val}. " for key, val in detailed_error_message.items() ] ) diff --git a/app/grandchallenge/components/forms.py b/app/grandchallenge/components/forms.py index 5c2c9dedec..25dcc8ad32 100644 --- a/app/grandchallenge/components/forms.py +++ b/app/grandchallenge/components/forms.py @@ -279,7 +279,7 @@ def __init__( widget = autocomplete.ModelSelect2 attrs.update( { - "data-placeholder": "Search for an interface ...", + "data-placeholder": "Search for a socket ...", "data-minimum-input-length": 3, "data-theme": settings.CRISPY_TEMPLATE_PACK, "data-html": True, @@ -296,12 +296,12 @@ def __init__( initial=selected_interface, queryset=qs, widget=widget(**widget_kwargs), - label="Interface", + label="Socket", help_text=format_lazy( ( - 'See the list of interfaces for more ' - "information about each interface. " - "Please contact support if your desired interface is missing." + 'See the list of sockets for more ' + "information about each socket. " + "Please contact support if your desired socket is missing." ), reverse_lazy(base_obj.interface_viewname), ), diff --git a/app/grandchallenge/components/templates/components/civ_set_form.html b/app/grandchallenge/components/templates/components/civ_set_form.html index bc89e38d77..dd393ca023 100644 --- a/app/grandchallenge/components/templates/components/civ_set_form.html +++ b/app/grandchallenge/components/templates/components/civ_set_form.html @@ -54,7 +54,7 @@

    >
    - Add value for new interface + Add value for new socket
    diff --git a/app/grandchallenge/components/templates/components/componentinterface_io_switch.html b/app/grandchallenge/components/templates/components/componentinterface_io_switch.html index 932cbf8b26..229561d540 100644 --- a/app/grandchallenge/components/templates/components/componentinterface_io_switch.html +++ b/app/grandchallenge/components/templates/components/componentinterface_io_switch.html @@ -3,39 +3,40 @@ {% load static %} {% block title %} - Interface Options - {{ block.super }} + Socket Options - {{ block.super }} {% endblock %} {% block breadcrumbs %}
      -
    1. Interface Options
      2. +
    2. Socket Options
    {% endblock %} {% block content %} -

    Algorithm Interface Options

    +

    Algorithm Socket Options

    - You will need to define the inputs to, and the outputs from, your algorithm. - The set of inputs and outputs that your algorithm uses is known as its interface. - You can build up your algorithm's interface by selecting from the current set of interface options. - Use the links below to see the options that are available for your algorithm's interface. + You will need to define the inputs and outputs for your algorithm. + A set of inputs and outputs that your algorithm uses is known as its interface. + The inputs and outputs that make up an algorithm interface are called sockets. + You can build up an algorithm interface by selecting from the current set of socket options. + Use the links below to see the socket options that are available.

    For example, lets say that you are developing an algorithm that predicts COVID-19 from a CT scan, which also produces a segmentation of COVID-19 lesions. - You would look in the list of interface options and select the ones to use. - You would then define an algorithm that has one input, a ct-image, + You would look in the list of socket options and select the ones to use. + You would then define an algorithm that has one interface with one input, a ct-image, and two outputs, probability-covid-19 (a number) and covid-19-lesion-segementation (a segmentation of covid 19 lesions).

    - Example algorithm with one input interface and two output interfaces + Example algorithm with one interface consisting of one input and two output sockets

    - You can select as many inputs and outputs as your algorithm needs. - However, the same interface option cannot be used for both input to and output from your algorithm. + You can select as many inputs and outputs for a single interface as you need. + However, the same socket option cannot be used for both input to and output from your algorithm. All inputs and outputs are required to be set when your algorithm runs. The inputs will be set by the user running your algorithm. Your algorithm must write valid data to all the outputs you set. @@ -57,7 +58,7 @@

    Algorithm Interface Options

    If an option does not exist for your use case please contact support with the title, description, and - kind for your new interface option. + kind for your new socket option.

    {% endblock %} diff --git a/app/grandchallenge/components/templates/components/componentinterface_list.html b/app/grandchallenge/components/templates/components/componentinterface_list.html index 0698044602..a9cc15d046 100644 --- a/app/grandchallenge/components/templates/components/componentinterface_list.html +++ b/app/grandchallenge/components/templates/components/componentinterface_list.html @@ -6,19 +6,19 @@ {% block title %} {% if object_type == object_type_options.ALGORITHM %} - {{ list_type|title }} - Interface Options - {{ block.super }} + {{ list_type|title }} - Socket Options - {{ block.super }} {% else %} - Interface Options - {{ block.super }} + Socket Options - {{ block.super }} {% endif %} {% endblock %} {% block breadcrumbs %}
      {% if object_type == object_type_options.ALGORITHM %} -
    1. Interface Options
      2. +
    2. Socket Options
    3. {{ list_type|title }}
      4. {% else %} -
    4. Interface Options
      5. +
    5. Socket Options
      6. {% endif %}
    {% endblock %} @@ -39,7 +39,7 @@

    {{ object_type|title }} {{ list_type|title }} Options

    times within one {{ list_type|lower }}. {% endif %} If an option does not exist for your use case please contact support with the title, description, and kind - for your new interface option. + for your new socket option.

    diff --git a/app/grandchallenge/core/error_handlers.py b/app/grandchallenge/core/error_handlers.py index 3bc87b4bed..b73496828b 100644 --- a/app/grandchallenge/core/error_handlers.py +++ b/app/grandchallenge/core/error_handlers.py @@ -100,8 +100,8 @@ def __init__(self, *args, user_upload, **kwargs): def handle_error(self, *, error_message, user, interface): Notification.send( kind=NotificationType.NotificationTypeChoices.FILE_COPY_STATUS, - message=f"Validation for interface {interface.title} failed.", - description=f"Validation for interface {interface.title} failed: {error_message}", + message=f"Validation for socket {interface.title} failed.", + description=f"Validation for socket {interface.title} failed: {error_message}", actor=user, ) @@ -118,8 +118,8 @@ def handle_error(self, *, error_message, user, interface=None): if interface: Notification.send( kind=NotificationType.NotificationTypeChoices.CIV_VALIDATION, - message=f"Validation for interface {interface.title} failed.", - description=f"Validation for interface {interface.title} failed: {error_message}", + message=f"Validation for socket {interface.title} failed.", + description=f"Validation for socket {interface.title} failed: {error_message}", actor=user, ) else: diff --git a/app/grandchallenge/evaluation/migrations/0027_auto_20220707_0933.py b/app/grandchallenge/evaluation/migrations/0027_auto_20220707_0933.py index 2ff4a17980..e58c5a10ca 100644 --- a/app/grandchallenge/evaluation/migrations/0027_auto_20220707_0933.py +++ b/app/grandchallenge/evaluation/migrations/0027_auto_20220707_0933.py @@ -21,7 +21,7 @@ class Migration(migrations.Migration): name="hanging_protocol", field=models.ForeignKey( blank=True, - help_text='Indicate which Component Interfaces need to be displayed in which image port. E.g. {"main": ["interface1"]}. The first item in the list of interfaces will be the main image in the image port. The first overlay type interface thereafter will be rendered as an overlay. For now, any other items will be ignored by the viewer.', + help_text='Indicate which sockets need to be displayed in which image port. E.g. {"main": ["socket1"]}. The first item in the list of sockets will be the main image in the image port. The first overlay type socket thereafter will be rendered as an overlay. For now, any other items will be ignored by the viewer.', null=True, on_delete=django.db.models.deletion.SET_NULL, to="hanging_protocols.hangingprotocol", diff --git a/app/grandchallenge/hanging_protocols/forms.py b/app/grandchallenge/hanging_protocols/forms.py index c9846ffea3..6e96e4989e 100644 --- a/app/grandchallenge/hanging_protocols/forms.py +++ b/app/grandchallenge/hanging_protocols/forms.py @@ -159,7 +159,7 @@ def __init__(self, *args, **kwargs): if len(interface_slugs) > 0: self.fields[ "view_content" - ].help_text += f"The following interfaces are used in your {self.instance._meta.verbose_name}: {oxford_comma(interface_slugs)}. " + ].help_text += f"The following sockets are used in your {self.instance._meta.verbose_name}: {oxford_comma(interface_slugs)}. " view_content_example = self.generate_view_content_example() @@ -170,7 +170,7 @@ def __init__(self, *args, **kwargs): else: self.fields[ "view_content" - ].help_text += "No interfaces of type image, chart, pdf, mp4, thumbnail_jpg or thumbnail_png are used. At least one interface of those types is needed to configure the viewer. " + ].help_text += "No sockets of type image, chart, pdf, mp4, thumbnail_jpg or thumbnail_png are used. At least one socket of those types is needed to configure the viewer. " self.fields["view_content"].help_text += format_lazy( 'Refer to the documentation for more information', diff --git a/app/grandchallenge/hanging_protocols/models.py b/app/grandchallenge/hanging_protocols/models.py index 13286887dd..729f154cfa 100644 --- a/app/grandchallenge/hanging_protocols/models.py +++ b/app/grandchallenge/hanging_protocols/models.py @@ -316,10 +316,10 @@ class HangingProtocolMixin(models.Model): blank=True, on_delete=models.SET_NULL, help_text=( - "Indicate which Component Interfaces need to be displayed in " - 'which image port. E.g. {"main": ["interface1"]}. The first ' - "item in the list of interfaces will be the main image in " - "the image port. The first overlay type interface thereafter " + "Indicate which sockets need to be displayed in " + 'which image port. E.g. {"main": ["socket1"]}. The first ' + "item in the list of sockets will be the main image in " + "the image port. The first overlay type socket thereafter " "will be rendered as an overlay. For now, any other items " "will be ignored by the viewer." ), @@ -356,7 +356,7 @@ def check_all_interfaces_in_view_content_exist(*, view_content): if set(slugs) != {i.slug for i in viewport_interfaces}: raise ValidationError( - f"Unknown interfaces in view content for viewport {viewport}: {', '.join(slugs)}" + f"Unknown sockets in view content for viewport {viewport}: {', '.join(slugs)}" ) image_interfaces = [ @@ -367,7 +367,7 @@ def check_all_interfaces_in_view_content_exist(*, view_content): if len(image_interfaces) > 1: raise ValidationError( - "Maximum of one image interface is allowed per viewport, " + "Maximum of one image socket is allowed per viewport, " f"got {len(image_interfaces)} for viewport {viewport}: " f"{', '.join(i.slug for i in image_interfaces)}" ) @@ -383,7 +383,7 @@ def check_all_interfaces_in_view_content_exist(*, view_content): and len(viewport_interfaces) > 1 ): raise ValidationError( - "Some of the selected interfaces can only be displayed in isolation, " + "Some of the selected sockets can only be displayed in isolation, " f"found {len(mandatory_isolation_interfaces)} for viewport {viewport}: " f"{', '.join(i.slug for i in mandatory_isolation_interfaces)}" ) @@ -396,7 +396,7 @@ def check_all_interfaces_in_view_content_exist(*, view_content): if len(undisplayable_interfaces) > 0: raise ValidationError( - "Some of the selected interfaces cannot be displayed, " + "Some of the selected sockets cannot be displayed, " f"found {len(undisplayable_interfaces)} for viewport {viewport}: " f"{', '.join(i.slug for i in undisplayable_interfaces)}" ) diff --git a/app/grandchallenge/reader_studies/forms.py b/app/grandchallenge/reader_studies/forms.py index e8876f4aa1..dafe7cbcbd 100644 --- a/app/grandchallenge/reader_studies/forms.py +++ b/app/grandchallenge/reader_studies/forms.py @@ -615,10 +615,11 @@ class Meta: interface = DynamicField( ModelChoiceField, + label="Socket", queryset=interface_choices, initial=None, required=False, - help_text="Select component interface to use as a default answer for this " + help_text="Select socket to use as a default answer for this " "question.", ) diff --git a/app/grandchallenge/reader_studies/migrations/0019_readerstudy_hanging_protocol.py b/app/grandchallenge/reader_studies/migrations/0019_readerstudy_hanging_protocol.py index 528605f82e..6e98c84532 100644 --- a/app/grandchallenge/reader_studies/migrations/0019_readerstudy_hanging_protocol.py +++ b/app/grandchallenge/reader_studies/migrations/0019_readerstudy_hanging_protocol.py @@ -17,7 +17,7 @@ class Migration(migrations.Migration): name="hanging_protocol", field=models.ForeignKey( blank=True, - help_text='Indicate which Component Interfaces need to be displayed in which image port. E.g. {"main": ["interface1"]}. The first item in the list of interfaces will be the main image in the image port. The first overlay type interface thereafter will be rendered as an overlay. For now, any other items will be ignored by the viewer.', + help_text='Indicate which sockets need to be displayed in which image port. E.g. {"main": ["socket1"]}. The first item in the list of sockets will be the main image in the image port. The first overlay type socket thereafter will be rendered as an overlay. For now, any other items will be ignored by the viewer.', null=True, on_delete=django.db.models.deletion.SET_NULL, to="hanging_protocols.hangingprotocol", diff --git a/app/grandchallenge/reader_studies/models.py b/app/grandchallenge/reader_studies/models.py index f90bcb7e01..d912894601 100644 --- a/app/grandchallenge/reader_studies/models.py +++ b/app/grandchallenge/reader_studies/models.py @@ -1513,7 +1513,7 @@ def _clean_interface(self): and self.interface not in self.allowed_component_interfaces ): raise ValidationError( - f"The interface {self.interface} is not allowed for this " + f"The socket {self.interface} is not allowed for this " f"question type ({self.answer_type})" ) diff --git a/app/grandchallenge/reader_studies/templates/reader_studies/readerstudy_add_object.html b/app/grandchallenge/reader_studies/templates/reader_studies/readerstudy_add_object.html index ca193bf078..8d348e90b3 100644 --- a/app/grandchallenge/reader_studies/templates/reader_studies/readerstudy_add_object.html +++ b/app/grandchallenge/reader_studies/templates/reader_studies/readerstudy_add_object.html @@ -20,6 +20,6 @@ {% block form_heading %}

    Add {{ type_to_add }} to {{ object }}

    {% if type_to_add == "images" %} -

    This will create one new display set for each uploaded image using the interface you choose below.

    +

    This will create one new display set for each uploaded image using the socket you choose below.

    {% endif %} {% endblock %} diff --git a/app/tests/cases_tests/test_forms.py b/app/tests/cases_tests/test_forms.py index ad938d52a7..76d35352d9 100644 --- a/app/tests/cases_tests/test_forms.py +++ b/app/tests/cases_tests/test_forms.py @@ -46,7 +46,7 @@ def test_upload_some_images( response = get_view_for_user( data={ "user_uploads": [user_upload.pk], - "interface": ComponentInterface.objects.first().pk, + "socket": ComponentInterface.objects.first().pk, }, client=client, viewname="reader-studies:display-sets-create", diff --git a/app/tests/components_tests/test_tasks.py b/app/tests/components_tests/test_tasks.py index 33769e05b5..47fe1f0c7e 100644 --- a/app/tests/components_tests/test_tasks.py +++ b/app/tests/components_tests/test_tasks.py @@ -423,7 +423,7 @@ def test_add_image_to_object_updates_upload_session_on_validation_fail( us = RawImageUploadSessionFactory(status=RawImageUploadSession.SUCCESS) ci = ComponentInterfaceFactory(kind="IMG") - error_message = f"Image validation for interface {ci.title} failed with error: Image imports should result in a single image. " + error_message = f"Image validation for socket {ci.title} failed with error: Image imports should result in a single image. " linked_task = some_async_task.signature( kwargs={"foo": "bar"}, immutable=True @@ -458,7 +458,7 @@ def test_add_image_to_object_marks_job_as_failed_on_validation_fail( us = RawImageUploadSessionFactory(status=RawImageUploadSession.SUCCESS) ci = ComponentInterfaceFactory(kind="IMG") - error_message = f"Image validation for interface {ci.title} failed with error: Image imports should result in a single image. " + error_message = f"Image validation for socket {ci.title} failed with error: Image imports should result in a single image. " linked_task = some_async_task.signature( kwargs={"foo": "bar"}, immutable=True @@ -597,7 +597,7 @@ def test_add_file_to_object_sends_notification_on_validation_fail( us.refresh_from_db() assert Notification.objects.count() == 1 assert ( - f"Validation for interface {ci.title} failed." + f"Validation for socket {ci.title} failed." in Notification.objects.first().message ) assert "some_async_task" not in str(callbacks) diff --git a/app/tests/hanging_protocols_tests/test_forms.py b/app/tests/hanging_protocols_tests/test_forms.py index 589e7ee427..0443468a05 100644 --- a/app/tests/hanging_protocols_tests/test_forms.py +++ b/app/tests/hanging_protocols_tests/test_forms.py @@ -52,9 +52,7 @@ def test_view_content_mixin(): assert not form.is_valid() assert form.errors == { - "__all__": [ - "Unknown interfaces in view content for viewport main: test" - ] + "__all__": ["Unknown sockets in view content for viewport main: test"] } i = ComponentInterfaceFactory( @@ -450,7 +448,7 @@ def make_ci_list( 0, 0, ( - "No interfaces of type image, chart, pdf, mp4, thumbnail_jpg or thumbnail_png are used. At least one interface of those types is needed to configure the viewer. " + "No sockets of type image, chart, pdf, mp4, thumbnail_jpg or thumbnail_png are used. At least one socket of those types is needed to configure the viewer. " 'Refer to the documentation for more information' ), ), @@ -460,8 +458,8 @@ def make_ci_list( 0, 1, ( - "The following interfaces are used in your {}: test-ci-overlay-0 and test-ci-undisplayable-0. " - "No interfaces of type image, chart, pdf, mp4, thumbnail_jpg or thumbnail_png are used. At least one interface of those types is needed to configure the viewer. " + "The following sockets are used in your {}: test-ci-overlay-0 and test-ci-undisplayable-0. " + "No sockets of type image, chart, pdf, mp4, thumbnail_jpg or thumbnail_png are used. At least one socket of those types is needed to configure the viewer. " 'Refer to the documentation for more information' ), ), @@ -471,7 +469,7 @@ def make_ci_list( 1, 1, ( - "The following interfaces are used in your {}: test-ci-isolated-0, test-ci-image-0, test-ci-overlay-0, and test-ci-undisplayable-0. " + "The following sockets are used in your {}: test-ci-isolated-0, test-ci-image-0, test-ci-overlay-0, and test-ci-undisplayable-0. " 'Example usage: {{"main": ["test-ci-isolated-0"], "secondary": ["test-ci-image-0", "test-ci-overlay-0"]}}. ' 'Refer to the documentation for more information' ), @@ -522,7 +520,7 @@ def test_archive_and_reader_study_forms_view_content_help_text( 0, 0, ( - "No interfaces of type image, chart, pdf, mp4, thumbnail_jpg or thumbnail_png are used. At least one interface of those types is needed to configure the viewer. " + "No sockets of type image, chart, pdf, mp4, thumbnail_jpg or thumbnail_png are used. At least one socket of those types is needed to configure the viewer. " 'Refer to the documentation for more information' ), ), @@ -532,7 +530,7 @@ def test_archive_and_reader_study_forms_view_content_help_text( 0, 1, ( - "No interfaces of type image, chart, pdf, mp4, thumbnail_jpg or thumbnail_png are used. At least one interface of those types is needed to configure the viewer. " + "No sockets of type image, chart, pdf, mp4, thumbnail_jpg or thumbnail_png are used. At least one socket of those types is needed to configure the viewer. " 'Refer to the documentation for more information' ), ), @@ -586,7 +584,7 @@ def test_algorithm_form_view_content_help_text( 0, 0, ( - "No interfaces of type image, chart, pdf, mp4, thumbnail_jpg or thumbnail_png are used. At least one interface of those types is needed to configure the viewer. " + "No sockets of type image, chart, pdf, mp4, thumbnail_jpg or thumbnail_png are used. At least one socket of those types is needed to configure the viewer. " 'Refer to the documentation for more information' ), ), @@ -596,7 +594,7 @@ def test_algorithm_form_view_content_help_text( 0, 1, ( - "No interfaces of type image, chart, pdf, mp4, thumbnail_jpg or thumbnail_png are used. At least one interface of those types is needed to configure the viewer. " + "No sockets of type image, chart, pdf, mp4, thumbnail_jpg or thumbnail_png are used. At least one socket of those types is needed to configure the viewer. " 'Refer to the documentation for more information' ), ), diff --git a/app/tests/hanging_protocols_tests/test_models.py b/app/tests/hanging_protocols_tests/test_models.py index 9239366e41..91bba684ad 100644 --- a/app/tests/hanging_protocols_tests/test_models.py +++ b/app/tests/hanging_protocols_tests/test_models.py @@ -497,7 +497,7 @@ def test_view_content_validation(): with pytest.raises(ValidationError) as err: hp.full_clean() - assert "Unknown interfaces in view content for viewport main: test" in str( + assert "Unknown sockets in view content for viewport main: test" in str( err.value ) @@ -553,7 +553,7 @@ def test_at_most_two_images(): hp.full_clean() assert ( - "Maximum of one image interface is allowed per viewport, got 2 for viewport main:" + "Maximum of one image socket is allowed per viewport, got 2 for viewport main:" in str(err.value) ) @@ -591,7 +591,7 @@ def test_interfaces_that_must_be_isolated(interface_kind): hp.full_clean() assert ( - "Some of the selected interfaces can only be displayed in isolation, found 2 for viewport main" + "Some of the selected sockets can only be displayed in isolation, found 2 for viewport main" in str(err.value) ) @@ -603,7 +603,7 @@ def test_interfaces_that_must_be_isolated(interface_kind): hp.full_clean() assert ( - "Some of the selected interfaces can only be displayed in isolation, found 1 for viewport main" + "Some of the selected sockets can only be displayed in isolation, found 1 for viewport main" in str(err.value) ) @@ -625,7 +625,7 @@ def test_interfaces_that_cannot_be_displayed(interface_kind): hp.full_clean() assert ( - "Some of the selected interfaces cannot be displayed, found 1 for viewport main:" + "Some of the selected sockets cannot be displayed, found 1 for viewport main:" in str(err.value) ) diff --git a/app/tests/reader_studies_tests/test_models.py b/app/tests/reader_studies_tests/test_models.py index d30400ee12..ffed75e1d8 100644 --- a/app/tests/reader_studies_tests/test_models.py +++ b/app/tests/reader_studies_tests/test_models.py @@ -749,7 +749,7 @@ def test_question_interface(): q.clean() assert e.value.message == ( - f"The interface {ci_img} is not allowed for this " + f"The socket {ci_img} is not allowed for this " f"question type ({AnswerType.TEXT})" )