diff --git a/app/config/urls/challenge_subdomain.py b/app/config/urls/challenge_subdomain.py index 71dfc9b92..55c56aada 100644 --- a/app/config/urls/challenge_subdomain.py +++ b/app/config/urls/challenge_subdomain.py @@ -14,6 +14,10 @@ path( "", include("grandchallenge.well_known.urls", namespace="well-known") ), + path( + "components/", + include("grandchallenge.components.urls", namespace="components"), + ), path( "evaluation/", include("grandchallenge.evaluation.urls", namespace="evaluation"), diff --git a/app/grandchallenge/algorithms/admin.py b/app/grandchallenge/algorithms/admin.py index b9ac6ff3f..b504f3e0d 100644 --- a/app/grandchallenge/algorithms/admin.py +++ b/app/grandchallenge/algorithms/admin.py @@ -8,13 +8,14 @@ from django.utils.html import format_html from guardian.admin import GuardedModelAdmin -from grandchallenge.algorithms.forms import AlgorithmIOValidationMixin from grandchallenge.algorithms.models import ( Algorithm, + AlgorithmAlgorithmInterface, AlgorithmGroupObjectPermission, AlgorithmImage, AlgorithmImageGroupObjectPermission, AlgorithmImageUserObjectPermission, + AlgorithmInterface, AlgorithmModel, AlgorithmModelGroupObjectPermission, AlgorithmModelUserObjectPermission, @@ -36,12 +37,13 @@ UserObjectPermissionAdmin, ) from grandchallenge.core.templatetags.costs import millicents_to_euro +from grandchallenge.core.templatetags.remove_whitespace import oxford_comma from grandchallenge.core.utils.grand_challenge_forge import ( get_forge_algorithm_template_context, ) -class AlgorithmAdminForm(AlgorithmIOValidationMixin, ModelForm): +class AlgorithmAdminForm(ModelForm): class Meta: model = Algorithm fields = "__all__" @@ -208,6 +210,7 @@ class JobAdmin(GuardedModelAdmin): "task_on_success", "task_on_failure", "runtime_metrics", + "algorithm_interface", ) search_fields = ( "creator__username", @@ -235,6 +238,57 @@ class AlgorithmModelAdmin(GuardedModelAdmin): readonly_fields = ("creator", "algorithm", "sha256", "size_in_storage") +@admin.register(AlgorithmInterface) +class AlgorithmInterfaceAdmin(GuardedModelAdmin): + readonly_fields = ("algorithm_inputs", "algorithm_outputs") + list_display = ( + "pk", + "algorithm_inputs", + "algorithm_outputs", + ) + search_fields = ( + "pk", + "inputs__slug", + "outputs__slug", + ) + + def algorithm_inputs(self, obj): + return oxford_comma(obj.inputs.all()) + + def algorithm_outputs(self, obj): + return oxford_comma(obj.outputs.all()) + + def has_change_permission(self, request, obj=None): + # interfaces cannot be modified + return False + + def has_delete_permission(self, request, obj=None): + # interfaces cannot be deleted + return False + + def has_add_permission(self, request, obj=None): + # interfaces should only be created through the UI + return False + + +@admin.register(AlgorithmAlgorithmInterface) +class AlgorithmAlgorithmInterfaceAdmin(GuardedModelAdmin): + list_display = ( + "pk", + "interface", + "algorithm", + ) + list_filter = ("algorithm",) + + def has_add_permission(self, request, obj=None): + # through table entries should only be created through the UI + return False + + def has_change_permission(self, request, obj=None): + # through table entries should only be updated through the UI + return False + + admin.site.register(AlgorithmUserObjectPermission, UserObjectPermissionAdmin) admin.site.register(AlgorithmGroupObjectPermission, GroupObjectPermissionAdmin) admin.site.register(AlgorithmImage, ComponentImageAdmin) diff --git a/app/grandchallenge/algorithms/forms.py b/app/grandchallenge/algorithms/forms.py index b310f90ed..3e82b4cf1 100644 --- a/app/grandchallenge/algorithms/forms.py +++ b/app/grandchallenge/algorithms/forms.py @@ -36,7 +36,11 @@ TextInput, URLField, ) -from django.forms.widgets import MultipleHiddenInput, PasswordInput +from django.forms.widgets import ( + MultipleHiddenInput, + PasswordInput, + RadioSelect, +) from django.urls import Resolver404, resolve from django.utils.functional import cached_property from django.utils.html import format_html @@ -46,6 +50,7 @@ from grandchallenge.algorithms.models import ( Algorithm, AlgorithmImage, + AlgorithmInterface, AlgorithmModel, AlgorithmPermissionRequest, Job, @@ -117,10 +122,18 @@ class JobCreateForm(SaveFormInitMixin, Form): creator = ModelChoiceField( queryset=None, disabled=True, required=False, widget=HiddenInput ) + algorithm_interface = ModelChoiceField( + queryset=None, + disabled=True, + required=True, + widget=HiddenInput, + ) - def __init__(self, *args, algorithm, user, **kwargs): + def __init__(self, *args, algorithm, user, interface, **kwargs): super().__init__(*args, **kwargs) + self._algorithm = algorithm + self.helper = FormHelper() self._user = user @@ -129,7 +142,11 @@ def __init__(self, *args, algorithm, user, **kwargs): ) self.fields["creator"].initial = self._user - self._algorithm = algorithm + self.fields["algorithm_interface"].queryset = ( + self._algorithm.interfaces.all() + ) + self.fields["algorithm_interface"].initial = interface + self._algorithm_image = self._algorithm.active_image active_model = self._algorithm.active_model @@ -146,7 +163,7 @@ def __init__(self, *args, algorithm, user, **kwargs): ) self.fields["algorithm_model"].initial = active_model - for inp in self._algorithm.inputs.all(): + for inp in interface.inputs.all(): prefixed_interface_slug = ( f"{INTERFACE_FORM_FIELD_PREFIX}{inp.slug}" ) @@ -168,7 +185,7 @@ def __init__(self, *args, algorithm, user, **kwargs): instance=inp, initial=initial if initial else inp.default_value, user=self._user, - required=True, + required=inp.value_required, help_text=clean(inp.description) if inp.description else "", form_data=self.data, ).field @@ -233,23 +250,6 @@ def reformat_inputs(self, *, cleaned_data): ] -class AlgorithmIOValidationMixin: - def clean(self): - cleaned_data = super().clean() - - duplicate_interfaces = {*cleaned_data.get("inputs", [])}.intersection( - {*cleaned_data.get("outputs", [])} - ) - - if duplicate_interfaces: - raise ValidationError( - f"The sets of Inputs and Outputs must be unique: " - f"{oxford_comma(duplicate_interfaces)} present in both" - ) - - return cleaned_data - - class PhaseSelectForm(Form): phase = ModelChoiceField( label="Please select the phase for which you are creating an algorithm", @@ -294,43 +294,11 @@ def clean_phase(self): class AlgorithmForm( - AlgorithmIOValidationMixin, WorkstationUserFilterMixin, SaveFormInitMixin, ViewContentExampleMixin, ModelForm, ): - inputs = ModelMultipleChoiceField( - queryset=ComponentInterface.objects.exclude( - slug__in=[*NON_ALGORITHM_INTERFACES, "results-json-file"] - ), - widget=Select2MultipleWidget, - help_text=format_lazy( - ( - "The inputs to this algorithm. " - 'See the list of interfaces for more ' - "information about each interface. " - "Please contact support if your desired input is missing." - ), - reverse_lazy("components:component-interface-list-algorithms"), - ), - ) - outputs = ModelMultipleChoiceField( - queryset=ComponentInterface.objects.exclude( - slug__in=NON_ALGORITHM_INTERFACES - ), - widget=Select2MultipleWidget, - help_text=format_lazy( - ( - "The outputs to this algorithm. " - 'See the list of interfaces for more ' - "information about each interface. " - "Please contact support if your desired output is missing." - ), - reverse_lazy("components:component-interface-list-algorithms"), - ), - ) - class Meta: model = Algorithm fields = ( @@ -350,8 +318,6 @@ class Meta: "hanging_protocol", "optional_hanging_protocols", "view_content", - "inputs", - "outputs", "minimum_credits_per_job", "job_requires_gpu_type", "job_requires_memory_gb", @@ -440,33 +406,22 @@ def job_requirement_properties_from_phases(self): qs = get_objects_for_user( self._user, "evaluation.create_phase_submission" ) - inputs = self.instance.inputs.all() - outputs = self.instance.outputs.all() + interfaces = self.instance.interfaces.all() return ( qs.annotate( - total_algorithm_input_count=Count( - "algorithm_inputs", distinct=True - ), - total_algorithm_output_count=Count( - "algorithm_outputs", distinct=True + total_algorithm_interface_count=Count( + "algorithm_interfaces", distinct=True ), - relevant_algorithm_input_count=Count( - "algorithm_inputs", - filter=Q(algorithm_inputs__in=inputs), - distinct=True, - ), - relevant_algorithm_output_count=Count( - "algorithm_outputs", - filter=Q(algorithm_outputs__in=outputs), + relevant_algorithm_interface_count=Count( + "algorithm_interfaces", + filter=Q(algorithm_interfaces__in=interfaces), distinct=True, ), ) .filter( submission_kind=SubmissionKindChoices.ALGORITHM, - total_algorithm_input_count=len(inputs), - total_algorithm_output_count=len(outputs), - relevant_algorithm_input_count=len(inputs), - relevant_algorithm_output_count=len(outputs), + total_algorithm_interface_count=len(interfaces), + relevant_algorithm_interface_count=len(interfaces), ) .aggregate( max_memory=Max("algorithm_maximum_settable_memory_gb"), @@ -529,32 +484,31 @@ def __init__(self, *args, user, phase, **kwargs): self._user = user self._phase = phase - def get_phase_algorithm_inputs_outputs(self): - return ( - self._phase.algorithm_inputs.all(), - self._phase.algorithm_outputs.all(), - ) - @cached_property def user_algorithms_for_phase(self): - inputs, outputs = self.get_phase_algorithm_inputs_outputs() + interfaces = self._phase.algorithm_interfaces.all() desired_image_subquery = AlgorithmImage.objects.filter( algorithm=OuterRef("pk"), is_desired_version=True ) desired_model_subquery = AlgorithmModel.objects.filter( algorithm=OuterRef("pk"), is_desired_version=True ) + return ( get_objects_for_user(self._user, "algorithms.change_algorithm") .annotate( - total_input_count=Count("inputs", distinct=True), - total_output_count=Count("outputs", distinct=True), - relevant_input_count=Count( - "inputs", filter=Q(inputs__in=inputs), distinct=True - ), - relevant_output_count=Count( - "outputs", filter=Q(outputs__in=outputs), distinct=True + interface_count=Count("interfaces", distinct=True), + relevant_interfaces_count=Count( + "interfaces", + filter=Q(interfaces__in=interfaces), + distinct=True, ), + ) + .filter( + interface_count=len(interfaces), + relevant_interfaces_count=len(interfaces), + ) + .annotate( has_active_image=Exists(desired_image_subquery), active_image_pk=desired_image_subquery.values_list( "pk", flat=True @@ -569,12 +523,6 @@ def user_algorithms_for_phase(self): "comment", flat=True ), ) - .filter( - total_input_count=len(inputs), - total_output_count=len(outputs), - relevant_input_count=len(inputs), - relevant_output_count=len(outputs), - ) ) @cached_property @@ -592,8 +540,7 @@ class Meta: "description", "modalities", "structures", - "inputs", - "outputs", + "interfaces", "workstation", "workstation_config", "hanging_protocol", @@ -615,8 +562,7 @@ class Meta: "display_editors": HiddenInput(), "contact_email": HiddenInput(), "workstation": HiddenInput(), - "inputs": MultipleHiddenInput(), - "outputs": MultipleHiddenInput(), + "interfaces": MultipleHiddenInput(), "modalities": MultipleHiddenInput(), "structures": MultipleHiddenInput(), "logo": HiddenInput(), @@ -639,8 +585,7 @@ def __init__( display_editors, contact_email, workstation, - inputs, - outputs, + interfaces, structures, modalities, logo, @@ -671,10 +616,8 @@ def __init__( ) ) self.fields["workstation"].disabled = True - self.fields["inputs"].initial = inputs - self.fields["inputs"].disabled = True - self.fields["outputs"].initial = outputs - self.fields["outputs"].disabled = True + self.fields["interfaces"].initial = interfaces + self.fields["interfaces"].disabled = True self.fields["modalities"].initial = modalities self.fields["modalities"].disabled = True self.fields["structures"].initial = structures @@ -772,15 +715,6 @@ def __init__(self, *args, **kwargs): ) -class AlgorithmUpdateForm(AlgorithmForm): - def __init__(self, *args, interfaces_editable, **kwargs): - super().__init__(*args, **kwargs) - - if not interfaces_editable: - for field_key in ("inputs", "outputs"): - self.fields.pop(field_key) - - class AlgorithmImageForm(ContainerImageForm): algorithm = ModelChoiceField(widget=HiddenInput(), queryset=None) @@ -1068,7 +1002,8 @@ def __init__(self, *args, user, **kwargs): self.algorithm_serializer = None self.algorithm_image_serializer = None self.algorithm = None - self.new_interfaces = None + self.algorithm_interfaces = [] + self.new_component_interfaces = [] @property def remote_instance_client(self): @@ -1174,27 +1109,16 @@ def _build_algorithm_image(self, headers, netloc): self.algorithm_image_serializer = algorithm_image_serializer def _build_interfaces(self): - remote_interfaces = { - interface["slug"]: interface - for interface in chain( - self.algorithm_serializer.initial_data["inputs"], - self.algorithm_serializer.initial_data["outputs"], + for remote_interface in self.algorithm_serializer.initial_data[ + "interfaces" + ]: + self._validate_interface_inputs_and_outputs( + remote_interface=remote_interface ) - } - - self.new_interfaces = [] - for slug, remote_interface in remote_interfaces.items(): - try: - self._validate_existing_interface( - slug=slug, remote_interface=remote_interface - ) - except ObjectDoesNotExist: - # The remote interface does not exist locally, create it - self._create_new_interface( - slug=slug, remote_interface=remote_interface - ) - def _validate_existing_interface(self, *, remote_interface, slug): + def _validate_existing_component_interface( + self, *, remote_component_interface, slug + ): serialized_local_interface = ComponentInterfaceSerializer( instance=ComponentInterface.objects.get(slug=slug) ) @@ -1203,27 +1127,72 @@ def _validate_existing_interface(self, *, remote_interface, slug): # Check all the values match, some are allowed to differ if ( key not in {"pk", "description"} - and value != remote_interface[key] + and value != remote_component_interface[key] ): raise ValidationError( f"Interface {key} does not match for `{slug}`" ) - def _create_new_interface(self, *, remote_interface, slug): - new_interface = ComponentInterfaceSerializer(data=remote_interface) + def _create_new_component_interface( + self, *, remote_component_interface, slug + ): + new_interface = ComponentInterfaceSerializer( + data=remote_component_interface + ) if not new_interface.is_valid(): raise ValidationError(f"New interface {slug!r} is invalid") - self.new_interfaces.append(new_interface) + self.new_component_interfaces.append(new_interface) + + def _validate_interface_inputs_and_outputs(self, *, remote_interface): + for input in remote_interface["inputs"]: + try: + self._validate_existing_component_interface( + remote_component_interface=input, slug=input["slug"] + ) + except ObjectDoesNotExist: + self._create_new_component_interface( + remote_component_interface=input, slug=input["slug"] + ) + + for output in remote_interface["outputs"]: + try: + self._validate_existing_component_interface( + remote_component_interface=output, slug=output["slug"] + ) + except ObjectDoesNotExist: + self._create_new_component_interface( + remote_component_interface=output, slug=output["slug"] + ) def save(self): + # first save new ComponentInterfaces + self._save_new_component_interfaces() + # then get or create algorithm interfaces self._save_new_interfaces() self._save_new_algorithm() self._save_new_algorithm_image() def _save_new_interfaces(self): - for interface in self.new_interfaces: + for interface in self.algorithm_serializer.initial_data["interfaces"]: + inputs = [ + ComponentInterface.objects.get(slug=input["slug"]) + for input in interface["inputs"] + ] + outputs = [ + ComponentInterface.objects.get(slug=output["slug"]) + for output in interface["outputs"] + ] + # either get or create an AlgorithmInterface + # with the given inputs / outputs using the custom model manager + interface = AlgorithmInterface.objects.create( + inputs=inputs, outputs=outputs + ) + self.algorithm_interfaces.append(interface) + + def _save_new_component_interfaces(self): + for interface in self.new_component_interfaces: interface.save( # The interface kind is a read only display value, this could # be better solved with a custom DRF Field but deadlines... @@ -1256,26 +1225,7 @@ def _save_new_algorithm(self): self.algorithm.add_editor(user=self.user) - self.algorithm.inputs.set( - ComponentInterface.objects.filter( - slug__in={ - interface["slug"] - for interface in self.algorithm_serializer.initial_data[ - "inputs" - ] - } - ) - ) - self.algorithm.outputs.set( - ComponentInterface.objects.filter( - slug__in={ - interface["slug"] - for interface in self.algorithm_serializer.initial_data[ - "outputs" - ] - } - ) - ) + self.algorithm.interfaces.set(self.algorithm_interfaces) if logo_url := self.algorithm_serializer.initial_data["logo"]: response = requests.get( @@ -1423,6 +1373,7 @@ def __init__( ) if hide_algorithm_model_input: + self.fields["algorithm_model"].widget = HiddenInput() self.helper = FormHelper(self) if activate: @@ -1451,3 +1402,124 @@ def clean_algorithm_model(self): raise ValidationError("Model updating already in progress.") return algorithm_model + + +class AlgorithmInterfaceForm(SaveFormInitMixin, ModelForm): + inputs = ModelMultipleChoiceField( + queryset=ComponentInterface.objects.exclude( + slug__in=[*NON_ALGORITHM_INTERFACES, "results-json-file"] + ), + widget=Select2MultipleWidget, + ) + outputs = ModelMultipleChoiceField( + queryset=ComponentInterface.objects.exclude( + slug__in=[*NON_ALGORITHM_INTERFACES, "results-json-file"] + ), + widget=Select2MultipleWidget, + ) + + class Meta: + model = AlgorithmInterface + fields = ( + "inputs", + "outputs", + ) + + def __init__(self, *args, base_obj, **kwargs): + super().__init__(*args, **kwargs) + self._base_obj = base_obj + + def clean_inputs(self): + inputs = self.cleaned_data.get("inputs", []) + + if not inputs: + raise ValidationError("You must provide at least 1 input.") + + if ( + self._base_obj.algorithm_interface_through_model_manager.annotate( + input_count=Count("interface__inputs", distinct=True), + relevant_input_count=Count( + "interface__inputs", + filter=Q(interface__inputs__in=inputs), + distinct=True, + ), + ) + .filter(input_count=len(inputs), relevant_input_count=len(inputs)) + .exists() + ): + raise ValidationError( + f"An AlgorithmInterface for this {self._base_obj._meta.verbose_name} with the " + "same inputs already exists. " + "Algorithm interfaces need to have unique sets of inputs." + ) + return inputs + + def clean_outputs(self): + outputs = self.cleaned_data.get("outputs", []) + + if not outputs: + raise ValidationError("You must provide at least 1 output.") + + return outputs + + def clean(self): + cleaned_data = super().clean() + + # there should always be at least one input and one output, + # this is checked in the individual fields clean methods + inputs = cleaned_data.get("inputs") + outputs = cleaned_data.get("outputs") + + # if either of the two fields is not provided, no need to check for + # duplicates here + if inputs and outputs: + duplicate_interfaces = {*inputs}.intersection({*outputs}) + + if duplicate_interfaces: + raise ValidationError( + f"The sets of Inputs and Outputs must be unique: " + f"{oxford_comma(duplicate_interfaces)} present in both" + ) + + return cleaned_data + + def save(self): + interface = AlgorithmInterface.objects.create( + inputs=self.cleaned_data["inputs"], + outputs=self.cleaned_data["outputs"], + ) + self._base_obj.algorithm_interface_manager.add(interface) + return interface + + +class JobInterfaceSelectForm(SaveFormInitMixin, Form): + algorithm_interface = ModelChoiceField( + queryset=None, + required=True, + help_text="Select an input-output combination to use for this job.", + widget=RadioSelect, + ) + + def __init__(self, *args, algorithm, **kwargs): + super().__init__(*args, **kwargs) + + self._algorithm = algorithm + + self.fields["algorithm_interface"].queryset = ( + self._algorithm.interfaces.all() + ) + self.fields["algorithm_interface"].initial = ( + self._algorithm.interfaces.first() + ) + self.fields["algorithm_interface"].widget.choices = { + ( + interface.pk, + format_html( + "
Inputs: {inputs}
" + "
Outputs: {outputs}
", + inputs=oxford_comma(interface.inputs.all()), + outputs=oxford_comma(interface.outputs.all()), + ), + ) + for interface in self._algorithm.interfaces.all() + } diff --git a/app/grandchallenge/algorithms/migrations/0025_algorithm_hanging_protocol.py b/app/grandchallenge/algorithms/migrations/0025_algorithm_hanging_protocol.py index b872664de..d7d50e23a 100644 --- a/app/grandchallenge/algorithms/migrations/0025_algorithm_hanging_protocol.py +++ b/app/grandchallenge/algorithms/migrations/0025_algorithm_hanging_protocol.py @@ -17,7 +17,7 @@ class Migration(migrations.Migration): name="hanging_protocol", field=models.ForeignKey( blank=True, - help_text='Indicate which Component Interfaces need to be displayed in which image port. E.g. {"main": ["interface1"]}. The first item in the list of interfaces will be the main image in the image port. The first overlay type interface thereafter will be rendered as an overlay. For now, any other items will be ignored by the viewer.', + help_text='Indicate which sockets need to be displayed in which image port. E.g. {"main": ["socket1"]}. The first item in the list of sockets will be the main image in the image port. The first overlay type socket thereafter will be rendered as an overlay. For now, any other items will be ignored by the viewer.', null=True, on_delete=django.db.models.deletion.SET_NULL, to="hanging_protocols.hangingprotocol", diff --git a/app/grandchallenge/algorithms/migrations/0064_algorithminterface_algorithminterfaceoutput_and_more.py b/app/grandchallenge/algorithms/migrations/0064_algorithminterface_algorithminterfaceoutput_and_more.py new file mode 100644 index 000000000..b79bff954 --- /dev/null +++ b/app/grandchallenge/algorithms/migrations/0064_algorithminterface_algorithminterfaceoutput_and_more.py @@ -0,0 +1,166 @@ +# Generated by Django 4.2.16 on 2024-12-04 14:06 + +import uuid + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("components", "0022_alter_componentinterface_kind_and_more"), + ( + "algorithms", + "0063_alter_optionalhangingprotocolalgorithm_unique_together", + ), + ] + + operations = [ + migrations.CreateModel( + name="AlgorithmInterface", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("created", models.DateTimeField(auto_now_add=True)), + ("modified", models.DateTimeField(auto_now=True)), + ], + options={ + "abstract": False, + }, + ), + migrations.CreateModel( + name="AlgorithmInterfaceOutput", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "interface", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="algorithms.algorithminterface", + ), + ), + ( + "output", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="components.componentinterface", + ), + ), + ], + ), + migrations.CreateModel( + name="AlgorithmInterfaceInput", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "input", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="components.componentinterface", + ), + ), + ( + "interface", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="algorithms.algorithminterface", + ), + ), + ], + ), + migrations.AddField( + model_name="algorithminterface", + name="inputs", + field=models.ManyToManyField( + related_name="inputs", + through="algorithms.AlgorithmInterfaceInput", + to="components.componentinterface", + ), + ), + migrations.AddField( + model_name="algorithminterface", + name="outputs", + field=models.ManyToManyField( + related_name="outputs", + through="algorithms.AlgorithmInterfaceOutput", + to="components.componentinterface", + ), + ), + migrations.CreateModel( + name="AlgorithmAlgorithmInterface", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "algorithm", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="algorithms.algorithm", + ), + ), + ( + "interface", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="algorithms.algorithminterface", + ), + ), + ], + ), + migrations.AddField( + model_name="algorithm", + name="interfaces", + field=models.ManyToManyField( + related_name="algorithm_interfaces", + through="algorithms.AlgorithmAlgorithmInterface", + to="algorithms.algorithminterface", + ), + ), + migrations.AddConstraint( + model_name="algorithmalgorithminterface", + constraint=models.UniqueConstraint( + fields=("algorithm", "interface"), + name="unique_algorithm_interface_combination", + ), + ), + migrations.AddField( + model_name="job", + name="algorithm_interface", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.PROTECT, + to="algorithms.algorithminterface", + ), + ), + ] diff --git a/app/grandchallenge/algorithms/migrations/0065_create_algorithm_interfaces.py b/app/grandchallenge/algorithms/migrations/0065_create_algorithm_interfaces.py new file mode 100644 index 000000000..3caf518a7 --- /dev/null +++ b/app/grandchallenge/algorithms/migrations/0065_create_algorithm_interfaces.py @@ -0,0 +1,44 @@ +from django.db import migrations + +from grandchallenge.algorithms.models import ( + get_existing_interface_for_inputs_and_outputs, +) + + +def create_algorithm_interfaces(apps, _schema_editor): + Algorithm = apps.get_model("algorithms", "Algorithm") # noqa: N806 + AlgorithmInterface = apps.get_model( # noqa: N806 + "algorithms", "AlgorithmInterface" + ) + + for algorithm in Algorithm.objects.prefetch_related( + "inputs", "outputs" + ).all(): + inputs = algorithm.inputs.all() + outputs = algorithm.outputs.all() + + if not inputs or not outputs: + raise RuntimeError(f"{algorithm} is improperly configured.") + + io = get_existing_interface_for_inputs_and_outputs( + model=AlgorithmInterface, inputs=inputs, outputs=outputs + ) + if not io: + io = AlgorithmInterface.objects.create() + io.inputs.set(inputs) + io.outputs.set(outputs) + + algorithm.interfaces.add(io) + + +class Migration(migrations.Migration): + dependencies = [ + ( + "algorithms", + "0064_algorithminterface_algorithminterfaceoutput_and_more", + ), + ] + + operations = [ + migrations.RunPython(create_algorithm_interfaces, elidable=True), + ] diff --git a/app/grandchallenge/algorithms/migrations/0066_create_interfaces_for_jobs.py b/app/grandchallenge/algorithms/migrations/0066_create_interfaces_for_jobs.py new file mode 100644 index 000000000..55f7c3333 --- /dev/null +++ b/app/grandchallenge/algorithms/migrations/0066_create_interfaces_for_jobs.py @@ -0,0 +1,24 @@ +from django.db import migrations + + +def add_algorithm_interfaces_to_jobs(apps, _schema_editor): + Algorithm = apps.get_model("algorithms", "Algorithm") # noqa: N806 + Job = apps.get_model("algorithms", "Job") # noqa: N806 + + for algorithm in Algorithm.objects.prefetch_related("interfaces").all(): + Job.objects.filter(algorithm_image__algorithm=algorithm).update( + algorithm_interface=algorithm.interfaces.get() + ) + + +class Migration(migrations.Migration): + dependencies = [ + ( + "algorithms", + "0065_create_algorithm_interfaces", + ), + ] + + operations = [ + migrations.RunPython(add_algorithm_interfaces_to_jobs, elidable=True), + ] diff --git a/app/grandchallenge/algorithms/migrations/0067_alter_algorithm_inputs_alter_algorithm_outputs.py b/app/grandchallenge/algorithms/migrations/0067_alter_algorithm_inputs_alter_algorithm_outputs.py new file mode 100644 index 000000000..d1326ae49 --- /dev/null +++ b/app/grandchallenge/algorithms/migrations/0067_alter_algorithm_inputs_alter_algorithm_outputs.py @@ -0,0 +1,31 @@ +# Generated by Django 4.2.19 on 2025-02-17 09:48 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("components", "0024_alter_componentinterface_kind_and_more"), + ("algorithms", "0066_create_interfaces_for_jobs"), + ] + + operations = [ + migrations.AlterField( + model_name="algorithm", + name="inputs", + field=models.ManyToManyField( + null=True, + related_name="algorithm_inputs", + to="components.componentinterface", + ), + ), + migrations.AlterField( + model_name="algorithm", + name="outputs", + field=models.ManyToManyField( + null=True, + related_name="algorithm_outputs", + to="components.componentinterface", + ), + ), + ] diff --git a/app/grandchallenge/algorithms/models.py b/app/grandchallenge/algorithms/models.py index 930b70023..2b3e187b8 100644 --- a/app/grandchallenge/algorithms/models.py +++ b/app/grandchallenge/algorithms/models.py @@ -20,6 +20,7 @@ from django.utils.functional import cached_property from django.utils.text import get_valid_filename from django.utils.timezone import now +from django_deprecate_fields import deprecate_field from django_extensions.db.models import TitleSlugDescriptionModel from guardian.models import GroupObjectPermissionBase, UserObjectPermissionBase from guardian.shortcuts import assign_perm, remove_perm @@ -69,6 +70,96 @@ JINJA_ENGINE = sandbox.ImmutableSandboxedEnvironment() +def annotate_input_output_counts(queryset, inputs=None, outputs=None): + return queryset.annotate( + input_count=Count("inputs", distinct=True), + output_count=Count("outputs", distinct=True), + relevant_input_count=Count( + "inputs", + filter=Q(inputs__in=inputs) if inputs is not None else Q(), + distinct=True, + ), + relevant_output_count=Count( + "outputs", + filter=Q(outputs__in=outputs) if outputs is not None else Q(), + distinct=True, + ), + ) + + +class AlgorithmInterfaceManager(models.Manager): + + def create( + self, + *, + inputs, + outputs, + **kwargs, + ): + if not inputs or not outputs: + raise ValidationError( + "An interface must have at least one input and one output." + ) + + obj = get_existing_interface_for_inputs_and_outputs( + inputs=inputs, outputs=outputs + ) + if not obj: + obj = super().create(**kwargs) + obj.inputs.set(inputs) + obj.outputs.set(outputs) + + return obj + + def delete(self): + raise NotImplementedError("Bulk delete is not allowed.") + + +class AlgorithmInterface(UUIDModel): + inputs = models.ManyToManyField( + to=ComponentInterface, + related_name="inputs", + through="algorithms.AlgorithmInterfaceInput", + ) + outputs = models.ManyToManyField( + to=ComponentInterface, + related_name="outputs", + through="algorithms.AlgorithmInterfaceOutput", + ) + + objects = AlgorithmInterfaceManager() + + def delete(self, *args, **kwargs): + raise ValidationError("AlgorithmInterfaces cannot be deleted.") + + +class AlgorithmInterfaceInput(models.Model): + input = models.ForeignKey(ComponentInterface, on_delete=models.CASCADE) + interface = models.ForeignKey(AlgorithmInterface, on_delete=models.CASCADE) + + +class AlgorithmInterfaceOutput(models.Model): + output = models.ForeignKey(ComponentInterface, on_delete=models.CASCADE) + interface = models.ForeignKey(AlgorithmInterface, on_delete=models.CASCADE) + + +def get_existing_interface_for_inputs_and_outputs( + *, inputs, outputs, model=AlgorithmInterface +): + annotated_qs = annotate_input_output_counts( + model.objects.all(), inputs=inputs, outputs=outputs + ) + try: + return annotated_qs.get( + relevant_input_count=len(inputs), + relevant_output_count=len(outputs), + input_count=len(inputs), + output_count=len(outputs), + ) + except ObjectDoesNotExist: + return None + + class Algorithm(UUIDModel, TitleSlugDescriptionModel, HangingProtocolMixin): editors_group = models.OneToOneField( Group, @@ -148,11 +239,22 @@ class Algorithm(UUIDModel, TitleSlugDescriptionModel, HangingProtocolMixin): "{% endfor %}" ), ) - inputs = models.ManyToManyField( - to=ComponentInterface, related_name="algorithm_inputs", blank=False + interfaces = models.ManyToManyField( + to=AlgorithmInterface, + related_name="algorithm_interfaces", + through="algorithms.AlgorithmAlgorithmInterface", ) - outputs = models.ManyToManyField( - to=ComponentInterface, related_name="algorithm_outputs", blank=False + inputs = deprecate_field( + models.ManyToManyField( + to=ComponentInterface, related_name="algorithm_inputs", blank=False + ) + ) + outputs = deprecate_field( + models.ManyToManyField( + to=ComponentInterface, + related_name="algorithm_outputs", + blank=False, + ) ) publications = models.ManyToManyField( Publication, @@ -425,6 +527,28 @@ def default_workstation(self): return w + @property + def algorithm_interface_manager(self): + return self.interfaces + + @property + def algorithm_interface_through_model_manager(self): + return AlgorithmAlgorithmInterface.objects.filter(algorithm=self) + + @property + def algorithm_interface_create_url(self): + return reverse( + "algorithms:interface-create", kwargs={"slug": self.slug} + ) + + @property + def algorithm_interface_delete_viewname(self): + return "algorithms:interface-delete" + + @property + def algorithm_interface_list_url(self): + return reverse("algorithms:interface-list", kwargs={"slug": self.slug}) + def is_editor(self, user): return user.groups.filter(pk=self.editors_group.pk).exists() @@ -445,7 +569,11 @@ def remove_user(self, user): @cached_property def linked_component_interfaces(self): - return (self.inputs.all() | self.outputs.all()).distinct() + return { + ci + for interface in self.interfaces.all() + for ci in (interface.inputs.all() | interface.outputs.all()) + } @cached_property def user_statistics(self): @@ -525,6 +653,22 @@ def form_field_label(self): return title +class AlgorithmAlgorithmInterface(models.Model): + algorithm = models.ForeignKey(Algorithm, on_delete=models.CASCADE) + interface = models.ForeignKey(AlgorithmInterface, on_delete=models.CASCADE) + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["algorithm", "interface"], + name="unique_algorithm_interface_combination", + ), + ] + + def __str__(self): + return str(self.interface) + + class AlgorithmUserObjectPermission(UserObjectPermissionBase): content_object = models.ForeignKey(Algorithm, on_delete=models.CASCADE) @@ -857,21 +1001,23 @@ def get_jobs_with_same_inputs( unique_kwargs = { "algorithm_image": algorithm_image, } - input_interface_count = algorithm_image.algorithm.inputs.count() + input_interface_count = len(inputs) if algorithm_model: unique_kwargs["algorithm_model"] = algorithm_model else: unique_kwargs["algorithm_model__isnull"] = True - existing_jobs = ( - Job.objects.filter(**unique_kwargs) - .annotate( - inputs_match_count=Count( - "inputs", filter=Q(inputs__in=existing_civs) - ) - ) - .filter(inputs_match_count=input_interface_count) + # annotate the number of inputs and the number of inputs that match + # the existing civs and filter on both counts so as to not include jobs + # with partially overlapping inputs + # or jobs with more inputs than the existing civs + annotated_qs = annotate_input_output_counts( + queryset=Job.objects.filter(**unique_kwargs), inputs=existing_civs + ) + existing_jobs = annotated_qs.filter( + input_count=input_interface_count, + relevant_input_count=input_interface_count, ) return existing_jobs @@ -962,6 +1108,9 @@ class Job(CIVForObjectMixin, ComponentJob): algorithm_model = models.ForeignKey( AlgorithmModel, on_delete=models.PROTECT, null=True, blank=True ) + algorithm_interface = models.ForeignKey( + AlgorithmInterface, on_delete=models.PROTECT, null=True, blank=True + ) creator = models.ForeignKey( settings.AUTH_USER_MODEL, null=True, on_delete=models.SET_NULL ) @@ -1008,14 +1157,17 @@ def container(self): @property def output_interfaces(self): - return self.algorithm_image.algorithm.outputs + return self.algorithm_interface.outputs.all() @cached_property def inputs_complete(self): # check if all inputs are present and if they all have a value + # interfaces that do not require a value will be considered complete regardless return { - civ.interface for civ in self.inputs.all() if civ.has_value - } == {*self.algorithm_image.algorithm.inputs.all()} + civ.interface + for civ in self.inputs.all() + if civ.has_value or not civ.interface.value_required + } == {*self.algorithm_interface.inputs.all()} @cached_property def rendered_result_text(self) -> str: diff --git a/app/grandchallenge/algorithms/serializers.py b/app/grandchallenge/algorithms/serializers.py index 28ece083a..13f81ded7 100644 --- a/app/grandchallenge/algorithms/serializers.py +++ b/app/grandchallenge/algorithms/serializers.py @@ -1,6 +1,6 @@ import logging -from django.core.exceptions import ValidationError +from django.core.exceptions import ObjectDoesNotExist, ValidationError from rest_framework import serializers from rest_framework.fields import ( CharField, @@ -16,13 +16,15 @@ from grandchallenge.algorithms.models import ( Algorithm, AlgorithmImage, + AlgorithmInterface, AlgorithmModel, Job, + annotate_input_output_counts, ) from grandchallenge.components.backends.exceptions import ( CIVNotEditableException, ) -from grandchallenge.components.models import CIVData, ComponentInterface +from grandchallenge.components.models import CIVData from grandchallenge.components.serializers import ( ComponentInterfaceSerializer, ComponentInterfaceValuePostSerializer, @@ -30,6 +32,7 @@ HyperlinkedComponentInterfaceValueSerializer, ) from grandchallenge.core.guardian import filter_by_permission +from grandchallenge.core.templatetags.remove_whitespace import oxford_comma from grandchallenge.hanging_protocols.serializers import ( HangingProtocolSerializer, ) @@ -37,12 +40,25 @@ logger = logging.getLogger(__name__) -class AlgorithmSerializer(serializers.ModelSerializer): - average_duration = SerializerMethodField() +class AlgorithmInterfaceSerializer(serializers.ModelSerializer): + """Serializer without hyperlinks for internal use""" + inputs = ComponentInterfaceSerializer(many=True, read_only=True) outputs = ComponentInterfaceSerializer(many=True, read_only=True) + + class Meta: + model = AlgorithmInterface + fields = [ + "inputs", + "outputs", + ] + + +class AlgorithmSerializer(serializers.ModelSerializer): + average_duration = SerializerMethodField() logo = URLField(source="logo.x20.url", read_only=True) url = URLField(source="get_absolute_url", read_only=True) + interfaces = AlgorithmInterfaceSerializer(many=True, read_only=True) class Meta: model = Algorithm @@ -55,8 +71,7 @@ class Meta: "logo", "slug", "average_duration", - "inputs", - "outputs", + "interfaces", ] def get_average_duration(self, obj: Algorithm) -> float | None: @@ -158,7 +173,6 @@ class HyperlinkedJobSerializer(JobSerializer): view_name="api:algorithm-detail", read_only=True, ) - inputs = HyperlinkedComponentInterfaceValueSerializer(many=True) outputs = HyperlinkedComponentInterfaceValueSerializer(many=True) @@ -215,45 +229,10 @@ def validate(self, data): "You have run out of algorithm credits" ) - # validate that no inputs are provided that are not configured for the - # algorithm and that all interfaces without defaults are provided - algorithm_input_pks = {a.pk for a in self._algorithm.inputs.all()} - input_pks = {i["interface"].pk for i in data["inputs"]} - - # surplus inputs: provided but interfaces not configured for the algorithm - surplus = ComponentInterface.objects.filter( - id__in=list(input_pks - algorithm_input_pks) - ) - if surplus: - titles = ", ".join(ci.title for ci in surplus) - raise serializers.ValidationError( - f"Provided inputs(s) {titles} are not defined for this algorithm" - ) - - # missing inputs - missing = self._algorithm.inputs.filter( - id__in=list(algorithm_input_pks - input_pks), - default_value__isnull=True, - ) - if missing: - titles = ", ".join(ci.title for ci in missing) - raise serializers.ValidationError( - f"Interface(s) {titles} do not have a default value and should be provided." - ) - inputs = data.pop("inputs") - - default_inputs = self._algorithm.inputs.filter( - id__in=list(algorithm_input_pks - input_pks), - default_value__isnull=False, + data["algorithm_interface"] = ( + self.validate_inputs_and_return_matching_interface(inputs=inputs) ) - # Use default interface values if not present - for interface in default_inputs: - if interface.default_value: - inputs.append( - {"interface": interface, "value": interface.default_value} - ) - self.inputs = self.reformat_inputs(serialized_civs=inputs) if Job.objects.get_jobs_with_same_inputs( @@ -298,6 +277,29 @@ def create(self, validated_data): return job + def validate_inputs_and_return_matching_interface(self, *, inputs): + """ + Validates that the provided inputs match one of the configured interfaces of + the algorithm and returns that AlgorithmInterface + """ + provided_inputs = {i["interface"] for i in inputs} + annotated_qs = annotate_input_output_counts( + self._algorithm.interfaces, inputs=provided_inputs + ) + try: + interface = annotated_qs.get( + relevant_input_count=len(provided_inputs), + input_count=len(provided_inputs), + ) + return interface + except ObjectDoesNotExist: + raise serializers.ValidationError( + f"The set of inputs provided does not match " + f"any of the algorithm's interfaces. This algorithm supports the " + f"following input combinations: " + f"{oxford_comma([f'Interface {n}: {oxford_comma(interface.inputs.all())}' for n, interface in enumerate(self._algorithm.interfaces.all(), start=1)])}" + ) + @staticmethod def reformat_inputs(*, serialized_civs): """Takes serialized CIV data and returns list of CIVData objects.""" diff --git a/app/grandchallenge/algorithms/tasks.py b/app/grandchallenge/algorithms/tasks.py index 9622c27ca..15db440c4 100644 --- a/app/grandchallenge/algorithms/tasks.py +++ b/app/grandchallenge/algorithms/tasks.py @@ -8,7 +8,6 @@ from django.core.cache import cache from django.core.files.base import File from django.db import transaction -from django.db.models import Count, Q from django.db.transaction import on_commit from django.utils._os import safe_join @@ -97,12 +96,9 @@ def create_algorithm_jobs_for_archive( create_algorithm_jobs( algorithm_image=algorithm.active_image, algorithm_model=algorithm.active_model, - civ_sets=[ - {*ai.values.all()} - for ai in archive_items.prefetch_related( - "values__interface" - ) - ], + archive_items=archive_items.prefetch_related( + "values__interface" + ), extra_viewer_groups=archive_groups, # NOTE: no emails in case the logs leak data # to the algorithm editors @@ -116,7 +112,7 @@ def create_algorithm_jobs_for_archive( def create_algorithm_jobs( *, algorithm_image, - civ_sets, + archive_items, time_limit, requires_gpu_type, requires_memory_gb, @@ -134,8 +130,8 @@ def create_algorithm_jobs( ---------- algorithm_image The algorithm image to use - civ_sets - The sets of component interface values that will be used as input + archive_items + Archive items whose values will be used as input for the algorithm image time_limit The time limit for the Job @@ -163,54 +159,61 @@ def create_algorithm_jobs( if not algorithm_image: raise RuntimeError("Algorithm image required to create jobs.") - civ_sets = filter_civs_for_algorithm( - civ_sets=civ_sets, + valid_job_inputs = filter_archive_items_for_algorithm( + archive_items=archive_items, algorithm_image=algorithm_image, algorithm_model=algorithm_model, ) - if max_jobs is not None: - civ_sets = civ_sets[:max_jobs] - if time_limit is None: time_limit = settings.ALGORITHMS_JOB_DEFAULT_TIME_LIMIT_SECONDS jobs = [] - for civ_set in civ_sets: - - if len(jobs) >= settings.ALGORITHMS_JOB_BATCH_LIMIT: - raise TooManyJobsScheduled - - with transaction.atomic(): - job = Job.objects.create( - creator=None, # System jobs, so no creator - algorithm_image=algorithm_image, - algorithm_model=algorithm_model, - task_on_success=task_on_success, - task_on_failure=task_on_failure, - time_limit=time_limit, - requires_gpu_type=requires_gpu_type, - requires_memory_gb=requires_memory_gb, - extra_viewer_groups=extra_viewer_groups, - extra_logs_viewer_groups=extra_logs_viewer_groups, - input_civ_set=civ_set, - ) - on_commit(job.execute) + for interface, archive_items in valid_job_inputs.items(): + for ai in archive_items: + if max_jobs is not None and len(jobs) >= max_jobs: + # only schedule max_jobs amount of jobs + # the rest will be scheduled only after these have succeeded + # we do not want to retry the task here, so just stop the loop + break + + if len(jobs) >= settings.ALGORITHMS_JOB_BATCH_LIMIT: + # raise exception so that we retry the task + raise TooManyJobsScheduled + + with transaction.atomic(): + job = Job.objects.create( + creator=None, # System jobs, so no creator + algorithm_image=algorithm_image, + algorithm_model=algorithm_model, + algorithm_interface=interface, + task_on_success=task_on_success, + task_on_failure=task_on_failure, + time_limit=time_limit, + requires_gpu_type=requires_gpu_type, + requires_memory_gb=requires_memory_gb, + extra_viewer_groups=extra_viewer_groups, + extra_logs_viewer_groups=extra_logs_viewer_groups, + input_civ_set=ai.values.all(), + ) + on_commit(job.execute) - jobs.append(job) + jobs.append(job) return jobs -def filter_civs_for_algorithm(*, civ_sets, algorithm_image, algorithm_model): +def filter_archive_items_for_algorithm( + *, archive_items, algorithm_image, algorithm_model=None +): """ - Removes sets of civs that are invalid for new jobs + Removes archive items that are invalid for new jobs. + The archive items need to contain values for all inputs of one of the algorithm's interfaces. Parameters ---------- - civ_sets - Iterable of sets of ComponentInterfaceValues that are candidate for - new Jobs + archive_items + Archive items whose values are candidates for new jobs' inputs algorithm_image The algorithm image to use for new job algorithm_model @@ -218,50 +221,48 @@ def filter_civs_for_algorithm(*, civ_sets, algorithm_image, algorithm_model): Returns ------- - Filtered set of ComponentInterfaceValues + Dictionary of valid ArchiveItems for new jobs, grouped by AlgorithmInterface """ - from grandchallenge.algorithms.models import Job - - input_interfaces = {*algorithm_image.algorithm.inputs.all()} + from grandchallenge.evaluation.models import ( + get_archive_items_for_interfaces, + get_valid_jobs_for_interfaces_and_archive_items, + ) - existing_jobs = { - frozenset(j.inputs.all()) - for j in Job.objects.filter( - algorithm_image=algorithm_image, algorithm_model=algorithm_model - ) - .annotate( - inputs_match_count=Count( - "inputs", - filter=Q( - inputs__in={civ for civ_set in civ_sets for civ in civ_set} - ), - ) - ) - .filter(inputs_match_count=len(input_interfaces), creator=None) - .prefetch_related("inputs") - } - - valid_job_inputs = [] - - for civ_set in civ_sets: - # Check interfaces are complete - civ_interfaces = {civ.interface for civ in civ_set} - if input_interfaces.issubset(civ_interfaces): - # If the algorithm works with a subset of the interfaces - # present in the set then only feed these through to the algorithm - valid_input = { - civ for civ in civ_set if civ.interface in input_interfaces - } - else: - continue + algorithm_interfaces = ( + algorithm_image.algorithm.interfaces.prefetch_related("inputs").all() + ) - # Check job has not been run - if frozenset(valid_input) in existing_jobs: - continue + # First, sort archive items by algorithm interface: + # An archive item is only valid for an interface if it has values + # for all inputs of the interface + valid_job_inputs = get_archive_items_for_interfaces( + algorithm_interfaces=algorithm_interfaces, archive_items=archive_items + ) - valid_job_inputs.append(valid_input) + # Next, group all system jobs that have been run for the provided archive items + # with the same model and image by interface + existing_jobs_for_interfaces = ( + get_valid_jobs_for_interfaces_and_archive_items( + algorithm_image=algorithm_image, + algorithm_model=algorithm_model, + algorithm_interfaces=algorithm_interfaces, + valid_archive_items_per_interface=valid_job_inputs, + ) + ) + # Finally, exclude archive items for which there already is a job + filtered_valid_job_inputs = {} + for interface, archive_items in valid_job_inputs.items(): + job_input_sets_for_interface = { + frozenset(j.inputs.all()) + for j in existing_jobs_for_interfaces[interface] + } + filtered_valid_job_inputs[interface] = [ + ai + for ai in archive_items + if not frozenset(ai.values.all()) in job_input_sets_for_interface + ] - return valid_job_inputs + return filtered_valid_job_inputs @acks_late_micro_short_task diff --git a/app/grandchallenge/algorithms/templates/algorithms/algorithm_detail.html b/app/grandchallenge/algorithms/templates/algorithms/algorithm_detail.html index 63bfd9d9e..896403f46 100644 --- a/app/grandchallenge/algorithms/templates/algorithms/algorithm_detail.html +++ b/app/grandchallenge/algorithms/templates/algorithms/algorithm_detail.html @@ -36,6 +36,13 @@ {% if "change_algorithm" in algorithm_perms %} {% if perms.algorithms.add_algorithm %} +  Interfaces + {% if not object.interfaces.all %}  + + {% endif %} + + href="{% url 'algorithms:job-interface-select' slug=object.slug %}">  Try-out Algorithm {% endif %} @@ -121,7 +128,7 @@ {% if object.public and not object.public_test_case %} {% endif %} {% endif %} @@ -220,29 +227,10 @@

About

{% endif %} -
-
Inputs:
-
- -
-
+

Interfaces

+

This algorithm implements all of the following input-output combinations:

+ {% include 'algorithms/partials/algorithminterface_table.html' with base_obj=object interfaces=object.interfaces.all delete_option=False %} -
-
Outputs:
-
- -
-
{% if best_evaluation_per_phase %}

Challenge Performance

@@ -602,7 +590,7 @@
Make this algorithm public
Update Settings Update Description - Add Test Case + Add Test Case
Publish algorithm diff --git a/app/grandchallenge/algorithms/templates/algorithms/algorithmalgorithminterface_confirm_delete.html b/app/grandchallenge/algorithms/templates/algorithms/algorithmalgorithminterface_confirm_delete.html new file mode 100644 index 000000000..4d0c05151 --- /dev/null +++ b/app/grandchallenge/algorithms/templates/algorithms/algorithmalgorithminterface_confirm_delete.html @@ -0,0 +1,26 @@ +{% extends "base.html" %} +{% load remove_whitespace %} + +{% block title %} + Delete Interface - {{ algorithm.title }} - {{ block.super }} +{% endblock %} + +{% block breadcrumbs %} +
    +
  1. Algorithms +
    2. +
  2. {{ algorithm.title }} +
    3. +
  3. Interfaces +
    4. +
  4. Delete Interface +
    5. +
+{% endblock %} + +{% block content %} + {% include 'algorithms/partials/algorithminterface_confirm_delete.html' with base_obj=algorithm %} +{% endblock %} diff --git a/app/grandchallenge/algorithms/templates/algorithms/algorithmalgorithminterface_list.html b/app/grandchallenge/algorithms/templates/algorithms/algorithmalgorithminterface_list.html new file mode 100644 index 000000000..883c635a4 --- /dev/null +++ b/app/grandchallenge/algorithms/templates/algorithms/algorithmalgorithminterface_list.html @@ -0,0 +1,24 @@ +{% extends "base.html" %} + +{% block title %} + Interfaces - {{ algorithm.title }} - {{ block.super }} +{% endblock %} + +{% block breadcrumbs %} +
    +
  1. Algorithms +
    2. +
  2. {{ algorithm.title }} + +
    3. +
  3. Interfaces +
    4. +
+{% endblock %} + +{% block content %} + {% include 'algorithms/partials/algorithminterface_list.html' with base_obj=algorithm %} +{% endblock %} diff --git a/app/grandchallenge/algorithms/templates/algorithms/algorithmimage_form.html b/app/grandchallenge/algorithms/templates/algorithms/algorithmimage_form.html index 5343922a9..8f1f18e61 100644 --- a/app/grandchallenge/algorithms/templates/algorithms/algorithmimage_form.html +++ b/app/grandchallenge/algorithms/templates/algorithms/algorithmimage_form.html @@ -24,53 +24,20 @@

Create An Algorithm Container Image

- Upload a container image that implements the selected inputs and outputs. - In order to work this container will need to read the following input: + Upload a container image that implements all of the configured interfaces (i.e. input-output combinations):

- - -

- Please see the - list of input options - more information and examples. -

- -

- The container will also need to write the following output: -

- - - -

- Please see the - list of output options - more information and examples. -

+ {% include 'algorithms/partials/algorithminterface_table.html' with base_obj=algorithm interfaces=algorithm.interfaces.all delete_option=False read_write_paths=True %} + + {% if perms.algorithms.add_algorithm %} +

+ To add or update interfaces for your algorithm, go here. +

+ {% else %} +

+ To add or update interfaces for your algorithm, please contact support@grand-challenge.org. +

+ {% endif %}

Container Image Options

diff --git a/app/grandchallenge/algorithms/templates/algorithms/algorithminterface_form.html b/app/grandchallenge/algorithms/templates/algorithms/algorithminterface_form.html new file mode 100644 index 000000000..02614e610 --- /dev/null +++ b/app/grandchallenge/algorithms/templates/algorithms/algorithminterface_form.html @@ -0,0 +1,25 @@ +{% extends "base.html" %} + +{% block title %} + Create Interface - {{ base_obj.title }} - {{ block.super }} +{% endblock %} + +{% block breadcrumbs %} +
    +
  1. Algorithms +
    2. +
  2. {{ base_obj.title }} +
    3. +
  3. Interfaces +
    4. +
  4. Create Interface +
    5. +
+{% endblock %} + +{% block content %} + {% include 'algorithms/partials/algorithminterface_form.html' %} +{% endblock %} diff --git a/app/grandchallenge/algorithms/templates/algorithms/job_detail.html b/app/grandchallenge/algorithms/templates/algorithms/job_detail.html index 2eb9edf58..bce180ec4 100644 --- a/app/grandchallenge/algorithms/templates/algorithms/job_detail.html +++ b/app/grandchallenge/algorithms/templates/algorithms/job_detail.html @@ -52,7 +52,6 @@ aria-selected="false"> Logs {% endif %} - {% if object.status == object.SUCCESS and perms.reader_studies.add_readerstudy %} diff --git a/app/grandchallenge/algorithms/templates/algorithms/job_form_create.html b/app/grandchallenge/algorithms/templates/algorithms/job_form_create.html index 1914cf8ad..47527a490 100644 --- a/app/grandchallenge/algorithms/templates/algorithms/job_form_create.html +++ b/app/grandchallenge/algorithms/templates/algorithms/job_form_create.html @@ -36,59 +36,66 @@ {% block content %}

Try-out Algorithm

- {{ algorithm.job_create_page_markdown|md2html }} - {% get_obj_perms request.user for algorithm as "algorithm_perms" %} - {% if not algorithm.active_image %} -

- This algorithm is not ready to be used. - {% if 'change_algorithm' in algorithm_perms %} - Please upload a valid container image for this algorithm. - {% endif %} -

- {% elif form.jobs_limit < 1 %} -

- You have run out of credits to try this algorithm. - You can request more credits by sending an e-mail to - - support@grand-challenge.org. -

+ {% if not algorithm.interfaces.all and 'change_algorithm' in algorithm_perms %} +

Your algorithm does not have any interfaces yet. You need to define at least one interface (i.e. input - output combination) before you can try it out.

+

To define an interface, navigate here.

+ {% elif not algorithm.interfaces.all %} +

This algorithm is not fully configured yet and hence cannot be used.

{% else %} -

- Select the data that you would like to run the algorithm on. -

-

- {% if 'change_algorithm' in algorithm_perms %} - As an editor for this algorithm you can test and debug your algorithm in total {{ editors_job_limit }} times per unique algorithm image. - You share these credits with all other editors of this algorithm. - Once you have reached the limit, any extra jobs will be deducted from your personal algorithm credits, - of which you get {{ request.user.user_credit.credits }} per month. - {% else %} - You receive {{ request.user.user_credit.credits }} credits per month. - {% endif %} - Using this algorithm requires {{ algorithm.credits_per_job }} - credit{{ algorithm.credits_per_job|pluralize }} per job. - You can currently create up to {{ form.jobs_limit }} job{{ form.jobs_limit|pluralize }} for this algorithm. -

+ {{ algorithm.job_create_page_markdown|md2html }} + + {% if not algorithm.active_image %} +

+ This algorithm is not ready to be used. + {% if 'change_algorithm' in algorithm_perms %} + Please upload a valid container image for this algorithm. + {% endif %} +

+ {% elif form.jobs_limit < 1 %} +

+ You have run out of credits to try this algorithm. + You can request more credits by sending an e-mail to + + support@grand-challenge.org. +

+ {% else %} +

+ Select the data that you would like to run the algorithm on. +

+

+ {% if 'change_algorithm' in algorithm_perms %} + As an editor for this algorithm you can test and debug your algorithm in total {{ editors_job_limit }} times per unique algorithm image. + You share these credits with all other editors of this algorithm. + Once you have reached the limit, any extra jobs will be deducted from your personal algorithm credits, + of which you get {{ request.user.user_credit.credits }} per month. + {% else %} + You receive {{ request.user.user_credit.credits }} credits per month. + {% endif %} + Using this algorithm requires {{ algorithm.credits_per_job }} + credit{{ algorithm.credits_per_job|pluralize }} per job. + You can currently create up to {{ form.jobs_limit }} job{{ form.jobs_limit|pluralize }} for this algorithm. +

-
- {% csrf_token %} - {{ form|crispy }} - -
+
+ {% csrf_token %} + {{ form|crispy }} + +
-

- By running this algorithm you agree to the - General - Terms of Service{% if algorithm.additional_terms_markdown %}, - as well as this algorithm's specific Terms of Service: - {% else %}. - {% endif %} -

+

+ By running this algorithm you agree to the + General + Terms of Service{% if algorithm.additional_terms_markdown %}, + as well as this algorithm's specific Terms of Service: + {% else %}. + {% endif %} +

- {{ algorithm.additional_terms_markdown|md2html }} + {{ algorithm.additional_terms_markdown|md2html }} + {% endif %} {% endif %} {% endblock %} diff --git a/app/grandchallenge/algorithms/templates/algorithms/job_list.html b/app/grandchallenge/algorithms/templates/algorithms/job_list.html index 5f216ec98..ab3d5657b 100644 --- a/app/grandchallenge/algorithms/templates/algorithms/job_list.html +++ b/app/grandchallenge/algorithms/templates/algorithms/job_list.html @@ -23,7 +23,7 @@

Results for {{ algorithm.title }}

{% if "execute_algorithm" in algorithm_perms and algorithm.active_image %}

+ href="{% url 'algorithms:job-interface-select' slug=algorithm.slug %}">  Try-out Algorithm

diff --git a/app/grandchallenge/algorithms/templates/algorithms/job_progress_detail.html b/app/grandchallenge/algorithms/templates/algorithms/job_progress_detail.html index 2d8f2d8d2..39e4351db 100644 --- a/app/grandchallenge/algorithms/templates/algorithms/job_progress_detail.html +++ b/app/grandchallenge/algorithms/templates/algorithms/job_progress_detail.html @@ -35,7 +35,7 @@
Creating inputs
- diff --git a/app/grandchallenge/algorithms/templates/algorithms/partials/algorithminterface_confirm_delete.html b/app/grandchallenge/algorithms/templates/algorithms/partials/algorithminterface_confirm_delete.html new file mode 100644 index 000000000..2f166af3b --- /dev/null +++ b/app/grandchallenge/algorithms/templates/algorithms/partials/algorithminterface_confirm_delete.html @@ -0,0 +1,20 @@ +{% load remove_whitespace %} + +

Confirm Algorithm Interface Deletion

+
+ {% csrf_token %} +

Are you sure that you want to delete the following algorithm interface from this {{ base_obj.verbose_name }}?

+ +

+ WARNING: You are not able to undo this action. Once the interface is deleted, it is deleted forever. +

+ Cancel + +
diff --git a/app/grandchallenge/algorithms/templates/algorithms/partials/algorithminterface_form.html b/app/grandchallenge/algorithms/templates/algorithms/partials/algorithminterface_form.html new file mode 100644 index 000000000..40ed27f74 --- /dev/null +++ b/app/grandchallenge/algorithms/templates/algorithms/partials/algorithminterface_form.html @@ -0,0 +1,15 @@ +{% load crispy_forms_tags %} +{% load meta_attr %} + +

Create An Algorithm Interface

+

+ Create an interface: define any combination of inputs and outputs, and optionally mark the interface as default for the {{ base_obj|verbose_name }}.

+

+ Please see the list of input options and the + list of output options for more information and examples. +

+

+ If you cannot find suitable inputs or outputs, please contact support@grand-challenge.org. +

+ +{% crispy form %} diff --git a/app/grandchallenge/algorithms/templates/algorithms/partials/algorithminterface_list.html b/app/grandchallenge/algorithms/templates/algorithms/partials/algorithminterface_list.html new file mode 100644 index 000000000..e3ce744b9 --- /dev/null +++ b/app/grandchallenge/algorithms/templates/algorithms/partials/algorithminterface_list.html @@ -0,0 +1,12 @@ +{% load crispy_forms_tags %} +{% load remove_whitespace %} +{% load meta_attr %} + +

Algorithm Interfaces for {{ base_obj }}

+ +

The following interfaces (i.e. input-output combinations) are configured for your {{ base_obj|verbose_name }}:

+

+ Add new interface +

+ +{% include 'algorithms/partials/algorithminterface_table.html' with base_obj=base_obj interfaces=interfaces delete_option=True %} diff --git a/app/grandchallenge/algorithms/templates/algorithms/partials/algorithminterface_table.html b/app/grandchallenge/algorithms/templates/algorithms/partials/algorithminterface_table.html new file mode 100644 index 000000000..54671a19d --- /dev/null +++ b/app/grandchallenge/algorithms/templates/algorithms/partials/algorithminterface_table.html @@ -0,0 +1,61 @@ +{% load meta_attr %} + +
+ + + + + + {% if delete_option %} + + {% endif %} + + + {% for interface in interfaces %} + + + + + {% if delete_option %} + + {% endif %} + + {% empty %} + + {% endfor %} + +
InputsOutputsDelete
{{ forloop.counter }} + {% for input in interface.inputs.all %} +
  • {{ input }} + {% if read_write_paths %} + at + {% if input.is_image_kind %} + /input{% if input.relative_path %}/{{ input.relative_path }}{% endif %}/<uuid>.mha or + /input{% if input.relative_path %}/{{ input.relative_path }}{% endif %}/<uuid>.tif + {% else %} + /input/{{ input.relative_path }} + {% endif %} + {% endif %} +
    • + {% endfor %} +     		+ {% for output in interface.outputs.all %} +
  • {{ output }} + {% if read_write_paths %} + at + {% if output.is_image_kind %} + /output{% if output.relative_path %}/{{ output.relative_path }}{% endif %}/<uuid>.mha or + /output{% if output.relative_path %}/{{ output.relative_path }}{% endif %}/<uuid>.tif + {% else %} + /output/{{ output.relative_path }} + {% endif %} + {% endif %} +
    • + {% endfor %} +     		+ + + +
    This {{ base_obj|verbose_name }} does not have any interfaces defined yet.
    +
    diff --git a/app/grandchallenge/algorithms/urls.py b/app/grandchallenge/algorithms/urls.py index b77711330..daff36463 100644 --- a/app/grandchallenge/algorithms/urls.py +++ b/app/grandchallenge/algorithms/urls.py @@ -13,6 +13,9 @@ AlgorithmImageTemplate, AlgorithmImageUpdate, AlgorithmImportView, + AlgorithmInterfaceForAlgorithmCreate, + AlgorithmInterfaceForAlgorithmDelete, + AlgorithmInterfacesForAlgorithmList, AlgorithmList, AlgorithmModelCreate, AlgorithmModelDetail, @@ -30,6 +33,7 @@ EditorsUpdate, JobCreate, JobDetail, + JobInterfaceSelect, JobProgressDetail, JobsList, JobStatusDetail, @@ -56,6 +60,21 @@ AlgorithmDescriptionUpdate.as_view(), name="description-update", ), + path( + "/interfaces/", + AlgorithmInterfacesForAlgorithmList.as_view(), + name="interface-list", + ), + path( + "/interfaces/create/", + AlgorithmInterfaceForAlgorithmCreate.as_view(), + name="interface-create", + ), + path( + "/interfaces//delete/", + AlgorithmInterfaceForAlgorithmDelete.as_view(), + name="interface-delete", + ), path( "/repository/", AlgorithmRepositoryUpdate.as_view(), @@ -127,7 +146,16 @@ name="model-update", ), path("/jobs/", JobsList.as_view(), name="job-list"), - path("/jobs/create/", JobCreate.as_view(), name="job-create"), + path( + "/jobs/interface-select/", + JobInterfaceSelect.as_view(), + name="job-interface-select", + ), + path( + "//jobs/create/", + JobCreate.as_view(), + name="job-create", + ), path("/jobs//", JobDetail.as_view(), name="job-detail"), path( "/jobs//status/", diff --git a/app/grandchallenge/algorithms/views.py b/app/grandchallenge/algorithms/views.py index fa9227dbe..15bfdbcd0 100644 --- a/app/grandchallenge/algorithms/views.py +++ b/app/grandchallenge/algorithms/views.py @@ -5,6 +5,7 @@ from django.conf import settings from django.contrib import messages from django.contrib.auth.mixins import ( + AccessMixin, PermissionRequiredMixin, UserPassesTestMixin, ) @@ -21,6 +22,7 @@ from django.utils.safestring import mark_safe from django.views.generic import ( CreateView, + DeleteView, DetailView, FormView, ListView, @@ -46,24 +48,27 @@ AlgorithmImageForm, AlgorithmImageUpdateForm, AlgorithmImportForm, + AlgorithmInterfaceForm, AlgorithmModelForm, AlgorithmModelUpdateForm, AlgorithmModelVersionControlForm, AlgorithmPermissionRequestUpdateForm, AlgorithmPublishForm, AlgorithmRepoForm, - AlgorithmUpdateForm, DisplaySetFromJobForm, ImageActivateForm, JobCreateForm, JobForm, + JobInterfaceSelectForm, PhaseSelectForm, UsersForm, ViewersForm, ) from grandchallenge.algorithms.models import ( Algorithm, + AlgorithmAlgorithmInterface, AlgorithmImage, + AlgorithmInterface, AlgorithmModel, AlgorithmPermissionRequest, Job, @@ -286,26 +291,10 @@ class AlgorithmUpdate( UpdateView, ): model = Algorithm - form_class = AlgorithmUpdateForm + form_class = AlgorithmForm permission_required = "algorithms.change_algorithm" raise_exception = True - def get_form_kwargs(self): - kwargs = super().get_form_kwargs() - - # Only users with the add_algorithm permission can change - # the input and output interfaces, other users must use - # the interfaces pre-set by the Phase - kwargs.update( - { - "interfaces_editable": self.request.user.has_perm( - "algorithms.add_algorithm" - ) - } - ) - - return kwargs - class AlgorithmDescriptionUpdate( LoginRequiredMixin, @@ -523,28 +512,84 @@ def get_success_url(self): return self.algorithm.get_absolute_url() +class JobCreatePermissionMixin( + LoginRequiredMixin, VerificationRequiredMixin, AccessMixin +): + @cached_property + def algorithm(self) -> Algorithm: + return get_object_or_404(Algorithm, slug=self.kwargs["slug"]) + + def dispatch(self, request, *args, **kwargs): + if not request.user.has_perm( + "algorithms.execute_algorithm", self.algorithm + ): + return self.handle_no_permission() + return super().dispatch(request, *args, **kwargs) + + +class JobInterfaceSelect( + JobCreatePermissionMixin, + FormView, +): + form_class = JobInterfaceSelectForm + template_name = "algorithms/job_form_create.html" + selected_interface = None + + def get(self, request, *args, **kwargs): + if self.algorithm.interfaces.count() == 1: + self.selected_interface = self.algorithm.interfaces.get() + return HttpResponseRedirect(self.get_success_url()) + else: + return super().get(request, *args, **kwargs) + + def get_form_kwargs(self): + kwargs = super().get_form_kwargs() + kwargs.update({"algorithm": self.algorithm}) + return kwargs + + def get_context_data(self, *args, **kwargs): + context = super().get_context_data(*args, **kwargs) + context.update( + { + "algorithm": self.algorithm, + "editors_job_limit": settings.ALGORITHM_IMAGES_COMPLIMENTARY_EDITOR_JOBS, + } + ) + return context + + def form_valid(self, form): + self.selected_interface = form.cleaned_data["algorithm_interface"] + return super().form_valid(form) + + def get_success_url(self): + return reverse( + "algorithms:job-create", + kwargs={ + "slug": self.algorithm.slug, + "interface_pk": self.selected_interface.pk, + }, + ) + + class JobCreate( - LoginRequiredMixin, - ObjectPermissionRequiredMixin, - VerificationRequiredMixin, + JobCreatePermissionMixin, UserFormKwargsMixin, FormView, ): form_class = JobCreateForm template_name = "algorithms/job_form_create.html" - permission_required = "algorithms.execute_algorithm" - raise_exception = True @cached_property - def algorithm(self) -> Algorithm: - return get_object_or_404(Algorithm, slug=self.kwargs["slug"]) - - def get_permission_object(self): - return self.algorithm + def interface(self): + return get_object_or_404( + AlgorithmInterface, pk=self.kwargs["interface_pk"] + ) def get_form_kwargs(self): kwargs = super().get_form_kwargs() - kwargs.update({"algorithm": self.algorithm}) + kwargs.update( + {"algorithm": self.algorithm, "interface": self.interface} + ) return kwargs def get_context_data(self, *args, **kwargs): @@ -778,7 +823,7 @@ def form_valid(self, form): class AlgorithmViewSet(ReadOnlyModelViewSet): - queryset = Algorithm.objects.all().prefetch_related("outputs", "inputs") + queryset = Algorithm.objects.all().prefetch_related("interfaces") serializer_class = AlgorithmSerializer permission_classes = [DjangoObjectPermissions] filter_backends = [DjangoFilterBackend, ObjectPermissionsFilter] @@ -1113,8 +1158,7 @@ class AlgorithmImageTemplate(ObjectPermissionRequiredMixin, DetailView): permission_required = "algorithms.change_algorithm" raise_exception = True queryset = Algorithm.objects.prefetch_related( - "inputs", - "outputs", + "interfaces", ) def get(self, *_, **__): @@ -1140,3 +1184,106 @@ def get(self, *_, **__): filename=f"{algorithm.slug}-template.zip", content_type="application/zip", ) + + +class AlgorithmInterfacePermissionMixin(AccessMixin): + @property + def algorithm(self): + return get_object_or_404(Algorithm, slug=self.kwargs["slug"]) + + def dispatch(self, request, *args, **kwargs): + if request.user.has_perm( + "change_algorithm", self.algorithm + ) and request.user.has_perm("algorithms.add_algorithm"): + return super().dispatch(request, *args, **kwargs) + else: + return self.handle_no_permission() + + +class AlgorithmInterfaceCreateBase(CreateView): + model = AlgorithmInterface + form_class = AlgorithmInterfaceForm + success_message = "Algorithm interface successfully added" + + def get_success_url(self): + return NotImplementedError + + @property + def base_obj(self): + return NotImplementedError + + def get_context_data(self, *args, **kwargs): + context = super().get_context_data(*args, **kwargs) + context.update({"base_obj": self.base_obj}) + return context + + def get_form_kwargs(self): + kwargs = super().get_form_kwargs() + kwargs.update({"base_obj": self.base_obj}) + return kwargs + + +class AlgorithmInterfaceForAlgorithmCreate( + AlgorithmInterfacePermissionMixin, AlgorithmInterfaceCreateBase +): + @property + def base_obj(self): + return self.algorithm + + def get_success_url(self): + return reverse( + "algorithms:interface-list", + kwargs={"slug": self.algorithm.slug}, + ) + + +class AlgorithmInterfacesForAlgorithmList( + AlgorithmInterfacePermissionMixin, ListView +): + model = AlgorithmAlgorithmInterface + + def get_queryset(self): + qs = super().get_queryset() + return qs.filter(algorithm=self.algorithm) + + def get_context_data(self, *args, **kwargs): + context = super().get_context_data(*args, **kwargs) + context.update( + { + "algorithm": self.algorithm, + "interfaces": [obj.interface for obj in self.object_list], + } + ) + return context + + +class AlgorithmInterfaceForAlgorithmDelete( + AlgorithmInterfacePermissionMixin, DeleteView +): + model = AlgorithmAlgorithmInterface + + @property + def algorithm_interface(self): + return get_object_or_404( + klass=AlgorithmAlgorithmInterface, + algorithm=self.algorithm, + interface__pk=self.kwargs["interface_pk"], + ) + + def get_object(self, queryset=None): + return self.algorithm_interface + + def get_success_url(self): + return reverse( + "algorithms:interface-list", + kwargs={"slug": self.algorithm.slug}, + ) + + def get_context_data(self, *args, **kwargs): + context = super().get_context_data(*args, **kwargs) + context.update( + { + "algorithm": self.algorithm, + } + ) + return context diff --git a/app/grandchallenge/archives/forms.py b/app/grandchallenge/archives/forms.py index 59212637a..29e33d6cc 100644 --- a/app/grandchallenge/archives/forms.py +++ b/app/grandchallenge/archives/forms.py @@ -206,14 +206,14 @@ def __init__(self, *args, user, archive, **kwargs): class AddCasesForm(UploadRawImagesForm): - interface = ModelChoiceField( + socket = ModelChoiceField( queryset=ComponentInterface.objects.filter( kind__in=InterfaceKind.interface_type_image() ).order_by("title"), widget=autocomplete.ModelSelect2( url="components:component-interface-autocomplete", attrs={ - "data-placeholder": "Search for an interface ...", + "data-placeholder": "Search for a socket ...", "data-minimum-input-length": 3, "data-theme": settings.CRISPY_TEMPLATE_PACK, "data-html": True, @@ -223,18 +223,18 @@ class AddCasesForm(UploadRawImagesForm): def __init__(self, *args, interface_viewname, **kwargs): super().__init__(*args, **kwargs) - self.fields["interface"].help_text = format_lazy( + self.fields["socket"].help_text = format_lazy( ( - 'See the list of interfaces for more ' - "information about each interface. " - "Please contact support if your desired interface is missing." + 'See the list of sockets for more ' + "information about each socket. " + "Please contact support if your desired socket is missing." ), reverse_lazy(interface_viewname), ) def save(self, *args, **kwargs): self._linked_task.kwargs.update( - {"interface_pk": self.cleaned_data["interface"].pk} + {"interface_pk": self.cleaned_data["socket"].pk} ) return super().save(*args, **kwargs) diff --git a/app/grandchallenge/archives/migrations/0011_archive_hanging_protocol.py b/app/grandchallenge/archives/migrations/0011_archive_hanging_protocol.py index d9d243a7b..1349a209c 100644 --- a/app/grandchallenge/archives/migrations/0011_archive_hanging_protocol.py +++ b/app/grandchallenge/archives/migrations/0011_archive_hanging_protocol.py @@ -17,7 +17,7 @@ class Migration(migrations.Migration): name="hanging_protocol", field=models.ForeignKey( blank=True, - help_text='Indicate which Component Interfaces need to be displayed in which image port. E.g. {"main": ["interface1"]}. The first item in the list of interfaces will be the main image in the image port. The first overlay type interface thereafter will be rendered as an overlay. For now, any other items will be ignored by the viewer.', + help_text='Indicate which sockets need to be displayed in which image port. E.g. {"main": ["socket1"]}. The first item in the list of sockets will be the main image in the image port. The first overlay type socket thereafter will be rendered as an overlay. For now, any other items will be ignored by the viewer.', null=True, on_delete=django.db.models.deletion.SET_NULL, to="hanging_protocols.hangingprotocol", diff --git a/app/grandchallenge/cases/models.py b/app/grandchallenge/cases/models.py index 6b6f31f9f..8eee6fe8e 100644 --- a/app/grandchallenge/cases/models.py +++ b/app/grandchallenge/cases/models.py @@ -153,7 +153,7 @@ def update_status( if detailed_error_message: notification_description = oxford_comma( [ - f"Image validation for interface {key} failed with error: {val}. " + f"Image validation for socket {key} failed with error: {val}. " for key, val in detailed_error_message.items() ] ) diff --git a/app/grandchallenge/components/forms.py b/app/grandchallenge/components/forms.py index 5c2c9dede..25dcc8ad3 100644 --- a/app/grandchallenge/components/forms.py +++ b/app/grandchallenge/components/forms.py @@ -279,7 +279,7 @@ def __init__( widget = autocomplete.ModelSelect2 attrs.update( { - "data-placeholder": "Search for an interface ...", + "data-placeholder": "Search for a socket ...", "data-minimum-input-length": 3, "data-theme": settings.CRISPY_TEMPLATE_PACK, "data-html": True, @@ -296,12 +296,12 @@ def __init__( initial=selected_interface, queryset=qs, widget=widget(**widget_kwargs), - label="Interface", + label="Socket", help_text=format_lazy( ( - 'See the list of interfaces for more ' - "information about each interface. " - "Please contact support if your desired interface is missing." + 'See the list of sockets for more ' + "information about each socket. " + "Please contact support if your desired socket is missing." ), reverse_lazy(base_obj.interface_viewname), ), diff --git a/app/grandchallenge/components/templates/components/civ_set_form.html b/app/grandchallenge/components/templates/components/civ_set_form.html index bc89e38d7..dd393ca02 100644 --- a/app/grandchallenge/components/templates/components/civ_set_form.html +++ b/app/grandchallenge/components/templates/components/civ_set_form.html @@ -54,7 +54,7 @@

    >
    - Add value for new interface + Add value for new socket
    diff --git a/app/grandchallenge/components/templates/components/componentinterface_io_switch.html b/app/grandchallenge/components/templates/components/componentinterface_io_switch.html index 932cbf8b2..229561d54 100644 --- a/app/grandchallenge/components/templates/components/componentinterface_io_switch.html +++ b/app/grandchallenge/components/templates/components/componentinterface_io_switch.html @@ -3,39 +3,40 @@ {% load static %} {% block title %} - Interface Options - {{ block.super }} + Socket Options - {{ block.super }} {% endblock %} {% block breadcrumbs %}
      -
    1. Interface Options
      2. +
    2. Socket Options
    {% endblock %} {% block content %} -

    Algorithm Interface Options

    +

    Algorithm Socket Options

    - You will need to define the inputs to, and the outputs from, your algorithm. - The set of inputs and outputs that your algorithm uses is known as its interface. - You can build up your algorithm's interface by selecting from the current set of interface options. - Use the links below to see the options that are available for your algorithm's interface. + You will need to define the inputs and outputs for your algorithm. + A set of inputs and outputs that your algorithm uses is known as its interface. + The inputs and outputs that make up an algorithm interface are called sockets. + You can build up an algorithm interface by selecting from the current set of socket options. + Use the links below to see the socket options that are available.

    For example, lets say that you are developing an algorithm that predicts COVID-19 from a CT scan, which also produces a segmentation of COVID-19 lesions. - You would look in the list of interface options and select the ones to use. - You would then define an algorithm that has one input, a ct-image, + You would look in the list of socket options and select the ones to use. + You would then define an algorithm that has one interface with one input, a ct-image, and two outputs, probability-covid-19 (a number) and covid-19-lesion-segementation (a segmentation of covid 19 lesions).

    - Example algorithm with one input interface and two output interfaces + Example algorithm with one interface consisting of one input and two output sockets

    - You can select as many inputs and outputs as your algorithm needs. - However, the same interface option cannot be used for both input to and output from your algorithm. + You can select as many inputs and outputs for a single interface as you need. + However, the same socket option cannot be used for both input to and output from your algorithm. All inputs and outputs are required to be set when your algorithm runs. The inputs will be set by the user running your algorithm. Your algorithm must write valid data to all the outputs you set. @@ -57,7 +58,7 @@

    Algorithm Interface Options

    If an option does not exist for your use case please contact support with the title, description, and - kind for your new interface option. + kind for your new socket option.

    {% endblock %} diff --git a/app/grandchallenge/components/templates/components/componentinterface_list.html b/app/grandchallenge/components/templates/components/componentinterface_list.html index 069804460..a9cc15d04 100644 --- a/app/grandchallenge/components/templates/components/componentinterface_list.html +++ b/app/grandchallenge/components/templates/components/componentinterface_list.html @@ -6,19 +6,19 @@ {% block title %} {% if object_type == object_type_options.ALGORITHM %} - {{ list_type|title }} - Interface Options - {{ block.super }} + {{ list_type|title }} - Socket Options - {{ block.super }} {% else %} - Interface Options - {{ block.super }} + Socket Options - {{ block.super }} {% endif %} {% endblock %} {% block breadcrumbs %}
      {% if object_type == object_type_options.ALGORITHM %} -
    1. Interface Options
      2. +
    2. Socket Options
    3. {{ list_type|title }}
      4. {% else %} -
    4. Interface Options
      5. +
    5. Socket Options
      6. {% endif %}
    {% endblock %} @@ -39,7 +39,7 @@

    {{ object_type|title }} {{ list_type|title }} Options

    times within one {{ list_type|lower }}. {% endif %} If an option does not exist for your use case please contact support with the title, description, and kind - for your new interface option. + for your new socket option.

    diff --git a/app/grandchallenge/core/error_handlers.py b/app/grandchallenge/core/error_handlers.py index 3bc87b4be..b73496828 100644 --- a/app/grandchallenge/core/error_handlers.py +++ b/app/grandchallenge/core/error_handlers.py @@ -100,8 +100,8 @@ def __init__(self, *args, user_upload, **kwargs): def handle_error(self, *, error_message, user, interface): Notification.send( kind=NotificationType.NotificationTypeChoices.FILE_COPY_STATUS, - message=f"Validation for interface {interface.title} failed.", - description=f"Validation for interface {interface.title} failed: {error_message}", + message=f"Validation for socket {interface.title} failed.", + description=f"Validation for socket {interface.title} failed: {error_message}", actor=user, ) @@ -118,8 +118,8 @@ def handle_error(self, *, error_message, user, interface=None): if interface: Notification.send( kind=NotificationType.NotificationTypeChoices.CIV_VALIDATION, - message=f"Validation for interface {interface.title} failed.", - description=f"Validation for interface {interface.title} failed: {error_message}", + message=f"Validation for socket {interface.title} failed.", + description=f"Validation for socket {interface.title} failed: {error_message}", actor=user, ) else: diff --git a/app/grandchallenge/core/utils/grand_challenge_forge.py b/app/grandchallenge/core/utils/grand_challenge_forge.py index e300caeda..25223d924 100644 --- a/app/grandchallenge/core/utils/grand_challenge_forge.py +++ b/app/grandchallenge/core/utils/grand_challenge_forge.py @@ -25,7 +25,7 @@ def get_forge_challenge_pack_context(challenge, phase_pks=None): phases = challenge.phase_set.filter( archive__isnull=False, submission_kind=SubmissionKindChoices.ALGORITHM, - ).prefetch_related("archive", "algorithm_inputs", "algorithm_outputs") + ).prefetch_related("archive", "algorithm_interfaces") if phase_pks is not None: phases = phases.filter(pk__in=phase_pks) @@ -39,16 +39,21 @@ def process_archive(archive): } def process_phase(phase): + interfaces = phase.algorithm_interfaces.all() + inputs = { + ci for interface in interfaces for ci in interface.inputs.all() + } + outputs = { + ci for interface in interfaces for ci in interface.outputs.all() + } return { "slug": phase.slug, "archive": process_archive(phase.archive), "algorithm_inputs": [ - _process_component_interface(ci) - for ci in phase.algorithm_inputs.all() + _process_component_interface(ci) for ci in inputs ], "algorithm_outputs": [ - _process_component_interface(ci) - for ci in phase.algorithm_outputs.all() + _process_component_interface(ci) for ci in outputs ], } @@ -63,18 +68,17 @@ def process_phase(phase): def get_forge_algorithm_template_context(algorithm): + interfaces = algorithm.interfaces.all() + inputs = {ci for interface in interfaces for ci in interface.inputs.all()} + outputs = { + ci for interface in interfaces for ci in interface.outputs.all() + } return { "algorithm": { "title": algorithm.title, "slug": algorithm.slug, "url": algorithm.get_absolute_url(), - "inputs": [ - _process_component_interface(ci) - for ci in algorithm.inputs.all() - ], - "outputs": [ - _process_component_interface(ci) - for ci in algorithm.outputs.all() - ], + "inputs": [_process_component_interface(ci) for ci in inputs], + "outputs": [_process_component_interface(ci) for ci in outputs], } } diff --git a/app/grandchallenge/evaluation/admin.py b/app/grandchallenge/evaluation/admin.py index aad7d674c..94ab1effd 100644 --- a/app/grandchallenge/evaluation/admin.py +++ b/app/grandchallenge/evaluation/admin.py @@ -1,9 +1,9 @@ import json from django.contrib import admin -from django.core.exceptions import ValidationError from django.forms import ModelForm from django.utils.html import format_html +from guardian.admin import GuardedModelAdmin from grandchallenge.components.admin import ( ComponentImageAdmin, @@ -15,7 +15,6 @@ GroupObjectPermissionAdmin, UserObjectPermissionAdmin, ) -from grandchallenge.core.templatetags.remove_whitespace import oxford_comma from grandchallenge.core.utils.grand_challenge_forge import ( get_forge_challenge_pack_context, ) @@ -29,6 +28,7 @@ MethodGroupObjectPermission, MethodUserObjectPermission, Phase, + PhaseAlgorithmInterface, PhaseGroupObjectPermission, PhaseUserObjectPermission, Submission, @@ -52,21 +52,6 @@ def __init__(self, *args, **kwargs): ) in self.instance.read_only_fields_for_dependent_phases: self.fields[field_name].disabled = True - def clean(self): - cleaned_data = super().clean() - - duplicate_interfaces = { - *cleaned_data.get("algorithm_inputs", []) - }.intersection({*cleaned_data.get("algorithm_outputs", [])}) - - if duplicate_interfaces: - raise ValidationError( - f"The sets of Algorithm Inputs and Algorithm Outputs must be unique: " - f"{oxford_comma(duplicate_interfaces)} present in both" - ) - - return cleaned_data - @admin.register(Phase) class PhaseAdmin(admin.ModelAdmin): @@ -97,13 +82,12 @@ class PhaseAdmin(admin.ModelAdmin): autocomplete_fields = ( "inputs", "outputs", - "algorithm_inputs", - "algorithm_outputs", "archive", ) readonly_fields = ( "give_algorithm_editors_job_view_permissions", "challenge_forge_json", + "algorithm_interfaces", ) form = PhaseAdminForm @@ -226,6 +210,24 @@ class EvaluationGroundTruthAdmin(admin.ModelAdmin): readonly_fields = ("creator", "phase", "sha256", "size_in_storage") +@admin.register(PhaseAlgorithmInterface) +class PhaseAlgorithmInterfaceAdmin(GuardedModelAdmin): + list_display = ( + "pk", + "interface", + "phase", + ) + list_filter = ("phase",) + + def has_add_permission(self, request, obj=None): + # through table entries should only be created through the UI + return False + + def has_change_permission(self, request, obj=None): + # through table entries should only be updated through the UI + return False + + admin.site.register(PhaseUserObjectPermission, UserObjectPermissionAdmin) admin.site.register(PhaseGroupObjectPermission, GroupObjectPermissionAdmin) admin.site.register(Method, ComponentImageAdmin) diff --git a/app/grandchallenge/evaluation/forms.py b/app/grandchallenge/evaluation/forms.py index 1bef795ad..f21094713 100644 --- a/app/grandchallenge/evaluation/forms.py +++ b/app/grandchallenge/evaluation/forms.py @@ -21,16 +21,12 @@ ) from django.utils.html import format_html from django.utils.text import format_lazy -from django_select2.forms import Select2MultipleWidget from grandchallenge.algorithms.forms import UserAlgorithmsForPhaseMixin from grandchallenge.algorithms.models import Job from grandchallenge.challenges.models import Challenge, ChallengeRequest from grandchallenge.components.forms import ContainerImageForm -from grandchallenge.components.models import ( - ComponentInterface, - ImportStatusChoices, -) +from grandchallenge.components.models import ImportStatusChoices from grandchallenge.components.schemas import GPUTypeChoices from grandchallenge.components.tasks import assign_tarball_from_upload from grandchallenge.core.forms import ( @@ -41,7 +37,6 @@ filter_by_permission, get_objects_for_user, ) -from grandchallenge.core.templatetags.remove_whitespace import oxford_comma from grandchallenge.core.widgets import ( JSONEditorWidget, MarkdownEditorInlineWidget, @@ -308,7 +303,10 @@ class SubmissionForm( label="Predictions File", queryset=None, ) - algorithm = AlgorithmChoiceField(queryset=None) + algorithm = AlgorithmChoiceField( + queryset=None, + help_text="Select one of your algorithms to submit as a solution to this phase. See above for information regarding the necessary configuration of the algorithm.", + ) confirm_submission = forms.BooleanField( required=True, label="I understand that by submitting my algorithm image and model " @@ -427,24 +425,6 @@ def __init__(self, *args, user, phase: Phase, **kwargs): # noqa: C901 self.fields["algorithm_image"].required = False self.fields["algorithm_model"].widget = HiddenInput() - self._algorithm_inputs = self._phase.algorithm_inputs.all() - self._algorithm_outputs = self._phase.algorithm_outputs.all() - self.fields["algorithm"].help_text = format_lazy( - "Select one of your algorithms to submit as a solution to this " - "challenge. The algorithms need to work with the following inputs: {} " - "and the following outputs: {}. If you have not created your " - "algorithm yet you can " - "do so on this page.", - oxford_comma(self._algorithm_inputs), - oxford_comma(self._algorithm_outputs), - reverse( - "evaluation:phase-algorithm-create", - kwargs={ - "slug": phase.slug, - "challenge_short_name": phase.challenge.short_name, - }, - ), - ) if ( not self._phase.active_image and not self._phase.external_evaluation @@ -486,7 +466,7 @@ def clean_phase(self): if ( phase.submission_kind == SubmissionKindChoices.ALGORITHM and not phase.external_evaluation - and phase.count_valid_archive_items == 0 + and phase.jobs_to_schedule_per_submission == 0 ): self.add_error( None, @@ -742,14 +722,6 @@ class ConfigureAlgorithmPhasesForm(SaveFormInitMixin, Form): queryset=None, widget=CheckboxSelectMultiple, ) - algorithm_inputs = ModelMultipleChoiceField( - queryset=ComponentInterface.objects.all(), - widget=Select2MultipleWidget, - ) - algorithm_outputs = ModelMultipleChoiceField( - queryset=ComponentInterface.objects.all(), - widget=Select2MultipleWidget, - ) algorithm_time_limit = IntegerField( widget=forms.HiddenInput(), disabled=True, diff --git a/app/grandchallenge/evaluation/migrations/0027_auto_20220707_0933.py b/app/grandchallenge/evaluation/migrations/0027_auto_20220707_0933.py index 2ff4a1798..e58c5a10c 100644 --- a/app/grandchallenge/evaluation/migrations/0027_auto_20220707_0933.py +++ b/app/grandchallenge/evaluation/migrations/0027_auto_20220707_0933.py @@ -21,7 +21,7 @@ class Migration(migrations.Migration): name="hanging_protocol", field=models.ForeignKey( blank=True, - help_text='Indicate which Component Interfaces need to be displayed in which image port. E.g. {"main": ["interface1"]}. The first item in the list of interfaces will be the main image in the image port. The first overlay type interface thereafter will be rendered as an overlay. For now, any other items will be ignored by the viewer.', + help_text='Indicate which sockets need to be displayed in which image port. E.g. {"main": ["socket1"]}. The first item in the list of sockets will be the main image in the image port. The first overlay type socket thereafter will be rendered as an overlay. For now, any other items will be ignored by the viewer.', null=True, on_delete=django.db.models.deletion.SET_NULL, to="hanging_protocols.hangingprotocol", diff --git a/app/grandchallenge/evaluation/migrations/0072_phasealgorithminterface_phase_algorithm_interfaces_and_more.py b/app/grandchallenge/evaluation/migrations/0072_phasealgorithminterface_phase_algorithm_interfaces_and_more.py new file mode 100644 index 000000000..6e3e24e17 --- /dev/null +++ b/app/grandchallenge/evaluation/migrations/0072_phasealgorithminterface_phase_algorithm_interfaces_and_more.py @@ -0,0 +1,62 @@ +# Generated by Django 4.2.18 on 2025-01-29 10:23 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("algorithms", "0066_create_interfaces_for_jobs"), + ( + "evaluation", + "0071_alter_combinedleaderboardphase_unique_together_and_more", + ), + ] + + operations = [ + migrations.CreateModel( + name="PhaseAlgorithmInterface", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "interface", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="algorithms.algorithminterface", + ), + ), + ( + "phase", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="evaluation.phase", + ), + ), + ], + ), + migrations.AddField( + model_name="phase", + name="algorithm_interfaces", + field=models.ManyToManyField( + blank=True, + help_text="The interfaces that an algorithm for this phase must implement.", + through="evaluation.PhaseAlgorithmInterface", + to="algorithms.algorithminterface", + ), + ), + migrations.AddConstraint( + model_name="phasealgorithminterface", + constraint=models.UniqueConstraint( + fields=("phase", "interface"), + name="unique_phase_interface_combination", + ), + ), + ] diff --git a/app/grandchallenge/evaluation/migrations/0073_migrate_interfaces_for_phases.py b/app/grandchallenge/evaluation/migrations/0073_migrate_interfaces_for_phases.py new file mode 100644 index 000000000..56a0e90bb --- /dev/null +++ b/app/grandchallenge/evaluation/migrations/0073_migrate_interfaces_for_phases.py @@ -0,0 +1,49 @@ +from django.db import migrations + +from grandchallenge.algorithms.models import ( + get_existing_interface_for_inputs_and_outputs, +) +from grandchallenge.evaluation.utils import SubmissionKindChoices + + +def add_algorithm_interfaces_for_phases(apps, _schema_editor): + Phase = apps.get_model("evaluation", "Phase") # noqa: N806 + AlgorithmInterface = apps.get_model( # noqa: N806 + "algorithms", "AlgorithmInterface" + ) + + for phase in ( + Phase.objects.filter(submission_kind=SubmissionKindChoices.ALGORITHM) + .prefetch_related("algorithm_inputs", "algorithm_outputs") + .all() + ): + inputs = phase.algorithm_inputs.all() + outputs = phase.algorithm_outputs.all() + + if not inputs or not outputs: + raise RuntimeError(f"{phase} is improperly configured.") + + io = get_existing_interface_for_inputs_and_outputs( + model=AlgorithmInterface, inputs=inputs, outputs=outputs + ) + if not io: + io = AlgorithmInterface.objects.create() + io.inputs.set(inputs) + io.outputs.set(outputs) + + phase.algorithm_interfaces.add(io) + + +class Migration(migrations.Migration): + dependencies = [ + ( + "evaluation", + "0072_phasealgorithminterface_phase_algorithm_interfaces_and_more", + ), + ] + + operations = [ + migrations.RunPython( + add_algorithm_interfaces_for_phases, elidable=True + ), + ] diff --git a/app/grandchallenge/evaluation/migrations/0074_alter_phase_algorithm_inputs_and_more.py b/app/grandchallenge/evaluation/migrations/0074_alter_phase_algorithm_inputs_and_more.py new file mode 100644 index 000000000..644f8e782 --- /dev/null +++ b/app/grandchallenge/evaluation/migrations/0074_alter_phase_algorithm_inputs_and_more.py @@ -0,0 +1,35 @@ +# Generated by Django 4.2.19 on 2025-02-17 09:48 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("components", "0024_alter_componentinterface_kind_and_more"), + ("evaluation", "0073_migrate_interfaces_for_phases"), + ] + + operations = [ + migrations.AlterField( + model_name="phase", + name="algorithm_inputs", + field=models.ManyToManyField( + blank=True, + help_text="The input interfaces that the algorithms for this phase must use", + null=True, + related_name="+", + to="components.componentinterface", + ), + ), + migrations.AlterField( + model_name="phase", + name="algorithm_outputs", + field=models.ManyToManyField( + blank=True, + help_text="The output interfaces that the algorithms for this phase must use", + null=True, + related_name="+", + to="components.componentinterface", + ), + ), + ] diff --git a/app/grandchallenge/evaluation/models.py b/app/grandchallenge/evaluation/models.py index 090f00d1d..a2b9f670e 100644 --- a/app/grandchallenge/evaluation/models.py +++ b/app/grandchallenge/evaluation/models.py @@ -18,16 +18,18 @@ from django.utils.html import format_html from django.utils.text import get_valid_filename from django.utils.timezone import localtime +from django_deprecate_fields import deprecate_field from django_extensions.db.fields import AutoSlugField from guardian.models import GroupObjectPermissionBase, UserObjectPermissionBase from guardian.shortcuts import assign_perm, remove_perm from grandchallenge.algorithms.models import ( AlgorithmImage, + AlgorithmInterface, AlgorithmModel, Job, ) -from grandchallenge.archives.models import Archive, ArchiveItem +from grandchallenge.archives.models import Archive from grandchallenge.challenges.models import Challenge from grandchallenge.components.models import ( ComponentImage, @@ -138,6 +140,79 @@ } +def get_archive_items_for_interfaces(*, algorithm_interfaces, archive_items): + valid_archive_items_per_interface = {} + for interface in algorithm_interfaces: + inputs = interface.inputs.all() + valid_archive_items_per_interface[interface] = ( + archive_items.annotate( + input_count=Count("values", distinct=True), + relevant_input_count=Count( + "values", + filter=Q(values__interface__in=inputs), + distinct=True, + ), + ) + .filter(input_count=len(inputs), relevant_input_count=len(inputs)) + .prefetch_related("values") + ) + return valid_archive_items_per_interface + + +def get_valid_jobs_for_interfaces_and_archive_items( + *, + algorithm_image, + algorithm_interfaces, + valid_archive_items_per_interface, + algorithm_model=None, + successful_jobs_only=False, +): + if algorithm_model: + extra_filter = {"algorithm_model": algorithm_model} + else: + extra_filter = {"algorithm_model__isnull": True} + + if successful_jobs_only: + extra_filter["status"] = Job.SUCCESS + + jobs = Job.objects.filter( + algorithm_image=algorithm_image, + creator=None, + **extra_filter, + ) + + jobs_per_interface = {} + for interface in algorithm_interfaces: + jobs_per_interface[interface] = [] + jobs_for_interface = ( + jobs.filter( + algorithm_interface=interface, + inputs__archive_items__in=valid_archive_items_per_interface[ + interface + ], + ) + .distinct() + .prefetch_related("inputs") + .select_related("algorithm_image__algorithm") + ) + + archive_item_value_sets = { + frozenset(value.pk for value in item.values.all()) + for item in valid_archive_items_per_interface[interface] + } + + for job in jobs_for_interface: + # subset to jobs whose input set exactly matches + # one of the valid archive items' value sets + if ( + frozenset(inpt.pk for inpt in job.inputs.all()) + in archive_item_value_sets + ): + jobs_per_interface[interface].append(job) + + return jobs_per_interface + + class PhaseManager(models.Manager): def get_queryset(self): return ( @@ -459,24 +534,33 @@ class Phase(FieldChangeMixin, HangingProtocolMixin, UUIDModel): "metrics.json over the API." ), ) - + algorithm_interfaces = models.ManyToManyField( + to=AlgorithmInterface, + through="evaluation.PhaseAlgorithmInterface", + blank=True, + help_text="The interfaces that an algorithm for this phase must implement.", + ) inputs = models.ManyToManyField( to=ComponentInterface, related_name="evaluation_inputs" ) outputs = models.ManyToManyField( to=ComponentInterface, related_name="evaluation_outputs" ) - algorithm_inputs = models.ManyToManyField( - to=ComponentInterface, - related_name="+", - blank=True, - help_text="The input interfaces that the algorithms for this phase must use", + algorithm_inputs = deprecate_field( + models.ManyToManyField( + to=ComponentInterface, + related_name="+", + blank=True, + help_text="The input interfaces that the algorithms for this phase must use", + ) ) - algorithm_outputs = models.ManyToManyField( - to=ComponentInterface, - related_name="+", - blank=True, - help_text="The output interfaces that the algorithms for this phase must use", + algorithm_outputs = deprecate_field( + models.ManyToManyField( + to=ComponentInterface, + related_name="+", + blank=True, + help_text="The output interfaces that the algorithms for this phase must use", + ) ) algorithm_selectable_gpu_type_choices = models.JSONField( default=get_default_gpu_type_choices, @@ -718,15 +802,11 @@ def _clean_algorithm_submission_settings(self): if ( self.submissions_limit_per_user_per_period > 0 and not self.external_evaluation - and ( - not self.archive - or not self.algorithm_inputs - or not self.algorithm_outputs - ) + and (not self.archive or not self.algorithm_interfaces) ): raise ValidationError( "To change the submission limit to above 0, you need to first link an archive containing the secret " - "test data to this phase and define the inputs and outputs that the submitted algorithms need to " + "test data to this phase and define the interfaces (input-output combinations) that the submitted algorithms need to " "read/write. To configure these settings, please get in touch with support@grand-challenge.org." ) if ( @@ -842,7 +922,7 @@ def valid_metrics(self): def read_only_fields_for_dependent_phases(self): common_fields = ["submission_kind"] if self.submission_kind == SubmissionKindChoices.ALGORITHM: - common_fields += ["algorithm_inputs", "algorithm_outputs"] + common_fields += ["algorithm_interfaces"] return common_fields def _clean_parent_phase(self): @@ -857,7 +937,7 @@ def _clean_parent_phase(self): f"the current phase's children set as its parent." ) - if self.parent.count_valid_archive_items < 1: + if self.parent.jobs_to_schedule_per_submission < 1: raise ValidationError( "The parent phase needs to have at least 1 valid archive item." ) @@ -879,9 +959,14 @@ def set_default_interfaces(self): @cached_property def linked_component_interfaces(self): - return ( - self.algorithm_inputs.all() | self.algorithm_outputs.all() - ).distinct() + return { + ci + for interface in self.algorithm_interfaces.all() + for ci in ( + interface.inputs.order_by("pk") + | interface.outputs.order_by("pk") + ) + } def assign_permissions(self): assign_perm("view_phase", self.challenge.admins_group, self) @@ -1091,23 +1176,33 @@ def ground_truth_upload_in_progress(self): ).exists() @cached_property - def valid_archive_items(self): - """Returns the archive items that are valid for this phase""" - if self.archive and self.algorithm_inputs: - return self.archive.items.annotate( - interface_match_count=Count( - "values", - filter=Q( - values__interface__in={*self.algorithm_inputs.all()} - ), - ) - ).filter(interface_match_count=len(self.algorithm_inputs.all())) + def valid_archive_items_per_interface(self): + """ + Returns the archive items that are valid for + each interface configured for this phase + """ + if self.archive and self.algorithm_interfaces: + return get_archive_items_for_interfaces( + algorithm_interfaces=self.algorithm_interfaces.prefetch_related( + "inputs" + ), + archive_items=self.archive.items.prefetch_related( + "values__interface" + ), + ) else: - return ArchiveItem.objects.none() + return {} @cached_property - def count_valid_archive_items(self): - return self.valid_archive_items.count() + def valid_archive_item_count_per_interface(self): + return { + interface: len(valid_archive_items) + for interface, valid_archive_items in self.valid_archive_items_per_interface.items() + } + + @cached_property + def jobs_to_schedule_per_submission(self): + return sum(self.valid_archive_item_count_per_interface.values()) def send_give_algorithm_editors_job_view_permissions_changed_email(self): message = format_html( @@ -1158,29 +1253,20 @@ def parent_phase_choices(self): extra_filters = {} extra_annotations = {} if self.submission_kind == SubmissionKindChoices.ALGORITHM: - algorithm_inputs = self.algorithm_inputs.all() - algorithm_outputs = self.algorithm_outputs.all() + algorithm_interfaces = self.algorithm_interfaces.all() extra_annotations = { - "total_input_count": Count("algorithm_inputs", distinct=True), - "total_output_count": Count( - "algorithm_outputs", distinct=True - ), - "relevant_input_count": Count( - "algorithm_inputs", - filter=Q(algorithm_inputs__in=algorithm_inputs), - distinct=True, + "total_interface_count": Count( + "algorithm_interfaces", distinct=True ), - "relevant_output_count": Count( - "algorithm_outputs", - filter=Q(algorithm_outputs__in=algorithm_outputs), + "relevant_interface_count": Count( + "algorithm_interfaces", + filter=Q(algorithm_interfaces__in=algorithm_interfaces), distinct=True, ), } extra_filters = { - "total_input_count": len(algorithm_inputs), - "total_output_count": len(algorithm_outputs), - "relevant_input_count": len(algorithm_inputs), - "relevant_output_count": len(algorithm_outputs), + "total_interface_count": len(algorithm_interfaces), + "relevant_interface_count": len(algorithm_interfaces), } return ( Phase.objects.annotate(**extra_annotations) @@ -1197,6 +1283,32 @@ def parent_phase_choices(self): ) ) + @property + def algorithm_interface_manager(self): + return self.algorithm_interfaces + + @property + def algorithm_interface_through_model_manager(self): + return PhaseAlgorithmInterface.objects.filter(phase=self) + + @property + def algorithm_interface_create_url(self): + return reverse( + "evaluation:interface-create", + kwargs={"challenge_short_name": self.challenge, "slug": self.slug}, + ) + + @property + def algorithm_interface_delete_viewname(self): + return "evaluation:interface-delete" + + @property + def algorithm_interface_list_url(self): + return reverse( + "evaluation:interface-list", + kwargs={"challenge_short_name": self.challenge, "slug": self.slug}, + ) + class PhaseUserObjectPermission(UserObjectPermissionBase): content_object = models.ForeignKey(Phase, on_delete=models.CASCADE) @@ -1206,6 +1318,19 @@ class PhaseGroupObjectPermission(GroupObjectPermissionBase): content_object = models.ForeignKey(Phase, on_delete=models.CASCADE) +class PhaseAlgorithmInterface(models.Model): + phase = models.ForeignKey(Phase, on_delete=models.CASCADE) + interface = models.ForeignKey(AlgorithmInterface, on_delete=models.CASCADE) + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["phase", "interface"], + name="unique_phase_interface_combination", + ), + ] + + class Method(UUIDModel, ComponentImage): """Store the methods for performing an evaluation.""" @@ -1622,63 +1747,50 @@ def output_interfaces(self): return self.submission.phase.outputs @cached_property - def algorithm_inputs(self): - return self.submission.phase.algorithm_inputs.all() + def successful_jobs_per_interface(self): + algorithm_interfaces = ( + self.submission.phase.algorithm_interfaces.prefetch_related( + "inputs" + ) + ) + + successful_jobs_per_interface = get_valid_jobs_for_interfaces_and_archive_items( + successful_jobs_only=True, + algorithm_image=self.submission.algorithm_image, + algorithm_model=self.submission.algorithm_model, + algorithm_interfaces=algorithm_interfaces, + valid_archive_items_per_interface=self.submission.phase.valid_archive_items_per_interface, + ) + + return successful_jobs_per_interface @cached_property - def valid_archive_item_values(self): + def successful_job_count_per_interface(self): return { - i.values.all() - for i in self.submission.phase.archive.items.annotate( - interface_match_count=Count( - "values", - filter=Q(values__interface__in=self.algorithm_inputs), - ) - ) - .filter(interface_match_count=len(self.algorithm_inputs)) - .prefetch_related("values") + interface: len(successful_jobs) + for interface, successful_jobs in self.successful_jobs_per_interface.items() } @cached_property - def successful_jobs(self): - if self.submission.algorithm_model: - extra_filter = {"algorithm_model": self.submission.algorithm_model} - else: - extra_filter = {"algorithm_model__isnull": True} + def total_successful_jobs(self): + return sum(self.successful_job_count_per_interface.values()) - successful_jobs = ( - Job.objects.filter( - algorithm_image=self.submission.algorithm_image, - status=Job.SUCCESS, - **extra_filter, - ) - .annotate( - inputs_match_count=Count( - "inputs", - filter=Q( - inputs__in={ - civ - for civ_set in self.valid_archive_item_values - for civ in civ_set - } - ), - ), - ) - .filter( - inputs_match_count=self.algorithm_inputs.count(), - creator=None, - ) - .distinct() - .prefetch_related("outputs__interface", "inputs__interface") - .select_related("algorithm_image__algorithm") + @cached_property + def successful_jobs(self): + return Job.objects.filter( + pk__in=[ + j.pk + for sublist in self.successful_jobs_per_interface.values() + for j in sublist + ] ) - return successful_jobs @cached_property def inputs_complete(self): if self.submission.algorithm_image: - return self.successful_jobs.count() == len( - self.valid_archive_item_values + return ( + self.total_successful_jobs + == self.submission.phase.jobs_to_schedule_per_submission ) elif self.submission.predictions_file: return True diff --git a/app/grandchallenge/evaluation/tasks.py b/app/grandchallenge/evaluation/tasks.py index de8e67837..ef6078488 100644 --- a/app/grandchallenge/evaluation/tasks.py +++ b/app/grandchallenge/evaluation/tasks.py @@ -212,20 +212,17 @@ def create_algorithm_jobs_for_evaluation(*, evaluation_pk, max_jobs=1): jobs = create_algorithm_jobs( algorithm_image=evaluation.submission.algorithm_image, algorithm_model=evaluation.submission.algorithm_model, - civ_sets=[ - {*ai.values.all()} - for ai in evaluation.submission.phase.archive.items.prefetch_related( - "values__interface" - ) - .annotate( - has_title=Case( - When(title="", then=Value(1)), - default=Value(0), - output_field=IntegerField(), - ) + archive_items=evaluation.submission.phase.archive.items.prefetch_related( + "values__interface" + ) + .annotate( + has_title=Case( + When(title="", then=Value(1)), + default=Value(0), + output_field=IntegerField(), ) - .order_by("has_title", "title", "created") - ], + ) + .order_by("has_title", "title", "created"), extra_viewer_groups=viewer_groups, extra_logs_viewer_groups=viewer_groups, task_on_success=task_on_success, diff --git a/app/grandchallenge/evaluation/templates/evaluation/algorithminterface_for_phase_form.html b/app/grandchallenge/evaluation/templates/evaluation/algorithminterface_for_phase_form.html new file mode 100644 index 000000000..85aa24ef3 --- /dev/null +++ b/app/grandchallenge/evaluation/templates/evaluation/algorithminterface_for_phase_form.html @@ -0,0 +1,26 @@ +{% extends "pages/challenge_settings_base.html" %} +{% load url %} +{% load static %} + +{% block title %} + Create Interface - {{ base_obj.title }} - {% firstof challenge.title challenge.short_name %} - {{ block.super }} +{% endblock %} + +{% block breadcrumbs %} +
      +
    1. Challenges +
      2. +
    2. {% firstof challenge.title challenge.short_name %}
      3. +
    3. Interfaces +
      4. +
    4. Create interface
      5. +
    +{% endblock %} + +{% block content %} + {% include 'algorithms/partials/algorithminterface_form.html' %} +{% endblock %} diff --git a/app/grandchallenge/evaluation/templates/evaluation/configure_algorithm_phases_form.html b/app/grandchallenge/evaluation/templates/evaluation/configure_algorithm_phases_form.html index 612cb945d..7a62485ac 100644 --- a/app/grandchallenge/evaluation/templates/evaluation/configure_algorithm_phases_form.html +++ b/app/grandchallenge/evaluation/templates/evaluation/configure_algorithm_phases_form.html @@ -20,6 +20,9 @@ {% block content %}

    Configure algorithm submission phases

    +

    Select the phases you would like to turn into algorithm submission phases.

    +

    This will create an archive for each selected phase, and populate the inference time limit, gpu type choices and memory requirements based on the corresponding challenge request.

    +

    To configure algorithm interfaces for each phase, please visit the phase settings after this step.

    {% crispy form %} {% endblock %} diff --git a/app/grandchallenge/evaluation/templates/evaluation/partials/algorithm_interfaces_information.html b/app/grandchallenge/evaluation/templates/evaluation/partials/algorithm_interfaces_information.html new file mode 100644 index 000000000..795d8f5c7 --- /dev/null +++ b/app/grandchallenge/evaluation/templates/evaluation/partials/algorithm_interfaces_information.html @@ -0,0 +1,21 @@ +{% load url %} + +
    +
    The algorithms for this phase need to implement all of the following input-output combinations:
    +
    + + Manage your algorithms + +
    +
    +{% include 'algorithms/partials/algorithminterface_table.html' with base_obj=phase interfaces=phase.algorithm_interfaces.all delete_option=False %} + +{% if phase.submission_kind == phase.SubmissionKindChoices.ALGORITHM and not form.fields.algorithm.queryset.exists %} +

    + Currently, you either do not have an algorithm that is compatible with this phase, + or none of your compatible algorithms have an active algorithm image. + Either upload a new algorithm image to one of your compatible algorithms, or create a new algorithm (see 'Manage your algorithms' button above). + As soon as your new algorithm image is active you can return to this page to submit it to this phase. +

    +{% endif %} diff --git a/app/grandchallenge/evaluation/templates/evaluation/phase_archive_info.html b/app/grandchallenge/evaluation/templates/evaluation/phase_archive_info.html index 5335adc6d..70bc120f1 100644 --- a/app/grandchallenge/evaluation/templates/evaluation/phase_archive_info.html +++ b/app/grandchallenge/evaluation/templates/evaluation/phase_archive_info.html @@ -23,24 +23,35 @@

    Linked archive for {{ object.title }}

    The (hidden) input data for this phase needs to be uploaded to an archive on Grand Challenge.

    - {% if object.archive and object.submission_kind == object.SubmissionKindChoices.ALGORITHM and not object.external_evaluation %} + + {% if object.archive and object.submission_kind == object.SubmissionKindChoices.ALGORITHM %}

    This phase is linked to archive {{ object.archive }}.

    - {% if object.algorithm_inputs.all %} -

    Each item in the archive needs to contain data for the following interfaces, which are defined as algorithm inputs:

    -
      - {% for input in object.algorithm_inputs.all %} -
    • {{ input }}
      • - {% endfor %} -
    + {% if object.algorithm_interfaces.all %} +

    Each item in the archive needs to contain data for one of the configured interfaces (i.e. input combinations):

    + {% for interface in object.algorithm_interfaces.all %} +
      +
    • Interface {{ forloop.counter }}
      • +
        + {% for input in interface.inputs.all %} +
      • {{ input }}
        • + {% endfor %} +
      +
    + {% endfor %}

    - For each submission, {{ object.count_valid_archive_items }} - algorithm job{{ object.count_valid_archive_items|pluralize }} will be created - from the {{ object.count_valid_archive_items }} valid item{{ object.count_valid_archive_items|pluralize }}. + Currently, for each submission, {{ object.jobs_to_schedule_per_submission }} + algorithm job{{ object.jobs_to_schedule_per_submission|pluralize }} will be created from the {{ object.jobs_to_schedule_per_submission }} valid archive item{{ object.jobs_to_schedule_per_submission|pluralize }}.

    +

    The jobs are spread across the configured algorithm interfaces as follows: +

      + {% for interface, count in object.valid_archive_item_count_per_interface.items %} +
    • Interface {{ forloop.counter }}: {{ count }} job{{ count|pluralize }}
      • + {% endfor %} +

    Upload data to {{ object.archive }} {% else %} -

    Before you can upload data to your archive, you need to define the algorithm inputs and outputs. Contact Grand Challenge Support for help with getting this set up.

    +

    Before you can upload data to your archive, you need to define the interfaces (i.e. possible input and output combinations) for algorithms submitted to this phase. Contact Grand Challenge Support for help with getting this set up.

    {% endif %} {% elif object.submission_kind == object.SubmissionKindChoices.ALGORITHM and not object.external_evaluation and not object.archive %} diff --git a/app/grandchallenge/evaluation/templates/evaluation/phasealgorithminterface_confirm_delete.html b/app/grandchallenge/evaluation/templates/evaluation/phasealgorithminterface_confirm_delete.html new file mode 100644 index 000000000..5ac02e4a6 --- /dev/null +++ b/app/grandchallenge/evaluation/templates/evaluation/phasealgorithminterface_confirm_delete.html @@ -0,0 +1,26 @@ +{% extends "pages/challenge_settings_base.html" %} +{% load url %} +{% load static %} + +{% block title %} + Delete interface - {{ phase.title }} - {% firstof challenge.title challenge.short_name %} - {{ block.super }} +{% endblock %} + +{% block breadcrumbs %} +
      +
    1. Challenges +
      2. +
    2. {% firstof challenge.title challenge.short_name %}
      3. +
    3. Interfaces +
      4. +
    4. Delete interface
      5. +
    +{% endblock %} + +{% block content %} + {% include 'algorithms/partials/algorithminterface_confirm_delete.html' with base_obj=phase %} +{% endblock %} diff --git a/app/grandchallenge/evaluation/templates/evaluation/phasealgorithminterface_list.html b/app/grandchallenge/evaluation/templates/evaluation/phasealgorithminterface_list.html new file mode 100644 index 000000000..793b5b8f0 --- /dev/null +++ b/app/grandchallenge/evaluation/templates/evaluation/phasealgorithminterface_list.html @@ -0,0 +1,23 @@ +{% extends "pages/challenge_settings_base.html" %} +{% load url %} +{% load static %} + +{% block title %} + Algorithm interfaces for {{ phase.title }} - {% firstof challenge.title challenge.short_name %} - {{ block.super }} +{% endblock %} + +{% block breadcrumbs %} +
      +
    1. Challenges +
      2. +
    2. {% firstof challenge.title challenge.short_name %}
      3. +
    3. Algorithm Interfaces for {{ phase.title }}
      4. +
    +{% endblock %} + +{% block content %} + {% include 'algorithms/partials/algorithminterface_list.html' with base_obj=phase %} +{% endblock %} diff --git a/app/grandchallenge/evaluation/templates/evaluation/submission_form.html b/app/grandchallenge/evaluation/templates/evaluation/submission_form.html index e7de89174..752d82cd1 100644 --- a/app/grandchallenge/evaluation/templates/evaluation/submission_form.html +++ b/app/grandchallenge/evaluation/templates/evaluation/submission_form.html @@ -69,9 +69,9 @@

    Create a new submission

    {% if phase.submission_kind == phase.SubmissionKindChoices.ALGORITHM and not phase.external_evaluation %}

    - For each submission, {{ phase.count_valid_archive_items }} - algorithm job{{ phase.count_valid_archive_items|pluralize }} will be created - from the {{ phase.count_valid_archive_items }} valid item{{ phase.count_valid_archive_items|pluralize }} + For each submission, {{ phase.jobs_to_schedule_per_submission }} + algorithm job{{ phase.jobs_to_schedule_per_submission|pluralize }} will be created + from the {{ phase.jobs_to_schedule_per_submission }} valid item{{ phase.jobs_to_schedule_per_submission|pluralize }} in {{ phase.archive }}. Each individual algorithm job will have a time limit of {{ phase.algorithm_time_limit|naturaldelta }}.

    @@ -81,6 +81,8 @@

    Create a new submission

    + {% include 'evaluation/partials/algorithm_interfaces_information.html' with phase=phase form=form %} + {% crispy form %} {% else %} @@ -112,19 +114,6 @@

    Create a new submission

    {% endif %}
    - {% elif phase.submission_kind == phase.SubmissionKindChoices.ALGORITHM and not form.fields.algorithm_image.queryset.exists %} - -

    - Currently, you do not have any Algorithm Image that can be submitted to this phase. - Either upload a new Algorithm Image to one of your existing Algorithms, or create a new Algorithm. - As soon as your new Algorithm Image is active you can return to this page to submit it to this phase. -

    - - - Add a new Algorithm - - {% else %} {% endif %} + {% include 'evaluation/partials/algorithm_interfaces_information.html' with phase=phase form=form %} + {% crispy form %} {% endif %} diff --git a/app/grandchallenge/evaluation/urls.py b/app/grandchallenge/evaluation/urls.py index 43fbdfbf5..7cfff1936 100644 --- a/app/grandchallenge/evaluation/urls.py +++ b/app/grandchallenge/evaluation/urls.py @@ -1,6 +1,9 @@ from django.urls import path from grandchallenge.evaluation.views import ( + AlgorithmInterfaceForPhaseCreate, + AlgorithmInterfaceForPhaseDelete, + AlgorithmInterfacesForPhaseList, CombinedLeaderboardCreate, CombinedLeaderboardDelete, CombinedLeaderboardDetail, @@ -64,109 +67,128 @@ name="combined-leaderboard-create", ), path( - "combined-leaderboards//", + "combined-leaderboards//", CombinedLeaderboardDetail.as_view(), name="combined-leaderboard-detail", ), path( - "combined-leaderboards//update/", + "combined-leaderboards//update/", CombinedLeaderboardUpdate.as_view(), name="combined-leaderboard-update", ), path( - "combined-leaderboards//delete/", + "combined-leaderboards//delete/", CombinedLeaderboardDelete.as_view(), name="combined-leaderboard-delete", ), - path("/", EvaluationList.as_view(), name="list"), + path("/", EvaluationList.as_view(), name="list"), path( - "/admin/", + "/admin/", EvaluationAdminList.as_view(), name="evaluation-admin-list", ), path( - "/algorithms/create/", + "/algorithms/create/", PhaseAlgorithmCreate.as_view(), name="phase-algorithm-create", ), path( - "/linked-archive/", + "/interfaces/", + AlgorithmInterfacesForPhaseList.as_view(), + name="interface-list", + ), + path( + "/interfaces/create/", + AlgorithmInterfaceForPhaseCreate.as_view(), + name="interface-create", + ), + path( + "/interfaces//delete/", + AlgorithmInterfaceForPhaseDelete.as_view(), + name="interface-delete", + ), + path( + "/linked-archive/", PhaseArchiveInfo.as_view(), name="phase-archive-info", ), path( - "/ground-truths/", + "/ground-truths/", EvaluationGroundTruthList.as_view(), name="ground-truth-list", ), path( - "/ground-truths/activate/", + "/ground-truths/activate/", EvaluationGroundTruthVersionManagement.as_view(activate=True), name="ground-truth-activate", ), path( - "/ground-truths/create/", + "/ground-truths/create/", EvaluationGroundTruthCreate.as_view(), name="ground-truth-create", ), path( - "/ground-truths/deactivate/", + "/ground-truths/deactivate/", EvaluationGroundTruthVersionManagement.as_view(activate=False), name="ground-truth-deactivate", ), path( - "/ground-truths//", + "/ground-truths//", EvaluationGroundTruthDetail.as_view(), name="ground-truth-detail", ), path( - "/ground-truths//import-status/", + "/ground-truths//import-status/", EvaluationGroundTruthImportStatusDetail.as_view(), name="ground-truth-import-status-detail", ), path( - "/ground-truths//update/", + "/ground-truths//update/", EvaluationGroundTruthUpdate.as_view(), name="ground-truth-update", ), path( - "/leaderboard/", LeaderboardDetail.as_view(), name="leaderboard" + "/leaderboard/", + LeaderboardDetail.as_view(), + name="leaderboard", ), - path("/methods/", MethodList.as_view(), name="method-list"), + path("/methods/", MethodList.as_view(), name="method-list"), path( - "/methods/create/", MethodCreate.as_view(), name="method-create" + "/methods/create/", + MethodCreate.as_view(), + name="method-create", ), path( - "/methods//", + "/methods//", MethodDetail.as_view(), name="method-detail", ), path( - "/methods//import-status/", + "/methods//import-status/", MethodImportStatusDetail.as_view(), name="method-import-status-detail", ), path( - "/methods//update/", + "/methods//update/", MethodUpdate.as_view(), name="method-update", ), path( - "/submissions/create/", + "/submissions/create/", SubmissionCreate.as_view(), name="submission-create", ), path( - "/submissions//", + "/submissions//", SubmissionDetail.as_view(), name="submission-detail", ), path( - "/submissions//evaluations/create/", + "/submissions//evaluations/create/", EvaluationCreate.as_view(), name="evaluation-create", ), - path("/update/", PhaseUpdate.as_view(), name="phase-update"), + path("/update/", PhaseUpdate.as_view(), name="phase-update"), path("results/", LeaderboardRedirect.as_view()), path("leaderboard/", LeaderboardRedirect.as_view()), ] diff --git a/app/grandchallenge/evaluation/views/__init__.py b/app/grandchallenge/evaluation/views/__init__.py index dbb94d17c..e5eee8583 100644 --- a/app/grandchallenge/evaluation/views/__init__.py +++ b/app/grandchallenge/evaluation/views/__init__.py @@ -29,6 +29,7 @@ from grandchallenge.algorithms.forms import AlgorithmForPhaseForm from grandchallenge.algorithms.models import Algorithm, Job +from grandchallenge.algorithms.views import AlgorithmInterfaceCreateBase from grandchallenge.archives.models import Archive from grandchallenge.challenges.views import ActiveChallengeRequiredMixin from grandchallenge.components.models import ImportStatusChoices @@ -60,6 +61,7 @@ EvaluationGroundTruth, Method, Phase, + PhaseAlgorithmInterface, Submission, ) from grandchallenge.evaluation.tasks import create_evaluation @@ -137,8 +139,7 @@ def dispatch(self, request, *args, **kwargs): return self.handle_no_permission() elif ( not self.phase.submission_kind == SubmissionKindChoices.ALGORITHM - or not self.phase.algorithm_inputs - or not self.phase.algorithm_outputs + or not self.phase.algorithm_interfaces or not self.phase.archive ): error_message = ( @@ -889,8 +890,7 @@ def get_form_kwargs(self): "display_editors": True, "contact_email": self.request.user.email, "workstation": self.phase.workstation, - "inputs": self.phase.algorithm_inputs.all(), - "outputs": self.phase.algorithm_outputs.all(), + "interfaces": self.phase.algorithm_interfaces.all(), "modalities": self.phase.challenge.modalities.all(), "structures": self.phase.challenge.structures.all(), "logo": self.phase.challenge.logo, @@ -1022,9 +1022,14 @@ def get_permission_object(self): return self.get_object().challenge -class ConfigureAlgorithmPhasesView(PermissionRequiredMixin, FormView): - form_class = ConfigureAlgorithmPhasesForm +class ConfigureAlgorithmPhasesPermissionMixin(PermissionRequiredMixin): permission_required = "evaluation.configure_algorithm_phase" + + +class ConfigureAlgorithmPhasesView( + ConfigureAlgorithmPhasesPermissionMixin, FormView +): + form_class = ConfigureAlgorithmPhasesForm template_name = "evaluation/configure_algorithm_phases_form.html" raise_exception = True @@ -1037,8 +1042,6 @@ def form_valid(self, form): for phase in form.cleaned_data["phases"]: self.turn_phase_into_algorithm_phase( phase=phase, - inputs=form.cleaned_data["algorithm_inputs"], - outputs=form.cleaned_data["algorithm_outputs"], algorithm_time_limit=form.cleaned_data["algorithm_time_limit"], algorithm_selectable_gpu_type_choices=form.cleaned_data[ "algorithm_selectable_gpu_type_choices" @@ -1059,8 +1062,6 @@ def turn_phase_into_algorithm_phase( self, *, phase, - inputs, - outputs, algorithm_time_limit, algorithm_selectable_gpu_type_choices, algorithm_maximum_settable_memory_gb, @@ -1097,8 +1098,6 @@ def turn_phase_into_algorithm_phase( phase.submission_kind = phase.SubmissionKindChoices.ALGORITHM phase.creator_must_be_verified = True phase.save() - phase.algorithm_outputs.add(*outputs) - phase.algorithm_inputs.add(*inputs) class EvaluationGroundTruthCreate( @@ -1258,3 +1257,95 @@ def get_object(self, queryset=None): challenge=self.request.challenge, slug=self.kwargs["slug"], ) + + +class AlgorithmInterfaceForPhaseMixin: + + @property + def phase(self): + return get_object_or_404( + Phase, + challenge=self.request.challenge, + challenge__phase__submission_kind=SubmissionKindChoices.ALGORITHM, + slug=self.kwargs["slug"], + ) + + +class AlgorithmInterfaceForPhaseCreate( + ConfigureAlgorithmPhasesPermissionMixin, + AlgorithmInterfaceForPhaseMixin, + AlgorithmInterfaceCreateBase, +): + template_name = "evaluation/algorithminterface_for_phase_form.html" + + @property + def base_obj(self): + return self.phase + + def get_success_url(self): + return reverse( + "evaluation:interface-list", + kwargs={ + "slug": self.phase.slug, + "challenge_short_name": self.request.challenge.short_name, + }, + ) + + +class AlgorithmInterfacesForPhaseList( + ConfigureAlgorithmPhasesPermissionMixin, + AlgorithmInterfaceForPhaseMixin, + ListView, +): + model = PhaseAlgorithmInterface + + def get_queryset(self): + qs = super().get_queryset() + return qs.filter(phase=self.phase) + + def get_context_data(self, *args, **kwargs): + context = super().get_context_data(*args, **kwargs) + context.update( + { + "phase": self.phase, + "interfaces": [obj.interface for obj in self.object_list], + } + ) + return context + + +class AlgorithmInterfaceForPhaseDelete( + ConfigureAlgorithmPhasesPermissionMixin, + AlgorithmInterfaceForPhaseMixin, + DeleteView, +): + model = PhaseAlgorithmInterface + + @property + def algorithm_interface(self): + return get_object_or_404( + klass=PhaseAlgorithmInterface, + phase=self.phase, + interface__pk=self.kwargs["interface_pk"], + ) + + def get_object(self, queryset=None): + return self.algorithm_interface + + def get_success_url(self): + return reverse( + "evaluation:interface-list", + kwargs={ + "slug": self.phase.slug, + "challenge_short_name": self.request.challenge.short_name, + }, + ) + + def get_context_data(self, *args, **kwargs): + context = super().get_context_data(*args, **kwargs) + context.update( + { + "phase": self.phase, + } + ) + return context diff --git a/app/grandchallenge/hanging_protocols/forms.py b/app/grandchallenge/hanging_protocols/forms.py index ad032fd3c..6e96e4989 100644 --- a/app/grandchallenge/hanging_protocols/forms.py +++ b/app/grandchallenge/hanging_protocols/forms.py @@ -159,7 +159,7 @@ def __init__(self, *args, **kwargs): if len(interface_slugs) > 0: self.fields[ "view_content" - ].help_text += f"The following interfaces are used in your {self.instance._meta.verbose_name}: {oxford_comma(interface_slugs)}. " + ].help_text += f"The following sockets are used in your {self.instance._meta.verbose_name}: {oxford_comma(interface_slugs)}. " view_content_example = self.generate_view_content_example() @@ -170,7 +170,7 @@ def __init__(self, *args, **kwargs): else: self.fields[ "view_content" - ].help_text += "No interfaces of type image, chart, pdf, mp4, thumbnail_jpg or thumbnail_png are used. At least one interface of those types is needed to configure the viewer. " + ].help_text += "No sockets of type image, chart, pdf, mp4, thumbnail_jpg or thumbnail_png are used. At least one socket of those types is needed to configure the viewer. " self.fields["view_content"].help_text += format_lazy( 'Refer to the documentation for more information', @@ -183,20 +183,24 @@ def __init__(self, *args, **kwargs): ) def _get_interface_lists(self): + sorted_interfaces = sorted( + list(self.instance.linked_component_interfaces), + key=lambda x: x.slug, + ) images = [ interface.slug - for interface in self.instance.linked_component_interfaces + for interface in sorted_interfaces if interface.kind == InterfaceKindChoices.IMAGE ] mandatory_isolation_interfaces = [ interface.slug - for interface in self.instance.linked_component_interfaces + for interface in sorted_interfaces if interface.kind in InterfaceKind.interface_type_mandatory_isolation() ] overlays = [ interface.slug - for interface in self.instance.linked_component_interfaces + for interface in sorted_interfaces if interface.kind not in ( *InterfaceKind.interface_type_undisplayable(), @@ -266,7 +270,6 @@ def _build_view_content( def generate_view_content_example(self): interface_lists = self._get_interface_lists() - if ( not interface_lists.images and not interface_lists.mandatory_isolation_interfaces diff --git a/app/grandchallenge/hanging_protocols/models.py b/app/grandchallenge/hanging_protocols/models.py index 13286887d..729f154cf 100644 --- a/app/grandchallenge/hanging_protocols/models.py +++ b/app/grandchallenge/hanging_protocols/models.py @@ -316,10 +316,10 @@ class HangingProtocolMixin(models.Model): blank=True, on_delete=models.SET_NULL, help_text=( - "Indicate which Component Interfaces need to be displayed in " - 'which image port. E.g. {"main": ["interface1"]}. The first ' - "item in the list of interfaces will be the main image in " - "the image port. The first overlay type interface thereafter " + "Indicate which sockets need to be displayed in " + 'which image port. E.g. {"main": ["socket1"]}. The first ' + "item in the list of sockets will be the main image in " + "the image port. The first overlay type socket thereafter " "will be rendered as an overlay. For now, any other items " "will be ignored by the viewer." ), @@ -356,7 +356,7 @@ def check_all_interfaces_in_view_content_exist(*, view_content): if set(slugs) != {i.slug for i in viewport_interfaces}: raise ValidationError( - f"Unknown interfaces in view content for viewport {viewport}: {', '.join(slugs)}" + f"Unknown sockets in view content for viewport {viewport}: {', '.join(slugs)}" ) image_interfaces = [ @@ -367,7 +367,7 @@ def check_all_interfaces_in_view_content_exist(*, view_content): if len(image_interfaces) > 1: raise ValidationError( - "Maximum of one image interface is allowed per viewport, " + "Maximum of one image socket is allowed per viewport, " f"got {len(image_interfaces)} for viewport {viewport}: " f"{', '.join(i.slug for i in image_interfaces)}" ) @@ -383,7 +383,7 @@ def check_all_interfaces_in_view_content_exist(*, view_content): and len(viewport_interfaces) > 1 ): raise ValidationError( - "Some of the selected interfaces can only be displayed in isolation, " + "Some of the selected sockets can only be displayed in isolation, " f"found {len(mandatory_isolation_interfaces)} for viewport {viewport}: " f"{', '.join(i.slug for i in mandatory_isolation_interfaces)}" ) @@ -396,7 +396,7 @@ def check_all_interfaces_in_view_content_exist(*, view_content): if len(undisplayable_interfaces) > 0: raise ValidationError( - "Some of the selected interfaces cannot be displayed, " + "Some of the selected sockets cannot be displayed, " f"found {len(undisplayable_interfaces)} for viewport {viewport}: " f"{', '.join(i.slug for i in undisplayable_interfaces)}" ) diff --git a/app/grandchallenge/pages/templates/pages/phase_menu_sidebar.html b/app/grandchallenge/pages/templates/pages/phase_menu_sidebar.html index c7f0b0fb4..55e71a9e2 100644 --- a/app/grandchallenge/pages/templates/pages/phase_menu_sidebar.html +++ b/app/grandchallenge/pages/templates/pages/phase_menu_sidebar.html @@ -1,15 +1,17 @@ +{% load guardian_tags %} +
  • {% if not phase.public %}{% endif %} {{ phase.title }} -

    • {% if type_to_add == "images" %} -

    This will create one new display set for each uploaded image using the interface you choose below.

    +

    This will create one new display set for each uploaded image using the socket you choose below.

    {% endif %} {% endblock %} diff --git a/app/tests/algorithms_tests/factories.py b/app/tests/algorithms_tests/factories.py index 3259c8a2d..78c8125b8 100644 --- a/app/tests/algorithms_tests/factories.py +++ b/app/tests/algorithms_tests/factories.py @@ -3,13 +3,17 @@ from grandchallenge.algorithms.models import ( Algorithm, AlgorithmImage, + AlgorithmInterface, AlgorithmModel, AlgorithmPermissionRequest, AlgorithmUserCredit, Job, ) from grandchallenge.components.schemas import GPUTypeChoices -from tests.components_tests.factories import ComponentInterfaceValueFactory +from tests.components_tests.factories import ( + ComponentInterfaceFactory, + ComponentInterfaceValueFactory, +) from tests.factories import ( ImageFactory, UserFactory, @@ -84,3 +88,19 @@ class Meta: user = factory.SubFactory(UserFactory) algorithm = factory.SubFactory(AlgorithmFactory) + + +class AlgorithmInterfaceFactory(factory.django.DjangoModelFactory): + class Meta: + model = AlgorithmInterface + + @classmethod + def _create(cls, model_class, *args, **kwargs): + manager = cls._get_manager(model_class) + inputs = kwargs.pop("inputs", None) + outputs = kwargs.pop("outputs", None) + if not inputs: + inputs = [ComponentInterfaceFactory()] + if not outputs: + outputs = [ComponentInterfaceFactory()] + return manager.create(*args, inputs=inputs, outputs=outputs, **kwargs) diff --git a/app/tests/algorithms_tests/test_admin.py b/app/tests/algorithms_tests/test_admin.py index bf51cc2f6..73031993a 100644 --- a/app/tests/algorithms_tests/test_admin.py +++ b/app/tests/algorithms_tests/test_admin.py @@ -3,12 +3,16 @@ import pytest from django.core.files.base import ContentFile -from grandchallenge.algorithms.admin import AlgorithmAdmin from grandchallenge.algorithms.models import Job from grandchallenge.algorithms.tasks import create_algorithm_jobs +from grandchallenge.archives.models import ArchiveItem from grandchallenge.components.admin import requeue_jobs from grandchallenge.components.models import ComponentInterface -from tests.algorithms_tests.factories import AlgorithmImageFactory +from tests.algorithms_tests.factories import ( + AlgorithmImageFactory, + AlgorithmInterfaceFactory, +) +from tests.archives_tests.factories import ArchiveItemFactory from tests.components_tests.factories import ( ComponentInterfaceFactory, ComponentInterfaceValueFactory, @@ -17,14 +21,6 @@ from tests.utils import recurse_callbacks -@pytest.mark.django_db -def test_disjoint_interfaces(): - i = ComponentInterfaceFactory() - form = AlgorithmAdmin.form(data={"inputs": [i.pk], "outputs": [i.pk]}) - assert form.is_valid() is False - assert "The sets of Inputs and Outputs must be unique" in str(form.errors) - - @pytest.mark.django_db def test_job_updated_start_and_complete_times_after_admin_requeue( algorithm_image, settings, django_capture_on_commit_callbacks @@ -45,30 +41,32 @@ def test_job_updated_start_and_complete_times_after_admin_requeue( ai.refresh_from_db() # Make sure the job fails when trying to upload an invalid file - input_interface = ComponentInterface.objects.get( - slug="generic-medical-image" - ) + ci = ComponentInterface.objects.get(slug="generic-medical-image") detection_interface = ComponentInterfaceFactory( store_in_database=False, relative_path="some_text.txt", slug="detection-json-file", kind=ComponentInterface.Kind.ANY, ) - ai.algorithm.inputs.add(input_interface) - ai.algorithm.outputs.add(detection_interface) + interface = AlgorithmInterfaceFactory( + inputs=[ci], outputs=[detection_interface] + ) + ai.algorithm.interfaces.add(interface) image_file = ImageFileFactory( file__from_path=Path(__file__).parent / "resources" / "input_file.tif" ) civ = ComponentInterfaceValueFactory( - image=image_file.image, interface=input_interface, file=None + image=image_file.image, interface=ci, file=None ) + item = ArchiveItemFactory() + item.values.add(civ) with django_capture_on_commit_callbacks() as callbacks: create_algorithm_jobs( algorithm_image=ai, - civ_sets=[{civ}], + archive_items=ArchiveItem.objects.all(), time_limit=ai.algorithm.time_limit, requires_gpu_type=ai.algorithm.job_requires_gpu_type, requires_memory_gb=ai.algorithm.job_requires_memory_gb, diff --git a/app/tests/algorithms_tests/test_api.py b/app/tests/algorithms_tests/test_api.py index 2a3a229ff..78540a4f9 100644 --- a/app/tests/algorithms_tests/test_api.py +++ b/app/tests/algorithms_tests/test_api.py @@ -18,6 +18,7 @@ from grandchallenge.uploads.models import UserUpload from tests.algorithms_tests.factories import ( AlgorithmImageFactory, + AlgorithmInterfaceFactory, AlgorithmJobFactory, AlgorithmModelFactory, ) @@ -150,16 +151,18 @@ def test_create_job_with_multiple_new_inputs( algorithm_with_multiple_inputs, ): # configure multiple inputs - algorithm_with_multiple_inputs.algorithm.inputs.set( - [ + interface = AlgorithmInterfaceFactory( + inputs=[ algorithm_with_multiple_inputs.ci_json_in_db_with_schema, algorithm_with_multiple_inputs.ci_existing_img, algorithm_with_multiple_inputs.ci_str, algorithm_with_multiple_inputs.ci_bool, algorithm_with_multiple_inputs.ci_json_file, algorithm_with_multiple_inputs.ci_img_upload, - ] + ], + outputs=[ComponentInterfaceFactory()], ) + algorithm_with_multiple_inputs.algorithm.interfaces.add(interface) assert ComponentInterfaceValue.objects.count() == 0 @@ -216,12 +219,9 @@ def test_create_job_with_multiple_new_inputs( pk=algorithm_with_multiple_inputs.file_upload.pk ).exists() - assert sorted( - [ - int.pk - for int in algorithm_with_multiple_inputs.algorithm.inputs.all() - ] - ) == sorted([civ.interface.pk for civ in job.inputs.all()]) + assert sorted([int.pk for int in interface.inputs.all()]) == sorted( + [civ.interface.pk for civ in job.inputs.all()] + ) value_inputs = [civ.value for civ in job.inputs.all() if civ.value] assert "Foo" in value_inputs @@ -244,14 +244,16 @@ def test_create_job_with_existing_inputs( algorithm_with_multiple_inputs, ): # configure multiple inputs - algorithm_with_multiple_inputs.algorithm.inputs.set( - [ + interface = AlgorithmInterfaceFactory( + inputs=[ algorithm_with_multiple_inputs.ci_json_in_db_with_schema, algorithm_with_multiple_inputs.ci_existing_img, algorithm_with_multiple_inputs.ci_str, algorithm_with_multiple_inputs.ci_bool, - ] + ], + outputs=[ComponentInterfaceFactory()], ) + algorithm_with_multiple_inputs.algorithm.interfaces.add(interface) civ1, civ2, civ3, civ4 = self.create_existing_civs( interface_data=algorithm_with_multiple_inputs @@ -301,14 +303,16 @@ def test_create_job_is_idempotent( algorithm_with_multiple_inputs, ): # configure multiple inputs - algorithm_with_multiple_inputs.algorithm.inputs.set( - [ + interface = AlgorithmInterfaceFactory( + inputs=[ algorithm_with_multiple_inputs.ci_str, algorithm_with_multiple_inputs.ci_bool, algorithm_with_multiple_inputs.ci_existing_img, algorithm_with_multiple_inputs.ci_json_in_db_with_schema, - ] + ], + outputs=[ComponentInterfaceFactory()], ) + algorithm_with_multiple_inputs.algorithm.interfaces.add(interface) civ1, civ2, civ3, civ4 = self.create_existing_civs( interface_data=algorithm_with_multiple_inputs ) @@ -365,9 +369,11 @@ def test_create_job_with_faulty_file_input( algorithm_with_multiple_inputs, ): # configure file input - algorithm_with_multiple_inputs.algorithm.inputs.set( - [algorithm_with_multiple_inputs.ci_json_file] + interface = AlgorithmInterfaceFactory( + inputs=[algorithm_with_multiple_inputs.ci_json_file], + outputs=[ComponentInterfaceFactory()], ) + algorithm_with_multiple_inputs.algorithm.interfaces.add(interface) file_upload = UserUploadFactory( filename="file.json", creator=algorithm_with_multiple_inputs.editor ) @@ -413,9 +419,11 @@ def test_create_job_with_faulty_json_input( django_capture_on_commit_callbacks, algorithm_with_multiple_inputs, ): - algorithm_with_multiple_inputs.algorithm.inputs.set( - [algorithm_with_multiple_inputs.ci_json_in_db_with_schema] + interface = AlgorithmInterfaceFactory( + inputs=[algorithm_with_multiple_inputs.ci_json_in_db_with_schema], + outputs=[ComponentInterfaceFactory()], ) + algorithm_with_multiple_inputs.algorithm.interfaces.add(interface) response = self.create_job( client=client, @@ -444,9 +452,11 @@ def test_create_job_with_faulty_image_input( django_capture_on_commit_callbacks, algorithm_with_multiple_inputs, ): - algorithm_with_multiple_inputs.algorithm.inputs.set( - [algorithm_with_multiple_inputs.ci_img_upload] + interface = AlgorithmInterfaceFactory( + inputs=[algorithm_with_multiple_inputs.ci_img_upload], + outputs=[ComponentInterfaceFactory()], ) + algorithm_with_multiple_inputs.algorithm.interfaces.add(interface) user_upload = create_upload_from_file( creator=algorithm_with_multiple_inputs.editor, file_path=RESOURCE_PATH / "corrupt.png", @@ -500,12 +510,10 @@ def test_create_job_with_multiple_faulty_existing_image_inputs( ] ci.save() - algorithm_with_multiple_inputs.algorithm.inputs.set( - [ - ci1, - ci2, - ] + interface = AlgorithmInterfaceFactory( + inputs=[ci1, ci2], outputs=[ComponentInterfaceFactory()] ) + algorithm_with_multiple_inputs.algorithm.interfaces.add(interface) im = ImageFactory() im.files.set([ImageFileFactoryWithMHDFile()]) diff --git a/app/tests/algorithms_tests/test_forms.py b/app/tests/algorithms_tests/test_forms.py index e8595eb4d..4a2937bb6 100644 --- a/app/tests/algorithms_tests/test_forms.py +++ b/app/tests/algorithms_tests/test_forms.py @@ -8,6 +8,7 @@ from grandchallenge.algorithms.forms import ( AlgorithmForm, AlgorithmForPhaseForm, + AlgorithmInterfaceForm, AlgorithmModelForm, AlgorithmModelVersionControlForm, AlgorithmPublishForm, @@ -36,6 +37,7 @@ from tests.algorithms_tests.factories import ( AlgorithmFactory, AlgorithmImageFactory, + AlgorithmInterfaceFactory, AlgorithmJobFactory, AlgorithmModelFactory, AlgorithmPermissionRequestFactory, @@ -157,7 +159,6 @@ def test_algorithm_create(client, uploaded_image): VerificationFactory(user=creator, is_verified=True) ws = WorkstationFactory() - ci = ComponentInterface.objects.get(slug="generic-medical-image") def try_create_algorithm(): return get_view_for_user( @@ -168,8 +169,7 @@ def try_create_algorithm(): "title": "foo bar", "logo": uploaded_image(), "workstation": ws.pk, - "inputs": [ci.pk], - "outputs": [ComponentInterfaceFactory().pk], + "interfaces": AlgorithmInterfaceFactory(), "minimum_credits_per_job": 20, "job_requires_gpu_type": GPUTypeChoices.NO_GPU, "job_requires_memory_gb": 4, @@ -320,7 +320,10 @@ def test_create_job_input_fields( response = get_view_for_user( viewname="algorithms:job-create", client=client, - reverse_kwargs={"slug": alg.slug}, + reverse_kwargs={ + "slug": alg.slug, + "interface_pk": alg.interfaces.first().pk, + }, follow=True, user=creator, ) @@ -352,7 +355,10 @@ def test_create_job_json_input_field_validation( response = get_view_for_user( viewname="algorithms:job-create", client=client, - reverse_kwargs={"slug": alg.slug}, + reverse_kwargs={ + "slug": alg.slug, + "interface_pk": alg.interfaces.first().pk, + }, method=client.post, follow=True, user=creator, @@ -383,7 +389,10 @@ def test_create_job_simple_input_field_validation( response = get_view_for_user( viewname="algorithms:job-create", client=client, - reverse_kwargs={"slug": alg.slug}, + reverse_kwargs={ + "slug": alg.slug, + "interface_pk": alg.interfaces.first().pk, + }, method=client.post, follow=True, user=creator, @@ -399,7 +408,11 @@ def create_algorithm_with_input(slug): VerificationFactory(user=creator, is_verified=True) alg = AlgorithmFactory() alg.add_editor(user=creator) - alg.inputs.set([ComponentInterface.objects.get(slug=slug)]) + interface = AlgorithmInterfaceFactory( + inputs=[ComponentInterface.objects.get(slug=slug)], + outputs=[ComponentInterfaceFactory()], + ) + alg.interfaces.add(interface) AlgorithmImageFactory( algorithm=alg, is_manifest_valid=True, @@ -409,16 +422,6 @@ def create_algorithm_with_input(slug): return alg, creator -@pytest.mark.django_db -def test_disjoint_interfaces(): - i = ComponentInterfaceFactory() - form = AlgorithmForm( - user=UserFactory(), data={"inputs": [i.pk], "outputs": [i.pk]} - ) - assert form.is_valid() is False - assert "The sets of Inputs and Outputs must be unique" in str(form.errors) - - @pytest.mark.django_db def test_publish_algorithm(): algorithm = AlgorithmFactory() @@ -509,13 +512,34 @@ def test_only_publish_successful_jobs(): @pytest.mark.django_db class TestJobCreateLimits: + + def create_form(self, algorithm, user, algorithm_image=None): + ci = ComponentInterfaceFactory(kind=ComponentInterface.Kind.STRING) + interface = AlgorithmInterfaceFactory(inputs=[ci]) + algorithm.interfaces.add(interface) + + algorithm_image_kwargs = {} + if algorithm_image: + algorithm_image_kwargs = { + "algorithm_image": str(algorithm_image.pk) + } + + return JobCreateForm( + algorithm=algorithm, + user=user, + interface=interface, + data={ + **algorithm_image_kwargs, + **get_interface_form_data(interface_slug=ci.slug, data="Foo"), + }, + ) + def test_form_invalid_without_enough_credits(self, settings): algorithm = AlgorithmFactory( minimum_credits_per_job=( settings.ALGORITHMS_GENERAL_CREDITS_PER_MONTH_PER_USER + 1 ), ) - algorithm.inputs.clear() user = UserFactory() AlgorithmImageFactory( algorithm=algorithm, @@ -523,13 +547,11 @@ def test_form_invalid_without_enough_credits(self, settings): is_in_registry=True, is_desired_version=True, ) - - form = JobCreateForm(algorithm=algorithm, user=user, data={}) - + form = self.create_form(algorithm=algorithm, user=user) assert not form.is_valid() - assert form.errors == { - "__all__": ["You have run out of algorithm credits"], - } + assert "You have run out of algorithm credits" in str( + form.errors["__all__"] + ) def test_form_valid_for_editor(self, settings): algorithm = AlgorithmFactory( @@ -537,7 +559,6 @@ def test_form_valid_for_editor(self, settings): settings.ALGORITHMS_GENERAL_CREDITS_PER_MONTH_PER_USER + 1 ), ) - algorithm.inputs.clear() algorithm_image = AlgorithmImageFactory( algorithm=algorithm, is_manifest_valid=True, @@ -548,17 +569,15 @@ def test_form_valid_for_editor(self, settings): algorithm.add_editor(user=user) - form = JobCreateForm( + form = self.create_form( algorithm=algorithm, user=user, - data={"algorithm_image": str(algorithm_image.pk)}, + algorithm_image=algorithm_image, ) - assert form.is_valid() def test_form_valid_with_credits(self): algorithm = AlgorithmFactory(minimum_credits_per_job=1) - algorithm.inputs.clear() algorithm_image = AlgorithmImageFactory( algorithm=algorithm, is_manifest_valid=True, @@ -567,12 +586,11 @@ def test_form_valid_with_credits(self): ) user = UserFactory() - form = JobCreateForm( + form = self.create_form( algorithm=algorithm, user=user, - data={"algorithm_image": str(algorithm_image.pk)}, + algorithm_image=algorithm_image, ) - assert form.is_valid() @@ -719,7 +737,12 @@ def test_creator_queryset( ): algorithm = algorithm_with_image_and_model_and_two_inputs.algorithm editor = algorithm.editors_group.user_set.first() - form = JobCreateForm(algorithm=algorithm, user=editor, data={}) + form = JobCreateForm( + algorithm=algorithm, + user=editor, + interface=algorithm.interfaces.first(), + data={}, + ) assert list(form.fields["creator"].queryset.all()) == [editor] assert form.fields["creator"].initial == editor @@ -735,7 +758,12 @@ def test_algorithm_image_queryset( is_manifest_valid=True, is_in_registry=True, ) - form = JobCreateForm(algorithm=algorithm, user=editor, data={}) + form = JobCreateForm( + algorithm=algorithm, + user=editor, + interface=algorithm.interfaces.first(), + data={}, + ) ai_qs = form.fields["algorithm_image"].queryset.all() assert algorithm.active_image in ai_qs assert inactive_image not in ai_qs @@ -754,12 +782,14 @@ def test_cannot_create_job_with_same_inputs_twice( algorithm_model=algorithm.active_model, status=Job.SUCCESS, time_limit=123, + algorithm_interface=algorithm.interfaces.first(), ) job.inputs.set(civs) form = JobCreateForm( algorithm=algorithm, user=editor, + interface=algorithm.interfaces.first(), data={ "algorithm_image": algorithm.active_image, "algorithm_model": algorithm.active_model, @@ -779,23 +809,40 @@ def test_cannot_create_job_with_same_inputs_twice( @pytest.mark.django_db -def test_all_inputs_required_on_job_creation(algorithm_with_multiple_inputs): +def test_inputs_required_on_job_creation(algorithm_with_multiple_inputs): ci_json_in_db_without_schema = ComponentInterfaceFactory( kind=InterfaceKind.InterfaceKindChoices.ANY, store_in_database=True, ) - algorithm_with_multiple_inputs.algorithm.inputs.add( - ci_json_in_db_without_schema + interface = AlgorithmInterfaceFactory( + inputs=[ + ci_json_in_db_without_schema, + algorithm_with_multiple_inputs.ci_bool, + algorithm_with_multiple_inputs.ci_str, + algorithm_with_multiple_inputs.ci_json_in_db_with_schema, + algorithm_with_multiple_inputs.ci_existing_img, + algorithm_with_multiple_inputs.ci_json_file, + ], + outputs=[ComponentInterfaceFactory()], ) + algorithm_with_multiple_inputs.algorithm.interfaces.set([interface]) form = JobCreateForm( algorithm=algorithm_with_multiple_inputs.algorithm, user=algorithm_with_multiple_inputs.editor, + interface=interface, data={}, ) for name, field in form.fields.items(): - if name not in ["algorithm_model", "creator"]: + # boolean and json inputs that allow None should not be required, + # all other inputs should be + if name not in [ + "algorithm_model", + "creator", + f"{INTERFACE_FORM_FIELD_PREFIX}{algorithm_with_multiple_inputs.ci_bool.slug}", + f"{INTERFACE_FORM_FIELD_PREFIX}{ci_json_in_db_without_schema.slug}", + ]: assert field.required @@ -820,8 +867,8 @@ def test_algorithm_form_gpu_choices_from_phases(): ci1, ci2, ci3, ci4, ci5, ci6 = ComponentInterfaceFactory.create_batch(6) inputs = [ci1, ci2] outputs = [ci3, ci4] - algorithm.inputs.set(inputs) - algorithm.outputs.set(outputs) + interface = AlgorithmInterfaceFactory(inputs=inputs, outputs=outputs) + algorithm.interfaces.set([interface]) def assert_gpu_type_choices(expected_choices): form = AlgorithmForm(instance=algorithm, user=user) @@ -852,8 +899,7 @@ def assert_gpu_type_choices(expected_choices): GPUTypeChoices.T4, ], ) - phase.algorithm_inputs.set(inputs) - phase.algorithm_outputs.set(outputs) + phase.algorithm_interfaces.set([interface]) phase.challenge.add_participant(user) phases.append(phase) @@ -903,7 +949,19 @@ def assert_gpu_type_choices(expected_choices): ] ) - phases[0].algorithm_inputs.set([ci1, ci5]) + interface2 = AlgorithmInterfaceFactory(inputs=[ci1, ci5], outputs=outputs) + # add additional interface + phases[0].algorithm_interfaces.add(interface2) + + assert_gpu_type_choices( + [ + (GPUTypeChoices.NO_GPU, "No GPU"), + (GPUTypeChoices.T4, "NVIDIA T4 Tensor Core GPU"), + ] + ) + + # replace with different interface + phases[0].algorithm_interfaces.set([interface2]) assert_gpu_type_choices( [ @@ -921,12 +979,21 @@ def assert_gpu_type_choices(expected_choices): (GPUTypeChoices.T4, "NVIDIA T4 Tensor Core GPU"), ] ) + interface3 = AlgorithmInterfaceFactory(inputs=inputs, outputs=[ci4, ci6]) + phases[3].algorithm_interfaces.add(interface3) - phases[3].algorithm_outputs.set([ci4, ci6]) + assert_gpu_type_choices( + [ + (GPUTypeChoices.NO_GPU, "No GPU"), + (GPUTypeChoices.T4, "NVIDIA T4 Tensor Core GPU"), + ] + ) + algorithm.interfaces.add(interface3) assert_gpu_type_choices( [ (GPUTypeChoices.NO_GPU, "No GPU"), + (GPUTypeChoices.K80, "NVIDIA K80 GPU"), (GPUTypeChoices.T4, "NVIDIA T4 Tensor Core GPU"), ] ) @@ -993,8 +1060,9 @@ def test_algorithm_form_gpu_choices_from_organizations_and_phases(): ci1, ci2, ci3, ci4, ci5, ci6 = ComponentInterfaceFactory.create_batch(6) inputs = [ci1, ci2] outputs = [ci3, ci4] - algorithm.inputs.set(inputs) - algorithm.outputs.set(outputs) + interface = AlgorithmInterfaceFactory(inputs=inputs, outputs=outputs) + algorithm.interfaces.set([interface]) + org1 = OrganizationFactory( algorithm_selectable_gpu_type_choices=[ GPUTypeChoices.NO_GPU, @@ -1037,8 +1105,7 @@ def assert_gpu_type_choices(expected_choices): GPUTypeChoices.T4, ], ) - phase.algorithm_inputs.set(inputs) - phase.algorithm_outputs.set(outputs) + phase.algorithm_interfaces.set([interface]) phase.challenge.add_participant(user) phases.append(phase) @@ -1075,8 +1142,7 @@ def test_algorithm_for_phase_form_gpu_limited_choices(): display_editors=True, contact_email="test@test.com", workstation=WorkstationFactory.build(), - inputs=[ComponentInterfaceFactory.build()], - outputs=[ComponentInterfaceFactory.build()], + interfaces=[AlgorithmInterfaceFactory.build()], structures=[], modalities=[], logo=ImageField(filename="test.jpeg"), @@ -1103,8 +1169,7 @@ def test_algorithm_for_phase_form_gpu_additional_choices(): display_editors=True, contact_email="test@test.com", workstation=WorkstationFactory.build(), - inputs=[ComponentInterfaceFactory.build()], - outputs=[ComponentInterfaceFactory.build()], + interfaces=[AlgorithmInterfaceFactory.build()], structures=[], modalities=[], logo=ImageField(filename="test.jpeg"), @@ -1155,16 +1220,16 @@ def test_algorithm_form_max_memory_from_phases_for_admins(): ci1, ci2, ci3, ci4, ci5, ci6 = ComponentInterfaceFactory.create_batch(6) inputs = [ci1, ci2] outputs = [ci3, ci4] - algorithm.inputs.set(inputs) - algorithm.outputs.set(outputs) + interface = AlgorithmInterfaceFactory(inputs=inputs, outputs=outputs) + algorithm.interfaces.set([interface]) + phases = [] for max_memory in [42, 100, 200, 300, 400]: phase = PhaseFactory( submission_kind=SubmissionKindChoices.ALGORITHM, algorithm_maximum_settable_memory_gb=max_memory, ) - phase.algorithm_inputs.set(inputs) - phase.algorithm_outputs.set(outputs) + phase.algorithm_interfaces.set([interface]) phase.challenge.add_admin(user) phases.append(phase) @@ -1195,11 +1260,13 @@ def assert_max_value_validator(value): assert_max_value_validator(200) - phases[2].algorithm_inputs.set([ci1, ci5]) + interface2 = AlgorithmInterfaceFactory(inputs=[ci1, ci5], outputs=outputs) + phases[2].algorithm_interfaces.set([interface2]) assert_max_value_validator(100) - phases[1].algorithm_outputs.set([ci4, ci6]) + interface3 = AlgorithmInterfaceFactory(inputs=inputs, outputs=[ci4, ci6]) + phases[1].algorithm_interfaces.set([interface3]) assert_max_value_validator(42) @@ -1211,16 +1278,16 @@ def test_algorithm_form_max_memory_from_phases(): ci1, ci2, ci3, ci4, ci5, ci6 = ComponentInterfaceFactory.create_batch(6) inputs = [ci1, ci2] outputs = [ci3, ci4] - algorithm.inputs.set(inputs) - algorithm.outputs.set(outputs) + interface = AlgorithmInterfaceFactory(inputs=inputs, outputs=outputs) + algorithm.interfaces.set([interface]) + phases = [] for max_memory in [42, 100, 200, 300, 400, 500]: phase = PhaseFactory( submission_kind=SubmissionKindChoices.ALGORITHM, algorithm_maximum_settable_memory_gb=max_memory, ) - phase.algorithm_inputs.set(inputs) - phase.algorithm_outputs.set(outputs) + phase.algorithm_interfaces.set([interface]) phase.challenge.add_participant(user) phases.append(phase) @@ -1251,14 +1318,23 @@ def assert_max_value_validator(value): assert_max_value_validator(200) - phases[2].algorithm_inputs.set([ci1, ci5]) - + interface2 = AlgorithmInterfaceFactory(inputs=[ci1, ci5], outputs=outputs) + # adding an interface + phases[2].algorithm_interfaces.add(interface2) + assert_max_value_validator(100) + # replacing the interface + phases[2].algorithm_interfaces.set([interface2]) assert_max_value_validator(100) - phases[1].algorithm_outputs.set([ci4, ci6]) + interface3 = AlgorithmInterfaceFactory(inputs=inputs, outputs=[ci4, ci6]) + phases[1].algorithm_interfaces.set([interface3]) assert_max_value_validator(42) + # updating the algorithm's interface + algorithm.interfaces.set([interface3]) + assert_max_value_validator(100) + @pytest.mark.django_db def test_algorithm_form_max_memory_from_organizations(): @@ -1295,8 +1371,8 @@ def test_algorithm_form_max_memory_from_organizations_and_phases(): ci1, ci2, ci3, ci4, ci5, ci6 = ComponentInterfaceFactory.create_batch(6) inputs = [ci1, ci2] outputs = [ci3, ci4] - algorithm.inputs.set(inputs) - algorithm.outputs.set(outputs) + interface = AlgorithmInterfaceFactory(inputs=inputs, outputs=outputs) + algorithm.interfaces.set([interface]) org1 = OrganizationFactory(algorithm_maximum_settable_memory_gb=42) org2 = OrganizationFactory(algorithm_maximum_settable_memory_gb=1337) @@ -1318,8 +1394,7 @@ def assert_max_value_validator(value): submission_kind=SubmissionKindChoices.ALGORITHM, algorithm_maximum_settable_memory_gb=max_memory, ) - phase.algorithm_inputs.set(inputs) - phase.algorithm_outputs.set(outputs) + phase.algorithm_interfaces.set([interface]) phase.challenge.add_participant(user) assert_max_value_validator(42) @@ -1339,8 +1414,7 @@ def test_algorithm_for_phase_form_memory_limited(): display_editors=True, contact_email="test@test.com", workstation=WorkstationFactory.build(), - inputs=[ComponentInterfaceFactory.build()], - outputs=[ComponentInterfaceFactory.build()], + interfaces=[AlgorithmInterfaceFactory.build()], structures=[], modalities=[], logo=ImageField(filename="test.jpeg"), @@ -1363,6 +1437,33 @@ def test_algorithm_for_phase_form_memory_limited(): assert max_validator.limit_value == 32 +@pytest.mark.django_db +def test_algorithm_interface_disjoint_interfaces(): + ci = ComponentInterfaceFactory() + form = AlgorithmInterfaceForm( + base_obj=AlgorithmFactory(), data={"inputs": [ci], "outputs": [ci]} + ) + assert form.is_valid() is False + assert "The sets of Inputs and Outputs must be unique" in str(form.errors) + + +@pytest.mark.django_db +def test_algorithm_interface_unique_inputs_required(): + ci1, ci2 = ComponentInterfaceFactory.create_batch(2) + alg = AlgorithmFactory() + interface = AlgorithmInterfaceFactory(inputs=[ci1]) + alg.interfaces.add(interface) + + form = AlgorithmInterfaceForm( + base_obj=alg, data={"inputs": [ci1], "outputs": [ci2]} + ) + assert form.is_valid() is False + assert ( + "An AlgorithmInterface for this algorithm with the same inputs already exists" + in str(form.errors) + ) + + def test_algorithm_for_phase_form_memory(): form = AlgorithmForPhaseForm( workstation_config=WorkstationConfigFactory.build(), @@ -1372,8 +1473,7 @@ def test_algorithm_for_phase_form_memory(): display_editors=True, contact_email="test@test.com", workstation=WorkstationFactory.build(), - inputs=[ComponentInterfaceFactory.build()], - outputs=[ComponentInterfaceFactory.build()], + interfaces=[AlgorithmInterfaceFactory.build()], structures=[], modalities=[], logo=ImageField(filename="test.jpeg"), @@ -1388,3 +1488,27 @@ def test_algorithm_for_phase_form_memory(): ) assert max_validator is not None assert max_validator.limit_value == 42 + + +class TestAlgorithmInterfaceForm: + @pytest.mark.django_db + def test_existing_io_is_reused(self): + inp = ComponentInterfaceFactory() + out = ComponentInterfaceFactory() + io = AlgorithmInterfaceFactory() + io.inputs.set([inp]) + io.outputs.set([out]) + + alg = AlgorithmFactory() + + form = AlgorithmInterfaceForm( + base_obj=alg, + data={ + "inputs": [inp.pk], + "outputs": [out.pk], + }, + ) + assert form.is_valid() + new_io = form.save() + + assert io == new_io diff --git a/app/tests/algorithms_tests/test_models.py b/app/tests/algorithms_tests/test_models.py index d9f050932..96af3ff59 100644 --- a/app/tests/algorithms_tests/test_models.py +++ b/app/tests/algorithms_tests/test_models.py @@ -6,14 +6,18 @@ from django.contrib.auth import get_user_model from django.core.exceptions import ObjectDoesNotExist, ValidationError from django.core.files.base import ContentFile +from django.db import IntegrityError, transaction from django.db.models import ProtectedError from django.test import TestCase from django.utils.timezone import now from grandchallenge.algorithms.models import ( Algorithm, + AlgorithmAlgorithmInterface, + AlgorithmInterface, AlgorithmUserCredit, Job, + get_existing_interface_for_inputs_and_outputs, ) from grandchallenge.components.models import ( CIVData, @@ -24,6 +28,7 @@ from tests.algorithms_tests.factories import ( AlgorithmFactory, AlgorithmImageFactory, + AlgorithmInterfaceFactory, AlgorithmJobFactory, AlgorithmModelFactory, AlgorithmUserCreditFactory, @@ -72,14 +77,6 @@ def test_group_deletion_reverse(group): getattr(algorithm, group).delete() -@pytest.mark.django_db -def test_no_default_interfaces_created(): - a = AlgorithmFactory() - - assert {i.kind for i in a.inputs.all()} == set() - assert {o.kind for o in a.outputs.all()} == set() - - @pytest.mark.django_db def test_rendered_result_text(): def create_result(jb, result: dict): @@ -651,7 +648,9 @@ def test_job_with_same_image_different_model( data = self.get_civ_data(civs=civs) j = AlgorithmJobFactory( - algorithm_image=alg.active_image, time_limit=10 + algorithm_image=alg.active_image, + time_limit=10, + algorithm_interface=alg.interfaces.first(), ) j.inputs.set(civs) @@ -673,6 +672,7 @@ def test_job_with_same_model_different_image( algorithm_image=AlgorithmImageFactory(), algorithm_model=alg.active_model, time_limit=10, + algorithm_interface=alg.interfaces.first(), ) j.inputs.set(civs) jobs = Job.objects.get_jobs_with_same_inputs( @@ -693,6 +693,7 @@ def test_job_with_same_model_and_image( algorithm_model=alg.active_model, algorithm_image=alg.active_image, time_limit=10, + algorithm_interface=alg.interfaces.first(), ) j.inputs.set(civs) jobs = Job.objects.get_jobs_with_same_inputs( @@ -714,6 +715,7 @@ def test_job_with_different_image_and_model( algorithm_model=AlgorithmModelFactory(), algorithm_image=AlgorithmImageFactory(), time_limit=10, + algorithm_interface=alg.interfaces.first(), ) j.inputs.set(civs) jobs = Job.objects.get_jobs_with_same_inputs( @@ -734,10 +736,13 @@ def test_job_with_same_image_no_model_provided( algorithm_model=alg.active_model, algorithm_image=alg.active_image, time_limit=10, + algorithm_interface=alg.interfaces.first(), ) j.inputs.set(civs) jobs = Job.objects.get_jobs_with_same_inputs( - inputs=data, algorithm_image=alg.active_image, algorithm_model=None + inputs=data, + algorithm_image=alg.active_image, + algorithm_model=None, ) assert len(jobs) == 0 @@ -749,11 +754,15 @@ def test_job_with_same_image_and_without_model( data = self.get_civ_data(civs=civs) j = AlgorithmJobFactory( - algorithm_image=alg.active_image, time_limit=10 + algorithm_image=alg.active_image, + time_limit=10, + algorithm_interface=alg.interfaces.first(), ) j.inputs.set(civs) jobs = Job.objects.get_jobs_with_same_inputs( - inputs=data, algorithm_image=alg.active_image, algorithm_model=None + inputs=data, + algorithm_image=alg.active_image, + algorithm_model=None, ) assert j in jobs assert len(jobs) == 1 @@ -766,7 +775,9 @@ def test_job_with_different_input( data = self.get_civ_data(civs=civs) j = AlgorithmJobFactory( - algorithm_image=alg.active_image, time_limit=10 + algorithm_image=alg.active_image, + time_limit=10, + algorithm_interface=alg.interfaces.first(), ) j.inputs.set( [ @@ -775,7 +786,46 @@ def test_job_with_different_input( ] ) jobs = Job.objects.get_jobs_with_same_inputs( - inputs=data, algorithm_image=alg.active_image, algorithm_model=None + inputs=data, + algorithm_image=alg.active_image, + algorithm_model=None, + ) + assert len(jobs) == 0 + + def test_job_with_partially_overlapping_input( + self, algorithm_with_image_and_model_and_two_inputs + ): + alg = algorithm_with_image_and_model_and_two_inputs.algorithm + civs = algorithm_with_image_and_model_and_two_inputs.civs + data = self.get_civ_data(civs=civs) + + j = AlgorithmJobFactory( + algorithm_image=alg.active_image, + time_limit=10, + algorithm_interface=alg.interfaces.first(), + ) + j.inputs.set( + [ + civs[0], + ComponentInterfaceValueFactory(), + ] + ) + j2 = AlgorithmJobFactory( + algorithm_image=alg.active_image, + time_limit=10, + algorithm_interface=alg.interfaces.first(), + ) + j2.inputs.set( + [ + civs[0], + civs[1], + ComponentInterfaceValueFactory(), + ] + ) + jobs = Job.objects.get_jobs_with_same_inputs( + inputs=data, + algorithm_image=alg.active_image, + algorithm_model=None, ) assert len(jobs) == 0 @@ -977,8 +1027,15 @@ def test_inputs_complete(): ci1, ci2, ci3 = ComponentInterfaceFactory.create_batch( 3, kind=ComponentInterface.Kind.STRING ) - alg.inputs.set([ci1, ci2, ci3]) - job = AlgorithmJobFactory(algorithm_image__algorithm=alg, time_limit=10) + interface = AlgorithmInterfaceFactory( + inputs=[ci1, ci2, ci3], outputs=[ComponentInterfaceFactory()] + ) + alg.interfaces.add(interface) + job = AlgorithmJobFactory( + algorithm_image__algorithm=alg, + time_limit=10, + algorithm_interface=alg.interfaces.first(), + ) civ_with_value_1 = ComponentInterfaceValueFactory( interface=ci1, value="Foo" ) @@ -1379,3 +1436,103 @@ def test_active_credits_with_spent_credits(self): algorithm_image.get_remaining_non_complimentary_jobs(user=user) == 5 ) + + +@pytest.mark.django_db +def test_algorithm_interface_cannot_be_deleted(): + interface, _, _ = AlgorithmInterfaceFactory.create_batch(3) + + with pytest.raises(ValidationError): + interface.delete() + + with pytest.raises(NotImplementedError): + AlgorithmInterface.objects.delete() + + +@pytest.mark.django_db +def test_algorithmalgorithminterface_unique_constraints(): + interface1, interface2 = AlgorithmInterfaceFactory.create_batch(2) + algorithm = AlgorithmFactory() + + AlgorithmAlgorithmInterface.objects.create( + interface=interface1, algorithm=algorithm + ) + + # cannot add a second time the same interface for the same algorithm + with pytest.raises(IntegrityError): + with transaction.atomic(): + AlgorithmAlgorithmInterface.objects.create( + interface=interface1, algorithm=algorithm + ) + + +@pytest.mark.parametrize( + "inputs, outputs, expected_output", + ( + ([1], [2], 1), + ([1, 2], [3, 4], 2), + ([3, 4, 5], [6], 3), + ([1], [3, 4, 6], 4), + ([5, 6], [2], 5), + ([1], [3], None), + ([1], [3, 4], None), + ([2], [6], None), + ([2], [1], None), + ([3, 4, 5], [1], None), + ([1], [3, 4], None), + ([1, 3], [4], None), + ), +) +@pytest.mark.django_db +def test_get_existing_interface_for_inputs_and_outputs( + inputs, outputs, expected_output +): + io1, io2, io3, io4, io5, io6 = AlgorithmInterfaceFactory.create_batch(6) + ci1, ci2, ci3, ci4, ci5, ci6 = ComponentInterfaceFactory.create_batch(6) + + interfaces = [io1, io2, io3, io4, io5, io6] + cis = [ci1, ci2, ci3, ci4, ci5, ci6] + + io1.inputs.set([ci1]) + io2.inputs.set([ci1, ci2]) + io3.inputs.set([ci3, ci4, ci5]) + io4.inputs.set([ci1]) + io5.inputs.set([ci5, ci6]) + io6.inputs.set([ci1, ci2]) + + io1.outputs.set([ci2]) + io2.outputs.set([ci3, ci4]) + io3.outputs.set([ci6]) + io4.outputs.set([ci3, ci4, ci6]) + io5.outputs.set([ci2]) + io6.outputs.set([ci4]) + + inputs = [cis[i - 1] for i in inputs] + outputs = [cis[i - 1] for i in outputs] + + existing_interface = get_existing_interface_for_inputs_and_outputs( + inputs=inputs, outputs=outputs + ) + + if expected_output: + assert existing_interface == interfaces[expected_output - 1] + else: + assert not existing_interface + + +@pytest.mark.django_db +def test_algorithminterface_create(): + inputs = [ComponentInterfaceFactory(), ComponentInterfaceFactory()] + outputs = [ComponentInterfaceFactory(), ComponentInterfaceFactory()] + + with pytest.raises(TypeError) as e: + AlgorithmInterface.objects.create() + + assert ( + "AlgorithmInterfaceManager.create() missing 2 required keyword-only arguments: 'inputs' and 'outputs'" + in str(e) + ) + + io = AlgorithmInterface.objects.create(inputs=inputs, outputs=outputs) + assert list(io.inputs.all()) == inputs + assert list(io.outputs.all()) == outputs diff --git a/app/tests/algorithms_tests/test_permissions.py b/app/tests/algorithms_tests/test_permissions.py index 4349bd47e..94bf5fc02 100644 --- a/app/tests/algorithms_tests/test_permissions.py +++ b/app/tests/algorithms_tests/test_permissions.py @@ -18,6 +18,7 @@ from tests.algorithms_tests.factories import ( AlgorithmFactory, AlgorithmImageFactory, + AlgorithmInterfaceFactory, AlgorithmJobFactory, ) from tests.algorithms_tests.utils import TwoAlgorithms @@ -313,7 +314,10 @@ def test_job_permissions_from_template(self, client): kind=InterfaceKind.InterfaceKindChoices.ANY, store_in_database=True, ) - algorithm_image.algorithm.inputs.set([ci]) + interface = AlgorithmInterfaceFactory( + inputs=[ci], outputs=[ComponentInterfaceFactory()] + ) + algorithm_image.algorithm.interfaces.add(interface) response = get_view_for_user( viewname="algorithms:job-create", @@ -321,6 +325,7 @@ def test_job_permissions_from_template(self, client): method=client.post, reverse_kwargs={ "slug": algorithm_image.algorithm.slug, + "interface_pk": algorithm_image.algorithm.interfaces.first().pk, }, user=user, follow=True, @@ -346,18 +351,21 @@ def test_job_permissions_from_api(self, rf): is_in_registry=True, is_desired_version=True, ) - interfaces = { - ComponentInterfaceFactory( - kind=ComponentInterface.Kind.STRING, - title="TestInterface 1", - default_value="default", - ), - } - algorithm_image.algorithm.inputs.set(interfaces) + ci = ComponentInterfaceFactory( + kind=ComponentInterface.Kind.STRING, + title="TestInterface 1", + default_value="default", + ) + + interface = AlgorithmInterfaceFactory(inputs=[ci]) + algorithm_image.algorithm.interfaces.add(interface) algorithm_image.algorithm.add_user(user) algorithm_image.algorithm.add_editor(UserFactory()) - job = {"algorithm": algorithm_image.algorithm.api_url, "inputs": []} + job = { + "algorithm": algorithm_image.algorithm.api_url, + "inputs": [{"interface": ci.slug, "value": "foo"}], + } # test request = rf.get("/foo") @@ -391,12 +399,10 @@ def test_job_permissions_for_archive( im = ImageFactory() s.image_set.set([im]) - input_interface = ComponentInterface.objects.get( - slug="generic-medical-image" - ) - civ = ComponentInterfaceValueFactory( - image=im, interface=input_interface - ) + ci = ComponentInterface.objects.get(slug="generic-medical-image") + interface = AlgorithmInterfaceFactory(inputs=[ci]) + ai.algorithm.interfaces.add(interface) + civ = ComponentInterfaceValueFactory(image=im, interface=ci) archive_item = ArchiveItemFactory(archive=archive) with django_capture_on_commit_callbacks(execute=True): @@ -451,12 +457,10 @@ def test_job_permissions_for_normal_phase( im = ImageFactory() s.image_set.set([im]) - input_interface = ComponentInterface.objects.get( - slug="generic-medical-image" - ) - civ = ComponentInterfaceValueFactory( - image=im, interface=input_interface - ) + ci = ComponentInterface.objects.get(slug="generic-medical-image") + interface = AlgorithmInterfaceFactory(inputs=[ci]) + ai.algorithm.interfaces.add(interface) + civ = ComponentInterfaceValueFactory(image=im, interface=ci) archive_item = ArchiveItemFactory(archive=archive) with django_capture_on_commit_callbacks(execute=True): @@ -509,16 +513,15 @@ def test_job_permissions_for_debug_phase( im = ImageFactory() s.image_set.set([im]) - input_interface = ComponentInterface.objects.get( - slug="generic-medical-image" - ) - civ = ComponentInterfaceValueFactory( - image=im, interface=input_interface - ) + ci = ComponentInterface.objects.get(slug="generic-medical-image") + civ = ComponentInterfaceValueFactory(image=im, interface=ci) archive_item = ArchiveItemFactory(archive=archive) with django_capture_on_commit_callbacks(execute=True): archive_item.values.add(civ) + interface = AlgorithmInterfaceFactory(inputs=[ci]) + ai.algorithm.interfaces.add(interface) + create_algorithm_jobs_for_evaluation(evaluation_pk=evaluation.pk) job = Job.objects.get() diff --git a/app/tests/algorithms_tests/test_serializers.py b/app/tests/algorithms_tests/test_serializers.py index 1f4c88a89..523bdc902 100644 --- a/app/tests/algorithms_tests/test_serializers.py +++ b/app/tests/algorithms_tests/test_serializers.py @@ -12,6 +12,7 @@ from tests.algorithms_tests.factories import ( AlgorithmFactory, AlgorithmImageFactory, + AlgorithmInterfaceFactory, AlgorithmJobFactory, ) from tests.cases_tests.factories import RawImageUploadSessionFactory @@ -78,7 +79,7 @@ def test_algorithm_relations_on_job_serializer(rf): True, ("TestInterface 1", "TestInterface 2"), ("testinterface-1",), - "Interface(s) TestInterface 2 do not have a default value and should be provided.", + "The set of inputs provided does not match any of the algorithm's interfaces.", False, ), ( @@ -87,7 +88,7 @@ def test_algorithm_relations_on_job_serializer(rf): True, ("TestInterface 1",), ("testinterface-1", "testinterface-2"), - "Provided inputs(s) TestInterface 2 are not defined for this algorithm", + "The set of inputs provided does not match any of the algorithm's interfaces.", False, ), ( @@ -136,9 +137,11 @@ def test_algorithm_job_post_serializer_validations( is_desired_version=image_ready, ) algorithm_image.algorithm.title = title - algorithm_image.algorithm.inputs.set( - [interfaces[title] for title in algorithm_interface_titles] + interface = AlgorithmInterfaceFactory( + inputs=[interfaces[title] for title in algorithm_interface_titles] ) + algorithm_image.algorithm.interfaces.add(interface) + if add_user: algorithm_image.algorithm.add_user(user) @@ -160,8 +163,7 @@ def test_algorithm_job_post_serializer_validations( job = { "algorithm": algorithm_image.algorithm.api_url, "inputs": [ - {"interface": interface, "value": "dummy"} - for interface in job_interface_slugs + {"interface": int, "value": "dummy"} for int in job_interface_slugs ], } @@ -210,7 +212,8 @@ def test_algorithm_job_post_serializer_create( ci_img1 = ComponentInterfaceFactory(kind=ComponentInterface.Kind.IMAGE) ci_img2 = ComponentInterfaceFactory(kind=ComponentInterface.Kind.IMAGE) - algorithm_image.algorithm.inputs.set([ci_string, ci_img2, ci_img1]) + interface = AlgorithmInterfaceFactory(inputs=[ci_string, ci_img2, ci_img1]) + algorithm_image.algorithm.interfaces.add(interface) algorithm_image.algorithm.add_editor(user) job = { @@ -226,8 +229,22 @@ def test_algorithm_job_post_serializer_create( request.user = user serializer = JobPostSerializer(data=job, context={"request": request}) - # verify + # all inputs need to be provided, also those with default value + assert not serializer.is_valid() + + # add missing input + job = { + "algorithm": algorithm_image.algorithm.api_url, + "inputs": [ + {"interface": ci_img1.slug, "upload_session": upload.api_url}, + {"interface": ci_img2.slug, "image": image2.api_url}, + {"interface": ci_string.slug, "value": "foo"}, + ], + } + serializer = JobPostSerializer(data=job, context={"request": request}) + assert serializer.is_valid() + # fake successful upload upload.status = RawImageUploadSession.SUCCESS upload.save() @@ -238,6 +255,7 @@ def test_algorithm_job_post_serializer_create( job = Job.objects.first() assert job.creator == user assert len(job.inputs.all()) == 3 + assert job.algorithm_interface == interface @pytest.mark.django_db @@ -251,7 +269,6 @@ def test_form_invalid_without_enough_credits(self, rf, settings): settings.ALGORITHMS_GENERAL_CREDITS_PER_MONTH_PER_USER + 1 ), ) - algorithm_image.algorithm.inputs.clear() user = UserFactory() algorithm_image.algorithm.add_user(user=user) @@ -285,17 +302,21 @@ def test_form_valid_for_editor(self, rf, settings): settings.ALGORITHMS_GENERAL_CREDITS_PER_MONTH_PER_USER + 1 ), ) - algorithm_image.algorithm.inputs.clear() user = UserFactory() algorithm_image.algorithm.add_editor(user=user) + ci = ComponentInterfaceFactory(kind=ComponentInterface.Kind.STRING) + interface = AlgorithmInterfaceFactory(inputs=[ci]) + algorithm_image.algorithm.interfaces.add(interface) request = rf.get("/foo") request.user = user serializer = JobPostSerializer( data={ "algorithm": algorithm_image.algorithm.api_url, - "inputs": [], + "inputs": [ + {"interface": ci.slug, "value": "foo"}, + ], }, context={"request": request}, ) @@ -312,7 +333,9 @@ def test_form_valid_for_editor(self, rf, settings): serializer = JobPostSerializer( data={ "algorithm": algorithm_image.algorithm.api_url, - "inputs": [], + "inputs": [ + {"interface": ci.slug, "value": "foo"}, + ], }, context={"request": request}, ) @@ -326,17 +349,21 @@ def test_form_valid_with_credits(self, rf): is_desired_version=True, algorithm__minimum_credits_per_job=1, ) - algorithm_image.algorithm.inputs.clear() user = UserFactory() algorithm_image.algorithm.add_user(user=user) + ci = ComponentInterfaceFactory(kind=ComponentInterface.Kind.STRING) + interface = AlgorithmInterfaceFactory(inputs=[ci]) + algorithm_image.algorithm.interfaces.add(interface) request = rf.get("/foo") request.user = user serializer = JobPostSerializer( data={ "algorithm": algorithm_image.algorithm.api_url, - "inputs": [], + "inputs": [ + {"interface": ci.slug, "value": "foo"}, + ], }, context={"request": request}, ) @@ -394,7 +421,9 @@ def test_algorithm_post_serializer_image_and_time_limit_fixed(rf): ) different_ai = AlgorithmImageFactory(algorithm=alg) ci = ComponentInterfaceFactory(kind=ComponentInterface.Kind.STRING) - alg.inputs.set([ci]) + interface = AlgorithmInterfaceFactory(inputs=[ci]) + alg.interfaces.add(interface) + serializer = JobPostSerializer( data={ "algorithm": alg.api_url, @@ -412,3 +441,85 @@ def test_algorithm_post_serializer_image_and_time_limit_fixed(rf): assert job.algorithm_image != different_ai assert not job.algorithm_model assert job.time_limit == 10 + + +@pytest.mark.parametrize( + "inputs, interface", + ( + ([1], 1), # matches interface 1 of algorithm + ([1, 2], 2), # matches interface 2 of algorithm + ([3, 4, 5], 3), # matches interface 3 of algorithm + ([4], None), # matches interface 4, but not configured for algorithm + ( + [1, 2, 3], + None, + ), # matches interface 5, but not configured for algorithm + ([2], None), # matches no interface (implements part of interface 2) + ( + [1, 3, 4], + None, + ), # matches no interface (implements interface 3 and an additional input) + ), +) +@pytest.mark.django_db +def test_validate_inputs_on_job_serializer(inputs, interface, rf): + user = UserFactory() + algorithm = AlgorithmFactory() + algorithm.add_editor(user) + AlgorithmImageFactory( + algorithm=algorithm, + is_desired_version=True, + is_manifest_valid=True, + is_in_registry=True, + ) + + io1, io2, io3, io4, io5 = AlgorithmInterfaceFactory.create_batch(5) + ci1, ci2, ci3, ci4, ci5, ci6 = ComponentInterfaceFactory.create_batch( + 6, kind=ComponentInterface.Kind.STRING + ) + + interfaces = [io1, io2, io3] + cis = [ci1, ci2, ci3, ci4, ci5, ci6] + + io1.inputs.set([ci1]) + io2.inputs.set([ci1, ci2]) + io3.inputs.set([ci3, ci4, ci5]) + io4.inputs.set([ci1, ci2, ci3]) + io5.inputs.set([ci4]) + io1.outputs.set([ci6]) + io2.outputs.set([ci3]) + io3.outputs.set([ci1]) + io4.outputs.set([ci1]) + io5.outputs.set([ci1]) + + algorithm.interfaces.add(io1) + algorithm.interfaces.add(io2) + algorithm.interfaces.add(io3) + + algorithm_interface = interfaces[interface - 1] if interface else None + inputs = [cis[i - 1] for i in inputs] + + job = { + "algorithm": algorithm.api_url, + "inputs": [ + {"interface": int.slug, "value": "dummy"} for int in inputs + ], + } + + request = rf.get("/foo") + request.user = user + serializer = JobPostSerializer(data=job, context={"request": request}) + + if interface: + assert serializer.is_valid() + assert ( + serializer.validated_data["algorithm_interface"] + == algorithm_interface + ) + else: + assert not serializer.is_valid() + assert ( + "The set of inputs provided does not match any of the algorithm's interfaces." + in str(serializer.errors) + ) + assert "algorithm_interface" not in serializer.validated_data diff --git a/app/tests/algorithms_tests/test_tasks.py b/app/tests/algorithms_tests/test_tasks.py index 2e81afe8b..ecb2ffb25 100644 --- a/app/tests/algorithms_tests/test_tasks.py +++ b/app/tests/algorithms_tests/test_tasks.py @@ -5,18 +5,21 @@ from actstream.models import Follow from django.core.exceptions import ObjectDoesNotExist from django.core.files.base import ContentFile +from guardian.shortcuts import assign_perm from grandchallenge.algorithms.models import Job from grandchallenge.algorithms.tasks import ( create_algorithm_jobs, create_algorithm_jobs_for_archive, execute_algorithm_job_for_inputs, - filter_civs_for_algorithm, + filter_archive_items_for_algorithm, send_failed_job_notification, ) +from grandchallenge.archives.models import ArchiveItem from grandchallenge.components.models import ( ComponentInterface, ComponentInterfaceValue, + InterfaceKindChoices, ) from grandchallenge.components.schemas import GPUTypeChoices from grandchallenge.components.tasks import ( @@ -27,6 +30,7 @@ from tests.algorithms_tests.factories import ( AlgorithmFactory, AlgorithmImageFactory, + AlgorithmInterfaceFactory, AlgorithmJobFactory, AlgorithmModelFactory, ) @@ -36,6 +40,7 @@ ComponentInterfaceFactory, ComponentInterfaceValueFactory, ) +from tests.conftest import get_interface_form_data from tests.factories import ( GroupFactory, ImageFactory, @@ -44,6 +49,7 @@ UserFactory, ) from tests.utils import get_view_for_user, recurse_callbacks +from tests.verification_tests.factories import VerificationFactory @pytest.mark.django_db @@ -54,9 +60,12 @@ def default_input_interface(self): def test_no_images_does_nothing(self): ai = AlgorithmImageFactory() + interface = AlgorithmInterfaceFactory() + ai.algorithm.interfaces.set([interface]) + create_algorithm_jobs( algorithm_image=ai, - civ_sets=[], + archive_items=ArchiveItem.objects.none(), time_limit=ai.algorithm.time_limit, requires_gpu_type=GPUTypeChoices.NO_GPU, requires_memory_gb=4, @@ -67,7 +76,7 @@ def test_no_algorithm_image_errors_out(self): with pytest.raises(RuntimeError): create_algorithm_jobs( algorithm_image=None, - civ_sets=[], + archive_items=ArchiveItem.objects.none(), time_limit=60, requires_gpu_type=GPUTypeChoices.NO_GPU, requires_memory_gb=4, @@ -76,15 +85,18 @@ def test_no_algorithm_image_errors_out(self): def test_creates_job_correctly(self): ai = AlgorithmImageFactory() image = ImageFactory() - interface = ComponentInterface.objects.get( - slug="generic-medical-image" - ) - ai.algorithm.inputs.set([interface]) - civ = ComponentInterfaceValueFactory(image=image, interface=interface) + ci = ComponentInterface.objects.get(slug="generic-medical-image") + interface = AlgorithmInterfaceFactory(inputs=[ci]) + ai.algorithm.interfaces.set([interface]) + + civ = ComponentInterfaceValueFactory(image=image, interface=ci) + item = ArchiveItemFactory() + item.values.add(civ) + assert Job.objects.count() == 0 jobs = create_algorithm_jobs( algorithm_image=ai, - civ_sets=[{civ}], + archive_items=ArchiveItem.objects.all(), time_limit=ai.algorithm.time_limit, requires_gpu_type=ai.algorithm.job_requires_gpu_type, requires_memory_gb=ai.algorithm.job_requires_memory_gb, @@ -93,25 +105,94 @@ def test_creates_job_correctly(self): j = Job.objects.first() assert j.algorithm_image == ai assert j.creator is None + assert j.algorithm_interface == interface assert ( j.inputs.get(interface__slug="generic-medical-image").image == image ) assert j.pk == jobs[0].pk - def test_is_idempotent(self): + def test_creates_job_for_multiple_interfaces_correctly(self): ai = AlgorithmImageFactory() image = ImageFactory() - interface = ComponentInterface.objects.get( - slug="generic-medical-image" + ci1 = ComponentInterfaceFactory(kind=InterfaceKindChoices.BOOL) + ci2 = ComponentInterfaceFactory(kind=InterfaceKindChoices.IMAGE) + ci3 = ComponentInterfaceFactory(kind=InterfaceKindChoices.STRING) + + interface1 = AlgorithmInterfaceFactory(inputs=[ci1]) + interface2 = AlgorithmInterfaceFactory(inputs=[ci2]) + interface3 = AlgorithmInterfaceFactory(inputs=[ci3]) + interface4 = AlgorithmInterfaceFactory(inputs=[ci1, ci3]) + interface5 = AlgorithmInterfaceFactory(inputs=[ci1, ci2, ci3]) + ai.algorithm.interfaces.set( + [interface1, interface2, interface3, interface4, interface5] + ) + + civ1 = ComponentInterfaceValueFactory(value=False, interface=ci1) + civ2 = ComponentInterfaceValueFactory(image=image, interface=ci2) + civ3 = ComponentInterfaceValueFactory(value="foo", interface=ci3) + civ4 = ComponentInterfaceValueFactory() + + archive = ArchiveFactory() + item1, item2, item3, item4, item5, item6 = ( + ArchiveItemFactory.create_batch(6, archive=archive) ) - civ = ComponentInterfaceValueFactory(image=image, interface=interface) + item1.values.add(civ1) # item for interface 1 only + item2.values.add(civ2) # item for interface 2 only + item3.values.add(civ3) # item for interface 3 only + item4.values.set([civ1, civ3]) # item for interface 4 only + item5.values.set([civ4]) # not a match for any interface + item6.values.set([civ2, civ3]) # not a match for any interface + + assert Job.objects.count() == 0 + create_algorithm_jobs( + algorithm_image=ai, + archive_items=ArchiveItem.objects.all(), + time_limit=ai.algorithm.time_limit, + requires_gpu_type=ai.algorithm.job_requires_gpu_type, + requires_memory_gb=ai.algorithm.job_requires_memory_gb, + ) + assert Job.objects.count() == 4 + + for j in Job.objects.all(): + assert j.algorithm_image == ai + assert j.creator is None + + assert not ( + Job.objects.get(algorithm_interface=interface1).inputs.get().value + ) + assert ( + Job.objects.get(algorithm_interface=interface2).inputs.get().image + == image + ) + assert ( + Job.objects.get(algorithm_interface=interface3).inputs.get().value + == "foo" + ) + assert ( + Job.objects.get(algorithm_interface=interface4).inputs.count() == 2 + ) + assert [False, "foo"] == list( + Job.objects.get(algorithm_interface=interface4).inputs.values_list( + "value", flat=True + ) + ) + + def test_is_idempotent(self): + ai = AlgorithmImageFactory() + image = ImageFactory() + ci = ComponentInterface.objects.get(slug="generic-medical-image") + interface = AlgorithmInterfaceFactory(inputs=[ci]) + ai.algorithm.interfaces.set([interface]) + civ = ComponentInterfaceValueFactory(image=image, interface=ci) + item = ArchiveItemFactory() + item.values.add(civ) assert Job.objects.count() == 0 create_algorithm_jobs( algorithm_image=ai, - civ_sets=[{civ}], + archive_items=ArchiveItem.objects.all(), time_limit=ai.algorithm.time_limit, requires_gpu_type=ai.algorithm.job_requires_gpu_type, requires_memory_gb=ai.algorithm.job_requires_memory_gb, @@ -121,7 +202,7 @@ def test_is_idempotent(self): jobs = create_algorithm_jobs( algorithm_image=ai, - civ_sets=[{civ}], + archive_items=ArchiveItem.objects.all(), time_limit=ai.algorithm.time_limit, requires_gpu_type=ai.algorithm.job_requires_gpu_type, requires_memory_gb=ai.algorithm.job_requires_memory_gb, @@ -132,14 +213,16 @@ def test_is_idempotent(self): def test_extra_viewer_groups(self): ai = AlgorithmImageFactory() - interface = ComponentInterface.objects.get( - slug="generic-medical-image" - ) - civ = ComponentInterfaceValueFactory(interface=interface) + ci = ComponentInterface.objects.get(slug="generic-medical-image") + interface = AlgorithmInterfaceFactory(inputs=[ci]) + ai.algorithm.interfaces.set([interface]) + civ = ComponentInterfaceValueFactory(interface=ci) + item = ArchiveItemFactory() + item.values.add(civ) groups = (GroupFactory(), GroupFactory(), GroupFactory()) jobs = create_algorithm_jobs( algorithm_image=ai, - civ_sets=[{civ}], + archive_items=ArchiveItem.objects.all(), extra_viewer_groups=groups, time_limit=ai.algorithm.time_limit, requires_gpu_type=ai.algorithm.job_requires_gpu_type, @@ -155,7 +238,7 @@ def test_no_jobs_workflow(django_capture_on_commit_callbacks): with django_capture_on_commit_callbacks() as callbacks: create_algorithm_jobs( algorithm_image=ai, - civ_sets=[], + archive_items=ArchiveItem.objects.none(), time_limit=ai.algorithm.time_limit, requires_gpu_type=ai.algorithm.job_requires_gpu_type, requires_memory_gb=ai.algorithm.job_requires_memory_gb, @@ -167,15 +250,19 @@ def test_no_jobs_workflow(django_capture_on_commit_callbacks): def test_jobs_workflow(django_capture_on_commit_callbacks): ai = AlgorithmImageFactory() images = [ImageFactory(), ImageFactory()] - interface = ComponentInterface.objects.get(slug="generic-medical-image") - civ_sets = [ - {ComponentInterfaceValueFactory(image=im, interface=interface)} - for im in images - ] + ci = ComponentInterface.objects.get(slug="generic-medical-image") + archive = ArchiveFactory() + for im in images: + item = ArchiveItemFactory(archive=archive) + item.values.add(ComponentInterfaceValueFactory(image=im, interface=ci)) + + interface = AlgorithmInterfaceFactory(inputs=[ci]) + ai.algorithm.interfaces.set([interface]) + with django_capture_on_commit_callbacks() as callbacks: create_algorithm_jobs( algorithm_image=ai, - civ_sets=civ_sets, + archive_items=ArchiveItem.objects.all(), time_limit=ai.algorithm.time_limit, requires_gpu_type=ai.algorithm.job_requires_gpu_type, requires_memory_gb=ai.algorithm.job_requires_memory_gb, @@ -186,7 +273,7 @@ def test_jobs_workflow(django_capture_on_commit_callbacks): @pytest.mark.flaky(reruns=3) @pytest.mark.django_db def test_algorithm( - algorithm_image, settings, django_capture_on_commit_callbacks + algorithm_image, client, settings, django_capture_on_commit_callbacks ): # Override the celery settings settings.task_eager_propagates = (True,) @@ -201,6 +288,10 @@ def test_algorithm( with open(algorithm_image, "rb") as f: ai.image.save(algorithm_image, ContentFile(f.read())) + user = UserFactory() + ai.algorithm.add_editor(user) + VerificationFactory(user=user, is_verified=True) + recurse_callbacks( callbacks=callbacks, django_capture_on_commit_callbacks=django_capture_on_commit_callbacks, @@ -211,6 +302,7 @@ def test_algorithm( image_file = ImageFileFactory( file__from_path=Path(__file__).parent / "resources" / "input_file.tif" ) + assign_perm("cases.view_image", user, image_file.image) input_interface = ComponentInterface.objects.get( slug="generic-medical-image" @@ -219,20 +311,29 @@ def test_algorithm( slug="results-json-file" ) heatmap_interface = ComponentInterface.objects.get(slug="generic-overlay") - ai.algorithm.inputs.set([input_interface]) - ai.algorithm.outputs.set([json_result_interface, heatmap_interface]) - - civ = ComponentInterfaceValueFactory( - image=image_file.image, interface=input_interface, file=None + interface = AlgorithmInterfaceFactory( + inputs=[input_interface], + outputs=[json_result_interface, heatmap_interface], ) + ai.algorithm.interfaces.add(interface) with django_capture_on_commit_callbacks() as callbacks: - create_algorithm_jobs( - algorithm_image=ai, - civ_sets=[{civ}], - time_limit=ai.algorithm.time_limit, - requires_gpu_type=ai.algorithm.job_requires_gpu_type, - requires_memory_gb=ai.algorithm.job_requires_memory_gb, + get_view_for_user( + viewname="algorithms:job-create", + client=client, + method=client.post, + user=user, + reverse_kwargs={ + "slug": ai.algorithm.slug, + "interface_pk": interface.pk, + }, + follow=True, + data={ + **get_interface_form_data( + interface_slug=input_interface.slug, + data=image_file.image.pk, + ), + }, ) recurse_callbacks( @@ -273,22 +374,37 @@ def test_algorithm( slug="detection-json-file", kind=ComponentInterface.Kind.ANY, ) - ai.algorithm.outputs.add(detection_interface) - ai.save() + interface2 = AlgorithmInterfaceFactory( + inputs=[input_interface], + outputs=[ + json_result_interface, + heatmap_interface, + detection_interface, + ], + ) + ai.algorithm.interfaces.add(interface2) image_file = ImageFileFactory( file__from_path=Path(__file__).parent / "resources" / "input_file.tif" ) - civ = ComponentInterfaceValueFactory( - image=image_file.image, interface=input_interface, file=None - ) + assign_perm("cases.view_image", user, image_file.image) with django_capture_on_commit_callbacks() as callbacks: - create_algorithm_jobs( - algorithm_image=ai, - civ_sets=[{civ}], - time_limit=ai.algorithm.time_limit, - requires_gpu_type=ai.algorithm.job_requires_gpu_type, - requires_memory_gb=ai.algorithm.job_requires_memory_gb, + get_view_for_user( + viewname="algorithms:job-create", + client=client, + method=client.post, + user=user, + reverse_kwargs={ + "slug": ai.algorithm.slug, + "interface_pk": interface2.pk, + }, + follow=True, + data={ + **get_interface_form_data( + interface_slug=input_interface.slug, + data=image_file.image.pk, + ), + }, ) recurse_callbacks( @@ -312,7 +428,7 @@ def test_algorithm( @pytest.mark.django_db def test_algorithm_with_invalid_output( - algorithm_image, settings, django_capture_on_commit_callbacks + algorithm_image, client, settings, django_capture_on_commit_callbacks ): # Override the celery settings settings.task_eager_propagates = (True,) @@ -327,6 +443,10 @@ def test_algorithm_with_invalid_output( with open(algorithm_image, "rb") as f: ai.image.save(algorithm_image, ContentFile(f.read())) + user = UserFactory() + ai.algorithm.add_editor(user) + VerificationFactory(user=user, is_verified=True) + recurse_callbacks( callbacks=callbacks, django_capture_on_commit_callbacks=django_capture_on_commit_callbacks, @@ -343,26 +463,35 @@ def test_algorithm_with_invalid_output( slug="detection-json-file", kind=ComponentInterface.Kind.ANY, ) - ai.algorithm.inputs.add(input_interface) - ai.algorithm.outputs.add(detection_interface) - ai.save() + interface = AlgorithmInterfaceFactory( + inputs=[input_interface], outputs=[detection_interface] + ) + ai.algorithm.interfaces.add(interface) image_file = ImageFileFactory( file__from_path=Path(__file__).parent / "resources" / "input_file.tif" ) - - civ = ComponentInterfaceValueFactory( - image=image_file.image, interface=input_interface, file=None - ) + assign_perm("cases.view_image", user, image_file.image) with django_capture_on_commit_callbacks() as callbacks: - create_algorithm_jobs( - algorithm_image=ai, - civ_sets=[{civ}], - time_limit=ai.algorithm.time_limit, - requires_gpu_type=ai.algorithm.job_requires_gpu_type, - requires_memory_gb=ai.algorithm.job_requires_memory_gb, + get_view_for_user( + viewname="algorithms:job-create", + client=client, + method=client.post, + user=user, + reverse_kwargs={ + "slug": ai.algorithm.slug, + "interface_pk": interface.pk, + }, + follow=True, + data={ + **get_interface_form_data( + interface_slug=input_interface.slug, + data=image_file.image.pk, + ), + }, ) + recurse_callbacks( callbacks=callbacks, django_capture_on_commit_callbacks=django_capture_on_commit_callbacks, @@ -422,11 +551,15 @@ def test_execute_algorithm_job_for_missing_inputs(settings): # create the job without value for the ComponentInterfaceValues ci = ComponentInterface.objects.get(slug="generic-medical-image") ComponentInterfaceValue.objects.create(interface=ci) - alg.algorithm.inputs.add(ci) + interface = AlgorithmInterfaceFactory( + inputs=[ci], outputs=[ComponentInterfaceFactory()] + ) + alg.algorithm.interfaces.add(interface) job = AlgorithmJobFactory( creator=creator, algorithm_image=alg, time_limit=alg.algorithm.time_limit, + algorithm_interface=interface, ) execute_algorithm_job_for_inputs(job_pk=job.pk) @@ -438,120 +571,154 @@ def test_execute_algorithm_job_for_missing_inputs(settings): @pytest.mark.django_db class TestJobCreation: - def test_unmatched_interface_filter(self): - ai = AlgorithmImageFactory() - cis = ComponentInterfaceFactory.create_batch(2) - ai.algorithm.inputs.set(cis) - - civ_sets = [ - {}, # No interfaces - { - ComponentInterfaceValueFactory(interface=cis[0]) - }, # Missing interface - { - # OK - ComponentInterfaceValueFactory(interface=cis[0]), - ComponentInterfaceValueFactory(interface=cis[1]), - }, - { - # Unmatched interface - ComponentInterfaceValueFactory(interface=cis[0]), + def test_interface_matching(self): + ai1, ai2, ai3, ai4 = AlgorithmImageFactory.create_batch(4) + ci1, ci2, ci3, ci4 = ComponentInterfaceFactory.create_batch(4) + interface1 = AlgorithmInterfaceFactory(inputs=[ci1]) + interface2 = AlgorithmInterfaceFactory(inputs=[ci1, ci2]) + interface3 = AlgorithmInterfaceFactory(inputs=[ci2, ci3, ci4]) + interface4 = AlgorithmInterfaceFactory(inputs=[ci2]) + + ai1.algorithm.interfaces.set([interface1]) + ai2.algorithm.interfaces.set([interface1, interface2]) + ai3.algorithm.interfaces.set([interface1, interface3, interface4]) + ai4.algorithm.interfaces.set([interface4]) + + archive = ArchiveFactory() + i1, i2, i3, i4 = ArchiveItemFactory.create_batch(4, archive=archive) + i1.values.add( + ComponentInterfaceValueFactory(interface=ci1) + ) # Valid for interface 1 + i2.values.set( + [ + ComponentInterfaceValueFactory(interface=ci1), + ComponentInterfaceValueFactory(interface=ci2), + ] + ) # valid for interface 2 + i3.values.set( + [ + ComponentInterfaceValueFactory(interface=ci1), ComponentInterfaceValueFactory( interface=ComponentInterfaceFactory() ), - }, - ] - - filtered_civ_sets = filter_civs_for_algorithm( - civ_sets=civ_sets, algorithm_image=ai, algorithm_model=None - ) - - assert filtered_civ_sets == [civ_sets[2]] - - def test_unmatched_interface_filter_subset(self): - ai = AlgorithmImageFactory() - cis = ComponentInterfaceFactory.create_batch(2) - ai.algorithm.inputs.set(cis) - - civ_sets = [ - { - # Extra interface - ComponentInterfaceValueFactory(interface=cis[0]), - ComponentInterfaceValueFactory(interface=cis[1]), + ] + ) # valid for no interface, because of additional / mismatching interface + i4.values.set( + [ + ComponentInterfaceValueFactory(interface=ci2), ComponentInterfaceValueFactory( interface=ComponentInterfaceFactory() ), - } - ] - - filtered_civ_sets = filter_civs_for_algorithm( - civ_sets=civ_sets, algorithm_image=ai, algorithm_model=None + ] + ) # valid for no interface, because of additional / mismatching interface + + # filtered archive items depend on defined interfaces and archive item values + filtered_archive_items = filter_archive_items_for_algorithm( + archive_items=ArchiveItem.objects.all(), + algorithm_image=ai1, + algorithm_model=None, ) + assert filtered_archive_items.keys() == {interface1} + assert filtered_archive_items[interface1] == [i1] - assert len(filtered_civ_sets) == 1 - assert {civ.interface for civ in filtered_civ_sets[0]} == {*cis} + filtered_archive_items = filter_archive_items_for_algorithm( + archive_items=ArchiveItem.objects.all(), + algorithm_image=ai2, + algorithm_model=None, + ) + assert filtered_archive_items.keys() == {interface1, interface2} + assert filtered_archive_items[interface1] == [i1] + assert filtered_archive_items[interface2] == [i2] + + filtered_archive_items = filter_archive_items_for_algorithm( + archive_items=ArchiveItem.objects.all(), + algorithm_image=ai3, + algorithm_model=None, + ) + assert filtered_archive_items.keys() == { + interface1, + interface3, + interface4, + } + assert filtered_archive_items[interface1] == [i1] + assert filtered_archive_items[interface3] == [] + assert filtered_archive_items[interface4] == [] + + filtered_archive_items = filter_archive_items_for_algorithm( + archive_items=ArchiveItem.objects.all(), + algorithm_image=ai4, + algorithm_model=None, + ) + assert filtered_archive_items.keys() == {interface4} + assert filtered_archive_items[interface4] == [] - def test_existing_jobs(self): + def test_jobs_with_creator_ignored(self): alg = AlgorithmFactory() ai = AlgorithmImageFactory(algorithm=alg) - am = AlgorithmModelFactory(algorithm=alg) cis = ComponentInterfaceFactory.create_batch(2) - ai.algorithm.inputs.set(cis) + interface = AlgorithmInterfaceFactory(inputs=cis) + ai.algorithm.interfaces.set([interface]) civs1 = [ComponentInterfaceValueFactory(interface=c) for c in cis] civs2 = [ComponentInterfaceValueFactory(interface=c) for c in cis] - civs3 = [ComponentInterfaceValueFactory(interface=c) for c in cis] j1 = AlgorithmJobFactory( creator=None, algorithm_image=ai, + algorithm_interface=interface, time_limit=ai.algorithm.time_limit, ) j1.inputs.set(civs1) + # non-system job j2 = AlgorithmJobFactory( - algorithm_image=ai, time_limit=ai.algorithm.time_limit - ) - j2.inputs.set(civs2) - j3 = AlgorithmJobFactory( - creator=None, + creator=UserFactory(), algorithm_image=ai, - algorithm_model=am, + algorithm_interface=interface, time_limit=ai.algorithm.time_limit, ) - j3.inputs.set(civs3) - - civ_sets = [ - {civ for civ in civs1}, # Job already exists (system job) - { - civ for civ in civs2 - }, # Job already exists but with a creator set and hence should be ignored - { - civ for civ in civs3 - }, # Job exists but with an algorithm model set and should be ignored - { - # New values - ComponentInterfaceValueFactory(interface=cis[0]), - ComponentInterfaceValueFactory(interface=cis[1]), - }, - { - # Changed values - civs1[0], - ComponentInterfaceValueFactory(interface=cis[1]), - }, - ] + j2.inputs.set(civs2) + + archive = ArchiveFactory() + item1, item2 = ArchiveItemFactory.create_batch(2, archive=archive) + item1.values.set(civs1) # non-system job already exists + item2.values.set(civs2) # non-system job should be ignored - filtered_civ_sets = filter_civs_for_algorithm( - civ_sets=civ_sets, algorithm_image=ai, algorithm_model=None + filtered_civ_sets = filter_archive_items_for_algorithm( + archive_items=ArchiveItem.objects.all(), + algorithm_image=ai, ) - assert sorted(filtered_civ_sets) == sorted(civ_sets[1:]) + assert filtered_civ_sets.keys() == {interface} + assert filtered_civ_sets[interface] == [item2] + + def test_existing_jobs(self, archive_items_and_jobs_for_interfaces): + # image used for all jobs + image = archive_items_and_jobs_for_interfaces.jobs_for_interface1[ + 0 + ].algorithm_image - def test_existing_jobs_with_algorithm_model(self): + filtered_civ_sets = filter_archive_items_for_algorithm( + archive_items=ArchiveItem.objects.all(), + algorithm_image=image, + ) + # this should return 1 archive item per interface + # for which there is no job yet + assert filtered_civ_sets == { + archive_items_and_jobs_for_interfaces.interface1: [ + archive_items_and_jobs_for_interfaces.items_for_interface1[1] + ], + archive_items_and_jobs_for_interfaces.interface2: [ + archive_items_and_jobs_for_interfaces.items_for_interface2[1] + ], + } + + def test_model_filter_for_jobs_works(self): alg = AlgorithmFactory() ai = AlgorithmImageFactory(algorithm=alg) am = AlgorithmModelFactory(algorithm=alg) cis = ComponentInterfaceFactory.create_batch(2) - ai.algorithm.inputs.set(cis) + interface = AlgorithmInterfaceFactory(inputs=cis) + ai.algorithm.interfaces.set([interface]) civs1 = [ComponentInterfaceValueFactory(interface=c) for c in cis] civs2 = [ComponentInterfaceValueFactory(interface=c) for c in cis] @@ -560,29 +727,31 @@ def test_existing_jobs_with_algorithm_model(self): creator=None, algorithm_image=ai, algorithm_model=am, + algorithm_interface=interface, time_limit=ai.algorithm.time_limit, ) j1.inputs.set(civs1) j2 = AlgorithmJobFactory( creator=None, algorithm_image=ai, + algorithm_interface=interface, time_limit=ai.algorithm.time_limit, ) j2.inputs.set(civs2) - civ_sets = [ - {civ for civ in civs1}, # Job already exists with image and model - { - civ - for civ in civs2 # Job exists but only with image, so should be ignored - }, - ] + archive = ArchiveFactory() + item1, item2 = ArchiveItemFactory.create_batch(2, archive=archive) + item1.values.set(civs1) # Job already exists with image and model + item2.values.set(civs2) # Job exists but only with image - filtered_civ_sets = filter_civs_for_algorithm( - civ_sets=civ_sets, algorithm_image=ai, algorithm_model=am + filtered_civ_sets = filter_archive_items_for_algorithm( + archive_items=ArchiveItem.objects.all(), + algorithm_image=ai, + algorithm_model=am, ) - assert filtered_civ_sets == sorted(civ_sets[1:]) + assert filtered_civ_sets.keys() == {interface} + assert filtered_civ_sets[interface] == [item2] @pytest.mark.django_db @@ -715,10 +884,11 @@ def test_archive_job_gets_gpu_and_memory_set( im = ImageFactory() session.image_set.set([im]) - input_interface = ComponentInterface.objects.get( - slug="generic-medical-image" - ) - civ = ComponentInterfaceValueFactory(image=im, interface=input_interface) + ci = ComponentInterface.objects.get(slug="generic-medical-image") + interface = AlgorithmInterfaceFactory(inputs=[ci]) + algorithm_image.algorithm.interfaces.set([interface]) + + civ = ComponentInterfaceValueFactory(image=im, interface=ci) archive_item = ArchiveItemFactory(archive=archive) with django_capture_on_commit_callbacks(execute=True): diff --git a/app/tests/algorithms_tests/test_views.py b/app/tests/algorithms_tests/test_views.py index 46577e694..8ce2f170f 100644 --- a/app/tests/algorithms_tests/test_views.py +++ b/app/tests/algorithms_tests/test_views.py @@ -15,7 +15,13 @@ from guardian.shortcuts import assign_perm, remove_perm from requests import put -from grandchallenge.algorithms.models import Algorithm, AlgorithmImage, Job +from grandchallenge.algorithms.models import ( + Algorithm, + AlgorithmAlgorithmInterface, + AlgorithmImage, + AlgorithmInterface, + Job, +) from grandchallenge.algorithms.views import JobsList from grandchallenge.components.models import ( ComponentInterface, @@ -33,6 +39,7 @@ from tests.algorithms_tests.factories import ( AlgorithmFactory, AlgorithmImageFactory, + AlgorithmInterfaceFactory, AlgorithmJobFactory, AlgorithmModelFactory, AlgorithmPermissionRequestFactory, @@ -355,6 +362,11 @@ def test_permission_required_views(self, client): time_limit=ai.algorithm.time_limit, ) p = AlgorithmPermissionRequestFactory(algorithm=ai.algorithm) + interface = AlgorithmInterfaceFactory( + inputs=[ComponentInterfaceFactory()], + outputs=[ComponentInterfaceFactory()], + ) + ai.algorithm.interfaces.add(interface) VerificationFactory(user=u, is_verified=True) @@ -428,7 +440,10 @@ def test_permission_required_views(self, client): ), ( "job-create", - {"slug": ai.algorithm.slug}, + { + "slug": ai.algorithm.slug, + "interface_pk": ai.algorithm.interfaces.first().pk, + }, "execute_algorithm", ai.algorithm, None, @@ -718,105 +733,109 @@ def list_algorithms(self, **__): "logo": "https://rumc-gcorg-p-public.s3.amazonaws.com/logos/algorithm/0d11fc7b-c63f-4fd7-b80b-51d2e21492c0/square_logo.x20.jpeg", "slug": "the-pi-cai-challenge-baseline-nndetection", "average_duration": 363.50596, - "inputs": [ - { - "title": "Coronal T2 Prostate MRI", - "description": "Coronal T2 MRI of the Prostate", - "slug": "coronal-t2-prostate-mri", - "kind": "Image", - "pk": 31, - "default_value": None, - "super_kind": "Image", - "relative_path": "images/coronal-t2-prostate-mri", - "overlay_segments": [], - "look_up_table": None, - }, - { - "title": "Transverse T2 Prostate MRI", - "description": "Transverse T2 MRI of the Prostate", - "slug": "transverse-t2-prostate-mri", - "kind": "Image", - "pk": 32, - "default_value": None, - "super_kind": "Image", - "relative_path": "images/transverse-t2-prostate-mri", - "overlay_segments": [], - "look_up_table": None, - }, - { - "title": "Sagittal T2 Prostate MRI", - "description": "Sagittal T2 MRI of the Prostate", - "slug": "sagittal-t2-prostate-mri", - "kind": "Image", - "pk": 33, - "default_value": None, - "super_kind": "Image", - "relative_path": "images/sagittal-t2-prostate-mri", - "overlay_segments": [], - "look_up_table": None, - }, - { - "title": "Transverse HBV Prostate MRI", - "description": "Transverse High B-Value Prostate MRI", - "slug": "transverse-hbv-prostate-mri", - "kind": "Image", - "pk": 47, - "default_value": None, - "super_kind": "Image", - "relative_path": "images/transverse-hbv-prostate-mri", - "overlay_segments": [], - "look_up_table": None, - }, - { - "title": "Transverse ADC Prostate MRI", - "description": "Transverse Apparent Diffusion Coefficient Prostate MRI", - "slug": "transverse-adc-prostate-mri", - "kind": "Image", - "pk": 48, - "default_value": None, - "super_kind": "Image", - "relative_path": "images/transverse-adc-prostate-mri", - "overlay_segments": [], - "look_up_table": None, - }, + "interfaces": [ { - "title": "Clinical Information Prostate MRI", - "description": "Clinical information to support clinically significant prostate cancer detection in prostate MRI. Provided information: patient age at time of examination (patient_age), PSA level in ng/mL as reported (PSA_report), PSA density in ng/mL^2 as reported (PSAD_report), prostate volume as reported (prostate_volume_report), prostate volume derived from automatic whole-gland segmentation (prostate_volume_automatic), scanner manufacturer (scanner_manufacturer), scanner model name (scanner_model_name), diffusion b-value of (calculated) high b-value diffusion map (diffusion_high_bvalue). Values acquired from radiology reports will be missing, if not reported.", - "slug": "clinical-information-prostate-mri", - "kind": "Anything", - "pk": 156, - "default_value": None, - "super_kind": "Value", - "relative_path": "clinical-information-prostate-mri.json", - "overlay_segments": [], - "look_up_table": None, - }, - ], - "outputs": [ - { - "title": "Case-level Cancer Likelihood Prostate MRI", - "description": "Case-level likelihood of harboring clinically significant prostate cancer, in range [0,1].", - "slug": "prostate-cancer-likelihood", - "kind": "Float", - "pk": 144, - "default_value": None, - "super_kind": "Value", - "relative_path": "cspca-case-level-likelihood.json", - "overlay_segments": [], - "look_up_table": None, - }, - { - "title": "Transverse Cancer Detection Map Prostate MRI", - "description": "Single-class, detection map of clinically significant prostate cancer lesions in 3D, where each voxel represents a floating point in range [0,1].", - "slug": "cspca-detection-map", - "kind": "Heat Map", - "pk": 151, - "default_value": None, - "super_kind": "Image", - "relative_path": "images/cspca-detection-map", - "overlay_segments": [], - "look_up_table": None, - }, + "inputs": [ + { + "title": "Coronal T2 Prostate MRI", + "description": "Coronal T2 MRI of the Prostate", + "slug": "coronal-t2-prostate-mri", + "kind": "Image", + "pk": 31, + "default_value": None, + "super_kind": "Image", + "relative_path": "images/coronal-t2-prostate-mri", + "overlay_segments": [], + "look_up_table": None, + }, + { + "title": "Transverse T2 Prostate MRI", + "description": "Transverse T2 MRI of the Prostate", + "slug": "transverse-t2-prostate-mri", + "kind": "Image", + "pk": 32, + "default_value": None, + "super_kind": "Image", + "relative_path": "images/transverse-t2-prostate-mri", + "overlay_segments": [], + "look_up_table": None, + }, + { + "title": "Sagittal T2 Prostate MRI", + "description": "Sagittal T2 MRI of the Prostate", + "slug": "sagittal-t2-prostate-mri", + "kind": "Image", + "pk": 33, + "default_value": None, + "super_kind": "Image", + "relative_path": "images/sagittal-t2-prostate-mri", + "overlay_segments": [], + "look_up_table": None, + }, + { + "title": "Transverse HBV Prostate MRI", + "description": "Transverse High B-Value Prostate MRI", + "slug": "transverse-hbv-prostate-mri", + "kind": "Image", + "pk": 47, + "default_value": None, + "super_kind": "Image", + "relative_path": "images/transverse-hbv-prostate-mri", + "overlay_segments": [], + "look_up_table": None, + }, + { + "title": "Transverse ADC Prostate MRI", + "description": "Transverse Apparent Diffusion Coefficient Prostate MRI", + "slug": "transverse-adc-prostate-mri", + "kind": "Image", + "pk": 48, + "default_value": None, + "super_kind": "Image", + "relative_path": "images/transverse-adc-prostate-mri", + "overlay_segments": [], + "look_up_table": None, + }, + { + "title": "Clinical Information Prostate MRI", + "description": "Clinical information to support clinically significant prostate cancer detection in prostate MRI. Provided information: patient age at time of examination (patient_age), PSA level in ng/mL as reported (PSA_report), PSA density in ng/mL^2 as reported (PSAD_report), prostate volume as reported (prostate_volume_report), prostate volume derived from automatic whole-gland segmentation (prostate_volume_automatic), scanner manufacturer (scanner_manufacturer), scanner model name (scanner_model_name), diffusion b-value of (calculated) high b-value diffusion map (diffusion_high_bvalue). Values acquired from radiology reports will be missing, if not reported.", + "slug": "clinical-information-prostate-mri", + "kind": "Anything", + "pk": 156, + "default_value": None, + "super_kind": "Value", + "relative_path": "clinical-information-prostate-mri.json", + "overlay_segments": [], + "look_up_table": None, + }, + ], + "outputs": [ + { + "title": "Case-level Cancer Likelihood Prostate MRI", + "description": "Case-level likelihood of harboring clinically significant prostate cancer, in range [0,1].", + "slug": "prostate-cancer-likelihood", + "kind": "Float", + "pk": 144, + "default_value": None, + "super_kind": "Value", + "relative_path": "cspca-case-level-likelihood.json", + "overlay_segments": [], + "look_up_table": None, + }, + { + "title": "Transverse Cancer Detection Map Prostate MRI", + "description": "Single-class, detection map of clinically significant prostate cancer lesions in 3D, where each voxel represents a floating point in range [0,1].", + "slug": "cspca-detection-map", + "kind": "Heat Map", + "pk": 151, + "default_value": None, + "super_kind": "Image", + "relative_path": "images/cspca-detection-map", + "overlay_segments": [], + "look_up_table": None, + }, + ], + } ], } ], @@ -893,7 +912,10 @@ def list_algorithm_images(self, **__): "logos/algorithm/0d11fc7b-c63f-4fd7-b80b-51d2e21492c0/square_logo" ) assert "Imported from [grand-challenge.org]" in algorithm.summary - assert {i.slug for i in algorithm.inputs.all()} == { + + assert algorithm.interfaces.count() == 1 + interface = algorithm.interfaces.get() + assert {i.slug for i in interface.inputs.all()} == { "clinical-information-prostate-mri", "coronal-t2-prostate-mri", "sagittal-t2-prostate-mri", @@ -901,7 +923,7 @@ def list_algorithm_images(self, **__): "transverse-hbv-prostate-mri", "transverse-t2-prostate-mri", } - assert {i.slug for i in algorithm.outputs.all()} == { + assert {i.slug for i in interface.outputs.all()} == { "cspca-detection-map", "prostate-cancer-likelihood", } @@ -953,7 +975,10 @@ def test_create_job_with_json_file( ci = ComponentInterfaceFactory( kind=InterfaceKind.InterfaceKindChoices.ANY, store_in_database=False ) - ai.algorithm.inputs.set([ci]) + interface = AlgorithmInterfaceFactory( + inputs=[ci], + ) + ai.algorithm.interfaces.add(interface) with tempfile.NamedTemporaryFile(mode="w+", suffix=".json") as file: json.dump('{"Foo": "bar"}', file) @@ -969,6 +994,7 @@ def test_create_job_with_json_file( method=client.post, reverse_kwargs={ "slug": ai.algorithm.slug, + "interface_pk": interface.pk, }, user=editor, follow=True, @@ -1006,7 +1032,10 @@ def test_algorithm_job_create_with_image_input( ci = ComponentInterfaceFactory( kind=InterfaceKind.InterfaceKindChoices.IMAGE, store_in_database=False ) - ai.algorithm.inputs.set([ci]) + interface = AlgorithmInterfaceFactory( + inputs=[ci], + ) + ai.algorithm.interfaces.add(interface) image1, image2 = ImageFactory.create_batch(2) assign_perm("cases.view_image", editor, image1) @@ -1021,6 +1050,7 @@ def test_algorithm_job_create_with_image_input( method=client.post, reverse_kwargs={ "slug": ai.algorithm.slug, + "interface_pk": interface.pk, }, user=editor, follow=True, @@ -1045,6 +1075,7 @@ def test_algorithm_job_create_with_image_input( method=client.post, reverse_kwargs={ "slug": ai.algorithm.slug, + "interface_pk": interface.pk, }, user=editor, follow=True, @@ -1072,6 +1103,7 @@ def test_algorithm_job_create_with_image_input( method=client.post, reverse_kwargs={ "slug": ai.algorithm.slug, + "interface_pk": interface.pk, }, user=editor, follow=True, @@ -1111,6 +1143,7 @@ def create_job( user=user, reverse_kwargs={ "slug": algorithm.slug, + "interface_pk": algorithm.interfaces.first().pk, }, follow=True, data=inputs, @@ -1149,16 +1182,18 @@ def test_create_job_with_multiple_new_inputs( algorithm_with_multiple_inputs, ): # configure multiple inputs - algorithm_with_multiple_inputs.algorithm.inputs.set( - [ + interface = AlgorithmInterfaceFactory( + inputs=[ algorithm_with_multiple_inputs.ci_json_in_db_with_schema, algorithm_with_multiple_inputs.ci_existing_img, algorithm_with_multiple_inputs.ci_str, algorithm_with_multiple_inputs.ci_bool, algorithm_with_multiple_inputs.ci_json_file, algorithm_with_multiple_inputs.ci_img_upload, - ] + ], + outputs=[ComponentInterfaceFactory()], ) + algorithm_with_multiple_inputs.algorithm.interfaces.add(interface) assert ComponentInterfaceValue.objects.count() == 0 @@ -1218,7 +1253,7 @@ def test_create_job_with_multiple_new_inputs( assert sorted( [ int.pk - for int in algorithm_with_multiple_inputs.algorithm.inputs.all() + for int in algorithm_with_multiple_inputs.algorithm.interfaces.first().inputs.all() ] ) == sorted([civ.interface.pk for civ in job.inputs.all()]) @@ -1245,15 +1280,17 @@ def test_create_job_with_existing_inputs( algorithm_with_multiple_inputs, ): # configure multiple inputs - algorithm_with_multiple_inputs.algorithm.inputs.set( - [ + interface = AlgorithmInterfaceFactory( + inputs=[ algorithm_with_multiple_inputs.ci_json_in_db_with_schema, algorithm_with_multiple_inputs.ci_existing_img, algorithm_with_multiple_inputs.ci_str, algorithm_with_multiple_inputs.ci_bool, algorithm_with_multiple_inputs.ci_json_file, - ] + ], + outputs=[ComponentInterfaceFactory()], ) + algorithm_with_multiple_inputs.algorithm.interfaces.add(interface) civ1, civ2, civ3, civ4, civ5 = self.create_existing_civs( interface_data=algorithm_with_multiple_inputs @@ -1319,14 +1356,16 @@ def test_create_job_is_idempotent( algorithm_with_multiple_inputs, ): # configure multiple inputs - algorithm_with_multiple_inputs.algorithm.inputs.set( - [ + interface = AlgorithmInterfaceFactory( + inputs=[ algorithm_with_multiple_inputs.ci_str, algorithm_with_multiple_inputs.ci_bool, algorithm_with_multiple_inputs.ci_existing_img, algorithm_with_multiple_inputs.ci_json_in_db_with_schema, - ] + ], + outputs=[ComponentInterfaceFactory()], ) + algorithm_with_multiple_inputs.algorithm.interfaces.add(interface) civ1, civ2, civ3, civ4, civ5 = self.create_existing_civs( interface_data=algorithm_with_multiple_inputs ) @@ -1383,9 +1422,11 @@ def test_create_job_with_faulty_file_input( algorithm_with_multiple_inputs, ): # configure file input - algorithm_with_multiple_inputs.algorithm.inputs.set( - [algorithm_with_multiple_inputs.ci_json_file] + interface = AlgorithmInterfaceFactory( + inputs=[algorithm_with_multiple_inputs.ci_json_file], + outputs=[ComponentInterfaceFactory()], ) + algorithm_with_multiple_inputs.algorithm.interfaces.add(interface) file_upload = UserUploadFactory( filename="file.json", creator=algorithm_with_multiple_inputs.editor ) @@ -1430,9 +1471,11 @@ def test_create_job_with_faulty_json_input( django_capture_on_commit_callbacks, algorithm_with_multiple_inputs, ): - algorithm_with_multiple_inputs.algorithm.inputs.set( - [algorithm_with_multiple_inputs.ci_json_in_db_with_schema] + interface = AlgorithmInterfaceFactory( + inputs=[algorithm_with_multiple_inputs.ci_json_in_db_with_schema], + outputs=[ComponentInterfaceFactory()], ) + algorithm_with_multiple_inputs.algorithm.interfaces.add(interface) response = self.create_job( client=client, django_capture_on_commit_callbacks=django_capture_on_commit_callbacks, @@ -1460,9 +1503,11 @@ def test_create_job_with_faulty_image_input( django_capture_on_commit_callbacks, algorithm_with_multiple_inputs, ): - algorithm_with_multiple_inputs.algorithm.inputs.set( - [algorithm_with_multiple_inputs.ci_img_upload] + interface = AlgorithmInterfaceFactory( + inputs=[algorithm_with_multiple_inputs.ci_img_upload], + outputs=[ComponentInterfaceFactory()], ) + algorithm_with_multiple_inputs.algorithm.interfaces.add(interface) user_upload = create_upload_from_file( creator=algorithm_with_multiple_inputs.editor, file_path=RESOURCE_PATH / "corrupt.png", @@ -1513,12 +1558,10 @@ def test_create_job_with_multiple_faulty_existing_image_inputs( ] ci.save() - algorithm_with_multiple_inputs.algorithm.inputs.set( - [ - ci1, - ci2, - ] + interface = AlgorithmInterfaceFactory( + inputs=[ci1, ci2], outputs=[ComponentInterfaceFactory()] ) + algorithm_with_multiple_inputs.algorithm.interfaces.add(interface) assert ComponentInterfaceValue.objects.count() == 0 @@ -1654,31 +1697,6 @@ def test_algorithm_image_activate( assert i2.is_in_registry -@pytest.mark.django_db -@pytest.mark.parametrize("interfaces_editable", (True, False)) -def test_algorithm_interfaces_editable(client, interfaces_editable): - creator = UserFactory() - VerificationFactory(user=creator, is_verified=True) - - if interfaces_editable: - assign_perm("algorithms.add_algorithm", creator) - - alg = AlgorithmFactory() - alg.add_editor(user=creator) - - response = get_view_for_user( - viewname="algorithms:update", - client=client, - reverse_kwargs={"slug": alg.slug}, - user=creator, - ) - - assert ("inputs" in response.context["form"].fields) is interfaces_editable - assert ( - "outputs" in response.context["form"].fields - ) is interfaces_editable - - @pytest.mark.django_db def test_job_time_limit(client): algorithm = AlgorithmFactory(time_limit=600) @@ -1695,7 +1713,10 @@ def test_job_time_limit(client): ci = ComponentInterfaceFactory( kind=InterfaceKind.InterfaceKindChoices.ANY, store_in_database=True ) - algorithm.inputs.set([ci]) + interface = AlgorithmInterfaceFactory( + inputs=[ci], outputs=[ComponentInterfaceFactory()] + ) + algorithm.interfaces.add(interface) response = get_view_for_user( viewname="algorithms:job-create", @@ -1703,6 +1724,7 @@ def test_job_time_limit(client): method=client.post, reverse_kwargs={ "slug": algorithm.slug, + "interface_pk": algorithm.interfaces.first().pk, }, user=user, follow=True, @@ -1742,7 +1764,10 @@ def test_job_gpu_type_set(client, settings): ci = ComponentInterfaceFactory( kind=InterfaceKind.InterfaceKindChoices.ANY, store_in_database=True ) - algorithm.inputs.set([ci]) + interface = AlgorithmInterfaceFactory( + inputs=[ci], outputs=[ComponentInterfaceFactory()] + ) + algorithm.interfaces.add(interface) response = get_view_for_user( viewname="algorithms:job-create", @@ -1750,6 +1775,7 @@ def test_job_gpu_type_set(client, settings): method=client.post, reverse_kwargs={ "slug": algorithm.slug, + "interface_pk": algorithm.interfaces.first().pk, }, user=user, follow=True, @@ -1792,7 +1818,10 @@ def test_job_gpu_type_set_with_api(client, settings): ci = ComponentInterfaceFactory( kind=InterfaceKind.InterfaceKindChoices.ANY, store_in_database=True ) - algorithm.inputs.set([ci]) + interface = AlgorithmInterfaceFactory( + inputs=[ci], + ) + algorithm.interfaces.add(interface) response = get_view_for_user( viewname="api:algorithms-job-list", @@ -1816,6 +1845,7 @@ def test_job_gpu_type_set_with_api(client, settings): job = Job.objects.get() + assert job.algorithm_interface == interface assert job.algorithm_image == algorithm_image assert job.requires_gpu_type == GPUTypeChoices.A10G assert job.requires_memory_gb == 64 @@ -1832,9 +1862,18 @@ def test_job_create_view_for_verified_users_only(client): alg.add_user(user) alg.add_user(editor) + interface = AlgorithmInterfaceFactory( + inputs=[ComponentInterfaceFactory()], + outputs=[ComponentInterfaceFactory()], + ) + alg.interfaces.add(interface) + response = get_view_for_user( viewname="algorithms:job-create", - reverse_kwargs={"slug": alg.slug}, + reverse_kwargs={ + "slug": alg.slug, + "interface_pk": alg.interfaces.first().pk, + }, client=client, user=user, ) @@ -1842,7 +1881,10 @@ def test_job_create_view_for_verified_users_only(client): response2 = get_view_for_user( viewname="algorithms:job-create", - reverse_kwargs={"slug": alg.slug}, + reverse_kwargs={ + "slug": alg.slug, + "interface_pk": alg.interfaces.first().pk, + }, client=client, user=editor, ) @@ -1940,7 +1982,10 @@ def test_job_create_denied_for_same_input_model_and_image(client): ci = ComponentInterfaceFactory( kind=InterfaceKind.InterfaceKindChoices.IMAGE ) - alg.inputs.set([ci]) + interface = AlgorithmInterfaceFactory( + inputs=[ci], outputs=[ComponentInterfaceFactory()] + ) + alg.interfaces.add(interface) ai = AlgorithmImageFactory( algorithm=alg, is_manifest_valid=True, @@ -1963,6 +2008,7 @@ def test_job_create_denied_for_same_input_model_and_image(client): method=client.post, reverse_kwargs={ "slug": alg.slug, + "interface_pk": alg.interfaces.first().pk, }, user=creator, data={ @@ -2184,6 +2230,246 @@ def test_algorithm_template_download(client): ), f"{file_name} is in the ZIP file" +@pytest.mark.parametrize( + "viewname", ["algorithms:interface-list", "algorithms:interface-create"] +) +@pytest.mark.django_db +def test_algorithm_interface_view_permission(client, viewname): + ( + user_with_alg_add_perm, + user_without_alg_add_perm, + algorithm_editor_with_alg_add, + algorithm_editor_without_alg_add, + ) = UserFactory.create_batch(4) + assign_perm("algorithms.add_algorithm", user_with_alg_add_perm) + assign_perm("algorithms.add_algorithm", algorithm_editor_with_alg_add) + + alg = AlgorithmFactory() + alg.add_editor(algorithm_editor_with_alg_add) + alg.add_editor(algorithm_editor_without_alg_add) + + for user, status in [ + [user_with_alg_add_perm, 403], + [user_without_alg_add_perm, 403], + [algorithm_editor_with_alg_add, 200], + [algorithm_editor_without_alg_add, 403], + ]: + response = get_view_for_user( + viewname=viewname, + client=client, + reverse_kwargs={"slug": alg.slug}, + user=user, + ) + assert response.status_code == status + + +@pytest.mark.django_db +def test_algorithm_interface_delete_permission(client): + ( + user_with_alg_add_perm, + user_without_alg_add_perm, + algorithm_editor_with_alg_add, + algorithm_editor_without_alg_add, + ) = UserFactory.create_batch(4) + assign_perm("algorithms.add_algorithm", user_with_alg_add_perm) + assign_perm("algorithms.add_algorithm", algorithm_editor_with_alg_add) + + alg = AlgorithmFactory() + alg.add_editor(algorithm_editor_with_alg_add) + alg.add_editor(algorithm_editor_without_alg_add) + + int1, int2 = AlgorithmInterfaceFactory.create_batch(2) + alg.interfaces.add(int1) + alg.interfaces.add(int2) + + for user, status in [ + [user_with_alg_add_perm, 403], + [user_without_alg_add_perm, 403], + [algorithm_editor_with_alg_add, 200], + [algorithm_editor_without_alg_add, 403], + ]: + response = get_view_for_user( + viewname="algorithms:interface-delete", + client=client, + reverse_kwargs={ + "slug": alg.slug, + "interface_pk": int2.pk, + }, + user=user, + ) + assert response.status_code == status + + +@pytest.mark.django_db +def test_algorithm_interface_create(client): + user = UserFactory() + assign_perm("algorithms.add_algorithm", user) + alg = AlgorithmFactory() + alg.add_editor(user) + + ci_1 = ComponentInterfaceFactory() + ci_2 = ComponentInterfaceFactory() + + response = get_view_for_user( + viewname="algorithms:interface-create", + client=client, + method=client.post, + reverse_kwargs={"slug": alg.slug}, + data={ + "inputs": [ci_1.pk], + "outputs": [ci_2.pk], + }, + user=user, + ) + assert response.status_code == 302 + + assert AlgorithmInterface.objects.count() == 1 + io = AlgorithmInterface.objects.get() + assert io.inputs.get() == ci_1 + assert io.outputs.get() == ci_2 + + assert AlgorithmAlgorithmInterface.objects.count() == 1 + io_through = AlgorithmAlgorithmInterface.objects.get() + assert io_through.algorithm == alg + assert io_through.interface == io + + +@pytest.mark.django_db +def test_algorithm_interfaces_list_queryset(client): + user = UserFactory() + assign_perm("algorithms.add_algorithm", user) + alg, alg2 = AlgorithmFactory.create_batch(2) + alg.add_editor(user) + VerificationFactory(user=user, is_verified=True) + + io1, io2, io3, io4 = AlgorithmInterfaceFactory.create_batch(4) + + alg.interfaces.set([io1, io2]) + alg2.interfaces.set([io3, io4]) + + iots = AlgorithmAlgorithmInterface.objects.order_by("id").all() + + response = get_view_for_user( + viewname="algorithms:interface-list", + client=client, + reverse_kwargs={"slug": alg.slug}, + user=user, + ) + assert response.status_code == 200 + assert response.context["object_list"].count() == 2 + assert iots[0] in response.context["object_list"] + assert iots[1] in response.context["object_list"] + assert iots[2] not in response.context["object_list"] + assert iots[3] not in response.context["object_list"] + + +@pytest.mark.django_db +def test_algorithm_interface_delete(client): + user = UserFactory() + assign_perm("algorithms.add_algorithm", user) + alg = AlgorithmFactory() + alg.add_editor(user) + + int1, int2 = AlgorithmInterfaceFactory.create_batch(2) + alg.interfaces.add(int1) + alg.interfaces.add(int2) + + response = get_view_for_user( + viewname="algorithms:interface-delete", + client=client, + method=client.post, + reverse_kwargs={ + "slug": alg.slug, + "interface_pk": int2.pk, + }, + user=user, + ) + assert response.status_code == 302 + # no interface was deleted + assert AlgorithmInterface.objects.count() == 2 + # only the relation between interface and algorithm was deleted + assert AlgorithmAlgorithmInterface.objects.count() == 1 + assert alg.interfaces.count() == 1 + assert alg.interfaces.get() == int1 + + +@pytest.mark.django_db +def test_interface_select_for_job_view_permission(client): + verified_user, unverified_user = UserFactory.create_batch(2) + VerificationFactory(user=verified_user, is_verified=True) + alg = AlgorithmFactory() + alg.add_user(verified_user) + alg.add_user(unverified_user) + + interface1 = AlgorithmInterfaceFactory( + inputs=[ComponentInterfaceFactory()], + outputs=[ComponentInterfaceFactory()], + ) + interface2 = AlgorithmInterfaceFactory( + inputs=[ComponentInterfaceFactory()], + outputs=[ComponentInterfaceFactory()], + ) + alg.interfaces.add(interface1) + alg.interfaces.add(interface2) + + response = get_view_for_user( + viewname="algorithms:job-interface-select", + reverse_kwargs={"slug": alg.slug}, + client=client, + user=unverified_user, + ) + assert response.status_code == 403 + + response = get_view_for_user( + viewname="algorithms:job-interface-select", + reverse_kwargs={"slug": alg.slug}, + client=client, + user=verified_user, + ) + assert response.status_code == 200 + + +@pytest.mark.django_db +def test_interface_select_automatic_redirect(client): + verified_user = UserFactory() + VerificationFactory(user=verified_user, is_verified=True) + alg = AlgorithmFactory() + alg.add_user(verified_user) + + interface = AlgorithmInterfaceFactory( + inputs=[ComponentInterfaceFactory()], + outputs=[ComponentInterfaceFactory()], + ) + alg.interfaces.add(interface) + + # with just 1 interface, user gets redirected immediately + response = get_view_for_user( + viewname="algorithms:job-interface-select", + reverse_kwargs={"slug": alg.slug}, + client=client, + user=verified_user, + ) + assert response.status_code == 302 + assert response.url == reverse( + "algorithms:job-create", + kwargs={"slug": alg.slug, "interface_pk": interface.pk}, + ) + + # with more than 1 interfaces, user has to choose the interface + interface2 = AlgorithmInterfaceFactory( + inputs=[ComponentInterfaceFactory()], + outputs=[ComponentInterfaceFactory()], + ) + alg.interfaces.add(interface2) + response = get_view_for_user( + viewname="algorithms:job-interface-select", + reverse_kwargs={"slug": alg.slug}, + client=client, + user=verified_user, + ) + assert response.status_code == 200 + + @pytest.mark.django_db def test_algorithm_statistics_view(client): alg = AlgorithmFactory() diff --git a/app/tests/archives_tests/test_forms.py b/app/tests/archives_tests/test_forms.py index 91cae1233..ed812df2b 100644 --- a/app/tests/archives_tests/test_forms.py +++ b/app/tests/archives_tests/test_forms.py @@ -20,6 +20,7 @@ from tests.algorithms_tests.factories import ( AlgorithmFactory, AlgorithmImageFactory, + AlgorithmInterfaceFactory, ) from tests.archives_tests.factories import ( ArchiveFactory, @@ -336,13 +337,14 @@ def test_archive_item_update_triggers_algorithm_jobs( assert Job.objects.count() == 0 alg = AlgorithmFactory() + interface = AlgorithmInterfaceFactory(inputs=[ci]) + alg.interfaces.add(interface) AlgorithmImageFactory( algorithm=alg, is_manifest_valid=True, is_in_registry=True, is_desired_version=True, ) - alg.inputs.set([ci]) with django_capture_on_commit_callbacks(execute=True): archive.algorithms.add(alg) diff --git a/app/tests/cases_tests/test_forms.py b/app/tests/cases_tests/test_forms.py index ad938d52a..76d35352d 100644 --- a/app/tests/cases_tests/test_forms.py +++ b/app/tests/cases_tests/test_forms.py @@ -46,7 +46,7 @@ def test_upload_some_images( response = get_view_for_user( data={ "user_uploads": [user_upload.pk], - "interface": ComponentInterface.objects.first().pk, + "socket": ComponentInterface.objects.first().pk, }, client=client, viewname="reader-studies:display-sets-create", diff --git a/app/tests/components_tests/test_tasks.py b/app/tests/components_tests/test_tasks.py index 33769e05b..47fe1f0c7 100644 --- a/app/tests/components_tests/test_tasks.py +++ b/app/tests/components_tests/test_tasks.py @@ -423,7 +423,7 @@ def test_add_image_to_object_updates_upload_session_on_validation_fail( us = RawImageUploadSessionFactory(status=RawImageUploadSession.SUCCESS) ci = ComponentInterfaceFactory(kind="IMG") - error_message = f"Image validation for interface {ci.title} failed with error: Image imports should result in a single image. " + error_message = f"Image validation for socket {ci.title} failed with error: Image imports should result in a single image. " linked_task = some_async_task.signature( kwargs={"foo": "bar"}, immutable=True @@ -458,7 +458,7 @@ def test_add_image_to_object_marks_job_as_failed_on_validation_fail( us = RawImageUploadSessionFactory(status=RawImageUploadSession.SUCCESS) ci = ComponentInterfaceFactory(kind="IMG") - error_message = f"Image validation for interface {ci.title} failed with error: Image imports should result in a single image. " + error_message = f"Image validation for socket {ci.title} failed with error: Image imports should result in a single image. " linked_task = some_async_task.signature( kwargs={"foo": "bar"}, immutable=True @@ -597,7 +597,7 @@ def test_add_file_to_object_sends_notification_on_validation_fail( us.refresh_from_db() assert Notification.objects.count() == 1 assert ( - f"Validation for interface {ci.title} failed." + f"Validation for socket {ci.title} failed." in Notification.objects.first().message ) assert "some_async_task" not in str(callbacks) diff --git a/app/tests/conftest.py b/app/tests/conftest.py index e6b560280..87b904756 100644 --- a/app/tests/conftest.py +++ b/app/tests/conftest.py @@ -12,6 +12,7 @@ from guardian.shortcuts import assign_perm from requests import put +from grandchallenge.algorithms.models import Job from grandchallenge.cases.widgets import ImageWidgetChoices from grandchallenge.components.backends import docker_client from grandchallenge.components.form_fields import INTERFACE_FORM_FIELD_PREFIX @@ -22,8 +23,11 @@ from tests.algorithms_tests.factories import ( AlgorithmFactory, AlgorithmImageFactory, + AlgorithmInterfaceFactory, + AlgorithmJobFactory, AlgorithmModelFactory, ) +from tests.archives_tests.factories import ArchiveFactory, ArchiveItemFactory from tests.cases_tests import RESOURCE_PATH from tests.cases_tests.factories import ( ImageFileFactoryWithMHDFile, @@ -33,7 +37,11 @@ ComponentInterfaceFactory, ComponentInterfaceValueFactory, ) -from tests.evaluation_tests.factories import MethodFactory, PhaseFactory +from tests.evaluation_tests.factories import ( + MethodFactory, + PhaseFactory, + SubmissionFactory, +) from tests.factories import ChallengeFactory, ImageFactory, UserFactory from tests.reader_studies_tests.factories import ( AnswerFactory, @@ -481,7 +489,10 @@ def algorithm_with_image_and_model_and_two_inputs(): ci1, ci2 = ComponentInterfaceFactory.create_batch( 2, kind=ComponentInterface.Kind.STRING ) - alg.inputs.set([ci1, ci2]) + interface = AlgorithmInterfaceFactory( + inputs=[ci1, ci2], outputs=[ComponentInterfaceFactory()] + ) + alg.interfaces.add(interface) civs = [ ComponentInterfaceValueFactory(interface=ci1, value="foo"), ComponentInterfaceValueFactory(interface=ci2, value="bar"), @@ -625,3 +636,252 @@ def get_interface_form_data( ] = FileWidgetChoices.FILE_UPLOAD.name return form_data + + +class InterfacesAndJobs(NamedTuple): + archive: ArchiveFactory + algorithm_image: AlgorithmImageFactory + interface1: AlgorithmInterfaceFactory + interface2: AlgorithmInterfaceFactory + interface3: AlgorithmInterfaceFactory + jobs_for_interface1: list[AlgorithmJobFactory] + jobs_for_interface2: list[AlgorithmJobFactory] + jobs_for_interface3: list[AlgorithmJobFactory] + items_for_interface1: list[ArchiveItemFactory] + items_for_interface2: list[ArchiveItemFactory] + items_for_interface3: list[ArchiveItemFactory] + civs_for_interface1: list[ComponentInterfaceValueFactory] + civs_for_interface2: list[ComponentInterfaceValueFactory] + civs_for_interface3: list[ComponentInterfaceValueFactory] + output_civs: list[ComponentInterfaceValueFactory] + + +@pytest.fixture +def archive_items_and_jobs_for_interfaces(): + ci1, ci2, ci3 = ComponentInterfaceFactory.create_batch(3) + + interface1 = AlgorithmInterfaceFactory(inputs=[ci1], outputs=[ci3]) + interface2 = AlgorithmInterfaceFactory(inputs=[ci1, ci2], outputs=[ci3]) + interface3 = AlgorithmInterfaceFactory(inputs=[ci1, ci3], outputs=[ci3]) + + archive = ArchiveFactory() + ai1, ai2, ai3, ai4, ai5, ai6 = ArchiveItemFactory.create_batch( + 6, archive=archive + ) + + civ_int_1a = ComponentInterfaceValueFactory(interface=ci1) + civ_int_1b = ComponentInterfaceValueFactory(interface=ci1) + civ_int_1c = ComponentInterfaceValueFactory(interface=ci1) + civ_int_2a = ComponentInterfaceValueFactory(interface=ci2) + civ_int_2b = ComponentInterfaceValueFactory(interface=ci2) + civ_int_3a = ComponentInterfaceValueFactory(interface=ci3) + + civs_out = ComponentInterfaceValueFactory.create_batch(6, interface=ci3) + + ai1.values.set([civ_int_1a]) # valid for interface 1 + ai2.values.set([civ_int_1b]) # valid for interface 1 + ai3.values.set([civ_int_1c, civ_int_2a]) # valid for interface 2 + ai4.values.set([civ_int_1b, civ_int_2b]) # valid for interface 2 + ai5.values.set([civ_int_1a, civ_int_3a]) # valid for interface 3 + ai6.values.set([civ_int_3a]) # not valid for any interface + + algorithm_image = AlgorithmImageFactory() + algorithm_image.algorithm.interfaces.set([interface1, interface2]) + + # create jobs for interface 1 + j1, j2, j3 = AlgorithmJobFactory.create_batch( + 3, + algorithm_image=algorithm_image, + algorithm_interface=interface1, + time_limit=algorithm_image.algorithm.time_limit, + creator=None, + ) + # outputs don't matter + j1.inputs.set([civ_int_1a]) # corresponds to item ai1 + j2.inputs.set( + [civ_int_1c] + ) # matches interface1, but does not match an item (uses value from an item for interface2) + j3.inputs.set( + [ComponentInterfaceValueFactory(interface=ci1)] + ) # matches interface1 but does not correspond to an item (new value) + + # create jobs for interface 2 + j4, j5, j6 = AlgorithmJobFactory.create_batch( + 3, + algorithm_image=algorithm_image, + algorithm_interface=interface2, + time_limit=algorithm_image.algorithm.time_limit, + creator=None, + ) + j4.inputs.set([civ_int_1c, civ_int_2a]) # corresponds to item ai3 + j5.inputs.set( + [civ_int_1a, civ_int_2a] + ) # valid for interface 2 but does not correspond to an item (mixes values from different items) + j6.inputs.set( + [ + ComponentInterfaceValueFactory(interface=ci1), + ComponentInterfaceValueFactory(interface=ci2), + ] + ) # valid for interface 2 but does not correspond to an item (new values) + + # create jobs for interface 3 (which is not configured for the algorithm) + ( + j7, + j8, + ) = AlgorithmJobFactory.create_batch( + 2, + algorithm_image=algorithm_image, + algorithm_interface=interface3, + time_limit=algorithm_image.algorithm.time_limit, + creator=None, + ) + j7.inputs.set( + [civ_int_1a, civ_int_3a] + ) # valid for interface3, corresponds to item ai5 + j8.inputs.set( + [civ_int_1b, civ_int_3a] + ) # valid for interface3, but does not match item + + return InterfacesAndJobs( + archive=archive, + algorithm_image=algorithm_image, + interface1=interface1, + interface2=interface2, + interface3=interface3, + jobs_for_interface1=[j1, j2, j3], + jobs_for_interface2=[j4, j5, j6], + jobs_for_interface3=[j7, j8], + items_for_interface1=[ai1, ai2], + items_for_interface2=[ai3, ai4], + items_for_interface3=[ai5], + civs_for_interface1=[civ_int_1a, civ_int_1b, civ_int_1c], + civs_for_interface2=[ + [civ_int_1c, civ_int_2a], + [civ_int_1b, civ_int_2b], + ], + civs_for_interface3=[civ_int_3a], + output_civs=civs_out, + ) + + +@pytest.fixture +def jobs_for_optional_inputs(archive_items_and_jobs_for_interfaces): + # delete existing jobs + Job.objects.all().delete() + + # create 2 successful jobs per interface, for each of the archive items + j1, j2 = AlgorithmJobFactory.create_batch( + 2, + status=Job.SUCCESS, + creator=None, + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_interface=archive_items_and_jobs_for_interfaces.interface1, + time_limit=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm.time_limit, + ) + j1.inputs.set( + [archive_items_and_jobs_for_interfaces.civs_for_interface1[0]] + ) + j1.outputs.set([archive_items_and_jobs_for_interfaces.output_civs[0]]) + j2.inputs.set( + [archive_items_and_jobs_for_interfaces.civs_for_interface1[1]] + ) + j2.outputs.set([archive_items_and_jobs_for_interfaces.output_civs[1]]) + + # create 2 jobs per interface, for each of the archive items + j3, j4 = AlgorithmJobFactory.create_batch( + 2, + status=Job.SUCCESS, + creator=None, + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_interface=archive_items_and_jobs_for_interfaces.interface2, + time_limit=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm.time_limit, + ) + j3.inputs.set(archive_items_and_jobs_for_interfaces.civs_for_interface2[0]) + j3.outputs.set([archive_items_and_jobs_for_interfaces.output_civs[2]]) + j4.inputs.set(archive_items_and_jobs_for_interfaces.civs_for_interface2[1]) + j4.outputs.set([archive_items_and_jobs_for_interfaces.output_civs[3]]) + + return InterfacesAndJobs( + archive=archive_items_and_jobs_for_interfaces.archive, + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + interface1=archive_items_and_jobs_for_interfaces.interface1, + interface2=archive_items_and_jobs_for_interfaces.interface2, + interface3=archive_items_and_jobs_for_interfaces.interface3, + jobs_for_interface1=[j1, j2], + jobs_for_interface2=[j3, j4], + jobs_for_interface3=[], + items_for_interface1=archive_items_and_jobs_for_interfaces.items_for_interface1, + items_for_interface2=archive_items_and_jobs_for_interfaces.items_for_interface2, + items_for_interface3=archive_items_and_jobs_for_interfaces.items_for_interface3, + civs_for_interface1=archive_items_and_jobs_for_interfaces.civs_for_interface1, + civs_for_interface2=archive_items_and_jobs_for_interfaces.civs_for_interface2, + civs_for_interface3=archive_items_and_jobs_for_interfaces.civs_for_interface3, + output_civs=archive_items_and_jobs_for_interfaces.output_civs[:4], + ) + + +class SubmissionWithJobs(NamedTuple): + submission: SubmissionFactory + jobs: list[AlgorithmJobFactory] + interface1: AlgorithmInterfaceFactory + interface2: AlgorithmInterfaceFactory + civs_for_interface1: list[ComponentInterfaceValueFactory] + civs_for_interface2: list[ComponentInterfaceValueFactory] + output_civs: list[ComponentInterfaceValueFactory] + + +@pytest.fixture +def submission_without_model_for_optional_inputs(jobs_for_optional_inputs): + submission = SubmissionFactory( + algorithm_image=jobs_for_optional_inputs.algorithm_image + ) + submission.phase.archive = jobs_for_optional_inputs.archive + submission.phase.save() + submission.phase.algorithm_interfaces.set( + [ + jobs_for_optional_inputs.interface1, + jobs_for_optional_inputs.interface2, + ] + ) + return SubmissionWithJobs( + submission=submission, + jobs=[ + *jobs_for_optional_inputs.jobs_for_interface1, + *jobs_for_optional_inputs.jobs_for_interface2, + ], + interface1=jobs_for_optional_inputs.interface1, + interface2=jobs_for_optional_inputs.interface2, + civs_for_interface1=jobs_for_optional_inputs.civs_for_interface1, + civs_for_interface2=jobs_for_optional_inputs.civs_for_interface2, + output_civs=jobs_for_optional_inputs.output_civs, + ) + + +@pytest.fixture +def submission_with_model_for_optional_inputs(jobs_for_optional_inputs): + submission_with_model = SubmissionFactory( + algorithm_image=jobs_for_optional_inputs.algorithm_image, + algorithm_model=AlgorithmModelFactory( + algorithm=jobs_for_optional_inputs.algorithm_image.algorithm + ), + ) + submission_with_model.phase.archive = jobs_for_optional_inputs.archive + submission_with_model.phase.save() + submission_with_model.phase.algorithm_interfaces.set( + [ + jobs_for_optional_inputs.interface1, + jobs_for_optional_inputs.interface2, + ] + ) + return SubmissionWithJobs( + submission=submission_with_model, + jobs=[ + *jobs_for_optional_inputs.jobs_for_interface1, + *jobs_for_optional_inputs.jobs_for_interface2, + ], + interface1=jobs_for_optional_inputs.interface1, + interface2=jobs_for_optional_inputs.interface2, + civs_for_interface1=jobs_for_optional_inputs.civs_for_interface1, + civs_for_interface2=jobs_for_optional_inputs.civs_for_interface2, + output_civs=jobs_for_optional_inputs.output_civs, + ) diff --git a/app/tests/core_tests/test_grand_challenge_forge.py b/app/tests/core_tests/test_grand_challenge_forge.py index 03803e72d..c48571d60 100644 --- a/app/tests/core_tests/test_grand_challenge_forge.py +++ b/app/tests/core_tests/test_grand_challenge_forge.py @@ -6,7 +6,10 @@ get_forge_challenge_pack_context, ) from grandchallenge.evaluation.utils import SubmissionKindChoices -from tests.algorithms_tests.factories import AlgorithmFactory +from tests.algorithms_tests.factories import ( + AlgorithmFactory, + AlgorithmInterfaceFactory, +) from tests.archives_tests.factories import ArchiveFactory from tests.components_tests.factories import ( ComponentInterfaceExampleValueFactory, @@ -21,12 +24,19 @@ def test_get_challenge_pack_context(): challenge = ChallengeFactory() inputs = [ ComponentInterfaceFactory(kind=ComponentInterface.Kind.INTEGER), - ComponentInterfaceFactory(), + ComponentInterfaceFactory(kind=ComponentInterface.Kind.IMAGE), ] outputs = [ ComponentInterfaceFactory(), ComponentInterfaceFactory(), + ComponentInterfaceFactory(), ] + interface1 = AlgorithmInterfaceFactory( + inputs=[inputs[0]], outputs=outputs[:2] + ) + interface2 = AlgorithmInterfaceFactory( + inputs=[inputs[1]], outputs=[outputs[2]] + ) # Add an example ComponentInterfaceExampleValueFactory( @@ -46,8 +56,7 @@ def test_get_challenge_pack_context(): submission_kind=SubmissionKindChoices.ALGORITHM, ) for phase in phase_1, phase_2: - phase.algorithm_inputs.set(inputs) - phase.algorithm_outputs.set(outputs) + phase.algorithm_interfaces.set([interface1, interface2]) # Setup phases that should not pass the filters phase_3 = PhaseFactory( @@ -89,24 +98,31 @@ def test_get_challenge_pack_context(): ]: assert ci_key in component_interface - # Quick check on CI input and outputs - for index, ci in enumerate( - context["challenge"]["phases"][0]["algorithm_inputs"] - ): - assert inputs[index].slug == ci["slug"] - # Test assigned example value - assert ( - context["challenge"]["phases"][0]["algorithm_inputs"][0][ - "example_value" - ] - == 87 - ) + example_values = [ + input["example_value"] + for input in context["challenge"]["phases"][0]["algorithm_inputs"] + if input["example_value"] + ] + assert example_values == [87] - for index, ci in enumerate( - context["challenge"]["phases"][0]["algorithm_outputs"] - ): - assert outputs[index].slug == ci["slug"] + # Quick check on CI input and outputs + input_slugs = [ + input["slug"] + for input in context["challenge"]["phases"][0]["algorithm_inputs"] + ] + assert len(input_slugs) == len(inputs) + assert inputs[0].slug in input_slugs + assert inputs[1].slug in input_slugs + + output_slugs = [ + output["slug"] + for output in context["challenge"]["phases"][0]["algorithm_outputs"] + ] + assert len(output_slugs) == len(outputs) + assert outputs[0].slug in output_slugs + assert outputs[1].slug in output_slugs + assert outputs[2].slug in output_slugs context = get_forge_challenge_pack_context( challenge, phase_pks=[phase_1.pk] @@ -125,25 +141,37 @@ def test_get_algorithm_template_context(): inputs = [ ComponentInterfaceFactory(kind=ComponentInterface.Kind.INTEGER), + ComponentInterfaceFactory(kind=ComponentInterface.Kind.STRING), ] - algorithm.inputs.set(inputs) - outputs = [ ComponentInterfaceFactory(), ComponentInterfaceFactory(), + ComponentInterfaceFactory(), ] - algorithm.outputs.set(outputs) + interface1 = AlgorithmInterfaceFactory( + inputs=[inputs[0]], outputs=outputs[:2] + ) + interface2 = AlgorithmInterfaceFactory( + inputs=[inputs[1]], outputs=[outputs[2]] + ) + algorithm.interfaces.set([interface1, interface2]) context = get_forge_algorithm_template_context(algorithm=algorithm) for key in ["title", "slug", "url", "inputs", "outputs"]: assert key in context["algorithm"] - for index, ci in enumerate(context["algorithm"]["inputs"]): - assert inputs[index].slug == ci["slug"] + input_slugs = [input["slug"] for input in context["algorithm"]["inputs"]] + assert len(input_slugs) == len(inputs) + assert inputs[0].slug in input_slugs + assert inputs[1].slug in input_slugs - for index, ci in enumerate(context["algorithm"]["outputs"]): - assert outputs[index].slug == ci["slug"] + output_slugs = [ + output["slug"] for output in context["algorithm"]["outputs"] + ] + assert len(output_slugs) == len(outputs) + assert outputs[0].slug in output_slugs + assert outputs[1].slug in output_slugs # Test adding default examples assert context["algorithm"]["inputs"][0]["example_value"] == 42 diff --git a/app/tests/evaluation_tests/test_admin.py b/app/tests/evaluation_tests/test_admin.py index 9037e67a3..9779ea94a 100644 --- a/app/tests/evaluation_tests/test_admin.py +++ b/app/tests/evaluation_tests/test_admin.py @@ -2,26 +2,10 @@ from grandchallenge.evaluation.admin import PhaseAdmin from grandchallenge.evaluation.utils import SubmissionKindChoices -from tests.components_tests.factories import ComponentInterfaceFactory from tests.evaluation_tests.factories import PhaseFactory from tests.factories import ChallengeFactory -@pytest.mark.django_db -def test_disjoint_interfaces(): - i = ComponentInterfaceFactory() - p = PhaseFactory(challenge=ChallengeFactory()) - form = PhaseAdmin.form( - instance=p, - data={"algorithm_inputs": [i.pk], "algorithm_outputs": [i.pk]}, - ) - assert form.is_valid() is False - assert ( - "The sets of Algorithm Inputs and Algorithm Outputs must be unique" - in str(form.errors) - ) - - @pytest.mark.django_db def test_read_only_fields_disabled(): p1, p2 = PhaseFactory.create_batch( @@ -34,8 +18,7 @@ def test_read_only_fields_disabled(): form = PhaseAdmin.form( instance=p1, ) - assert form.fields["algorithm_inputs"].disabled - assert form.fields["algorithm_outputs"].disabled + assert form.fields["algorithm_interfaces"].disabled assert form.fields["submission_kind"].disabled p3, p4 = PhaseFactory.create_batch( diff --git a/app/tests/evaluation_tests/test_api.py b/app/tests/evaluation_tests/test_api.py index 51c219cd4..787694304 100644 --- a/app/tests/evaluation_tests/test_api.py +++ b/app/tests/evaluation_tests/test_api.py @@ -14,6 +14,7 @@ from tests.algorithms_tests.factories import ( AlgorithmFactory, AlgorithmImageFactory, + AlgorithmInterfaceFactory, ) from tests.components_tests.factories import ( ComponentInterfaceFactory, @@ -42,9 +43,9 @@ def generate_claimable_evaluation(): 2, challenge=challenge, submission_kind=SubmissionKindChoices.ALGORITHM ) ci1, ci2 = ComponentInterfaceFactory.create_batch(2) + interface = AlgorithmInterfaceFactory(inputs=[ci1], outputs=[ci2]) for phase in [p1, p2]: - phase.algorithm_outputs.set([ci1]) - phase.algorithm_inputs.set([ci2]) + phase.algorithm_interfaces.set([interface]) p2.external_evaluation = True p2.parent = p1 p2.save() @@ -54,8 +55,7 @@ def generate_claimable_evaluation(): is_in_registry=True, is_desired_version=True, ) - ai.algorithm.inputs.set([ci1]) - ai.algorithm.inputs.set([ci2]) + ai.algorithm.interfaces.set([interface]) eval = EvaluationFactory( submission__algorithm_image=ai, diff --git a/app/tests/evaluation_tests/test_forms.py b/app/tests/evaluation_tests/test_forms.py index 50ae58c90..e1f3b3d4b 100644 --- a/app/tests/evaluation_tests/test_forms.py +++ b/app/tests/evaluation_tests/test_forms.py @@ -26,6 +26,7 @@ from tests.algorithms_tests.factories import ( AlgorithmFactory, AlgorithmImageFactory, + AlgorithmInterfaceFactory, AlgorithmJobFactory, AlgorithmModelFactory, ) @@ -101,12 +102,12 @@ def test_algorithm_queryset(self): alg2.add_editor(editor) alg4.add_editor(editor) ci1, ci2, ci3, ci4 = ComponentInterfaceFactory.create_batch(4) - alg1.inputs.set([ci1, ci2]) - alg1.outputs.set([ci3, ci4]) - alg3.inputs.set([ci1, ci2]) - alg3.outputs.set([ci3, ci4]) - alg4.inputs.set([ci1, ci2]) - alg4.outputs.set([ci3, ci4]) + interface = AlgorithmInterfaceFactory( + inputs=[ci1, ci2], outputs=[ci3, ci4] + ) + alg1.interfaces.set([interface]) + alg3.interfaces.set([interface]) + alg4.interfaces.set([interface]) for alg in [alg1, alg2, alg3]: AlgorithmImageFactory( algorithm=alg, @@ -116,8 +117,7 @@ def test_algorithm_queryset(self): ) AlgorithmImageFactory(algorithm=alg4) p = PhaseFactory(submission_kind=SubmissionKindChoices.ALGORITHM) - p.algorithm_inputs.set([ci1, ci2]) - p.algorithm_outputs.set([ci3, ci4]) + p.algorithm_interfaces.set([interface]) form = SubmissionForm( user=editor, phase=p, @@ -134,6 +134,9 @@ def test_algorithm_queryset_if_parent_phase_exists(self): AlgorithmFactory.create_batch(10) ) ci1, ci2, ci3, ci4 = ComponentInterfaceFactory.create_batch(4) + interface = AlgorithmInterfaceFactory( + inputs=[ci1, ci2], outputs=[ci3, ci4] + ) for alg in [ alg1, alg2, @@ -147,8 +150,7 @@ def test_algorithm_queryset_if_parent_phase_exists(self): alg10, ]: alg.add_editor(editor) - alg.inputs.set([ci1, ci2]) - alg.outputs.set([ci3, ci4]) + alg.interfaces.set([interface]) AlgorithmImageFactory( algorithm=alg, is_in_registry=True, @@ -164,6 +166,7 @@ def test_algorithm_queryset_if_parent_phase_exists(self): AlgorithmJobFactory( algorithm_image=alg.active_image, algorithm_model=alg.active_model, + algorithm_interface=interface, status=Job.SUCCESS, time_limit=alg.time_limit, ) @@ -174,8 +177,7 @@ def test_algorithm_queryset_if_parent_phase_exists(self): challenge=ChallengeFactory(), ) for p in [p_parent, p_child]: - p.algorithm_inputs.set([ci1, ci2]) - p.algorithm_outputs.set([ci3, ci4]) + p.algorithm_interfaces.set([interface]) p_child.parent = p_parent p_child.save() @@ -234,6 +236,7 @@ def test_algorithm_queryset_if_parent_phase_exists(self): AlgorithmJobFactory( algorithm_image=alg.active_image, algorithm_model=alg.active_model, + algorithm_interface=interface, status=Job.FAILURE, time_limit=alg.time_limit, ) @@ -248,6 +251,7 @@ def test_algorithm_queryset_if_parent_phase_exists(self): AlgorithmJobFactory( algorithm_image=AlgorithmImageFactory(algorithm=alg8), algorithm_model=alg.active_model, + algorithm_interface=interface, status=Job.SUCCESS, time_limit=alg.time_limit, ) @@ -262,6 +266,7 @@ def test_algorithm_queryset_if_parent_phase_exists(self): AlgorithmJobFactory( algorithm_image=alg9.active_image, algorithm_model=AlgorithmModelFactory(algorithm=alg9), + algorithm_interface=interface, status=Job.SUCCESS, time_limit=alg9.time_limit, ) @@ -388,16 +393,15 @@ def test_algorithm_with_permission(self): alg.add_editor(user=user) ci1 = ComponentInterfaceFactory() ci2 = ComponentInterfaceFactory() - alg.inputs.set([ci1]) - alg.outputs.set([ci2]) + interface = AlgorithmInterfaceFactory(inputs=[ci1], outputs=[ci2]) + alg.interfaces.set([interface]) archive = ArchiveFactory() p = PhaseFactory( submission_kind=SubmissionKindChoices.ALGORITHM, submissions_limit_per_user_per_period=10, archive=archive, ) - p.algorithm_inputs.set([ci1]) - p.algorithm_outputs.set([ci2]) + p.algorithm_interfaces.set([interface]) civ = ComponentInterfaceValueFactory(interface=ci1) i = ArchiveItemFactory(archive=p.archive) i.values.add(civ) @@ -419,7 +423,10 @@ def test_algorithm_with_permission(self): algorithm=alg, ) AlgorithmJobFactory( - algorithm_image=ai, status=4, time_limit=ai.algorithm.time_limit + algorithm_image=ai, + algorithm_interface=interface, + status=4, + time_limit=ai.algorithm.time_limit, ) MethodFactory( is_manifest_valid=True, @@ -444,16 +451,15 @@ def test_algorithm_image_and_model_set(self): alg.add_editor(user=user) ci1 = ComponentInterfaceFactory() ci2 = ComponentInterfaceFactory() - alg.inputs.set([ci1]) - alg.outputs.set([ci2]) + interface = AlgorithmInterfaceFactory(inputs=[ci1], outputs=[ci2]) + alg.interfaces.set([interface]) archive = ArchiveFactory() p = PhaseFactory( submission_kind=SubmissionKindChoices.ALGORITHM, submissions_limit_per_user_per_period=10, archive=archive, ) - p.algorithm_inputs.set([ci1]) - p.algorithm_outputs.set([ci2]) + p.algorithm_interfaces.set([interface]) civ = ComponentInterfaceValueFactory(interface=ci1) i = ArchiveItemFactory(archive=p.archive) i.values.add(civ) @@ -477,6 +483,7 @@ def test_algorithm_image_and_model_set(self): am = AlgorithmModelFactory(algorithm=alg, is_desired_version=True) AlgorithmJobFactory( algorithm_image=ai, + algorithm_interface=interface, status=Job.SUCCESS, time_limit=ai.algorithm.time_limit, ) @@ -549,16 +556,15 @@ def test_no_valid_archive_items(self): alg.add_editor(user=user) ci1 = ComponentInterfaceFactory() ci2 = ComponentInterfaceFactory() - alg.inputs.set([ci1]) - alg.outputs.set([ci2]) + interface = AlgorithmInterfaceFactory(inputs=[ci1], outputs=[ci2]) + alg.interfaces.set([interface]) archive = ArchiveFactory() p_alg = PhaseFactory( submission_kind=SubmissionKindChoices.ALGORITHM, submissions_limit_per_user_per_period=10, archive=archive, ) - p_alg.algorithm_inputs.set([ci1]) - p_alg.algorithm_outputs.set([ci2]) + p_alg.algorithm_interfaces.set([interface]) MethodFactory( phase=p_alg, is_manifest_valid=True, @@ -590,7 +596,10 @@ def test_no_valid_archive_items(self): algorithm=alg, ) AlgorithmJobFactory( - algorithm_image=ai, status=4, time_limit=ai.algorithm.time_limit + algorithm_image=ai, + algorithm_interface=interface, + status=4, + time_limit=ai.algorithm.time_limit, ) upload = UserUploadFactory(creator=user) @@ -652,16 +661,15 @@ def test_submission_or_eval_exists_for_image(self): alg.add_editor(user=user) ci1 = ComponentInterfaceFactory() ci2 = ComponentInterfaceFactory() - alg.inputs.set([ci1]) - alg.outputs.set([ci2]) + interface = AlgorithmInterfaceFactory(inputs=[ci1], outputs=[ci2]) + alg.interfaces.set([interface]) archive = ArchiveFactory() p = PhaseFactory( submission_kind=SubmissionKindChoices.ALGORITHM, submissions_limit_per_user_per_period=10, archive=archive, ) - p.algorithm_inputs.set([ci1]) - p.algorithm_outputs.set([ci2]) + p.algorithm_interfaces.set([interface]) civ = ComponentInterfaceValueFactory(interface=ci1) i = ArchiveItemFactory(archive=p.archive) i.values.add(civ) @@ -683,7 +691,10 @@ def test_submission_or_eval_exists_for_image(self): algorithm=alg, ) AlgorithmJobFactory( - algorithm_image=ai, status=4, time_limit=ai.algorithm.time_limit + algorithm_image=ai, + algorithm_interface=interface, + status=4, + time_limit=ai.algorithm.time_limit, ) SubmissionFactory( phase=p, @@ -744,16 +755,15 @@ def test_eval_exists_for_image_and_model( alg.add_editor(user=user) ci1 = ComponentInterfaceFactory() ci2 = ComponentInterfaceFactory() - alg.inputs.set([ci1]) - alg.outputs.set([ci2]) + interface = AlgorithmInterfaceFactory(inputs=[ci1], outputs=[ci2]) + alg.interfaces.set([interface]) archive = ArchiveFactory() p = PhaseFactory( submission_kind=SubmissionKindChoices.ALGORITHM, submissions_limit_per_user_per_period=10, archive=archive, ) - p.algorithm_inputs.set([ci1]) - p.algorithm_outputs.set([ci2]) + p.algorithm_interfaces.set([interface]) civ = ComponentInterfaceValueFactory(interface=ci1) i = ArchiveItemFactory(archive=p.archive) i.values.add(civ) @@ -894,8 +904,7 @@ def test_algorithm_for_phase_form(): display_editors=True, contact_email="test@test.com", workstation=WorkstationFactory.build(), - inputs=[ComponentInterfaceFactory.build()], - outputs=[ComponentInterfaceFactory.build()], + interfaces=[AlgorithmInterfaceFactory.build()], structures=[], modalities=[], logo=ImageField(filename="test.jpeg"), @@ -903,8 +912,7 @@ def test_algorithm_for_phase_form(): user=UserFactory.build(), ) - assert form.fields["inputs"].disabled - assert form.fields["outputs"].disabled + assert form.fields["interfaces"].disabled assert form.fields["workstation_config"].disabled assert form.fields["hanging_protocol"].disabled assert form.fields["optional_hanging_protocols"].disabled @@ -922,8 +930,7 @@ def test_algorithm_for_phase_form(): assert not form.fields["job_requires_memory_gb"].disabled assert { - form.fields["inputs"], - form.fields["outputs"], + form.fields["interfaces"], form.fields["workstation_config"], form.fields["hanging_protocol"], form.fields["optional_hanging_protocols"], @@ -951,12 +958,15 @@ def test_algorithm_for_phase_form_validation(): phase = PhaseFactory() alg1, alg2, alg3 = AlgorithmFactory.create_batch(3) ci1, ci2, ci3, ci4 = ComponentInterfaceFactory.create_batch(4) - phase.algorithm_inputs.set([ci1, ci2]) - phase.algorithm_outputs.set([ci3, ci4]) + + interface = AlgorithmInterfaceFactory( + inputs=[ci1, ci2], outputs=[ci3, ci4] + ) + + phase.algorithm_interfaces.set([interface]) for alg in [alg1, alg2]: alg.add_editor(user) - alg.inputs.set([ci1, ci2]) - alg.outputs.set([ci3, ci4]) + alg.interfaces.set([interface]) form = AlgorithmForPhaseForm( workstation_config=WorkstationConfigFactory(), @@ -966,8 +976,7 @@ def test_algorithm_for_phase_form_validation(): display_editors=True, contact_email="test@test.com", workstation=WorkstationFactory(), - inputs=[ci1, ci2], - outputs=[ci3, ci4], + interfaces=[interface], structures=[], modalities=[], logo=ImageField(filename="test.jpeg"), @@ -984,8 +993,7 @@ def test_algorithm_for_phase_form_validation(): ) alg3.add_editor(user) - alg3.inputs.set([ci1, ci2]) - alg3.outputs.set([ci3, ci4]) + alg3.interfaces.set([interface]) form = AlgorithmForPhaseForm( workstation_config=WorkstationConfigFactory(), @@ -995,8 +1003,7 @@ def test_algorithm_for_phase_form_validation(): display_editors=True, contact_email="test@test.com", workstation=WorkstationFactory(), - inputs=[ci1, ci2], - outputs=[ci3, ci4], + interfaces=[interface], structures=[], modalities=[], logo=ImageField(filename="test.jpeg"), @@ -1013,6 +1020,68 @@ def test_algorithm_for_phase_form_validation(): ) +@pytest.mark.django_db +def test_user_algorithms_for_phase(): + + def populate_form(interfaces): + return AlgorithmForPhaseForm( + workstation_config=WorkstationConfigFactory(), + hanging_protocol=HangingProtocolFactory(), + optional_hanging_protocols=[HangingProtocolFactory()], + view_content=None, + display_editors=True, + contact_email="test@test.com", + workstation=WorkstationFactory(), + interfaces=interfaces, + structures=[], + modalities=[], + logo=ImageField(filename="test.jpeg"), + phase=phase, + user=user, + data={ + "title": "foo", + }, + ) + + user = UserFactory() + phase = PhaseFactory() + alg1, alg2, alg3, alg4, alg5 = AlgorithmFactory.create_batch(5) + ci1, ci2, ci3, ci4 = ComponentInterfaceFactory.create_batch(4) + + interface1 = AlgorithmInterfaceFactory( + inputs=[ci1, ci2], outputs=[ci3, ci4] + ) + interface2 = AlgorithmInterfaceFactory(inputs=[ci1], outputs=[ci3]) + interface3 = AlgorithmInterfaceFactory(inputs=[ci3], outputs=[ci1, ci4]) + + for alg in [alg1, alg2, alg3, alg4, alg5]: + alg.add_editor(user) + + # phase has 1 interface + phase.algorithm_interfaces.set([interface1]) + # only algorithms that have this interface set only should match + # partial matches are not valid + alg1.interfaces.set([interface1]) # exact match + alg2.interfaces.set( + [interface3] + ) # same number of interfaces, but different interface + alg3.interfaces.set([interface1, interface3]) # additional interface + alg4.interfaces.set([interface2, interface3]) # diff num, diff interfaces + + form = populate_form(interfaces=[interface1]) + assert list(form.user_algorithms_for_phase) == [alg1] + + # phase with 2 interfaces + phase.algorithm_interfaces.set([interface1, interface3]) + form = populate_form(interfaces=[interface1, interface3]) + assert list(form.user_algorithms_for_phase) == [alg3] + + # user needs to be owner of algorithm + alg3.remove_editor(user) + form = populate_form(interfaces=[interface1, interface3]) + assert list(form.user_algorithms_for_phase) == [] + + @pytest.mark.django_db def test_configure_algorithm_phases_form(): ch = ChallengeFactory() @@ -1023,7 +1092,6 @@ def test_configure_algorithm_phases_form(): SubmissionFactory(phase=p1) MethodFactory(phase=p2) PhaseFactory(submission_kind=SubmissionKindChoices.ALGORITHM) - ci1, ci2 = ComponentInterfaceFactory.create_batch(2) form = ConfigureAlgorithmPhasesForm(challenge=ch) assert list(form.fields["phases"].queryset) == [p3] @@ -1032,8 +1100,6 @@ def test_configure_algorithm_phases_form(): challenge=ch, data={ "phases": [p3], - "algorithm_inputs": [ci1], - "algorithm_outputs": [ci2], }, ) assert form3.is_valid() diff --git a/app/tests/evaluation_tests/test_models.py b/app/tests/evaluation_tests/test_models.py index 96ea6f9d4..b09e237dd 100644 --- a/app/tests/evaluation_tests/test_models.py +++ b/app/tests/evaluation_tests/test_models.py @@ -5,11 +5,11 @@ import pytest from django.core import mail from django.core.exceptions import ValidationError -from django.test import TestCase from django.utils import timezone from django.utils.timezone import now from grandchallenge.algorithms.models import Job +from grandchallenge.archives.models import ArchiveItem from grandchallenge.components.models import ComponentInterface from grandchallenge.components.schemas import GPUTypeChoices from grandchallenge.evaluation.models import ( @@ -17,6 +17,8 @@ CombinedLeaderboard, Evaluation, Phase, + get_archive_items_for_interfaces, + get_valid_jobs_for_interfaces_and_archive_items, ) from grandchallenge.evaluation.tasks import ( calculate_ranks, @@ -28,6 +30,7 @@ from grandchallenge.invoices.models import PaymentStatusChoices from tests.algorithms_tests.factories import ( AlgorithmImageFactory, + AlgorithmInterfaceFactory, AlgorithmJobFactory, AlgorithmModelFactory, ) @@ -63,13 +66,14 @@ def algorithm_submission(): ) algorithm_image = AlgorithmImageFactory() - interface = ComponentInterfaceFactory() - algorithm_image.algorithm.inputs.set([interface]) + ci = ComponentInterfaceFactory() + interface = AlgorithmInterfaceFactory(inputs=[ci]) + algorithm_image.algorithm.interfaces.set([interface]) images = ImageFactory.create_batch(3) for image in images[:2]: - civ = ComponentInterfaceValueFactory(image=image, interface=interface) + civ = ComponentInterfaceValueFactory(image=image, interface=ci) ai = ArchiveItemFactory(archive=method.phase.archive) ai.values.add(civ) @@ -160,8 +164,8 @@ def test_create_algorithm_jobs_for_evaluation_sets_gpu_and_memory(): is_in_registry=True, is_desired_version=True, ) - algorithm_image.algorithm.inputs.set(inputs) - algorithm_image.algorithm.outputs.set(outputs) + interface = AlgorithmInterfaceFactory(inputs=inputs, outputs=outputs) + algorithm_image.algorithm.interfaces.set([interface]) archive = ArchiveFactory() archive_item = ArchiveItemFactory(archive=archive) @@ -177,8 +181,7 @@ def test_create_algorithm_jobs_for_evaluation_sets_gpu_and_memory(): submission_kind=SubmissionKindChoices.ALGORITHM, submissions_limit_per_user_per_period=1, ) - phase.algorithm_inputs.set(inputs) - phase.algorithm_outputs.set(outputs) + phase.algorithm_interfaces.set([interface]) method = MethodFactory( phase=phase, @@ -852,19 +855,19 @@ def test_evaluation_invalid_metrics( @pytest.mark.django_db -def test_count_valid_archive_items(): +def test_valid_archive_items_per_interface(): archive = ArchiveFactory() phase = PhaseFactory(archive=archive) i1, i2, i3 = ComponentInterfaceFactory.create_batch(3) - - phase.algorithm_inputs.set([i1, i2]) + interface = AlgorithmInterfaceFactory(inputs=[i1, i2]) + phase.algorithm_interfaces.set([interface]) # Valid archive item ai1 = ArchiveItemFactory(archive=archive) ai1.values.add(ComponentInterfaceValueFactory(interface=i1)) ai1.values.add(ComponentInterfaceValueFactory(interface=i2)) - # Valid, but with extra value + # Invalid, because it has extra value ai2 = ArchiveItemFactory(archive=archive) ai2.values.add(ComponentInterfaceValueFactory(interface=i1)) ai2.values.add(ComponentInterfaceValueFactory(interface=i2)) @@ -882,16 +885,26 @@ def test_count_valid_archive_items(): ai5.values.add(ComponentInterfaceValueFactory(interface=i1)) ai5.values.add(ComponentInterfaceValueFactory(interface=i3)) - cciv1 = ComponentInterfaceValueFactory(interface=i1) - cciv2 = ComponentInterfaceValueFactory(interface=i2) + civ1 = ComponentInterfaceValueFactory(interface=i1) + civ2 = ComponentInterfaceValueFactory(interface=i2) # Valid, reusing interfaces ai6 = ArchiveItemFactory(archive=archive) - ai6.values.set([cciv1, cciv2]) + ai6.values.set([civ1, civ2]) ai7 = ArchiveItemFactory(archive=archive) - ai7.values.set([cciv1, cciv2]) + ai7.values.set([civ1, civ2]) - assert {*phase.valid_archive_items} == {ai1, ai2, ai6, ai7} + assert phase.valid_archive_items_per_interface.keys() == {interface} + assert [ + item.pk + for qs in phase.valid_archive_items_per_interface.values() + for item in qs.order_by("pk") + ] == [ + item.pk + for item in ArchiveItem.objects.filter( + pk__in=[ai1.pk, ai6.pk, ai7.pk] + ).order_by("pk") + ] @pytest.mark.django_db @@ -963,14 +976,14 @@ def test_parent_phase_choices(): p5.submission_kind = SubmissionKindChoices.CSV ci1, ci2, ci3, ci4 = ComponentInterfaceFactory.create_batch(4) - - for phase in [p1, p2, p3, p4, p5, p6]: - phase.algorithm_inputs.set([ci1]) + interface1 = AlgorithmInterfaceFactory(inputs=[ci1], outputs=[ci2, ci3]) + interface2 = AlgorithmInterfaceFactory(inputs=[ci1], outputs=[ci2, ci4]) + interface3 = AlgorithmInterfaceFactory(inputs=[ci1], outputs=[ci2]) for phase in [p1, p4, p5]: - phase.algorithm_outputs.set([ci2, ci3]) - p2.algorithm_outputs.set([ci2, ci4]) - p3.algorithm_outputs.set([ci2]) + phase.algorithm_interfaces.set([interface1]) + p2.algorithm_interfaces.set([interface2]) + p3.algorithm_interfaces.set([interface3]) for phase in [p1, p2, p3, p4, p5, p6]: phase.save() @@ -986,10 +999,10 @@ def test_parent_phase_choices_no_circular_dependency(): submission_kind=SubmissionKindChoices.ALGORITHM, ) ci1, ci2 = ComponentInterfaceFactory.create_batch(2) + interface = AlgorithmInterfaceFactory(inputs=[ci1], outputs=[ci2]) for phase in [p1, p2, p3, p4]: - phase.algorithm_inputs.set([ci1]) - phase.algorithm_outputs.set([ci2]) + phase.algorithm_interfaces.set([interface]) p1.parent = p2 p2.parent = p3 @@ -1010,11 +1023,11 @@ def test_clean_parent_phase(): 4, challenge=ChallengeFactory(), creator_must_be_verified=True ) ci1, ci2 = ComponentInterfaceFactory.create_batch(2) + interface = AlgorithmInterfaceFactory(inputs=[ci1], outputs=[ci2]) for phase in [p1, p2, p3, p4]: phase.submission_kind = SubmissionKindChoices.ALGORITHM - phase.algorithm_inputs.set([ci1]) - phase.algorithm_outputs.set([ci2]) + phase.algorithm_interfaces.set([interface]) phase.save() ai = ArchiveItemFactory() @@ -1056,8 +1069,7 @@ def test_read_only_fields_for_dependent_phases(): ) assert p1.read_only_fields_for_dependent_phases == [ "submission_kind", - "algorithm_inputs", - "algorithm_outputs", + "algorithm_interfaces", ] assert p2.read_only_fields_for_dependent_phases == ["submission_kind"] @@ -1144,288 +1156,671 @@ def test_is_evaluated_with_active_image_and_ground_truth(): @pytest.mark.django_db -class TestInputsComplete(TestCase): +class TestInputsComplete: + def test_inputs_complete_for_prediction_submission(self): + eval_pred = EvaluationFactory( + submission__predictions_file=None, time_limit=10 + ) + assert not eval_pred.inputs_complete + + eval_pred2 = EvaluationFactory(time_limit=10) + assert eval_pred2.inputs_complete - def setUp(self): - self.interface = ComponentInterface.objects.get( - slug="generic-medical-image" + def test_non_successful_jobs_ignored( + self, archive_items_and_jobs_for_interfaces + ): + submission = SubmissionFactory( + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image + ) + submission.phase.archive = ( + archive_items_and_jobs_for_interfaces.archive + ) + submission.phase.save() + submission.phase.algorithm_interfaces.set( + [ + archive_items_and_jobs_for_interfaces.interface1, + archive_items_and_jobs_for_interfaces.interface2, + ] ) - archive = ArchiveFactory() - self.archive_items = ArchiveItemFactory.create_batch(2) - archive.items.set(self.archive_items) + eval_alg = EvaluationFactory(submission=submission, time_limit=10) + assert not eval_alg.inputs_complete - input_civs = ComponentInterfaceValueFactory.create_batch( - 2, interface=self.interface + # create 2 jobs per interface, for each of the archive items + j1, j2 = AlgorithmJobFactory.create_batch( + 2, + creator=None, + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_interface=archive_items_and_jobs_for_interfaces.interface1, + time_limit=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm.time_limit, + ) + j1.inputs.set( + [archive_items_and_jobs_for_interfaces.civs_for_interface1[0]] ) - output_civs = ComponentInterfaceValueFactory.create_batch( - 2, interface=self.interface + j2.inputs.set( + [archive_items_and_jobs_for_interfaces.civs_for_interface1[1]] ) - for ai, civ in zip(self.archive_items, input_civs, strict=True): - ai.values.set([civ]) + # create 2 jobs per interface, for each of the archive items + j3, j4 = AlgorithmJobFactory.create_batch( + 2, + creator=None, + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_interface=archive_items_and_jobs_for_interfaces.interface2, + time_limit=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm.time_limit, + ) + j3.inputs.set( + archive_items_and_jobs_for_interfaces.civs_for_interface2[0] + ) + j4.inputs.set( + archive_items_and_jobs_for_interfaces.civs_for_interface2[1] + ) - self.algorithm_image = AlgorithmImageFactory() - self.algorithm_model = AlgorithmModelFactory() - submission = SubmissionFactory(algorithm_image=self.algorithm_image) - submission.phase.archive = archive - submission.phase.save() - submission.phase.algorithm_inputs.set([self.interface]) + del eval_alg.successful_jobs_per_interface + del eval_alg.successful_job_count_per_interface + del eval_alg.total_successful_jobs + del eval_alg.inputs_complete + assert not eval_alg.inputs_complete - submission_with_model = SubmissionFactory( - algorithm_image=self.algorithm_image, - algorithm_model=self.algorithm_model, + def test_inputs_complete_for_algorithm_submission_without_model( + self, archive_items_and_jobs_for_interfaces + ): + submission = SubmissionFactory( + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image + ) + submission.phase.archive = ( + archive_items_and_jobs_for_interfaces.archive + ) + submission.phase.save() + submission.phase.algorithm_interfaces.set( + [ + archive_items_and_jobs_for_interfaces.interface1, + archive_items_and_jobs_for_interfaces.interface2, + ] ) - submission_with_model.phase.archive = archive - submission_with_model.phase.save() - submission_with_model.phase.algorithm_inputs.set([self.interface]) - self.submission = submission - self.submission_with_model = submission_with_model - self.input_civs = input_civs - self.output_civs = output_civs + eval_alg = EvaluationFactory(submission=submission, time_limit=10) + assert not eval_alg.inputs_complete - def test_inputs_complete_for_prediction_submission(self): - eval_pred = EvaluationFactory( - submission__predictions_file=None, time_limit=10 + # create 2 jobs per interface, for each of the archive items + j1, j2 = AlgorithmJobFactory.create_batch( + 2, + status=Job.SUCCESS, + creator=None, + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_interface=archive_items_and_jobs_for_interfaces.interface1, + time_limit=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm.time_limit, + ) + j1.inputs.set( + [archive_items_and_jobs_for_interfaces.civs_for_interface1[0]] + ) + j2.inputs.set( + [archive_items_and_jobs_for_interfaces.civs_for_interface1[1]] ) - assert not eval_pred.inputs_complete - eval_pred2 = EvaluationFactory(time_limit=10) - assert eval_pred2.inputs_complete + # create 2 jobs per interface, for each of the archive items + j3, j4 = AlgorithmJobFactory.create_batch( + 2, + status=Job.SUCCESS, + creator=None, + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_interface=archive_items_and_jobs_for_interfaces.interface2, + time_limit=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm.time_limit, + ) + j3.inputs.set( + archive_items_and_jobs_for_interfaces.civs_for_interface2[0] + ) + j4.inputs.set( + archive_items_and_jobs_for_interfaces.civs_for_interface2[1] + ) - def test_inputs_complete_for_algorithm_submission_without_model(self): - eval_alg = EvaluationFactory(submission=self.submission, time_limit=10) - assert not eval_alg.inputs_complete + # no need to set outputs, we assume that only a job with valid outputs has a + # status of SUCCESS - for inpt, output in zip( - self.input_civs, self.output_civs, strict=True - ): - j = AlgorithmJobFactory( - status=Job.SUCCESS, - algorithm_image=self.algorithm_image, - time_limit=self.algorithm_image.algorithm.time_limit, - ) - j.inputs.set([inpt]) - j.outputs.set([output]) - j.creator = None - j.save() - - del eval_alg.algorithm_inputs - del eval_alg.successful_jobs + del eval_alg.successful_jobs_per_interface + del eval_alg.successful_job_count_per_interface + del eval_alg.total_successful_jobs del eval_alg.inputs_complete assert eval_alg.inputs_complete - def test_inputs_complete_for_algorithm_submission_with_model(self): - eval_alg = EvaluationFactory( - submission=self.submission_with_model, time_limit=10 + def test_inputs_complete_for_algorithm_submission_with_model( + self, archive_items_and_jobs_for_interfaces + ): + algorithm_model = AlgorithmModelFactory( + algorithm=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm + ) + submission = SubmissionFactory( + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_model=algorithm_model, + ) + submission.phase.archive = ( + archive_items_and_jobs_for_interfaces.archive ) + submission.phase.save() + submission.phase.algorithm_interfaces.set( + [ + archive_items_and_jobs_for_interfaces.interface1, + archive_items_and_jobs_for_interfaces.interface2, + ] + ) + + eval_alg = EvaluationFactory(submission=submission, time_limit=10) assert not eval_alg.inputs_complete - for inpt, output in zip( - self.input_civs, self.output_civs, strict=True - ): - j = AlgorithmJobFactory( - status=Job.SUCCESS, - algorithm_image=self.algorithm_image, - algorithm_model=self.algorithm_model, - time_limit=self.algorithm_image.algorithm.time_limit, - ) - j.inputs.set([inpt]) - j.outputs.set([output]) - j.creator = None - j.save() - del eval_alg.algorithm_inputs - del eval_alg.successful_jobs + # create 2 jobs per interface, for each of the archive items + j1, j2 = AlgorithmJobFactory.create_batch( + 2, + status=Job.SUCCESS, + creator=None, + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_model=algorithm_model, + algorithm_interface=archive_items_and_jobs_for_interfaces.interface1, + time_limit=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm.time_limit, + ) + j1.inputs.set( + [archive_items_and_jobs_for_interfaces.civs_for_interface1[0]] + ) + j2.inputs.set( + [archive_items_and_jobs_for_interfaces.civs_for_interface1[1]] + ) + + # create 2 jobs per interface, for each of the archive items + j3, j4 = AlgorithmJobFactory.create_batch( + 2, + status=Job.SUCCESS, + creator=None, + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_model=algorithm_model, + algorithm_interface=archive_items_and_jobs_for_interfaces.interface2, + time_limit=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm.time_limit, + ) + j3.inputs.set( + archive_items_and_jobs_for_interfaces.civs_for_interface2[0] + ) + j4.inputs.set( + archive_items_and_jobs_for_interfaces.civs_for_interface2[1] + ) + + del eval_alg.successful_jobs_per_interface + del eval_alg.successful_job_count_per_interface + del eval_alg.total_successful_jobs del eval_alg.inputs_complete assert eval_alg.inputs_complete - def test_jobs_with_creator_ignored(self): - eval_alg = EvaluationFactory(submission=self.submission, time_limit=10) - assert not eval_alg.inputs_complete + def test_jobs_with_creator_ignored( + self, archive_items_and_jobs_for_interfaces + ): + submission = SubmissionFactory( + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image + ) + submission.phase.archive = ( + archive_items_and_jobs_for_interfaces.archive + ) + submission.phase.save() + submission.phase.algorithm_interfaces.set( + [ + archive_items_and_jobs_for_interfaces.interface1, + archive_items_and_jobs_for_interfaces.interface2, + ] + ) - for inpt, output in zip( - self.input_civs, self.output_civs, strict=True - ): - j_irrelevant = AlgorithmJobFactory( - status=Job.SUCCESS, - algorithm_image=self.algorithm_image, - creator=self.algorithm_image.creator, - time_limit=self.algorithm_image.algorithm.time_limit, - ) - j_irrelevant.inputs.set([inpt]) - j_irrelevant.outputs.set([output]) - del eval_alg.algorithm_inputs - del eval_alg.successful_jobs - del eval_alg.inputs_complete + eval_alg = EvaluationFactory(submission=submission, time_limit=10) assert not eval_alg.inputs_complete - def test_failed_jobs_ignored(self): - eval_alg = EvaluationFactory(submission=self.submission, time_limit=10) - assert not eval_alg.inputs_complete + # create 2 jobs per interface, for each of the archive items + j1, j2 = AlgorithmJobFactory.create_batch( + 2, + status=Job.SUCCESS, + creator=archive_items_and_jobs_for_interfaces.algorithm_image.creator, + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_interface=archive_items_and_jobs_for_interfaces.interface1, + time_limit=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm.time_limit, + ) + j1.inputs.set( + [archive_items_and_jobs_for_interfaces.civs_for_interface1[0]] + ) + j2.inputs.set( + [archive_items_and_jobs_for_interfaces.civs_for_interface1[1]] + ) + + # create 2 jobs per interface, for each of the archive items + j3, j4 = AlgorithmJobFactory.create_batch( + 2, + status=Job.SUCCESS, + creator=archive_items_and_jobs_for_interfaces.algorithm_image.creator, + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_interface=archive_items_and_jobs_for_interfaces.interface2, + time_limit=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm.time_limit, + ) + j3.inputs.set( + archive_items_and_jobs_for_interfaces.civs_for_interface2[0] + ) + j4.inputs.set( + archive_items_and_jobs_for_interfaces.civs_for_interface2[1] + ) - for inpt, output in zip( - self.input_civs, self.output_civs, strict=True - ): - j_irrelevant = AlgorithmJobFactory( - status=Job.FAILURE, - algorithm_image=self.algorithm_image, - time_limit=self.algorithm_image.algorithm.time_limit, - ) - j_irrelevant.inputs.set([inpt]) - j_irrelevant.outputs.set([output]) - del eval_alg.algorithm_inputs - del eval_alg.successful_jobs + del eval_alg.successful_jobs_per_interface + del eval_alg.successful_job_count_per_interface + del eval_alg.total_successful_jobs del eval_alg.inputs_complete assert not eval_alg.inputs_complete - def test_jobs_with_other_inputs_ignored(self): - eval_alg = EvaluationFactory(submission=self.submission, time_limit=10) + def test_jobs_with_other_inputs_ignored( + self, archive_items_and_jobs_for_interfaces + ): + submission = SubmissionFactory( + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image + ) + submission.phase.archive = ( + archive_items_and_jobs_for_interfaces.archive + ) + submission.phase.save() + submission.phase.algorithm_interfaces.set( + [ + archive_items_and_jobs_for_interfaces.interface1, + archive_items_and_jobs_for_interfaces.interface2, + ] + ) + + eval_alg = EvaluationFactory(submission=submission, time_limit=10) assert not eval_alg.inputs_complete - other_input_civs = ComponentInterfaceValueFactory.create_batch( - 2, interface=self.interface - ) - other_output_civs = ComponentInterfaceValueFactory.create_batch( - 2, interface=self.interface - ) - for inpt, output in zip( - other_input_civs, other_output_civs, strict=True - ): - j_irrelevant = AlgorithmJobFactory( - status=Job.SUCCESS, - algorithm_image=self.algorithm_image, - time_limit=self.algorithm_image.algorithm.time_limit, - ) - j_irrelevant.inputs.set([inpt]) - j_irrelevant.outputs.set([output]) - del eval_alg.algorithm_inputs - del eval_alg.successful_jobs + # create 2 jobs per interface, for each of the archive items + j1, j2 = AlgorithmJobFactory.create_batch( + 2, + status=Job.SUCCESS, + creator=None, + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_interface=archive_items_and_jobs_for_interfaces.interface1, + time_limit=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm.time_limit, + ) + j1.inputs.set( + [archive_items_and_jobs_for_interfaces.civs_for_interface1[0]] + ) + j2.inputs.set( + [archive_items_and_jobs_for_interfaces.civs_for_interface1[1]] + ) + + # create 2 jobs per interface, for each of the archive items + j3, j4 = AlgorithmJobFactory.create_batch( + 2, + status=Job.SUCCESS, + creator=None, + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_interface=archive_items_and_jobs_for_interfaces.interface2, + time_limit=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm.time_limit, + ) + j3.inputs.set( + archive_items_and_jobs_for_interfaces.civs_for_interface2[0] + ) + j4.inputs.set( + archive_items_and_jobs_for_interfaces.civs_for_interface2[1] + ) + + # now also create jobs with other inputs, those should be ignored + j5, j6 = AlgorithmJobFactory.create_batch( + 2, + status=Job.SUCCESS, + creator=None, + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_interface=archive_items_and_jobs_for_interfaces.interface1, + time_limit=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm.time_limit, + ) + j5.inputs.set( + [ + ComponentInterfaceValueFactory( + interface=archive_items_and_jobs_for_interfaces.interface1.inputs.get() + ) + ] + ) + j6.inputs.set( + [ + ComponentInterfaceValueFactory( + interface=archive_items_and_jobs_for_interfaces.interface1.inputs.get() + ) + ] + ) + + # create 2 jobs per interface, for each of the archive items + j7, j8 = AlgorithmJobFactory.create_batch( + 2, + status=Job.SUCCESS, + creator=None, + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_interface=archive_items_and_jobs_for_interfaces.interface2, + time_limit=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm.time_limit, + ) + j7.inputs.set( + [ + ComponentInterfaceValueFactory( + interface=archive_items_and_jobs_for_interfaces.interface2.inputs.first() + ), + ComponentInterfaceValueFactory( + interface=archive_items_and_jobs_for_interfaces.interface2.inputs.last() + ), + ] + ) + j8.inputs.set( + [ + ComponentInterfaceValueFactory( + interface=archive_items_and_jobs_for_interfaces.interface2.inputs.first() + ), + ComponentInterfaceValueFactory( + interface=archive_items_and_jobs_for_interfaces.interface2.inputs.last() + ), + ] + ) + + del eval_alg.successful_jobs_per_interface + del eval_alg.successful_job_count_per_interface + del eval_alg.total_successful_jobs del eval_alg.inputs_complete - assert not eval_alg.inputs_complete + assert eval_alg.inputs_complete - def test_jobs_with_partial_inputs_ignored(self): - eval_alg = EvaluationFactory(submission=self.submission, time_limit=10) + def test_jobs_with_partial_inputs_ignored( + self, archive_items_and_jobs_for_interfaces + ): + submission = SubmissionFactory( + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image + ) + submission.phase.archive = ( + archive_items_and_jobs_for_interfaces.archive + ) + submission.phase.save() + submission.phase.algorithm_interfaces.set( + [ + archive_items_and_jobs_for_interfaces.interface1, + archive_items_and_jobs_for_interfaces.interface2, + ] + ) + + eval_alg = EvaluationFactory(submission=submission, time_limit=10) assert not eval_alg.inputs_complete - # add values to archive items - new_interface = ComponentInterface.objects.get(slug="generic-overlay") - self.submission.phase.algorithm_inputs.add(new_interface) - self.submission.phase.save() - - new_input_civs = ComponentInterfaceValueFactory.create_batch( - 2, interface=new_interface - ) - for ai, civ in zip(self.archive_items, new_input_civs, strict=True): - ai.values.add(civ) - - for inpt, output in zip( - self.input_civs, self.output_civs, strict=True - ): - j = AlgorithmJobFactory( - status=Job.SUCCESS, - algorithm_image=self.algorithm_image, - time_limit=self.algorithm_image.algorithm.time_limit, - ) - j.inputs.set([inpt]) - j.outputs.set([output]) - j.creator = None - j.save() - - del eval_alg.algorithm_inputs - del eval_alg.successful_jobs + # create 2 complete jobs for interface1 + j1, j2 = AlgorithmJobFactory.create_batch( + 2, + status=Job.SUCCESS, + creator=None, + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_interface=archive_items_and_jobs_for_interfaces.interface1, + time_limit=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm.time_limit, + ) + j1.inputs.set( + [archive_items_and_jobs_for_interfaces.civs_for_interface1[0]] + ) + j2.inputs.set( + [archive_items_and_jobs_for_interfaces.civs_for_interface1[1]] + ) + + # create jobs for interface 2 with only part of the required inputs, + # those should not count as complete jobs + j3, j4 = AlgorithmJobFactory.create_batch( + 2, + status=Job.SUCCESS, + creator=None, + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_interface=archive_items_and_jobs_for_interfaces.interface2, + time_limit=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm.time_limit, + ) + j3.inputs.set( + [archive_items_and_jobs_for_interfaces.civs_for_interface2[0][0]] + ) + j4.inputs.set( + [archive_items_and_jobs_for_interfaces.civs_for_interface2[1][1]] + ) + + del eval_alg.successful_jobs_per_interface + del eval_alg.successful_job_count_per_interface + del eval_alg.total_successful_jobs del eval_alg.inputs_complete assert not eval_alg.inputs_complete def test_jobs_with_different_image_ignored_for_submission_without_model( - self, + self, archive_items_and_jobs_for_interfaces ): - eval_alg = EvaluationFactory(submission=self.submission, time_limit=10) + submission = SubmissionFactory( + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image + ) + submission.phase.archive = ( + archive_items_and_jobs_for_interfaces.archive + ) + submission.phase.save() + submission.phase.algorithm_interfaces.set( + [ + archive_items_and_jobs_for_interfaces.interface1, + archive_items_and_jobs_for_interfaces.interface2, + ] + ) + + eval_alg = EvaluationFactory(submission=submission, time_limit=10) assert not eval_alg.inputs_complete - for inpt, output in zip( - self.input_civs, self.output_civs, strict=True - ): - j_irrelevant = AlgorithmJobFactory( - status=Job.SUCCESS, - algorithm_image=AlgorithmImageFactory(), - time_limit=10, - ) - j_irrelevant.inputs.set([inpt]) - j_irrelevant.outputs.set([output]) - del eval_alg.algorithm_inputs - del eval_alg.successful_jobs + # create 2 jobs per interface, for each of the archive items + j1, j2 = AlgorithmJobFactory.create_batch( + 2, + status=Job.SUCCESS, + creator=None, + algorithm_image=AlgorithmImageFactory(), + algorithm_interface=archive_items_and_jobs_for_interfaces.interface1, + time_limit=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm.time_limit, + ) + j1.inputs.set( + [archive_items_and_jobs_for_interfaces.civs_for_interface1[0]] + ) + j2.inputs.set( + [archive_items_and_jobs_for_interfaces.civs_for_interface1[1]] + ) + + # create 2 jobs per interface, for each of the archive items + j3, j4 = AlgorithmJobFactory.create_batch( + 2, + status=Job.SUCCESS, + creator=None, + algorithm_image=AlgorithmImageFactory(), + algorithm_interface=archive_items_and_jobs_for_interfaces.interface2, + time_limit=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm.time_limit, + ) + j3.inputs.set( + archive_items_and_jobs_for_interfaces.civs_for_interface2[0] + ) + j4.inputs.set( + archive_items_and_jobs_for_interfaces.civs_for_interface2[1] + ) + + del eval_alg.successful_jobs_per_interface + del eval_alg.successful_job_count_per_interface + del eval_alg.total_successful_jobs del eval_alg.inputs_complete assert not eval_alg.inputs_complete def test_successful_job_with_model_ignored_for_submission_without_model( - self, + self, archive_items_and_jobs_for_interfaces ): - eval_alg = EvaluationFactory(submission=self.submission, time_limit=10) + submission = SubmissionFactory( + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image + ) + submission.phase.archive = ( + archive_items_and_jobs_for_interfaces.archive + ) + submission.phase.save() + submission.phase.algorithm_interfaces.set( + [ + archive_items_and_jobs_for_interfaces.interface1, + archive_items_and_jobs_for_interfaces.interface2, + ] + ) + + eval_alg = EvaluationFactory(submission=submission, time_limit=10) assert not eval_alg.inputs_complete - for inpt, output in zip( - self.input_civs, self.output_civs, strict=True - ): - j_irrelevant = AlgorithmJobFactory( - status=Job.SUCCESS, - algorithm_image=self.algorithm_image, - algorithm_model=AlgorithmModelFactory(), - time_limit=10, - ) - j_irrelevant.inputs.set([inpt]) - j_irrelevant.outputs.set([output]) - del eval_alg.algorithm_inputs - del eval_alg.successful_jobs + # create 2 jobs per interface, for each of the archive items + j1, j2 = AlgorithmJobFactory.create_batch( + 2, + status=Job.SUCCESS, + creator=None, + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_model=AlgorithmModelFactory(), + algorithm_interface=archive_items_and_jobs_for_interfaces.interface1, + time_limit=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm.time_limit, + ) + j1.inputs.set( + [archive_items_and_jobs_for_interfaces.civs_for_interface1[0]] + ) + j2.inputs.set( + [archive_items_and_jobs_for_interfaces.civs_for_interface1[1]] + ) + + # create 2 jobs per interface, for each of the archive items + j3, j4 = AlgorithmJobFactory.create_batch( + 2, + status=Job.SUCCESS, + creator=None, + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_model=AlgorithmModelFactory(), + algorithm_interface=archive_items_and_jobs_for_interfaces.interface2, + time_limit=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm.time_limit, + ) + j3.inputs.set( + archive_items_and_jobs_for_interfaces.civs_for_interface2[0] + ) + j4.inputs.set( + archive_items_and_jobs_for_interfaces.civs_for_interface2[1] + ) + + del eval_alg.successful_jobs_per_interface + del eval_alg.successful_job_count_per_interface + del eval_alg.total_successful_jobs del eval_alg.inputs_complete assert not eval_alg.inputs_complete - def test_jobs_with_different_model_ignored_for_submission_with_model(self): - eval_alg = EvaluationFactory( - submission=self.submission_with_model, time_limit=10 + def test_jobs_with_different_model_ignored_for_submission_with_model( + self, archive_items_and_jobs_for_interfaces + ): + algorithm_model = AlgorithmModelFactory( + algorithm=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm + ) + submission = SubmissionFactory( + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_model=algorithm_model, + ) + submission.phase.archive = ( + archive_items_and_jobs_for_interfaces.archive + ) + submission.phase.save() + submission.phase.algorithm_interfaces.set( + [ + archive_items_and_jobs_for_interfaces.interface1, + archive_items_and_jobs_for_interfaces.interface2, + ] ) + + eval_alg = EvaluationFactory(submission=submission, time_limit=10) assert not eval_alg.inputs_complete - for inpt, output in zip( - self.input_civs, self.output_civs, strict=True - ): - j_with_model = AlgorithmJobFactory( - status=Job.SUCCESS, - algorithm_image=eval_alg.submission.algorithm_image, - algorithm_model=AlgorithmModelFactory( - algorithm=eval_alg.submission.algorithm_image.algorithm - ), - time_limit=10, - ) - j_with_model.inputs.set([inpt]) - j_with_model.outputs.set([output]) - j_with_model.creator = None - j_with_model.save() - del eval_alg.algorithm_inputs - del eval_alg.successful_jobs + # create 2 jobs per interface, for each of the archive items + j1, j2 = AlgorithmJobFactory.create_batch( + 2, + status=Job.SUCCESS, + creator=None, + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_model=AlgorithmModelFactory( + algorithm=eval_alg.submission.algorithm_image.algorithm + ), + algorithm_interface=archive_items_and_jobs_for_interfaces.interface1, + time_limit=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm.time_limit, + ) + j1.inputs.set( + [archive_items_and_jobs_for_interfaces.civs_for_interface1[0]] + ) + j2.inputs.set( + [archive_items_and_jobs_for_interfaces.civs_for_interface1[1]] + ) + + # create 2 jobs per interface, for each of the archive items + j3, j4 = AlgorithmJobFactory.create_batch( + 2, + status=Job.SUCCESS, + creator=None, + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_model=AlgorithmModelFactory( + algorithm=eval_alg.submission.algorithm_image.algorithm + ), + algorithm_interface=archive_items_and_jobs_for_interfaces.interface2, + time_limit=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm.time_limit, + ) + j3.inputs.set( + archive_items_and_jobs_for_interfaces.civs_for_interface2[0] + ) + j4.inputs.set( + archive_items_and_jobs_for_interfaces.civs_for_interface2[1] + ) + + del eval_alg.successful_jobs_per_interface + del eval_alg.successful_job_count_per_interface + del eval_alg.total_successful_jobs del eval_alg.inputs_complete assert not eval_alg.inputs_complete - def test_jobs_without_model_ignored_for_submission_with_model(self): - eval_alg = EvaluationFactory( - submission=self.submission_with_model, time_limit=10 + def test_jobs_without_model_ignored_for_submission_with_model( + self, archive_items_and_jobs_for_interfaces + ): + algorithm_model = AlgorithmModelFactory( + algorithm=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm + ) + submission = SubmissionFactory( + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_model=algorithm_model, + ) + submission.phase.archive = ( + archive_items_and_jobs_for_interfaces.archive ) + submission.phase.save() + submission.phase.algorithm_interfaces.set( + [ + archive_items_and_jobs_for_interfaces.interface1, + archive_items_and_jobs_for_interfaces.interface2, + ] + ) + + eval_alg = EvaluationFactory(submission=submission, time_limit=10) assert not eval_alg.inputs_complete - for inpt, output in zip( - self.input_civs, self.output_civs, strict=True - ): - j_with_model = AlgorithmJobFactory( - status=Job.SUCCESS, - algorithm_image=eval_alg.submission.algorithm_image, - time_limit=10, - ) - j_with_model.inputs.set([inpt]) - j_with_model.outputs.set([output]) - j_with_model.creator = None - j_with_model.save() - del eval_alg.algorithm_inputs - del eval_alg.successful_jobs + # create 2 jobs per interface, for each of the archive items + j1, j2 = AlgorithmJobFactory.create_batch( + 2, + status=Job.SUCCESS, + creator=None, + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_interface=archive_items_and_jobs_for_interfaces.interface1, + time_limit=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm.time_limit, + ) + j1.inputs.set( + [archive_items_and_jobs_for_interfaces.civs_for_interface1[0]] + ) + j2.inputs.set( + [archive_items_and_jobs_for_interfaces.civs_for_interface1[1]] + ) + + # create 2 jobs per interface, for each of the archive items + j3, j4 = AlgorithmJobFactory.create_batch( + 2, + status=Job.SUCCESS, + creator=None, + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + algorithm_interface=archive_items_and_jobs_for_interfaces.interface2, + time_limit=archive_items_and_jobs_for_interfaces.algorithm_image.algorithm.time_limit, + ) + j3.inputs.set( + archive_items_and_jobs_for_interfaces.civs_for_interface2[0] + ) + j4.inputs.set( + archive_items_and_jobs_for_interfaces.civs_for_interface2[1] + ) + + del eval_alg.successful_jobs_per_interface + del eval_alg.successful_job_count_per_interface + del eval_alg.total_successful_jobs del eval_alg.inputs_complete assert not eval_alg.inputs_complete @@ -1452,3 +1847,135 @@ def test_algorithm_requires_memory_unchangable(): submission.save() assert "algorithm_requires_memory_gb cannot be changed" in str(error) + + +@pytest.mark.django_db +def test_archive_item_matching_to_interfaces(): + phase = PhaseFactory(submission_kind=SubmissionKindChoices.ALGORITHM) + + archive = ArchiveFactory() + phase.archive = archive + phase.save() + + i1, i2, i3, i4 = ArchiveItemFactory.create_batch(4, archive=archive) + + ci1, ci2, ci3, ci4 = ComponentInterfaceFactory.create_batch(4) + interface1 = AlgorithmInterfaceFactory(inputs=[ci1]) + interface2 = AlgorithmInterfaceFactory(inputs=[ci1, ci2]) + interface3 = AlgorithmInterfaceFactory(inputs=[ci2, ci3, ci4]) + + i1.values.add( + ComponentInterfaceValueFactory(interface=ci1) + ) # Valid for interface 1 + i2.values.set( + [ + ComponentInterfaceValueFactory(interface=ci1), + ComponentInterfaceValueFactory(interface=ci2), + ] + ) # valid for interface 2 + i3.values.set( + [ + ComponentInterfaceValueFactory(interface=ci1), + ComponentInterfaceValueFactory( + interface=ComponentInterfaceFactory() + ), + ] + ) # valid for no interface, because of additional / mismatching interface + i4.values.set( + [ + ComponentInterfaceValueFactory(interface=ci2), + ComponentInterfaceValueFactory( + interface=ComponentInterfaceFactory() + ), + ] + ) # valid for no interface, because of additional / mismatching interface + + phase.algorithm_interfaces.set([interface1]) + assert phase.valid_archive_items_per_interface.keys() == {interface1} + assert phase.valid_archive_items_per_interface[interface1].get() == i1 + assert phase.valid_archive_item_count_per_interface == {interface1: 1} + assert phase.jobs_to_schedule_per_submission == 1 + + del phase.valid_archive_items_per_interface + del phase.valid_archive_item_count_per_interface + del phase.jobs_to_schedule_per_submission + phase.algorithm_interfaces.set([interface2]) + assert phase.valid_archive_items_per_interface.keys() == {interface2} + assert phase.valid_archive_items_per_interface[interface2].get() == i2 + assert phase.valid_archive_item_count_per_interface == {interface2: 1} + assert phase.jobs_to_schedule_per_submission == 1 + + del phase.valid_archive_items_per_interface + del phase.valid_archive_item_count_per_interface + del phase.jobs_to_schedule_per_submission + phase.algorithm_interfaces.set([interface3]) + assert phase.valid_archive_items_per_interface.keys() == {interface3} + assert not phase.valid_archive_items_per_interface[interface3].exists() + assert phase.valid_archive_item_count_per_interface == {interface3: 0} + assert phase.jobs_to_schedule_per_submission == 0 + + del phase.valid_archive_items_per_interface + del phase.valid_archive_item_count_per_interface + del phase.jobs_to_schedule_per_submission + phase.algorithm_interfaces.set([interface1, interface3]) + assert phase.valid_archive_items_per_interface.keys() == { + interface1, + interface3, + } + assert phase.valid_archive_items_per_interface[interface1].get() == i1 + assert not phase.valid_archive_items_per_interface[interface3].exists() + assert phase.valid_archive_item_count_per_interface == { + interface1: 1, + interface3: 0, + } + assert phase.jobs_to_schedule_per_submission == 1 + + del phase.valid_archive_items_per_interface + del phase.valid_archive_item_count_per_interface + del phase.jobs_to_schedule_per_submission + phase.algorithm_interfaces.set([interface1, interface2, interface3]) + assert phase.valid_archive_items_per_interface.keys() == { + interface1, + interface2, + interface3, + } + assert phase.valid_archive_items_per_interface[interface1].get() == i1 + assert phase.valid_archive_items_per_interface[interface2].get() == i2 + assert not phase.valid_archive_items_per_interface[interface3].exists() + assert phase.valid_archive_item_count_per_interface == { + interface1: 1, + interface2: 1, + interface3: 0, + } + assert phase.jobs_to_schedule_per_submission == 2 + + +@pytest.mark.django_db +def test_get_valid_jobs_for_interfaces_and_archive_items( + archive_items_and_jobs_for_interfaces, +): + + valid_job_inputs = get_archive_items_for_interfaces( + algorithm_interfaces=[ + archive_items_and_jobs_for_interfaces.interface1, + archive_items_and_jobs_for_interfaces.interface2, + ], + archive_items=ArchiveItem.objects.all(), + ) + + jobs_per_interface = get_valid_jobs_for_interfaces_and_archive_items( + algorithm_interfaces=[ + archive_items_and_jobs_for_interfaces.interface1, + archive_items_and_jobs_for_interfaces.interface2, + ], + algorithm_image=archive_items_and_jobs_for_interfaces.algorithm_image, + valid_archive_items_per_interface=valid_job_inputs, + ) + assert jobs_per_interface == { + archive_items_and_jobs_for_interfaces.interface1: [ + archive_items_and_jobs_for_interfaces.jobs_for_interface1[0] + ], + archive_items_and_jobs_for_interfaces.interface2: [ + archive_items_and_jobs_for_interfaces.jobs_for_interface2[0] + ], + } diff --git a/app/tests/evaluation_tests/test_tasks.py b/app/tests/evaluation_tests/test_tasks.py index bbbf233ab..3493fe6e9 100644 --- a/app/tests/evaluation_tests/test_tasks.py +++ b/app/tests/evaluation_tests/test_tasks.py @@ -6,12 +6,11 @@ from actstream.actions import unfollow from django.conf import settings from django.core.cache import cache -from django.test import TestCase from django.utils.html import format_html from redis.exceptions import LockError from grandchallenge.algorithms.models import Job -from grandchallenge.components.models import ComponentInterface, InterfaceKind +from grandchallenge.components.models import InterfaceKind from grandchallenge.components.tasks import ( push_container_image, validate_docker_image, @@ -26,6 +25,7 @@ from grandchallenge.profiles.templatetags.profiles import user_profile_link from tests.algorithms_tests.factories import ( AlgorithmImageFactory, + AlgorithmInterfaceFactory, AlgorithmJobFactory, AlgorithmModelFactory, ) @@ -222,222 +222,287 @@ def test_method_validation_not_a_docker_tar(submission_file): assert method.status == "Could not decompress the container image file." -class TestSetEvaluationInputs(TestCase): - def setUp(self): - interface = ComponentInterface.objects.get( - slug="generic-medical-image" - ) - - archive = ArchiveFactory() - archive_items = ArchiveItemFactory.create_batch(2) - archive.items.set(archive_items) - - input_civs = ComponentInterfaceValueFactory.create_batch( - 2, interface=interface - ) - output_civs = ComponentInterfaceValueFactory.create_batch( - 2, interface=interface - ) - - for ai, civ in zip(archive_items, input_civs, strict=True): - ai.values.set([civ]) - - algorithm_image = AlgorithmImageFactory() - algorithm_model = AlgorithmModelFactory() - submission = SubmissionFactory(algorithm_image=algorithm_image) - submission.phase.archive = archive - submission.phase.save() - submission.phase.algorithm_inputs.set([interface]) - - submission_with_model = SubmissionFactory( - algorithm_image=algorithm_image, algorithm_model=algorithm_model - ) - submission_with_model.phase.archive = archive - submission_with_model.phase.save() - submission_with_model.phase.algorithm_inputs.set([interface]) - - jobs = [] - for inpt, output in zip(input_civs, output_civs, strict=True): - j = AlgorithmJobFactory( - status=Job.SUCCESS, - algorithm_image=algorithm_image, - time_limit=algorithm_image.algorithm.time_limit, - ) - j.inputs.set([inpt]) - j.outputs.set([output]) - j.creator = None - j.save() - jobs.append(j) - - # also create a job with the same input but a creator set, this job - # should be ignored - j_irrelevant = AlgorithmJobFactory( - status=Job.SUCCESS, - algorithm_image=algorithm_image, - creator=algorithm_image.creator, - time_limit=algorithm_image.algorithm.time_limit, - ) - j_irrelevant.inputs.set([input_civs[0]]) - j_irrelevant.outputs.set([output_civs[0]]) - - self.evaluation = EvaluationFactory( - submission=submission, - status=Evaluation.EXECUTING_PREREQUISITES, - time_limit=submission.phase.evaluation_time_limit, - ) - self.evaluation_with_model = EvaluationFactory( - submission=submission_with_model, +@pytest.mark.django_db +class TestSetEvaluationInputs: + def test_set_evaluation_inputs( + self, submission_without_model_for_optional_inputs + ): + eval = EvaluationFactory( + submission=submission_without_model_for_optional_inputs.submission, status=Evaluation.EXECUTING_PREREQUISITES, - time_limit=submission.phase.evaluation_time_limit, + time_limit=submission_without_model_for_optional_inputs.submission.phase.evaluation_time_limit, ) - self.input_civs = input_civs - self.output_civs = output_civs - self.jobs = jobs - self.output_civs = output_civs - - def test_set_evaluation_inputs(self): - set_evaluation_inputs(evaluation_pk=self.evaluation.pk) + set_evaluation_inputs(evaluation_pk=eval.pk) - self.evaluation.refresh_from_db() - assert self.evaluation.status == self.evaluation.PENDING - assert self.evaluation.error_message == "" - assert self.evaluation.inputs.count() == 3 - assert self.evaluation.input_prefixes == { + eval.refresh_from_db() + assert eval.status == eval.PENDING + assert eval.error_message == "" + assert eval.inputs.count() == 5 + assert eval.input_prefixes == { str(civ.pk): f"{alg.pk}/output/" - for alg, civ in zip(self.jobs, self.output_civs, strict=True) + for alg, civ in zip( + submission_without_model_for_optional_inputs.jobs, + submission_without_model_for_optional_inputs.output_civs, + strict=True, + ) } - def test_has_pending_jobs(self): + def test_has_pending_jobs( + self, submission_without_model_for_optional_inputs + ): + eval = EvaluationFactory( + submission=submission_without_model_for_optional_inputs.submission, + status=Evaluation.EXECUTING_PREREQUISITES, + time_limit=submission_without_model_for_optional_inputs.submission.phase.evaluation_time_limit, + ) AlgorithmJobFactory( status=Job.PENDING, creator=None, - algorithm_image=self.evaluation.submission.algorithm_image, - time_limit=self.evaluation.submission.algorithm_image.algorithm.time_limit, + algorithm_image=eval.submission.algorithm_image, + algorithm_interface=submission_without_model_for_optional_inputs.interface1, + time_limit=eval.submission.algorithm_image.algorithm.time_limit, ) # nothing happens because there are pending jobs - set_evaluation_inputs(evaluation_pk=self.evaluation.pk) - self.evaluation.refresh_from_db() - assert ( - self.evaluation.status == self.evaluation.EXECUTING_PREREQUISITES + set_evaluation_inputs(evaluation_pk=eval.pk) + eval.refresh_from_db() + assert eval.status == eval.EXECUTING_PREREQUISITES + assert eval.inputs.count() == 0 + assert eval.input_prefixes == {} + + def test_has_pending_jobs_with_image_and_model( + self, submission_with_model_for_optional_inputs + ): + evaluation_with_model = EvaluationFactory( + submission=submission_with_model_for_optional_inputs.submission, + status=Evaluation.EXECUTING_PREREQUISITES, + time_limit=submission_with_model_for_optional_inputs.submission.phase.evaluation_time_limit, ) - assert self.evaluation.inputs.count() == 0 - assert self.evaluation.input_prefixes == {} - - def test_has_pending_jobs_with_image_and_model(self): AlgorithmJobFactory( status=Job.PENDING, creator=None, - algorithm_image=self.evaluation_with_model.submission.algorithm_image, - algorithm_model=self.evaluation_with_model.submission.algorithm_model, - time_limit=self.evaluation_with_model.submission.algorithm_image.algorithm.time_limit, + algorithm_image=evaluation_with_model.submission.algorithm_image, + algorithm_model=evaluation_with_model.submission.algorithm_model, + algorithm_interface=submission_with_model_for_optional_inputs.interface1, + time_limit=evaluation_with_model.submission.algorithm_image.algorithm.time_limit, ) # nothing happens - set_evaluation_inputs(evaluation_pk=self.evaluation_with_model.pk) - self.evaluation_with_model.refresh_from_db() + set_evaluation_inputs(evaluation_pk=evaluation_with_model.pk) + evaluation_with_model.refresh_from_db() assert ( - self.evaluation_with_model.status - == self.evaluation_with_model.EXECUTING_PREREQUISITES + evaluation_with_model.status + == evaluation_with_model.EXECUTING_PREREQUISITES + ) + assert evaluation_with_model.inputs.count() == 0 + assert evaluation_with_model.input_prefixes == {} + + def test_has_pending_jobs_with_image_but_without_model( + self, submission_with_model_for_optional_inputs + ): + evaluation_with_model = EvaluationFactory( + submission=submission_with_model_for_optional_inputs.submission, + status=Evaluation.EXECUTING_PREREQUISITES, + time_limit=submission_with_model_for_optional_inputs.submission.phase.evaluation_time_limit, ) - assert self.evaluation_with_model.inputs.count() == 0 - assert self.evaluation_with_model.input_prefixes == {} - - def test_has_pending_jobs_with_image_but_without_model(self): # the pending job with only the image will be ignored, but task will still # fail because there are no successful jobs with both the image and model yet AlgorithmJobFactory( status=Job.PENDING, creator=None, - algorithm_image=self.evaluation_with_model.submission.algorithm_image, - time_limit=self.evaluation_with_model.submission.algorithm_image.algorithm.time_limit, + algorithm_image=evaluation_with_model.submission.algorithm_image, + algorithm_interface=submission_with_model_for_optional_inputs.interface1, + time_limit=evaluation_with_model.submission.algorithm_image.algorithm.time_limit, ) - set_evaluation_inputs(evaluation_pk=self.evaluation_with_model.pk) - self.evaluation_with_model.refresh_from_db() + set_evaluation_inputs(evaluation_pk=evaluation_with_model.pk) + evaluation_with_model.refresh_from_db() assert ( - self.evaluation_with_model.status - == self.evaluation_with_model.EXECUTING_PREREQUISITES - ) - assert self.evaluation_with_model.inputs.count() == 0 - assert self.evaluation_with_model.input_prefixes == {} - - # add jobs - jobs = [] - for inpt, output in zip( - self.input_civs, self.output_civs, strict=True - ): - j_with_model = AlgorithmJobFactory( - status=Job.SUCCESS, - algorithm_image=self.evaluation_with_model.submission.algorithm_image, - algorithm_model=self.evaluation_with_model.submission.algorithm_model, - time_limit=self.evaluation_with_model.submission.algorithm_image.algorithm.time_limit, - ) - j_with_model.inputs.set([inpt]) - j_with_model.outputs.set([output]) - j_with_model.creator = None - j_with_model.save() - jobs.append(j_with_model) - - set_evaluation_inputs(evaluation_pk=self.evaluation_with_model.pk) - self.evaluation_with_model.refresh_from_db() - assert ( - self.evaluation_with_model.status - == self.evaluation_with_model.PENDING + evaluation_with_model.status + == evaluation_with_model.EXECUTING_PREREQUISITES + ) + assert evaluation_with_model.inputs.count() == 0 + assert evaluation_with_model.input_prefixes == {} + + # add jobs, 2 for each interface with a model + j_with_model_1 = AlgorithmJobFactory( + status=Job.SUCCESS, + creator=None, + algorithm_image=evaluation_with_model.submission.algorithm_image, + algorithm_model=evaluation_with_model.submission.algorithm_model, + algorithm_interface=submission_with_model_for_optional_inputs.interface1, + time_limit=evaluation_with_model.submission.algorithm_image.algorithm.time_limit, + ) + j_with_model_2 = AlgorithmJobFactory( + status=Job.SUCCESS, + creator=None, + algorithm_image=evaluation_with_model.submission.algorithm_image, + algorithm_model=evaluation_with_model.submission.algorithm_model, + algorithm_interface=submission_with_model_for_optional_inputs.interface1, + time_limit=evaluation_with_model.submission.algorithm_image.algorithm.time_limit, + ) + j_with_model_1.inputs.set( + submission_with_model_for_optional_inputs.jobs[0].inputs.all() + ) + j_with_model_1.outputs.set( + submission_with_model_for_optional_inputs.jobs[0].outputs.all() + ) + j_with_model_2.inputs.set( + submission_with_model_for_optional_inputs.jobs[1].inputs.all() + ) + j_with_model_2.outputs.set( + submission_with_model_for_optional_inputs.jobs[1].outputs.all() + ) + + j_with_model_3 = AlgorithmJobFactory( + status=Job.SUCCESS, + creator=None, + algorithm_image=evaluation_with_model.submission.algorithm_image, + algorithm_model=evaluation_with_model.submission.algorithm_model, + algorithm_interface=submission_with_model_for_optional_inputs.interface2, + time_limit=evaluation_with_model.submission.algorithm_image.algorithm.time_limit, + ) + j_with_model_4 = AlgorithmJobFactory( + status=Job.SUCCESS, + creator=None, + algorithm_image=evaluation_with_model.submission.algorithm_image, + algorithm_model=evaluation_with_model.submission.algorithm_model, + algorithm_interface=submission_with_model_for_optional_inputs.interface2, + time_limit=evaluation_with_model.submission.algorithm_image.algorithm.time_limit, + ) + j_with_model_3.inputs.set( + submission_with_model_for_optional_inputs.jobs[2].inputs.all() ) - assert self.evaluation_with_model.inputs.count() == 3 - assert self.evaluation_with_model.input_prefixes == { + j_with_model_3.outputs.set( + submission_with_model_for_optional_inputs.jobs[2].outputs.all() + ) + j_with_model_4.inputs.set( + submission_with_model_for_optional_inputs.jobs[3].inputs.all() + ) + j_with_model_4.outputs.set( + submission_with_model_for_optional_inputs.jobs[3].outputs.all() + ) + + jobs = [j_with_model_1, j_with_model_2, j_with_model_3, j_with_model_4] + + set_evaluation_inputs(evaluation_pk=evaluation_with_model.pk) + evaluation_with_model.refresh_from_db() + assert evaluation_with_model.status == evaluation_with_model.PENDING + assert evaluation_with_model.inputs.count() == 5 + assert evaluation_with_model.input_prefixes == { str(civ.pk): f"{alg.pk}/output/" - for alg, civ in zip(jobs, self.output_civs, strict=True) + for alg, civ in zip( + jobs, + submission_with_model_for_optional_inputs.output_civs, + strict=True, + ) } - def test_has_pending_jobs_without_active_model(self): + def test_has_pending_jobs_without_active_model( + self, submission_without_model_for_optional_inputs + ): + eval = EvaluationFactory( + submission=submission_without_model_for_optional_inputs.submission, + status=Evaluation.EXECUTING_PREREQUISITES, + time_limit=submission_without_model_for_optional_inputs.submission.phase.evaluation_time_limit, + ) AlgorithmJobFactory( status=Job.PENDING, creator=None, - algorithm_image=self.evaluation.submission.algorithm_image, + algorithm_image=eval.submission.algorithm_image, algorithm_model=AlgorithmModelFactory( - algorithm=self.evaluation.submission.algorithm_image.algorithm + algorithm=eval.submission.algorithm_image.algorithm ), - time_limit=self.evaluation.submission.algorithm_image.algorithm.time_limit, + algorithm_interface=submission_without_model_for_optional_inputs.interface1, + time_limit=eval.submission.algorithm_image.algorithm.time_limit, ) # pending job for image with model should be ignored, # since active_model is None for the evaluation - set_evaluation_inputs(evaluation_pk=self.evaluation.pk) - self.evaluation.refresh_from_db() - assert self.evaluation.status == self.evaluation.PENDING - assert self.evaluation.inputs.count() == 3 - assert self.evaluation.input_prefixes == { + set_evaluation_inputs(evaluation_pk=eval.pk) + eval.refresh_from_db() + assert eval.status == eval.PENDING + assert eval.inputs.count() == 5 + assert eval.input_prefixes == { str(civ.pk): f"{alg.pk}/output/" - for alg, civ in zip(self.jobs, self.output_civs, strict=True) + for alg, civ in zip( + submission_without_model_for_optional_inputs.jobs, + submission_without_model_for_optional_inputs.output_civs, + strict=True, + ) } - def test_successful_jobs_without_model(self): + def test_successful_jobs_without_model( + self, submission_without_model_for_optional_inputs + ): # delete jobs created in setup(), # create new ones with a model, these should not count # towards successful jobs, and eval inputs should not be set Job.objects.all().delete() - jobs = [] - for inpt, output in zip( - self.input_civs, self.output_civs, strict=True - ): - j_with_model = AlgorithmJobFactory( - status=Job.SUCCESS, - algorithm_image=self.evaluation.submission.algorithm_image, - algorithm_model=AlgorithmModelFactory( - algorithm=self.evaluation.submission.algorithm_image.algorithm - ), - time_limit=self.evaluation.submission.algorithm_image.algorithm.time_limit, - ) - j_with_model.inputs.set([inpt]) - j_with_model.outputs.set([output]) - j_with_model.creator = None - j_with_model.save() - jobs.append(j_with_model) - set_evaluation_inputs(evaluation_pk=self.evaluation.pk) - self.evaluation.refresh_from_db() - assert ( - self.evaluation.status == self.evaluation.EXECUTING_PREREQUISITES + + eval = EvaluationFactory( + submission=submission_without_model_for_optional_inputs.submission, + status=Evaluation.EXECUTING_PREREQUISITES, + time_limit=submission_without_model_for_optional_inputs.submission.phase.evaluation_time_limit, + ) + + # create 2 jobs per interface, for each of the archive items + j1, j2 = AlgorithmJobFactory.create_batch( + 2, + creator=None, + algorithm_image=submission_without_model_for_optional_inputs.submission.algorithm_image, + algorithm_interface=submission_without_model_for_optional_inputs.interface1, + algorithm_model=AlgorithmModelFactory( + algorithm=eval.submission.algorithm_image.algorithm + ), + time_limit=submission_without_model_for_optional_inputs.submission.algorithm_image.algorithm.time_limit, + ) + j1.inputs.set( + [ + submission_without_model_for_optional_inputs.civs_for_interface1[ + 0 + ] + ] + ) + j1.outputs.set( + [submission_without_model_for_optional_inputs.output_civs[0]] + ) + j2.inputs.set( + [ + submission_without_model_for_optional_inputs.civs_for_interface1[ + 1 + ] + ] + ) + j2.outputs.set( + [submission_without_model_for_optional_inputs.output_civs[1]] ) - assert self.evaluation.inputs.count() == 0 - assert self.evaluation.input_prefixes == {} + + # create 2 jobs per interface, for each of the archive items + j3, j4 = AlgorithmJobFactory.create_batch( + 2, + creator=None, + algorithm_image=submission_without_model_for_optional_inputs.submission.algorithm_image, + algorithm_model=AlgorithmModelFactory( + algorithm=submission_without_model_for_optional_inputs.submission.algorithm_image.algorithm + ), + algorithm_interface=submission_without_model_for_optional_inputs.interface2, + time_limit=submission_without_model_for_optional_inputs.submission.algorithm_image.algorithm.time_limit, + ) + j3.inputs.set( + submission_without_model_for_optional_inputs.civs_for_interface2[0] + ) + j3.outputs.set( + [submission_without_model_for_optional_inputs.output_civs[2]] + ) + j4.inputs.set( + submission_without_model_for_optional_inputs.civs_for_interface2[1] + ) + j4.outputs.set( + [submission_without_model_for_optional_inputs.output_civs[3]] + ) + + set_evaluation_inputs(evaluation_pk=eval.pk) + eval.refresh_from_db() + assert eval.status == eval.EXECUTING_PREREQUISITES + assert eval.inputs.count() == 0 + assert eval.input_prefixes == {} @pytest.mark.django_db @@ -726,22 +791,18 @@ def test_evaluation_order_with_title(): time_limit=ai.algorithm.time_limit, ) - input_interface = ComponentInterfaceFactory( + input_ci = ComponentInterfaceFactory( kind=InterfaceKind.InterfaceKindChoices.BOOL ) - - evaluation.submission.phase.algorithm_inputs.set([input_interface]) - ai.algorithm.inputs.set([input_interface]) + interface = AlgorithmInterfaceFactory(inputs=[input_ci]) + ai.algorithm.interfaces.set([interface]) + evaluation.submission.phase.algorithm_interfaces.set([interface]) # Priority should be given to archive items with titles archive_item = ArchiveItemFactory(archive=archive) - archive_item.values.add( - ComponentInterfaceValueFactory(interface=input_interface) - ) + archive_item.values.add(ComponentInterfaceValueFactory(interface=input_ci)) - civs = ComponentInterfaceValueFactory.create_batch( - 5, interface=input_interface - ) + civs = ComponentInterfaceValueFactory.create_batch(5, interface=input_ci) for idx, civ in enumerate(civs): archive_item = ArchiveItemFactory(archive=archive, title=f"{5 - idx}") @@ -767,16 +828,15 @@ def test_evaluation_order_without_title(): time_limit=ai.algorithm.time_limit, ) - input_interface = ComponentInterfaceFactory( + input_ci = ComponentInterfaceFactory( kind=InterfaceKind.InterfaceKindChoices.BOOL ) + interface = AlgorithmInterfaceFactory(inputs=[input_ci]) + ai.algorithm.interfaces.set([interface]) - evaluation.submission.phase.algorithm_inputs.set([input_interface]) - ai.algorithm.inputs.set([input_interface]) + evaluation.submission.phase.algorithm_interfaces.set([interface]) - civs = ComponentInterfaceValueFactory.create_batch( - 5, interface=input_interface - ) + civs = ComponentInterfaceValueFactory.create_batch(5, interface=input_ci) for civ in civs: archive_item = ArchiveItemFactory(archive=archive) diff --git a/app/tests/evaluation_tests/test_views.py b/app/tests/evaluation_tests/test_views.py index d3b36396b..da6539ac6 100644 --- a/app/tests/evaluation_tests/test_views.py +++ b/app/tests/evaluation_tests/test_views.py @@ -10,7 +10,7 @@ from factory.django import ImageField from guardian.shortcuts import assign_perm, remove_perm -from grandchallenge.algorithms.models import Algorithm +from grandchallenge.algorithms.models import Algorithm, AlgorithmInterface from grandchallenge.components.models import ( ComponentInterface, ImportStatusChoices, @@ -21,6 +21,7 @@ from grandchallenge.evaluation.models import ( CombinedLeaderboard, Evaluation, + PhaseAlgorithmInterface, Submission, ) from grandchallenge.evaluation.tasks import update_combined_leaderboard @@ -30,6 +31,7 @@ from tests.algorithms_tests.factories import ( AlgorithmFactory, AlgorithmImageFactory, + AlgorithmInterfaceFactory, ) from tests.archives_tests.factories import ArchiveFactory, ArchiveItemFactory from tests.components_tests.factories import ( @@ -706,8 +708,8 @@ def test_create_algorithm_for_phase_permission(client, uploaded_image): phase.submission_kind = SubmissionKindChoices.ALGORITHM phase.creator_must_be_verified = True phase.archive = ArchiveFactory() - phase.algorithm_inputs.set([ComponentInterfaceFactory()]) - phase.algorithm_outputs.set([ComponentInterfaceFactory()]) + interface = AlgorithmInterfaceFactory() + phase.algorithm_interfaces.set([interface]) phase.save() response = get_view_for_user( @@ -810,10 +812,10 @@ def test_create_algorithm_for_phase_presets(client): phase.creator_must_be_verified = True phase.archive = ArchiveFactory() ci1 = ComponentInterfaceFactory(kind=InterfaceKindChoices.STRING) - ci2 = ComponentInterfaceFactory(kind=InterfaceKindChoices.STRING) optional_protocol = HangingProtocolFactory() - phase.algorithm_inputs.set([ci1]) - phase.algorithm_outputs.set([ci2]) + + interface1, interface2 = AlgorithmInterfaceFactory.create_batch(2) + phase.algorithm_interfaces.set([interface1, interface2]) phase.hanging_protocol = HangingProtocolFactory( json=[{"viewport_name": "main"}] ) @@ -832,8 +834,10 @@ def test_create_algorithm_for_phase_presets(client): client=client, user=admin, ) - assert response.context_data["form"]["inputs"].initial.get() == ci1 - assert response.context_data["form"]["outputs"].initial.get() == ci2 + assert list(response.context_data["form"]["interfaces"].initial.all()) == [ + interface1, + interface2, + ] assert response.context_data["form"][ "workstation" ].initial == Workstation.objects.get( @@ -879,11 +883,11 @@ def test_create_algorithm_for_phase_presets(client): data={ "title": "Test algorithm", "job_requires_memory_gb": 8, - "inputs": [ - response.context_data["form"]["inputs"].initial.get().pk - ], - "outputs": [ - response.context_data["form"]["outputs"].initial.get().pk + "interfaces": [ + interface.pk + for interface in response.context_data["form"][ + "interfaces" + ].initial.all() ], "workstation": response.context_data["form"][ "workstation" @@ -909,8 +913,7 @@ def test_create_algorithm_for_phase_presets(client): assert response.status_code == 302 assert Algorithm.objects.count() == 1 algorithm = Algorithm.objects.get() - assert algorithm.inputs.get() == ci1 - assert algorithm.outputs.get() == ci2 + assert list(algorithm.interfaces.all()) == [interface1, interface2] assert algorithm.hanging_protocol == phase.hanging_protocol assert algorithm.optional_hanging_protocols.get() == optional_protocol assert algorithm.workstation_config == phase.workstation_config @@ -943,8 +946,7 @@ def test_create_algorithm_for_phase_presets(client): data={ "title": "Test algorithm", "job_requires_memory_gb": 8, - "inputs": [ci3.pk], - "outputs": [ci2.pk], + "interfaces": [interface1.pk], "workstation": ws.pk, "hanging_protocol": hp.pk, "optional_hanging_protocols": [oph.pk], @@ -955,15 +957,12 @@ def test_create_algorithm_for_phase_presets(client): # created algorithm has the initial values set, not the modified ones alg2 = Algorithm.objects.last() - assert alg2.inputs.get() == ci1 - assert alg2.outputs.get() == ci2 + assert list(alg2.interfaces.all()) == [interface1, interface2] assert alg2.hanging_protocol == phase.hanging_protocol assert alg2.optional_hanging_protocols.get() == optional_protocol assert alg2.workstation_config == phase.workstation_config assert alg2.view_content == phase.view_content assert alg2.workstation.slug == settings.DEFAULT_WORKSTATION_SLUG - assert alg2.inputs.get() != ci3 - assert alg2.outputs.get() != ci4 assert alg2.hanging_protocol != hp assert alg2.workstation_config != wsc assert alg2.view_content != "{}" @@ -980,8 +979,10 @@ def test_create_algorithm_for_phase_limits(client): phase.archive = ArchiveFactory() ci1 = ComponentInterfaceFactory() ci2 = ComponentInterfaceFactory() - phase.algorithm_inputs.set([ci1]) - phase.algorithm_outputs.set([ci2]) + + interface = AlgorithmInterfaceFactory(inputs=[ci1], outputs=[ci2]) + phase.algorithm_interfaces.set([interface]) + phase.submissions_limit_per_user_per_period = 10 phase.save() @@ -1003,14 +1004,16 @@ def test_create_algorithm_for_phase_limits(client): alg3.add_editor(u1) alg4.add_editor(u2) alg5.add_editor(u1) - alg6.add_editor(u1) - for alg in [alg1, alg2, alg3, alg4, alg5]: - alg.inputs.set([ci1]) - alg.outputs.set([ci2]) + alg6.add_editor(u2) + for alg in [alg1, alg2, alg3, alg4]: + alg.interfaces.set([interface]) ci3 = ComponentInterfaceFactory() - alg5.inputs.add(ci3) - alg6.inputs.set([ci3]) - alg6.outputs.set([ci2]) + + interface2 = AlgorithmInterfaceFactory(inputs=[ci1, ci3], outputs=[ci2]) + alg5.interfaces.set([interface2]) + + interface3 = AlgorithmInterfaceFactory(inputs=[ci3], outputs=[ci2]) + alg6.interfaces.set([interface3]) response = get_view_for_user( viewname="evaluation:phase-algorithm-create", @@ -1373,8 +1376,6 @@ def test_configure_algorithm_phases_view(client): }, data={ "phases": [phase.pk], - "algorithm_inputs": [ci1.pk], - "algorithm_outputs": [ci2.pk], }, ) assert response.status_code == 302 @@ -1385,8 +1386,6 @@ def test_configure_algorithm_phases_view(client): phase.archive.title == f"{phase.challenge.short_name} {phase.title} dataset" ) - assert list(phase.algorithm_inputs.all()) == [ci1] - assert list(phase.algorithm_outputs.all()) == [ci2] assert ( phase.algorithm_time_limit == challenge_request.inference_time_limit_in_minutes * 60 @@ -1583,8 +1582,8 @@ def test_evaluation_details_zero_rank_message(client): @pytest.mark.django_db def test_submission_create_sets_limits_correctly_with_algorithm(client): - inputs = [ComponentInterfaceFactory()] - outputs = [ComponentInterfaceFactory()] + inputs = ComponentInterfaceFactory.create_batch(2) + interface = AlgorithmInterfaceFactory(inputs=inputs) algorithm_image = AlgorithmImageFactory( is_manifest_valid=True, @@ -1593,8 +1592,7 @@ def test_submission_create_sets_limits_correctly_with_algorithm(client): algorithm__job_requires_gpu_type=GPUTypeChoices.V100, algorithm__job_requires_memory_gb=1337, ) - algorithm_image.algorithm.inputs.set(inputs) - algorithm_image.algorithm.outputs.set(outputs) + algorithm_image.algorithm.interfaces.set([interface]) archive = ArchiveFactory() archive_item = ArchiveItemFactory(archive=archive) @@ -1612,8 +1610,7 @@ def test_submission_create_sets_limits_correctly_with_algorithm(client): algorithm_selectable_gpu_type_choices=[GPUTypeChoices.V100], algorithm_maximum_settable_memory_gb=1337, ) - phase.algorithm_inputs.set(inputs) - phase.algorithm_outputs.set(outputs) + phase.algorithm_interfaces.set([interface]) InvoiceFactory( challenge=phase.challenge, @@ -1758,6 +1755,196 @@ def test_phase_archive_info_permissions(client): assert response.status_code == 200 +@pytest.mark.parametrize( + "viewname", ["evaluation:interface-list", "evaluation:interface-create"] +) +@pytest.mark.django_db +def test_algorithm_interface_for_phase_view_permission(client, viewname): + (participant, admin, user, user_with_perm) = UserFactory.create_batch(4) + assign_perm("evaluation.configure_algorithm_phase", user_with_perm) + + prediction_phase = PhaseFactory(submission_kind=SubmissionKindChoices.CSV) + algorithm_phase = PhaseFactory( + submission_kind=SubmissionKindChoices.ALGORITHM + ) + + for phase in [prediction_phase, algorithm_phase]: + phase.challenge.add_admin(admin) + phase.challenge.add_participant(participant) + + for us, status1, status2 in [ + [user, 403, 403], + [participant, 403, 403], + [admin, 403, 403], + [user_with_perm, 404, 200], + ]: + response = get_view_for_user( + viewname=viewname, + client=client, + reverse_kwargs={ + "slug": prediction_phase.slug, + "challenge_short_name": prediction_phase.challenge.short_name, + }, + user=us, + ) + assert response.status_code == status1 + + response = get_view_for_user( + viewname=viewname, + client=client, + reverse_kwargs={ + "slug": algorithm_phase.slug, + "challenge_short_name": algorithm_phase.challenge.short_name, + }, + user=us, + ) + assert response.status_code == status2 + + +@pytest.mark.django_db +def test_algorithm_interface_for_phase_delete_permission(client): + (participant, admin, user, user_with_perm) = UserFactory.create_batch(4) + assign_perm("evaluation.configure_algorithm_phase", user_with_perm) + prediction_phase = PhaseFactory(submission_kind=SubmissionKindChoices.CSV) + algorithm_phase = PhaseFactory( + submission_kind=SubmissionKindChoices.ALGORITHM + ) + int1 = AlgorithmInterfaceFactory() + + for phase in [prediction_phase, algorithm_phase]: + phase.challenge.add_admin(admin) + phase.challenge.add_participant(participant) + phase.algorithm_interfaces.add(int1) + + for us, status1, status2 in [ + [user, 403, 403], + [participant, 403, 403], + [admin, 403, 403], + [user_with_perm, 404, 200], + ]: + response = get_view_for_user( + viewname="evaluation:interface-delete", + client=client, + reverse_kwargs={ + "slug": prediction_phase.slug, + "interface_pk": int1.pk, + "challenge_short_name": prediction_phase.challenge.short_name, + }, + user=us, + ) + assert response.status_code == status1 + + response = get_view_for_user( + viewname="evaluation:interface-delete", + client=client, + reverse_kwargs={ + "slug": algorithm_phase.slug, + "interface_pk": int1.pk, + "challenge_short_name": algorithm_phase.challenge.short_name, + }, + user=us, + ) + assert response.status_code == status2 + + +@pytest.mark.django_db +def test_algorithm_interface_for_phase_create(client): + user = UserFactory() + assign_perm("evaluation.configure_algorithm_phase", user) + phase = PhaseFactory(submission_kind=SubmissionKindChoices.ALGORITHM) + + ci_1 = ComponentInterfaceFactory() + ci_2 = ComponentInterfaceFactory() + + response = get_view_for_user( + viewname="evaluation:interface-create", + client=client, + method=client.post, + reverse_kwargs={ + "slug": phase.slug, + "challenge_short_name": phase.challenge.short_name, + }, + data={ + "inputs": [ci_1.pk], + "outputs": [ci_2.pk], + }, + user=user, + ) + assert response.status_code == 302 + + assert AlgorithmInterface.objects.count() == 1 + io = AlgorithmInterface.objects.get() + assert io.inputs.get() == ci_1 + assert io.outputs.get() == ci_2 + + assert PhaseAlgorithmInterface.objects.count() == 1 + io_through = PhaseAlgorithmInterface.objects.get() + assert io_through.phase == phase + assert io_through.interface == io + + +@pytest.mark.django_db +def test_algorithm_interfaces_for_phase_list_queryset(client): + user = UserFactory() + assign_perm("evaluation.configure_algorithm_phase", user) + phase1, phase2 = PhaseFactory.create_batch( + 2, submission_kind=SubmissionKindChoices.ALGORITHM + ) + + io1, io2, io3, io4 = AlgorithmInterfaceFactory.create_batch(4) + + phase1.algorithm_interfaces.set([io1, io2]) + phase2.algorithm_interfaces.set([io3, io4]) + + iots = PhaseAlgorithmInterface.objects.order_by("id").all() + + response = get_view_for_user( + viewname="evaluation:interface-list", + client=client, + reverse_kwargs={ + "slug": phase1.slug, + "challenge_short_name": phase1.challenge.short_name, + }, + user=user, + ) + assert response.status_code == 200 + assert response.context["object_list"].count() == 2 + assert iots[0] in response.context["object_list"] + assert iots[1] in response.context["object_list"] + assert iots[2] not in response.context["object_list"] + assert iots[3] not in response.context["object_list"] + + +@pytest.mark.django_db +def test_algorithm_interface_delete(client): + user = UserFactory() + assign_perm("evaluation.configure_algorithm_phase", user) + phase = PhaseFactory(submission_kind=SubmissionKindChoices.ALGORITHM) + + int1, int2 = AlgorithmInterfaceFactory.create_batch(2) + phase.algorithm_interfaces.add(int1) + phase.algorithm_interfaces.add(int2) + + response = get_view_for_user( + viewname="evaluation:interface-delete", + client=client, + method=client.post, + reverse_kwargs={ + "slug": phase.slug, + "challenge_short_name": phase.challenge.short_name, + "interface_pk": int2.pk, + }, + user=user, + ) + assert response.status_code == 302 + # no interface was deleted + assert AlgorithmInterface.objects.count() == 2 + # only the relation between interface and algorithm was deleted + assert PhaseAlgorithmInterface.objects.count() == 1 + assert phase.algorithm_interfaces.count() == 1 + assert phase.algorithm_interfaces.get() == int1 + + @pytest.mark.django_db def test_evaluation_details_error_message(client): evaluation_error_message = "Test evaluation error message" diff --git a/app/tests/hanging_protocols_tests/test_forms.py b/app/tests/hanging_protocols_tests/test_forms.py index 81fa7d73c..0443468a0 100644 --- a/app/tests/hanging_protocols_tests/test_forms.py +++ b/app/tests/hanging_protocols_tests/test_forms.py @@ -18,7 +18,10 @@ from grandchallenge.hanging_protocols.forms import HangingProtocolForm from grandchallenge.hanging_protocols.models import HangingProtocol from grandchallenge.reader_studies.forms import ReaderStudyUpdateForm -from tests.algorithms_tests.factories import AlgorithmFactory +from tests.algorithms_tests.factories import ( + AlgorithmFactory, + AlgorithmInterfaceFactory, +) from tests.archives_tests.factories import ArchiveItemFactory from tests.components_tests.factories import ( ComponentInterfaceFactory, @@ -49,9 +52,7 @@ def test_view_content_mixin(): assert not form.is_valid() assert form.errors == { - "__all__": [ - "Unknown interfaces in view content for viewport main: test" - ] + "__all__": ["Unknown sockets in view content for viewport main: test"] } i = ComponentInterfaceFactory( @@ -447,7 +448,7 @@ def make_ci_list( 0, 0, ( - "No interfaces of type image, chart, pdf, mp4, thumbnail_jpg or thumbnail_png are used. At least one interface of those types is needed to configure the viewer. " + "No sockets of type image, chart, pdf, mp4, thumbnail_jpg or thumbnail_png are used. At least one socket of those types is needed to configure the viewer. " 'Refer to the     documentation for more information' ), ), @@ -457,8 +458,8 @@ def make_ci_list( 0, 1, ( - "The following interfaces are used in your {}: test-ci-overlay-0 and test-ci-undisplayable-0. " - "No interfaces of type image, chart, pdf, mp4, thumbnail_jpg or thumbnail_png are used. At least one interface of those types is needed to configure the viewer. " + "The following sockets are used in your {}: test-ci-overlay-0 and test-ci-undisplayable-0. " + "No sockets of type image, chart, pdf, mp4, thumbnail_jpg or thumbnail_png are used. At least one socket of those types is needed to configure the viewer. " 'Refer to the documentation for more information' ), ), @@ -468,7 +469,7 @@ def make_ci_list( 1, 1, ( - "The following interfaces are used in your {}: test-ci-isolated-0, test-ci-image-0, test-ci-overlay-0, and test-ci-undisplayable-0. " + "The following sockets are used in your {}: test-ci-isolated-0, test-ci-image-0, test-ci-overlay-0, and test-ci-undisplayable-0. " 'Example usage: {{"main": ["test-ci-isolated-0"], "secondary": ["test-ci-image-0", "test-ci-overlay-0"]}}. ' 'Refer to the documentation for more information' ), @@ -519,7 +520,7 @@ def test_archive_and_reader_study_forms_view_content_help_text( 0, 0, ( - "No interfaces of type image, chart, pdf, mp4, thumbnail_jpg or thumbnail_png are used. At least one interface of those types is needed to configure the viewer. " + "No sockets of type image, chart, pdf, mp4, thumbnail_jpg or thumbnail_png are used. At least one socket of those types is needed to configure the viewer. " 'Refer to the documentation for more information' ), ), @@ -529,8 +530,7 @@ def test_archive_and_reader_study_forms_view_content_help_text( 0, 1, ( - "The following interfaces are used in your algorithm: test-ci-overlay-0 and test-ci-undisplayable-0. " - "No interfaces of type image, chart, pdf, mp4, thumbnail_jpg or thumbnail_png are used. At least one interface of those types is needed to configure the viewer. " + "No sockets of type image, chart, pdf, mp4, thumbnail_jpg or thumbnail_png are used. At least one socket of those types is needed to configure the viewer. " 'Refer to the documentation for more information' ), ), @@ -540,7 +540,6 @@ def test_archive_and_reader_study_forms_view_content_help_text( 1, 1, ( - "The following interfaces are used in your algorithm: test-ci-isolated-0, test-ci-image-0, test-ci-overlay-0, and test-ci-undisplayable-0. " 'Example usage: {"main": ["test-ci-isolated-0"], "secondary": ["test-ci-image-0", "test-ci-overlay-0"]}. ' 'Refer to the documentation for more information' ), @@ -562,11 +561,18 @@ def test_algorithm_form_view_content_help_text( number_of_undisplayable_interfaces=number_of_undisplayable_interfaces, ) algorithm = AlgorithmFactory() - algorithm.inputs.set(ci_list) + if ci_list: + interface = AlgorithmInterfaceFactory( + inputs=ci_list[:-1], outputs=[ci_list[-1]] + ) + algorithm.interfaces.set([interface]) form = AlgorithmForm(user=UserFactory(), instance=algorithm) - assert form.fields["view_content"].help_text == expected_help_text + assert sorted( + [interface.pk for interface in algorithm.linked_component_interfaces] + ) == [interface.pk for interface in ci_list] + assert expected_help_text in form.fields["view_content"].help_text @pytest.mark.parametrize( @@ -578,7 +584,7 @@ def test_algorithm_form_view_content_help_text( 0, 0, ( - "No interfaces of type image, chart, pdf, mp4, thumbnail_jpg or thumbnail_png are used. At least one interface of those types is needed to configure the viewer. " + "No sockets of type image, chart, pdf, mp4, thumbnail_jpg or thumbnail_png are used. At least one socket of those types is needed to configure the viewer. " 'Refer to the documentation for more information' ), ), @@ -588,8 +594,7 @@ def test_algorithm_form_view_content_help_text( 0, 1, ( - "The following interfaces are used in your phase: test-ci-overlay-0 and test-ci-undisplayable-0. " - "No interfaces of type image, chart, pdf, mp4, thumbnail_jpg or thumbnail_png are used. At least one interface of those types is needed to configure the viewer. " + "No sockets of type image, chart, pdf, mp4, thumbnail_jpg or thumbnail_png are used. At least one socket of those types is needed to configure the viewer. " 'Refer to the documentation for more information' ), ), @@ -599,7 +604,6 @@ def test_algorithm_form_view_content_help_text( 1, 1, ( - "The following interfaces are used in your phase: test-ci-isolated-0, test-ci-image-0, test-ci-overlay-0, and test-ci-undisplayable-0. " 'Example usage: {"main": ["test-ci-isolated-0"], "secondary": ["test-ci-image-0", "test-ci-overlay-0"]}. ' 'Refer to the documentation for more information' ), @@ -621,14 +625,20 @@ def test_phase_update_form_view_content_help_text( number_of_undisplayable_interfaces=number_of_undisplayable_interfaces, ) phase = PhaseFactory() - phase.algorithm_inputs.set(ci_list) - phase.algorithm_outputs.set([]) + if ci_list: + interface = AlgorithmInterfaceFactory( + inputs=ci_list[:-1], outputs=[ci_list[-1]] + ) + phase.algorithm_interfaces.set([interface]) form = PhaseUpdateForm( user=UserFactory(), instance=phase, **{"challenge": phase.challenge} ) - assert form.fields["view_content"].help_text == expected_help_text + assert sorted( + [interface.pk for interface in phase.linked_component_interfaces] + ) == [interface.pk for interface in ci_list] + assert expected_help_text in form.fields["view_content"].help_text @pytest.mark.django_db @@ -642,14 +652,14 @@ def test_phase_update_form_view_content_help_text( None, ), ( - 1, + 2, 0, 0, - {"main": ["test-ci-image-0"]}, + {"main": ["test-ci-image-0"], "secondary": ["test-ci-image-1"]}, ), ( 0, - 1, + 2, 0, None, ), @@ -741,9 +751,13 @@ def test_generate_view_content_example( number_of_isolated_interfaces=number_of_isolated_interfaces, number_of_undisplayable_interfaces=0, ) - base_obj = AlgorithmFactory() - base_obj.inputs.set(ci_list) - form = AlgorithmForm(user=UserFactory(), instance=base_obj) + alg = AlgorithmFactory() + if ci_list: + interface = AlgorithmInterfaceFactory( + inputs=ci_list[:-1], outputs=[ci_list[-1]] + ) + alg.interfaces.set([interface]) + form = AlgorithmForm(user=UserFactory(), instance=alg) view_content_example = form.generate_view_content_example() view_content_example_json = ( diff --git a/app/tests/hanging_protocols_tests/test_models.py b/app/tests/hanging_protocols_tests/test_models.py index 9239366e4..91bba684a 100644 --- a/app/tests/hanging_protocols_tests/test_models.py +++ b/app/tests/hanging_protocols_tests/test_models.py @@ -497,7 +497,7 @@ def test_view_content_validation(): with pytest.raises(ValidationError) as err: hp.full_clean() - assert "Unknown interfaces in view content for viewport main: test" in str( + assert "Unknown sockets in view content for viewport main: test" in str( err.value ) @@ -553,7 +553,7 @@ def test_at_most_two_images(): hp.full_clean() assert ( - "Maximum of one image interface is allowed per viewport, got 2 for viewport main:" + "Maximum of one image socket is allowed per viewport, got 2 for viewport main:" in str(err.value) ) @@ -591,7 +591,7 @@ def test_interfaces_that_must_be_isolated(interface_kind): hp.full_clean() assert ( - "Some of the selected interfaces can only be displayed in isolation, found 2 for viewport main" + "Some of the selected sockets can only be displayed in isolation, found 2 for viewport main" in str(err.value) ) @@ -603,7 +603,7 @@ def test_interfaces_that_must_be_isolated(interface_kind): hp.full_clean() assert ( - "Some of the selected interfaces can only be displayed in isolation, found 1 for viewport main" + "Some of the selected sockets can only be displayed in isolation, found 1 for viewport main" in str(err.value) ) @@ -625,7 +625,7 @@ def test_interfaces_that_cannot_be_displayed(interface_kind): hp.full_clean() assert ( - "Some of the selected interfaces cannot be displayed, found 1 for viewport main:" + "Some of the selected sockets cannot be displayed, found 1 for viewport main:" in str(err.value) ) diff --git a/app/tests/reader_studies_tests/test_models.py b/app/tests/reader_studies_tests/test_models.py index d30400ee1..ffed75e1d 100644 --- a/app/tests/reader_studies_tests/test_models.py +++ b/app/tests/reader_studies_tests/test_models.py @@ -749,7 +749,7 @@ def test_question_interface(): q.clean() assert e.value.message == ( - f"The interface {ci_img} is not allowed for this " + f"The socket {ci_img} is not allowed for this " f"question type ({AnswerType.TEXT})" ) diff --git a/app/tests/serving_tests/test_views.py b/app/tests/serving_tests/test_views.py index 80ecd2d05..cfa9858da 100644 --- a/app/tests/serving_tests/test_views.py +++ b/app/tests/serving_tests/test_views.py @@ -10,7 +10,10 @@ ComponentInterface, ComponentInterfaceValue, ) -from tests.algorithms_tests.factories import AlgorithmJobFactory +from tests.algorithms_tests.factories import ( + AlgorithmInterfaceFactory, + AlgorithmJobFactory, +) from tests.archives_tests.factories import ArchiveFactory, ArchiveItemFactory from tests.cases_tests import RESOURCE_PATH from tests.evaluation_tests.factories import ( @@ -172,7 +175,8 @@ def has_correct_access(user_allowed, user_denied, url): # test algorithm job = AlgorithmJobFactory(creator=user1, time_limit=60) - job.algorithm_image.algorithm.outputs.add(detection_interface) + interface = AlgorithmInterfaceFactory(outputs=[detection_interface]) + job.algorithm_image.algorithm.interfaces.add(interface) job.outputs.add(output_civ) has_correct_access(user1, user2, job.outputs.first().file.url) diff --git a/app/tests/workstations_tests/test_session_control.py b/app/tests/workstations_tests/test_session_control.py index 3e7a7dc79..8aecfe950 100644 --- a/app/tests/workstations_tests/test_session_control.py +++ b/app/tests/workstations_tests/test_session_control.py @@ -17,6 +17,7 @@ class SessionCreationView(TemplateView): template_name = "new_session.html" +@pytest.mark.xfail(reason="Still to be addressed for optional inputs pitch") @pytest.mark.playwright def test_viewer_session_control(live_server, page, settings): settings.WORKSTATIONS_EXTRA_BROADCAST_DOMAINS = [live_server.url] diff --git a/scripts/algorithm_evaluation_fixtures.py b/scripts/algorithm_evaluation_fixtures.py index 84b58b22f..add970d87 100644 --- a/scripts/algorithm_evaluation_fixtures.py +++ b/scripts/algorithm_evaluation_fixtures.py @@ -9,7 +9,11 @@ from django.contrib.auth import get_user_model from django.core.files.base import ContentFile -from grandchallenge.algorithms.models import Algorithm, AlgorithmImage +from grandchallenge.algorithms.models import ( + Algorithm, + AlgorithmImage, + AlgorithmInterface, +) from grandchallenge.archives.models import Archive, ArchiveItem from grandchallenge.cases.models import Image, ImageFile from grandchallenge.challenges.models import Challenge @@ -122,9 +126,10 @@ def _create_challenge( p = Phase.objects.create( challenge=c, title="Phase 1", algorithm_time_limit=300 ) - - p.algorithm_inputs.set(inputs) - p.algorithm_outputs.set(outputs) + interface = AlgorithmInterface.objects.create( + inputs=inputs, outputs=outputs + ) + p.algorithm_interfaces.set([interface]) p.title = "Algorithm Evaluation" p.submission_kind = SubmissionKindChoices.ALGORITHM @@ -146,8 +151,10 @@ def _create_algorithm(*, creator, inputs, outputs, suffix): title=f"Test Algorithm Evaluation {suffix}", logo=create_uploaded_image(), ) - algorithm.inputs.set(inputs) - algorithm.outputs.set(outputs) + interface = AlgorithmInterface.objects.create( + inputs=inputs, outputs=outputs + ) + algorithm.interfaces.set([interface]) algorithm.add_editor(creator) algorithm_image = AlgorithmImage(creator=creator, algorithm=algorithm) diff --git a/scripts/cost_fixtures.py b/scripts/cost_fixtures.py index feffc6702..9786f3f45 100644 --- a/scripts/cost_fixtures.py +++ b/scripts/cost_fixtures.py @@ -5,7 +5,12 @@ from django.contrib.auth import get_user_model from django.utils.timezone import now -from grandchallenge.algorithms.models import Algorithm, AlgorithmImage, Job +from grandchallenge.algorithms.models import ( + Algorithm, + AlgorithmImage, + AlgorithmInterface, + Job, +) from grandchallenge.archives.models import Archive, ArchiveItem from grandchallenge.cases.models import Image, ImageFile from grandchallenge.challenges.models import Challenge @@ -134,9 +139,10 @@ def _create_challenge( c.add_participant(participant) p = Phase.objects.create(challenge=c, title="Phase 1") - - p.algorithm_inputs.set(inputs) - p.algorithm_outputs.set(outputs) + interface = AlgorithmInterface.objects.create( + inputs=inputs, outputs=outputs + ) + p.algorithm_interfaces.set([interface]) p.title = f"Type 2 {suffix}" p.submission_kind = SubmissionKindChoices.ALGORITHM @@ -158,8 +164,10 @@ def _create_algorithm(*, creator, inputs, outputs, suffix): title=f"Test Algorithm Type 2 {suffix}", logo=create_uploaded_image(), ) - algorithm.inputs.set(inputs) - algorithm.outputs.set(outputs) + interface = AlgorithmInterface.objects.create( + inputs=inputs, outputs=outputs + ) + algorithm.interfaces.set([interface]) algorithm.add_editor(creator) algorithm_image = AlgorithmImage(creator=creator, algorithm=algorithm) @@ -176,6 +184,7 @@ def _create_submission(algorithm, challenge, archive_items): for _ in range(archive_items): job = Job.objects.create( algorithm_image=ai, + algorithm_interface=algorithm.interfaces.first(), started_at=now() - timedelta(minutes=random.randint(5, 120)), completed_at=now(), status=Job.SUCCESS, diff --git a/scripts/development_fixtures.py b/scripts/development_fixtures.py index ddc706674..24f8f3faf 100644 --- a/scripts/development_fixtures.py +++ b/scripts/development_fixtures.py @@ -17,7 +17,12 @@ from knox.settings import CONSTANTS from machina.apps.forum.models import Forum -from grandchallenge.algorithms.models import Algorithm, AlgorithmImage, Job +from grandchallenge.algorithms.models import ( + Algorithm, + AlgorithmImage, + AlgorithmInterface, + Job, +) from grandchallenge.anatomy.models import BodyRegion, BodyStructure from grandchallenge.archives.models import Archive, ArchiveItem from grandchallenge.challenges.models import Challenge, ChallengeSeries @@ -356,12 +361,11 @@ def _create_algorithm_demo(users): ) algorithm.editors_group.user_set.add(users["algorithm"], users["demo"]) algorithm.users_group.user_set.add(users["algorithmuser"]) - algorithm.inputs.set( - [ComponentInterface.objects.get(slug="generic-medical-image")] - ) - algorithm.outputs.set( - [ComponentInterface.objects.get(slug="results-json-file")] + interface = AlgorithmInterface.objects.create( + inputs=[ComponentInterface.objects.get(slug="generic-medical-image")], + outputs=[ComponentInterface.objects.get(slug="results-json-file")], ) + algorithm.interfaces.set([interface]) algorithm_image = AlgorithmImage( creator=users["algorithm"], algorithm=algorithm @@ -380,6 +384,7 @@ def _create_algorithm_demo(users): algorithms_job = Job.objects.create( creator=users["algorithm"], algorithm_image=algorithm_image, + algorithm_interface=interface, status=Evaluation.SUCCESS, time_limit=60, requires_gpu_type=algorithm_image.algorithm.job_requires_gpu_type, diff --git a/scripts/external_algorithm_evaluation_fixtures.py b/scripts/external_algorithm_evaluation_fixtures.py index d554df088..88dd48392 100644 --- a/scripts/external_algorithm_evaluation_fixtures.py +++ b/scripts/external_algorithm_evaluation_fixtures.py @@ -4,6 +4,7 @@ from knox.models import AuthToken, hash_token from knox.settings import CONSTANTS +from grandchallenge.algorithms.models import AlgorithmInterface from grandchallenge.challenges.models import Challenge from grandchallenge.evaluation.models import Phase from grandchallenge.evaluation.utils import SubmissionKindChoices @@ -63,9 +64,10 @@ def _create_external_evaluation_phase( existing_phase = challenge.phase_set.get() p = Phase.objects.create(challenge=challenge, title="Phase 2") - - p.algorithm_inputs.set(inputs) - p.algorithm_outputs.set(outputs) + interface = AlgorithmInterface.objects.create( + inputs=inputs, outputs=outputs + ) + p.algorithm_interfaces.set([interface]) p.title = "External Algorithm Evaluation" p.submission_kind = SubmissionKindChoices.ALGORITHM diff --git a/scripts/pentest_fixtures.py b/scripts/pentest_fixtures.py index 3fa757a37..f3ccb626a 100644 --- a/scripts/pentest_fixtures.py +++ b/scripts/pentest_fixtures.py @@ -25,7 +25,12 @@ from PIL import Image as PILImage from PIL import ImageDraw, ImageFont -from grandchallenge.algorithms.models import Algorithm, AlgorithmImage, Job +from grandchallenge.algorithms.models import ( + Algorithm, + AlgorithmImage, + AlgorithmInterface, + Job, +) from grandchallenge.archives.models import Archive, ArchiveItem from grandchallenge.cases.models import Image, ImageFile from grandchallenge.challenges.models import Challenge @@ -202,8 +207,10 @@ def create_challenge(*, challenge_num, container, public): give_algorithm_editors_job_view_permissions=give_algorithm_editors_job_view_permissions, ) - p.algorithm_inputs.set(_get_inputs()) - p.algorithm_outputs.set(_get_outputs()) + interface = AlgorithmInterface.objects.create( + inputs=_get_inputs(), outputs=_get_outputs() + ) + p.algorithm_interfaces.set([interface]) m = Method(creator=admin, phase=p) @@ -426,8 +433,10 @@ def create_algorithm( public=public, ) - algorithm.inputs.set(inputs) - algorithm.outputs.set(outputs) + interface = AlgorithmInterface.objects.create( + inputs=inputs, outputs=outputs + ) + algorithm.interfaces.set([interface]) if editor is None: editor = _create_user(f"algorithm-{algorithm_num}", "editor") @@ -465,6 +474,7 @@ def create_algorithm( job = Job.objects.create( algorithm_image=algorithm_image, + algorithm_interface=interface, creator=user, public=public, comment=job_title,