Unify the container templates (#342)

Working with pretty much a duplicate of the container template directory
was annoying, so this PR unifies the templates by introducing a
`template_kind` option.

evalutils/cli.py

@@ -30,12 +30,12 @@ def init():
 def validate_python_module_name_fn(option):
     def validate_python_module_name_string(ctx, param, arg):
         if len(arg.strip()) == 0:
-            click.echo(f"{option.upper()} should be non empty. Aborting...")
-            exit(1)
+            raise click.BadParameter(f"{option.upper()} should be non empty")
 
         if not re.match(MODULE_REGEX, arg) or arg in FORBIDDEN_NAMES:
-            click.echo(f"ERROR: {arg!r} is not a valid Python module name!")
-            exit(1)
+            raise click.BadParameter(
+                f"{arg!r} is not a valid Python module name"
+            )
 
         return arg
 
@@ -79,14 +79,15 @@ def convert(self, value, param, ctx):
 )
 @click.option("--dev", is_flag=True)
 def init_evaluation(challenge_name, kind, dev):
-    template_dir = Path(__file__).parent / "templates" / "evaluation"
+    template_dir = Path(__file__).parent / "templates" / "container"
     try:
         cookiecutter(
             template=str(template_dir.absolute()),
             no_input=True,
             extra_context={
-                "challenge_name": challenge_name,
-                "challenge_kind": kind,
+                "full_project_name": challenge_name,
+                "task_kind": kind,
+                "template_kind": "Evaluation",
                 **_get_cookiecutter_base_context(dev_build=dev),
             },
         )
@@ -167,14 +168,15 @@ def req_gpu_prompt(ctx, param, req_gpu_count):
 )
 @click.option("--dev", is_flag=True)
 def init_algorithm(algorithm_name, kind, dev):
-    template_dir = Path(__file__).parent / "templates" / "algorithm"
+    template_dir = Path(__file__).parent / "templates" / "container"
     try:
         cookiecutter(
             template=str(template_dir.absolute()),
             no_input=True,
             extra_context={
-                "algorithm_name": algorithm_name,
-                "algorithm_kind": kind,
+                "full_project_name": algorithm_name,
+                "task_kind": kind,
+                "template_kind": "Algorithm",
                 **_get_cookiecutter_base_context(dev_build=dev),
             },
         )

evalutils/evalutils.py

@@ -36,7 +36,7 @@
 DEFAULT_INPUT_PATH = Path("/input/")
 DEFAULT_ALGORITHM_OUTPUT_IMAGES_PATH = Path("/output/images/")
 DEFAULT_ALGORITHM_OUTPUT_FILE_PATH = Path("/output/results.json")
-DEFAULT_GROUND_TRUTH_PATH = Path("/opt/evaluation/ground-truth/")
+DEFAULT_GROUND_TRUTH_PATH = Path("/opt/app/ground-truth/")
 DEFAULT_EVALUATION_OUTPUT_FILE_PATH = Path("/output/metrics.json")
 
 

evalutils/templates/algorithm/cookiecutter.json → evalutils/templates/container/cookiecutter.json

@@ -1,11 +1,15 @@
 {
-    "algorithm_name": "",
-    "algorithm_kind": [
+    "template_kind": [
+        "Algorithm",
+        "Evaluation"
+    ],
+    "full_project_name": "",
+    "task_kind": [
         "Classification",
         "Segmentation",
         "Detection"
     ],
-    "package_name": "{{ cookiecutter.algorithm_name|replace(' ', '') }}",
+    "package_name": "{{ cookiecutter.full_project_name|replace(' ', '') }}",
     "evalutils_version": "",
     "dev_build": 0,
     "python_major_version": "",

evalutils/templates/container/hooks/post_gen_project.py

@@ -0,0 +1,77 @@
+import os
+import shutil
+from pathlib import Path
+
+from evalutils.utils import (
+    convert_line_endings,
+    generate_requirements_txt,
+    generate_source_wheel,
+)
+
+TASK_KIND = "{{ cookiecutter.task_kind }}"
+TEMPLATE_KIND = "{{ cookiecutter.template_kind }}"
+IS_DEV_BUILD = int("{{ cookiecutter.dev_build }}") == 1
+
+template_dir = Path(os.getcwd())
+
+templated_files = template_dir.glob("*.j2")
+for f in templated_files:
+    shutil.move(f.name, f.stem)
+
+if TEMPLATE_KIND == "Evaluation":  # noqa: C901
+    shutil.rmtree("algorithm_test")
+    os.remove("process.py")
+    os.rename("evaluation_test", "test")
+
+    def remove_classification_files():
+        os.remove(Path("ground-truth") / "reference.csv")
+        os.remove(Path("test") / "submission.csv")
+
+    def remove_segmentation_files():
+        files = []
+        for ext in ["mhd", "zraw"]:
+            files.extend(Path(".").glob(f"**/*.{ext}"))
+
+        for file in files:
+            os.remove(str(file))
+
+    def remove_detection_files():
+        os.remove(Path("ground-truth") / "detection-reference.csv")
+        os.remove(Path("test") / "detection-submission.csv")
+
+    if TASK_KIND.lower() != "segmentation":
+        remove_segmentation_files()
+
+    if TASK_KIND.lower() != "detection":
+        remove_detection_files()
+
+    if TASK_KIND.lower() != "classification":
+        remove_classification_files()
+
+elif TEMPLATE_KIND == "Algorithm":
+    shutil.rmtree("evaluation_test")
+    shutil.rmtree("ground-truth")
+    os.remove("evaluation.py")
+    os.rename("algorithm_test", "test")
+
+    template_test_dir = template_dir / "test"
+
+    def remove_result_files():
+        for task_kind in ["segmentation", "detection", "classification"]:
+            os.remove(template_test_dir / f"results_{task_kind}.json")
+
+    expected_output_file = (
+        template_test_dir / f"results_{TASK_KIND.lower()}.json"
+    )
+
+    shutil.copy(
+        str(expected_output_file), template_test_dir / "expected_output.json"
+    )
+
+    remove_result_files()
+
+if IS_DEV_BUILD:
+    generate_source_wheel(template_dir / "vendor")
+
+generate_requirements_txt()
+convert_line_endings()

evalutils/templates/algorithm/{{ cookiecutter.package_name }}/.github/workflows/ci.yml → evalutils/templates/container/{{ cookiecutter.package_name }}/.github/workflows/ci.yml

@@ -7,7 +7,7 @@ env:
 
 jobs:
 
-  algorithm-tests:
+  tests:
     runs-on: ubuntu-latest
     steps:
       - name: Install Python ${{ "{{" }} env.PYTHON_VERSION {{ "}}" }}

evalutils/templates/container/{{ cookiecutter.package_name }}/Dockerfile

@@ -0,0 +1,33 @@
+FROM python:{{ cookiecutter.python_major_version }}.{{ cookiecutter.python_minor_version }}-slim
+
+RUN groupadd -r user && useradd -m --no-log-init -r -g user user
+
+RUN mkdir -p /opt/app /input /output \
+    && chown user:user /opt/app /input /output
+
+USER user
+WORKDIR /opt/app
+
+ENV PATH="/home/user/.local/bin:${PATH}"
+
+RUN python -m pip install --user -U pip && python -m pip install --user pip-tools
+
+{% if cookiecutter.dev_build|int -%}
+COPY --chown=user:user vendor /opt/app/vendor
+{%- endif %}
+
+COPY --chown=user:user requirements.txt /opt/app/
+RUN python -m piptools sync requirements.txt
+
+{% if cookiecutter.template_kind == "Evaluation" -%}
+COPY --chown=user:user ground-truth /opt/app/ground-truth
+COPY --chown=user:user evaluation.py /opt/app/
+
+ENTRYPOINT [ "python", "-m", "evaluation" ]
+{%- endif %}
+
+{% if cookiecutter.template_kind == "Algorithm" -%}
+COPY --chown=user:user process.py /opt/app/
+
+ENTRYPOINT [ "python", "-m", "process" ]
+{%- endif %}

evalutils/templates/container/{{ cookiecutter.package_name }}/README.md

@@ -0,0 +1,6 @@
+# {{ cookiecutter.full_project_name }} {{ cookiecutter.template_kind }}
+
+The source code for the {{ cookiecutter.template_kind.lower() }} container for
+{{ cookiecutter.full_project_name }}, generated with
+evalutils version {{ cookiecutter.evalutils_version }}
+using Python {{ cookiecutter.python_major_version }}.{{ cookiecutter.python_minor_version }}.

evalutils/templates/evaluation/{{ cookiecutter.package_name }}/evaluation.py.j2 → evalutils/templates/container/{{ cookiecutter.package_name }}/evaluation.py.j2

@@ -1,20 +1,20 @@
-{%- if cookiecutter.challenge_kind == "Classification" -%}
+{%- if cookiecutter.task_kind == "Classification" -%}
 from sklearn.metrics import accuracy_score
 
 from evalutils import ClassificationEvaluation
 from evalutils.io import CSVLoader
 from evalutils.validators import (
     NumberOfCasesValidator, ExpectedColumnNamesValidator
 )
-{%- elif cookiecutter.challenge_kind == "Segmentation" -%}
+{%- elif cookiecutter.task_kind == "Segmentation" -%}
 import SimpleITK
 
 from evalutils import ClassificationEvaluation
 from evalutils.io import SimpleITKLoader
 from evalutils.validators import (
     NumberOfCasesValidator, UniquePathIndicesValidator, UniqueImagesValidator
 )
-{%- elif cookiecutter.challenge_kind == "Detection" -%}
+{%- elif cookiecutter.task_kind == "Detection" -%}
 from evalutils import DetectionEvaluation
 from evalutils.io import CSVLoader
 from evalutils.validators import ExpectedColumnNamesValidator
@@ -23,14 +23,14 @@ from evalutils.validators import ExpectedColumnNamesValidator
 
 class {{ cookiecutter.package_name|capitalize }}(
 
-{%- if cookiecutter.challenge_kind == "Detection" -%}
+{%- if cookiecutter.task_kind == "Detection" -%}
     DetectionEvaluation
 {%- else -%}
     ClassificationEvaluation
 {%- endif -%}
 
 ):
-{%- if cookiecutter.challenge_kind == "Classification" %}
+{%- if cookiecutter.task_kind == "Classification" %}
     def __init__(self):
         super().__init__(
             file_loader=CSVLoader(),
@@ -48,7 +48,7 @@ class {{ cookiecutter.package_name|capitalize }}(
                 self._cases["class_prediction"],
              ),
         }
-{% elif cookiecutter.challenge_kind == "Detection" %}
+{% elif cookiecutter.task_kind == "Detection" %}
     def __init__(self):
         super().__init__(
             file_loader=CSVLoader(),
@@ -78,7 +78,7 @@ class {{ cookiecutter.package_name|capitalize }}(
             for _, p in points.iterrows()
             if p["score"] > self._detection_threshold
         ]
-{% elif cookiecutter.challenge_kind == "Segmentation" %}
+{% elif cookiecutter.task_kind == "Segmentation" %}
     def __init__(self):
         super().__init__(
             file_loader=SimpleITKLoader(),

evalutils/templates/algorithm/{{ cookiecutter.package_name }}/process.py.j2 → evalutils/templates/container/{{ cookiecutter.package_name }}/process.py.j2

@@ -1,20 +1,20 @@
-{% if cookiecutter.algorithm_kind == "Classification" %}
+{% if cookiecutter.task_kind == "Classification" %}
 from typing import Dict
 {% endif %}
 import SimpleITK
 import numpy as np
-{% if cookiecutter.algorithm_kind == "Detection" %}
+{% if cookiecutter.task_kind == "Detection" %}
 from pandas import DataFrame
 from scipy.ndimage import center_of_mass, label
 {% endif %}
-from evalutils import {{ cookiecutter.algorithm_kind }}Algorithm
+from evalutils import {{ cookiecutter.task_kind }}Algorithm
 from evalutils.validators import (
     UniquePathIndicesValidator,
     UniqueImagesValidator,
 )
 
 
-class {{ cookiecutter.package_name|capitalize }}({{ cookiecutter.algorithm_kind }}Algorithm):
+class {{ cookiecutter.package_name|capitalize }}({{ cookiecutter.task_kind }}Algorithm):
     def __init__(self):
         super().__init__(
             validators=dict(
@@ -24,7 +24,7 @@ class {{ cookiecutter.package_name|capitalize }}({{ cookiecutter.algorithm_kind
                 )
             ),
         )
-{% if cookiecutter.algorithm_kind == "Detection" %}
+{% if cookiecutter.task_kind == "Detection" %}
     def predict(self, *, input_image: SimpleITK.Image) -> DataFrame:
         # Extract a numpy array with image data from the SimpleITK Image
         image_data = SimpleITK.GetArrayFromImage(input_image)
@@ -52,13 +52,13 @@ class {{ cookiecutter.package_name|capitalize }}({{ cookiecutter.algorithm_kind
 
         # Convert serialized candidates to a pandas.DataFrame
         return DataFrame(data)
-{% elif cookiecutter.algorithm_kind == "Segmentation" %}
+{% elif cookiecutter.task_kind == "Segmentation" %}
     def predict(self, *, input_image: SimpleITK.Image) -> SimpleITK.Image:
         # Segment all values greater than 2 in the input image
         return SimpleITK.BinaryThreshold(
             image1=input_image, lowerThreshold=2, insideValue=1, outsideValue=0
         )
-{% elif cookiecutter.algorithm_kind == "Classification" %}
+{% elif cookiecutter.task_kind == "Classification" %}
     def predict(self, *, input_image: SimpleITK.Image) -> Dict:
         # Checks if there are any nodules voxels (> 1) in the input image
         return dict(