[MAINT] use pytest-reporter-html1 to create test reports (nilearn#4444)

* use "pytest-reporter-html1" for creating reports

* add report template

* improve layout

* sort warning by types

* build report even on failure

* build report for test on codebase and not doc

* do not on doc

* reuse original template

* refactor template

.github/workflows/test_with_tox.yml

@@ -54,6 +54,12 @@ jobs:
             run: tox c
         -   name: Run test suite
             run: tox run -e ${{ matrix.env }} -- nilearn
+        -   name: Upload test report
+            if: success() || failure()
+            uses: actions/upload-artifact@v4
+            with:
+                name: ${{ matrix.os }}_${{ matrix.py }}_${{ matrix.description }}_report.html
+                path: report.html
         -   name: Upload coverage to CodeCov
             uses: codecov/codecov-action@v4
             with:

.gitignore

@@ -48,6 +48,7 @@ venv/
 
 # testing
 .pytest_cache
+report.html
 
 # CI benchmark
 maint_tools/*_runs_timing.*

.pre-commit-config.yaml

@@ -112,6 +112,11 @@ repos:
     hooks:
     -   id: prettier
         types_or: [css, html]
+        exclude: |
+            (?x)^(
+                maint_tools/templates/index.html
+            )$
+
 
 # Check that Python code complies with PEP8 guidelines
 # flake8 uses pydocstyle to check docstrings: https://flake8.pycqa.org/en/latest/

maint_tools/templates/index.html

@@ -0,0 +1,80 @@
+<!-- Print the warnings sorted by type first and then by filename -->
+
+{% extends "html1/index.html" %}
+<!-- URL of the extended template -->
+<!-- https://github.com/christiansandberg/pytest-reporter-html1/blob/master/pytest_reporter_html1/templates/html1/index.html -->
+
+{% block style %}
+  {{ super() }}
+
+  .warning-list {
+    margin-left: 24px;
+    margin-top: 12px;
+    widt: 90%;
+  }
+
+  #warnings ul {
+    padding: 0;
+    margin-left: 24px;
+    display: block;
+}
+
+  #warnings li {
+    display: block;
+    margin-bottom: 8px;
+  }
+
+{% endblock %}
+
+{% set title = "Nilearn test report" %}
+
+{% block warnings %}
+<div class="container">
+  <h1>Warnings</h1>
+  <details class="file">
+    <summary>
+      <h2 class="title file-title">
+        <span class="fspath"> WARNINGS </span>
+        <span class="counts">
+          <span title="{{ warnings|count }}"
+                class="status badge warning">
+            {{ warnings|count }}
+          </span>
+        </span>
+      </h2>
+    </summary>
+
+    {% set warning_types = warnings|map(attribute='category')|unique|list %}
+    {% for type in warning_types %}
+
+      {% set nb_warning = namespace(value=0) %}
+      {% for warning in warnings %}
+        {% if warning.category == type %}
+          {% set nb_warning.value = nb_warning.value + 1 %}
+        {% endif %}
+      {% endfor %}
+
+    <details class="warning-list">
+      <summary>
+        <div class="status">
+          {{ type.__name__  }}
+          <span class="badge warning counts">{{ nb_warning.value }}</span>
+        </div>
+      </summary>
+      <ul>
+        {% for warning in warnings|sort(attribute="filename") %}
+          {% if warning.category == type %}
+            <li>
+              <div>
+                <span class="filename">{{ warning.filename }}:{{ warning.lineno }}</span><br />
+                {{ warning.message }}
+              </div>
+            </li>
+          {% endif %}
+        {% endfor %}
+      </ul>
+    </details>
+    {% endfor %}
+  </details>
+</div>
+{% endblock %}

pyproject.toml

@@ -91,7 +91,8 @@ style = [
 test = [
     "coverage",
     "pytest>=6.0.0",
-    "pytest-cov"
+    "pytest-cov",
+    "pytest-reporter-html1>=0.9.0"
 ]
 
 [project.urls]
@@ -148,7 +149,7 @@ src_paths = [
 ]
 
 [tool.pytest.ini_options]
-addopts = "-ra --strict-config --strict-markers --doctest-modules --showlocals -s -vv --durations=0"
+addopts = "-ra --strict-config --strict-markers --doctest-modules --showlocals -s -vv --durations=0 --template=maint_tools/templates/index.html"
 doctest_optionflags = "NORMALIZE_WHITESPACE ELLIPSIS"
 junit_family = "xunit2"
 minversion = "6.0"

tox.ini

@@ -93,7 +93,7 @@ description = run tests on latest version of all dependencies (plotting not incl
 passenv = {[global_var]passenv}
 extras = test
 commands =
-    pytest --cov=nilearn --cov-report=xml {posargs:}
+    pytest --cov=nilearn --cov-report=xml --report=report.html {posargs:}
 
 [testenv:test_plotting]
 description = run tests on latest version of all dependencies
@@ -112,9 +112,9 @@ extras = test
 deps =
     {[plotting]deps}
 commands =
-	pytest doc/_additional_doctests.txt
+	pytest doc/_additional_doctests.txt --report=report_doc.html
     ; TODO find a way to rely on globbing instead of listing a specific folder
-	pytest --doctest-glob='*.rst' doc/manipulating_images/
+	pytest --doctest-glob='*.rst' doc/manipulating_images/  --report=report_doc.html
 
 [testenv:test_pre]
 description = run test_latest and test_doc on pre-release version of all dependencies