From df9ae989b01ee90f0e6541f5c4ca0ff9d5c38397 Mon Sep 17 00:00:00 2001
From: Philip Loche <ploche@physik.fu-berlin.de>
Date: Thu, 21 Sep 2023 18:56:21 +0200
Subject: [PATCH] Add coverage for Python torch bindings

---
 .codecov.yml                   | 12 ++++--------
 .github/workflows/coverage.yml | 20 ++++++++++++++++----
 .gitignore                     |  4 +++-
 CONTRIBUTING.rst               | 18 ++++++++++++++++++
 tox.ini                        | 27 +++++++++++++++++++++++----
 5 files changed, 64 insertions(+), 17 deletions(-)

diff --git a/.codecov.yml b/.codecov.yml
index 3b8bba5e6..94bab7aef 100644
--- a/.codecov.yml
+++ b/.codecov.yml
@@ -1,15 +1,11 @@
 coverage:
-  ignore:
-    - tests/.*
   status:
     project:
       default:
         target: 80%
     patch: off
+  ignore:
+    - "tests/.*"
+    - "examples/.*"
 
-comment: off
-
-fixes:
-  # map coverage collected inside tox virtual environments
-  # to the source dir in git
-  - ".tox/all-deps/lib/*/site-packages/::python/"
+comment: false
diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml
index 6949a6c0d..5858fe83e 100644
--- a/.github/workflows/coverage.yml
+++ b/.github/workflows/coverage.yml
@@ -38,7 +38,7 @@ jobs:
       - name: install python dependencies
         run: |
           python -m pip install --upgrade pip
-          python -m pip install tox
+          python -m pip install tox coverage
 
       - name: install lcov
         run: sudo apt install -y lcov
@@ -47,7 +47,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: .tox
-          key: tox-${{ matrix.os }}-${{ hashFiles('pyproject.toml', 'setup.cfg', 'tox.ini') }}
+          key: tox-${{ hashFiles('pyproject.toml', 'setup.cfg', 'tox.ini') }}
 
       - name: Setup sccache
         uses: mozilla-actions/sccache-action@v0.0.3
@@ -69,10 +69,22 @@ jobs:
           lcov --remove coverage.info '/usr/*' "$(pwd)/rascaline-c-api/tests/*" "$(pwd)/rascaline-c-api/examples/*" --output-file coverage.info
 
       - name: collect Python coverage
-        run: tox -e all-deps
+        run: |
+          tox -e all-deps
+          tox -e torch-tests
+        env:
+          # Use the CPU only version of torch when building/running the code
+          PIP_EXTRA_INDEX_URL: https://download.pytorch.org/whl/cpu
+
+      - name: combine Python coverage files
+        run: |
+            coverage combine \
+              ./.coverage \
+              ./python/rascaline-torch/.coverage
+            coverage xml
 
       - name: upload to codecov.io
         uses: codecov/codecov-action@v3
         with:
           fail_ci_if_error: true
-          files: target/tarpaulin/cobertura.xml,.tox/coverage.xml,coverage.info
+          files: target/tarpaulin/cobertura.xml,coverage.xml,coverage.info
diff --git a/.gitignore b/.gitignore
index 7bcc9343a..4f077544b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,10 +3,12 @@ target/
 Cargo.lock
 
 .tox/
-.coverage
 build/
 dist/
 *.egg-info
 __pycache__/
 .vscode
 *.DS_Store
+htmlcov/
+.coverage
+coverage.xml
diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
index 7b61cb772..05382c7db 100644
--- a/CONTRIBUTING.rst
+++ b/CONTRIBUTING.rst
@@ -176,6 +176,24 @@ of just testing it.
 .. _`cargo` : https://doc.rust-lang.org/cargo/
 .. _valgrind: https://valgrind.org/
 
+Inspecting Python code coverage
+-------------------------------
+
+The code coverage is reported at `codecov`_. You can also inspect the coverage locally.
+To get the full coverage first combine all reports and open produced html file in a
+browser
+
+.. code-block:: bash
+
+    tox
+    coverage combine \
+        ./.coverage \
+        ./python/rascaline-torch/.coverage
+    coverage html
+    firefox htmlcov/index.html
+
+.. _codecov: https://codecov.io/gh/lab-cosmo/metatensor
+
 Writing your own calculator
 ---------------------------
 
diff --git a/tox.ini b/tox.ini
index af80a8ed0..2eae7ef3d 100644
--- a/tox.ini
+++ b/tox.ini
@@ -4,8 +4,8 @@ min_version = 4.0
 # execute `tox` in the command-line without anything else
 envlist =
     lint
-    all-deps
     min-deps
+    all-deps
     docs-tests
     torch-tests
 
@@ -25,6 +25,7 @@ metatensor-torch-requirement =
     metatensor-torch @ git+https://github.com/lab-cosmo/metatensor@ad3eda5\#subdirectory=python/metatensor-torch
 
 build-single-wheel = --no-deps --no-build-isolation --check-build-dependencies
+test_options = --cov={env_site_packages_dir}/rascaline --cov-append --import-mode=append
 
 commands =
     # error if the user gives a wrong testenv name in `tox -e`
@@ -55,7 +56,7 @@ deps =
     pytest-cov
 
 commands =
-    pytest --cov={env_site_packages_dir}/rascaline --cov-report xml:.tox/coverage.xml --import-mode=append {posargs}
+    pytest {[testenv]test_options} {posargs}
 
 
 [testenv:min-deps]
@@ -63,9 +64,10 @@ commands =
 deps =
     {[testenv]metatensor-core-requirement}
     pytest
+    pytest-cov
 
 commands =
-    pytest --import-mode=append {posargs}
+    pytest {[testenv]test_options} {posargs}
 
 
 [testenv:torch-tests]
@@ -73,6 +75,7 @@ deps =
     {[testenv]metatensor-torch-requirement}
 
     pytest
+    pytest-cov
     numpy
     torch
     ase
@@ -85,7 +88,7 @@ commands =
     # install rascaline-torch
     pip install . {[testenv]build-single-wheel} --force-reinstall
     # run the unit tests
-    pytest --import-mode=append --assert=plain {posargs}
+    pytest {[testenv]test_options} --assert=plain {posargs}
 
 [testenv:docs]
 deps =
@@ -183,3 +186,19 @@ commands =
 [flake8]
 max_line_length = 88
 extend-ignore = E203
+
+[coverage:report]
+skip_covered = True
+show_missing = True
+omit =
+    tests/.*
+    examples/.*
+
+[coverage:paths]
+notorch =
+    python/rascaline/rascaline
+    .tox/*/lib/python*/site-packages/rascaline
+
+torch =
+    python/rascaline-torch/rascaline/torch
+    .tox/*/lib/python*/site-packages/rascaline/torch