From bd915ad5d9f317e586260dc1868b9a7de66c239c Mon Sep 17 00:00:00 2001
From: Daniel Weindl <dweindl@users.noreply.github.com>
Date: Mon, 26 Feb 2024 14:16:42 +0100
Subject: [PATCH] Cache remote files for tests (#2313)

Cache remote files for tests locally via pooch, and cache the pooch cache for GitHub actions.

Today, I had dozens of workflow failures due to rate-limiting or unavailability of biomodels. This is avoidable.
---
 .github/workflows/test_python_cplusplus.yml   | 14 ++++++++++
 python/sdist/setup.cfg                        |  1 +
 .../test_conserved_quantities_demartino.py    | 27 +++++++++----------
 python/tests/test_sbml_import.py              | 13 +++++----
 4 files changed, 34 insertions(+), 21 deletions(-)

diff --git a/.github/workflows/test_python_cplusplus.yml b/.github/workflows/test_python_cplusplus.yml
index fb90476eb8..58dd5480e5 100644
--- a/.github/workflows/test_python_cplusplus.yml
+++ b/.github/workflows/test_python_cplusplus.yml
@@ -18,6 +18,13 @@ jobs:
         python-version: [ "3.9" ]
 
     steps:
+    - name: Cache
+      uses: actions/cache@v3
+      with:
+        path: |
+          ~/.cache/pooch
+        key: ${{ runner.os }}-py${{ matrix.python-version }}-${{ github.job }}
+
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v5
       with:
@@ -265,6 +272,13 @@ jobs:
     runs-on: macos-latest
 
     steps:
+    - name: Cache
+      uses: actions/cache@v3
+      with:
+        path: |
+          ~/Library/Caches/pooch
+        key: ${{ runner.os }}-py${{ matrix.python-version }}-${{ github.job }}
+
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
diff --git a/python/sdist/setup.cfg b/python/sdist/setup.cfg
index 009d622a6b..0d27a0918e 100644
--- a/python/sdist/setup.cfg
+++ b/python/sdist/setup.cfg
@@ -62,6 +62,7 @@ test =
     #  unsupported x86_64 / x86_64h
     antimony!=2.14; platform_system=='Darwin' and platform_machine in 'x86_64h'
     scipy
+    pooch
 vis =
     matplotlib
     seaborn
diff --git a/python/tests/test_conserved_quantities_demartino.py b/python/tests/test_conserved_quantities_demartino.py
index 339743cc4e..ca40946db3 100644
--- a/python/tests/test_conserved_quantities_demartino.py
+++ b/python/tests/test_conserved_quantities_demartino.py
@@ -155,15 +155,15 @@
 def data_demartino2014():
     """Get tests from DeMartino2014 Suppl. Material"""
     import gzip
-    import io
-    import urllib.request
+    import pooch
 
     # stoichiometric matrix
-    response = urllib.request.urlopen(
-        r"https://github.com/AMICI-dev/AMICI/files/11430971/DeMartinoDe2014_test-ecoli.dat.gz",
-        timeout=10,
+    data = gzip.GzipFile(
+        pooch.retrieve(
+            "https://github.com/AMICI-dev/AMICI/files/11430971/DeMartinoDe2014_test-ecoli.dat.gz",
+            known_hash="md5:899873f8f1c413d13c3f8e94c1496b7e",
+        )
     )
-    data = gzip.GzipFile(fileobj=io.BytesIO(response.read()))
     S = [
         int(item)
         for sl in [
@@ -174,14 +174,13 @@ def data_demartino2014():
     ]
 
     # metabolite / row names
-    response = urllib.request.urlopen(
-        r"https://github.com/AMICI-dev/AMICI/files/11430970/test-ecoli-met.txt",
-        timeout=10,
-    )
-    row_names = [
-        entry.decode("ascii").strip() for entry in io.BytesIO(response.read())
-    ]
-
+    with open(
+        pooch.retrieve(
+            "https://github.com/AMICI-dev/AMICI/files/11430970/test-ecoli-met.txt",
+            known_hash="md5:d71e711a3655311390b38d00dcd6aa7f",
+        )
+    ) as f:
+        row_names = [entry.strip() for entry in f.readlines()]
     return S, row_names
 
 
diff --git a/python/tests/test_sbml_import.py b/python/tests/test_sbml_import.py
index aa343dfcc3..74a51d020a 100644
--- a/python/tests/test_sbml_import.py
+++ b/python/tests/test_sbml_import.py
@@ -3,7 +3,6 @@
 import re
 from numbers import Number
 from pathlib import Path
-from urllib.request import urlopen
 
 import amici
 import libsbml
@@ -543,13 +542,13 @@ def test_sympy_exp_monkeypatch():
     monkeypatching sympy.Pow._eval_derivative in order to be able to compute
     non-nan sensitivities
     """
-    url = (
-        "https://www.ebi.ac.uk/biomodels/model/download/BIOMD0000000529.2?"
-        "filename=BIOMD0000000529_url.xml"
-    )
-    importer = amici.SbmlImporter(
-        urlopen(url, timeout=20).read().decode("utf-8"), from_file=False
+    import pooch
+
+    model_file = pooch.retrieve(
+        url="https://www.ebi.ac.uk/biomodels/model/download/BIOMD0000000529.2?filename=BIOMD0000000529_url.xml",
+        known_hash="md5:c6e0b298397485b93d7acfab80b21fd4",
     )
+    importer = amici.SbmlImporter(model_file)
     module_name = "BIOMD0000000529"
 
     with TemporaryDirectory() as outdir: