handle pagination - needed for Stieger2021 ds

moabb/datasets/download.py

@@ -206,30 +206,39 @@ def fs_issue_request(method, url, headers, data=None, binary=False):
 
 def fs_get_file_list(article_id, version=None):
     """List all the files associated with a given article.
-
     Parameters
     ----------
     article_id : str or int
         Figshare article ID
     version : str or id, default is None
         Figshare article version. If None, selects the most recent version.
-
     Returns
     -------
     response : dict
         HTTP request response as a python dict
     """
     fsurl = "https://api.figshare.com/v2"
-    if version is None:
-        url = fsurl + "/articles/{}/files".format(article_id)
-        headers = {"Content-Type": "application/json"}
-        response = fs_issue_request("GET", url, headers=headers)
-        return response
-    else:
-        url = fsurl + "/articles/{}/versions/{}".format(article_id, version)
-        headers = {"Content-Type": "application/json"}
-        request = fs_issue_request("GET", url, headers=headers)
-        return request["files"]
+    all_files = []
+    page = 1
+
+    while True:
+        if version is None:
+            url = f"{fsurl}/articles/{article_id}/files?page={page}&page_size=100"
+            headers = {"Content-Type": "application/json"}
+            response = fs_issue_request("GET", url, headers=headers)
+
+            if not response:  # If response is empty, we've got all files
+                break
+
+            all_files.extend(response)
+            page += 1
+        else:
+            url = f"{fsurl}/articles/{article_id}/versions/{version}"
+            headers = {"Content-Type": "application/json"}
+            request = fs_issue_request("GET", url, headers=headers)
+            return request["files"]
+
+    return all_files
 
 
 def fs_get_file_hash(filelist):