Add |openneuro-py login" command to enable authentication (#74)

To download restricted datasets, we need to authenticate ourselves using
an API token. This PR adds a new command `openneuro login` to write the
API token to the `~/.openneuro` config file. The `openneuro-py download`
command has been modified to send the token along as a cookie.

fixes #60

Co-authored-by: Richard Höchenberger <[email protected]>

CHANGES.md

@@ -11,6 +11,7 @@
   entries.
 - Drop list of default excludes. OpenNeuro has fixed server response for the
   respective datasets, so excluding files by default is not necessary anymore.
+- Add ability to use an API token to access restricted datasets.
 
 ## 2022.1.0
 

README.md

@@ -94,3 +94,14 @@ openneuro-py download --dataset=ds000246 \
                       --include=sub-0001/meg/sub-0001_coordsystem.json \
                       --include=sub-0001/meg/sub-0001_acq-LPA_photo.jpg
 ```
+
+### Use an API token to log in
+
+To download private datasets, you will need an API key that grants you access
+permissions. Go to OpenNeuro.org, My Account → Obtain an API Key. Copy the key,
+and run:
+
+```shell
+openneuro-py login
+```
+Paste the API key and press return.

openneuro/__init__.py

@@ -10,4 +10,4 @@
     # package is not installed
     pass
 
-from .download import download  # noqa: F401
+from ._download import download, login  # noqa: F401

openneuro/download.py → openneuro/_download.py

@@ -33,7 +33,7 @@
 from sgqlc.endpoint.requests import RequestsEndpoint
 
 from . import __version__
-from .config import default_base_url
+from .config import BASE_URL, get_token, init_config
 
 
 if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding.lower() == 'utf-8':
@@ -45,6 +45,11 @@
     stdout_unicode = False
 
 
+def login():
+    """Login to OpenNeuro and store an access token."""
+    init_config()
+
+
 # HTTP server responses that indicate hopefully intermittent errors that
 # warrant a retry.
 allowed_retry_codes = (408, 500, 502, 503, 504, 522, 524)
@@ -113,6 +118,13 @@
 
 def _safe_query(query, *, timeout=None):
     with requests.Session() as session:
+        try:
+            token = get_token()
+            session.cookies.set_cookie(
+                requests.cookies.create_cookie('accessToken', token))
+            tqdm.write('🍪 Using API token to log in')
+        except ValueError:
+            pass  # No login
         gql_endpoint = RequestsEndpoint(
             url=gql_url, session=session, timeout=timeout)
         try:
@@ -155,7 +167,7 @@ def _check_snapshot_exists(*,
 
 
 def _get_download_metadata(*,
-                           base_url: str = default_base_url,
+                           base_url: str = BASE_URL,
                            dataset_id: str,
                            tag: Optional[str] = None,
                            tree: str = 'null',
@@ -204,8 +216,23 @@ def _get_download_metadata(*,
 
     if response_json is not None:
         if 'errors' in response_json:
-            raise RuntimeError(f'Query failed: '
-                               f'"{response_json["errors"][0]["message"]}"')
+            msg = response_json["errors"][0]["message"]
+            if msg == 'You do not have access to read this dataset.':
+                try:
+                    # Do we have an API token?
+                    get_token()
+                    raise RuntimeError('We were not permitted to download '
+                                       'this dataset. Perhaps your user '
+                                       'does not have access to it, or '
+                                       'your API token is wrong.')
+                except ValueError as e:
+                    # We don't have an API token.
+                    raise RuntimeError('It seems that this is a restricted '
+                                       'dataset. However, your API token is '
+                                       'not configured properly, so we could '
+                                       f'not log you in. {e}')
+            else:
+                raise RuntimeError(f'Query failed: "{msg}"')
         elif tag is None:
             return response_json['data']['dataset']['latestSnapshot']
         else:

openneuro/config.py

@@ -1,21 +1,46 @@
 from pathlib import Path
+import os
+import sys
+import stat
 import json
 import getpass
+if sys.version_info >= (3, 8):
+    from typing import TypedDict
+else:
+    from typing_extensions import TypedDict
 
+import appdirs
+from tqdm.auto import tqdm
 
-config_fname = Path('~/.openneuro').expanduser()
-default_base_url = 'https://openneuro.org/'
+
+CONFIG_DIR = Path(
+    appdirs.user_config_dir(appname='openneuro-py', appauthor=False, roaming=True)
+)
+CONFIG_DIR.mkdir(parents=True, exist_ok=True)
+CONFIG_PATH = CONFIG_DIR / 'config.json'
+BASE_URL = 'https://openneuro.org/'
+
+
+class Config(TypedDict):
+    endpoint: str
+    apikey: str
 
 
 def init_config() -> None:
     """Initialize a new OpenNeuro configuration file.
     """
+    tqdm.write('🙏 Please login to your OpenNeuro account and go to: '
+               'My Account → Obtain an API Key')
     api_key = getpass.getpass('OpenNeuro API key (input hidden): ')
-    config = dict(url=default_base_url,
-                  apikey=api_key,
-                  errorReporting=False)
-    with open(config_fname, 'w', encoding='utf-8') as f:
-        json.dump(config, f)
+
+    config: Config = {
+        'endpoint': BASE_URL,
+        'apikey': api_key,
+    }
+
+    with open(CONFIG_PATH, 'w', encoding='utf-8') as f:
+        json.dump(config, f, indent=2)
+    os.chmod(CONFIG_PATH, stat.S_IRUSR | stat.S_IWUSR)
 
 
 def load_config() -> dict:
@@ -26,6 +51,31 @@ def load_config() -> dict:
     dict
         The configuration options.
     """
-    with open(config_fname, 'r', encoding='utf-8') as f:
+    with open(CONFIG_PATH, 'r', encoding='utf-8') as f:
         config = json.load(f)
     return config
+
+
+def get_token() -> str:
+    """Get the OpenNeuro API token if configured with the 'login' command.
+
+    Returns
+    -------
+    The API token if configured.
+
+    Raises
+    ------
+    ValueError
+        When no token has been configured yet.
+    """
+    if not CONFIG_PATH.exists():
+        raise ValueError(
+            'Could not read API token as no openneuro-py configuration '
+            'file exists. Run "openneuro login" to generate it.'
+        )
+    config = load_config()
+    if 'apikey' not in config:
+        raise ValueError('An openneuro-py configuration file was found, but did not '
+                         'contain an "apikey" entry. Run "openneuro login" to '
+                         'add such an entry.')
+    return config['apikey']

openneuro/openneuro.py

@@ -7,7 +7,7 @@
 
 import click
 
-from .download import download_cli
+from ._download import login, download_cli
 from . import __version__
 
 
@@ -19,4 +19,11 @@ def cli() -> None:
     pass
 
 
+@click.command()
+def login_cli():
+    """Login to OpenNeuro and store an access token."""
+    login()
+
+
 cli.add_command(download_cli, name='download')
+cli.add_command(login_cli, name='login')

openneuro/tests/test_config.py

@@ -0,0 +1,19 @@
+from pathlib import Path
+from unittest import mock
+
+import openneuro
+from openneuro.config import init_config, load_config, get_token, Config
+
+
+def test_config(tmp_path: Path):
+    """Test creating and reading the config file."""
+    with mock.patch.object(openneuro.config, 'CONFIG_PATH', tmp_path / '.openneuro'):
+        assert not openneuro.config.CONFIG_PATH.exists()
+
+        with mock.patch('getpass.getpass', lambda _: 'test'):
+            init_config()
+        assert openneuro.config.CONFIG_PATH.exists()
+
+        expected_config = Config(endpoint='https://openneuro.org/', apikey='test')
+        assert load_config() == expected_config
+        assert get_token() == 'test'

openneuro/tests/test_download.py

@@ -2,6 +2,8 @@
 from pathlib import Path
 
 import pytest
+from unittest import mock
+import openneuro
 from openneuro import download
 
 
@@ -130,3 +132,19 @@ def test_doi_handling(tmp_path: Path):
         include=['participants.tsv'],
         target_dir=tmp_path
     )
+
+
+def test_restricted_dataset(tmp_path: Path):
+    """Test downloading a restricted dataset."""
+    # API token for dummy user [email protected]
+    token = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxOGNhNjE2ZS00OWQxLTRmOTUtODI1OS0xNzYwYzVhYjZjMDciLCJlbWFpbCI6ImFsaWpmbHNkdmJqaWVsc2Rsa2pmZWlsanN2akBnbWFpbC5jb20iLCJwcm92aWRlciI6Imdvb2dsZSIsIm5hbWUiOiJzZGZrbGVpamZsa3NkamYgc2xmZGRsa2phYWlmbCIsImFkbWluIjpmYWxzZSwiaWF0IjoxNjY1NDY4MjM4LCJleHAiOjE2OTcwMDQyMzh9.7YVL_Cagli84nTmumdcmrV1bW5hZMq3VJlMUDmTEpGU'  # noqa
+
+    with mock.patch.object(openneuro.config, 'CONFIG_PATH', tmp_path / '.openneuro'):
+        with mock.patch('getpass.getpass', lambda _: token):
+            openneuro.config.init_config()
+
+        # This is a restricted dataset that is only available if the API token
+        # was used correctly.
+        download(dataset='ds004287', target_dir=tmp_path)
+
+    assert (tmp_path / 'README').exists()

pyproject.toml

@@ -20,7 +20,8 @@ dependencies = [
     "aiofiles",
     "sgqlc",
     "importlib-metadata; python_version < '3.8'",
-    "typing-extensions; python_version < '3.8'"
+    "typing-extensions; python_version < '3.8'",
+    "appdirs",
 ]
 dynamic = ["version"]