From 8fbec8e9b414588157873c970cf02a2d66e4da2f Mon Sep 17 00:00:00 2001 From: poldrack Date: Thu, 18 Jul 2019 15:39:25 -0700 Subject: [PATCH 1/5] create README.rst, fix file saving --- README.rst | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 README.rst diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..9eed581 --- /dev/null +++ b/README.rst @@ -0,0 +1,26 @@ +NeuroVault Collection Downloader +================================ + +CLI tool for bulk downloads of NeuroVault data. + +* Free software: Apache Software License 2.0 + + +Installation +------------ + +`pip install git+https://github.com/NeuroVault/neurovault_collection_downloader.git` + +Usage +----- + +1. Create a text file with one collection ID (a number or string of capital letters in case of private collection) per line. +2. `neurovault_collection_downloader collections.txt /output_directory` + +Credits +------- + +This package was created with Cookiecutter_ and the `audreyr/cookiecutter-pypackage`_ project template. + +.. _Cookiecutter: https://github.com/audreyr/cookiecutter +.. _`audreyr/cookiecutter-pypackage`: https://github.com/audreyr/cookiecutter-pypackage From 75e272521655648072746430a1f640b624c37067 Mon Sep 17 00:00:00 2001 From: poldrack Date: Thu, 18 Jul 2019 15:49:12 -0700 Subject: [PATCH 2/5] updated to latest versions --- requirements_dev.txt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/requirements_dev.txt b/requirements_dev.txt index d977c87..19f6f0b 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,11 +1,11 @@ -pip==18.1 +pip==19.1.1 bumpversion==0.5.3 -wheel==0.32.1 +wheel==0.33.4 watchdog==0.9.0 -flake8==3.5.0 -tox==3.5.2 -coverage==4.5.1 -Sphinx==1.8.1 -twine==1.12.1 +flake8==3.7.8 +tox==3.13.2 +coverage==4.5.3 +Sphinx==2.1.2 +twine==1.13.1 From 512168d970445944364f501537cd0ccccd877ffd Mon Sep 17 00:00:00 2001 From: poldrack Date: Wed, 31 Jul 2019 08:31:14 -0700 Subject: [PATCH 3/5] add alt folder flag: --- neurovault_collection_downloader/cli.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/neurovault_collection_downloader/cli.py b/neurovault_collection_downloader/cli.py index 16b50ee..29e80e3 100644 --- a/neurovault_collection_downloader/cli.py +++ b/neurovault_collection_downloader/cli.py @@ -15,7 +15,7 @@ def mkdir_p(path): pass else: raise -def fetch_collection(col_id, destination_folder): +def fetch_collection(col_id, destination_folder, alt_label=None): images_url_template = "http://neurovault.org/api/collections/%s/images/" next_url = images_url_template%col_id images = [] @@ -26,13 +26,19 @@ def fetch_collection(col_id, destination_folder): images += data['results'] next_url = data['next'] - col_folder = os.path.join(destination_folder, "%s"%col_id) + if alt_label is not None: + col_folder = os.path.join(destination_folder, alt_label) + else: + col_folder = os.path.join(destination_folder, "%s"%col_id) + mkdir_p(col_folder) json.dump(images, open(os.path.join(col_folder, "images.json"), "w"), indent=4, sort_keys=True) for image in images: click.echo("fetching %s"%image['file']) - urllib.request.urlretrieve(image['file'], os.path.join(col_folder, str(image['id']) + ".nii.gz")) + fname = os.path.basename(image['file']) + print("saving",image['file'],"to",os.path.join(col_folder, fname)) + urllib.request.urlretrieve(image['file'], os.path.join(col_folder, fname)) @click.command() From f99ade008d232e99e11c5d5785bbf71399379adf Mon Sep 17 00:00:00 2001 From: poldrack Date: Wed, 31 Jul 2019 09:31:26 -0700 Subject: [PATCH 4/5] add exclude_tag --- neurovault_collection_downloader/cli.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/neurovault_collection_downloader/cli.py b/neurovault_collection_downloader/cli.py index 29e80e3..b028743 100644 --- a/neurovault_collection_downloader/cli.py +++ b/neurovault_collection_downloader/cli.py @@ -15,7 +15,8 @@ def mkdir_p(path): pass else: raise -def fetch_collection(col_id, destination_folder, alt_label=None): +def fetch_collection(col_id, destination_folder, + alt_label=None, exclude_tag=None): images_url_template = "http://neurovault.org/api/collections/%s/images/" next_url = images_url_template%col_id images = [] @@ -37,6 +38,9 @@ def fetch_collection(col_id, destination_folder, alt_label=None): for image in images: click.echo("fetching %s"%image['file']) fname = os.path.basename(image['file']) + if exclude_tag is not None: + if fname.find(exclude_tag)>-1: + continue print("saving",image['file'],"to",os.path.join(col_folder, fname)) urllib.request.urlretrieve(image['file'], os.path.join(col_folder, fname)) From f733fd88460230fc39b815574050082f319d7cb7 Mon Sep 17 00:00:00 2001 From: poldrack Date: Thu, 1 Aug 2019 17:48:19 -0700 Subject: [PATCH 5/5] fix exclude_tag --- neurovault_collection_downloader/cli.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/neurovault_collection_downloader/cli.py b/neurovault_collection_downloader/cli.py index b028743..eb6bae2 100644 --- a/neurovault_collection_downloader/cli.py +++ b/neurovault_collection_downloader/cli.py @@ -17,6 +17,9 @@ def mkdir_p(path): def fetch_collection(col_id, destination_folder, alt_label=None, exclude_tag=None): + if exclude_tag is not None: + if not isinstance(exclude_tag, list): + exclude_tag = [exclude_tag] images_url_template = "http://neurovault.org/api/collections/%s/images/" next_url = images_url_template%col_id images = [] @@ -33,16 +36,24 @@ def fetch_collection(col_id, destination_folder, col_folder = os.path.join(destination_folder, "%s"%col_id) mkdir_p(col_folder) - json.dump(images, open(os.path.join(col_folder, "images.json"), "w"), + json.dump(images, + open(os.path.join(col_folder, "images.json"), "w"), indent=4, sort_keys=True) for image in images: click.echo("fetching %s"%image['file']) fname = os.path.basename(image['file']) + exclude_img = False if exclude_tag is not None: - if fname.find(exclude_tag)>-1: - continue - print("saving",image['file'],"to",os.path.join(col_folder, fname)) - urllib.request.urlretrieve(image['file'], os.path.join(col_folder, fname)) + for et in exclude_tag: + if fname.find(et)>-1: + exclude_img = True + if not exclude_img: + print("saving", + image['file'],"to", + os.path.join(col_folder, fname)) + urllib.request.urlretrieve( + image['file'], + os.path.join(col_folder, fname)) @click.command()