From 9bb6ec20f70b7469b9fcd5e054e612f6ebfb7f71 Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Fri, 17 Sep 2021 06:24:08 +0530 Subject: [PATCH 01/11] Added google drive file source --- lib/galaxy/files/sources/googledrive.py | 28 +++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 lib/galaxy/files/sources/googledrive.py diff --git a/lib/galaxy/files/sources/googledrive.py b/lib/galaxy/files/sources/googledrive.py new file mode 100644 index 000000000000..ebb4b6756792 --- /dev/null +++ b/lib/galaxy/files/sources/googledrive.py @@ -0,0 +1,28 @@ +import json +try: + from fs.googledrivefs import GoogleDriveFS + from google.oauth2.credentials import Credentials +except ImportError: + GoogleDriveFS = None + +from ._pyfilesystem2 import PyFilesystem2FilesSource + + +class GoogleDriveFilesSource(PyFilesystem2FilesSource): + plugin_type = 'googledrive' + required_module = GoogleDriveFS + required_package = "fs.googledrivefs" + + def _open_fs(self, user_context): + props = self._serialization_props(user_context) + authorized_user_info = json.loads(props.get('credentials') or {}) + credentials = Credentials(authorized_user_info['access_token'], + refresh_token=authorized_user_info['refresh_token'], + token_uri="https://www.googleapis.com/oauth2/v4/token", + client_id=authorized_user_info['client_id'], + client_secret=authorized_user_info['client_secret']) + handle = GoogleDriveFS(credentials) + return handle + + +__all__ = ('GoogleDriveFilesSource',) From ae8a832b7b79b06c0721394eb98fe1df9841f30d Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Fri, 17 Sep 2021 06:24:32 +0530 Subject: [PATCH 02/11] Added unit tests for google drive file source --- test/unit/files/_util.py | 1 + test/unit/files/dropbox_file_sources_conf.yml | 2 +- .../files/googledrive_file_sources_conf.yml | 4 ++ test/unit/files/test_googledrive.py | 37 +++++++++++++++++++ 4 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 test/unit/files/googledrive_file_sources_conf.yml create mode 100644 test/unit/files/test_googledrive.py diff --git a/test/unit/files/_util.py b/test/unit/files/_util.py index eca9cf1fe708..881f986df6c6 100644 --- a/test/unit/files/_util.py +++ b/test/unit/files/_util.py @@ -55,6 +55,7 @@ def user_context_fixture(user_ftp_dir=None, role_names=None, group_names=None, i preferences={ 'webdav|password': 'secret1234', 'dropbox|access_token': os.environ.get('GALAXY_TEST_DROPBOX_ACCESS_TOKEN'), + 'googledrive|credentials_json': os.environ.get('GALAXY_TEST_GOOGLE_DRIVE_CREDENTIALS_JSON') }, role_names=role_names or set(), group_names=group_names or set(), diff --git a/test/unit/files/dropbox_file_sources_conf.yml b/test/unit/files/dropbox_file_sources_conf.yml index 9dac354b982d..99869accff89 100644 --- a/test/unit/files/dropbox_file_sources_conf.yml +++ b/test/unit/files/dropbox_file_sources_conf.yml @@ -1,4 +1,4 @@ - type: dropbox id: test1 - doc: Test WebDAV server. + doc: Test access to a dropbox account. accessToken: ${user.preferences['dropbox|access_token']} diff --git a/test/unit/files/googledrive_file_sources_conf.yml b/test/unit/files/googledrive_file_sources_conf.yml new file mode 100644 index 000000000000..8de9791dc674 --- /dev/null +++ b/test/unit/files/googledrive_file_sources_conf.yml @@ -0,0 +1,4 @@ +- type: googledrive + id: test1 + doc: Test access to a Google drive. + credentials: ${user.preferences['googledrive|credentials_json']} diff --git a/test/unit/files/test_googledrive.py b/test/unit/files/test_googledrive.py new file mode 100644 index 000000000000..831b6e6407f8 --- /dev/null +++ b/test/unit/files/test_googledrive.py @@ -0,0 +1,37 @@ +import os + +import pytest + +from galaxy.files import ConfiguredFileSources, ConfiguredFileSourcesConfig +from ._util import ( + assert_realizes_as, + find_file_a, + user_context_fixture, +) +SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__)) +FILE_SOURCES_CONF = os.path.join(SCRIPT_DIRECTORY, "googledrive_file_sources_conf.yml") + +skip_if_no_google_drive_access_token = pytest.mark.skipif( + not os.environ.get('GALAXY_TEST_GOOGLE_DRIVE_CREDENTIALS_JSON'), + reason="GALAXY_TEST_GOOGLE_DRIVE_CREDENTIALS_JSON not set" +) + + +@skip_if_no_google_drive_access_token +def test_file_source(): + user_context = user_context_fixture() + file_sources = _configured_file_sources() + file_source_pair = file_sources.get_file_source_path("gxfiles://test1") + + assert file_source_pair.path == "/" + file_source = file_source_pair.file_source + res = file_source.list("/", recursive=True, user_context=user_context) + a_file = find_file_a(res) + assert a_file + + assert_realizes_as(file_sources, "gxfiles://test1/a", "a\n", user_context=user_context) + + +def _configured_file_sources(conf_file=FILE_SOURCES_CONF): + file_sources_config = ConfiguredFileSourcesConfig() + return ConfiguredFileSources(file_sources_config, conf_file=conf_file) From 482fb2827ca2ed25d06abafc4269dcaea7fce81c Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Fri, 17 Sep 2021 06:31:10 +0530 Subject: [PATCH 03/11] Added google drive to conditional requirements --- lib/galaxy/dependencies/__init__.py | 3 +++ lib/galaxy/dependencies/conditional-requirements.txt | 1 + 2 files changed, 4 insertions(+) diff --git a/lib/galaxy/dependencies/__init__.py b/lib/galaxy/dependencies/__init__.py index 27a9ed53c8d9..68af3f95f178 100644 --- a/lib/galaxy/dependencies/__init__.py +++ b/lib/galaxy/dependencies/__init__.py @@ -232,6 +232,9 @@ def check_s3fs(self): # use s3fs directly (skipping pyfilesystem) for direct access to more options return 's3fs' in self.file_sources + def check_fs_googledrivefs(self): + return 'googledrive' in self.file_sources + def check_watchdog(self): install_set = {'auto', 'True', 'true', 'polling', True} return (self.config['watch_tools'] in install_set diff --git a/lib/galaxy/dependencies/conditional-requirements.txt b/lib/galaxy/dependencies/conditional-requirements.txt index 37e72d0f4686..92a396ef2a09 100644 --- a/lib/galaxy/dependencies/conditional-requirements.txt +++ b/lib/galaxy/dependencies/conditional-requirements.txt @@ -20,6 +20,7 @@ fs.sshfs # type: ssh fs-s3fs # type: s3 s3fs # type: s3fs fs.anvilfs # type: anvil +fs.googledrivefs # type: googledrive # Chronos client chronos-python==1.2.1 From 71862f540c2816c8d1b454824ab810aa611fe7f5 Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Fri, 17 Sep 2021 07:07:52 +0530 Subject: [PATCH 04/11] Use separate env vars for each required param in google drive credentials --- lib/galaxy/files/sources/googledrive.py | 8 +------- test/unit/files/_util.py | 5 ++++- test/unit/files/googledrive_file_sources_conf.yml | 6 +++++- test/unit/files/test_googledrive.py | 7 +++++-- 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/lib/galaxy/files/sources/googledrive.py b/lib/galaxy/files/sources/googledrive.py index ebb4b6756792..1c8da18849c7 100644 --- a/lib/galaxy/files/sources/googledrive.py +++ b/lib/galaxy/files/sources/googledrive.py @@ -1,4 +1,3 @@ -import json try: from fs.googledrivefs import GoogleDriveFS from google.oauth2.credentials import Credentials @@ -15,12 +14,7 @@ class GoogleDriveFilesSource(PyFilesystem2FilesSource): def _open_fs(self, user_context): props = self._serialization_props(user_context) - authorized_user_info = json.loads(props.get('credentials') or {}) - credentials = Credentials(authorized_user_info['access_token'], - refresh_token=authorized_user_info['refresh_token'], - token_uri="https://www.googleapis.com/oauth2/v4/token", - client_id=authorized_user_info['client_id'], - client_secret=authorized_user_info['client_secret']) + credentials = Credentials(**props) handle = GoogleDriveFS(credentials) return handle diff --git a/test/unit/files/_util.py b/test/unit/files/_util.py index 881f986df6c6..3566643d2fef 100644 --- a/test/unit/files/_util.py +++ b/test/unit/files/_util.py @@ -55,7 +55,10 @@ def user_context_fixture(user_ftp_dir=None, role_names=None, group_names=None, i preferences={ 'webdav|password': 'secret1234', 'dropbox|access_token': os.environ.get('GALAXY_TEST_DROPBOX_ACCESS_TOKEN'), - 'googledrive|credentials_json': os.environ.get('GALAXY_TEST_GOOGLE_DRIVE_CREDENTIALS_JSON') + 'googledrive|access_token': os.environ.get('GALAXY_TEST_GOOGLE_DRIVE_ACCESS_TOKEN'), + 'googledrive|refresh_token': os.environ.get('GALAXY_TEST_GOOGLE_DRIVE_REFRESH_TOKEN'), + 'googledrive|client_id': os.environ.get('GALAXY_TEST_GOOGLE_DRIVE_CLIENT_ID'), + 'googledrive|client_secret': os.environ.get('GALAXY_TEST_GOOGLE_DRIVE_CLIENT_SECRET'), }, role_names=role_names or set(), group_names=group_names or set(), diff --git a/test/unit/files/googledrive_file_sources_conf.yml b/test/unit/files/googledrive_file_sources_conf.yml index 8de9791dc674..809238465773 100644 --- a/test/unit/files/googledrive_file_sources_conf.yml +++ b/test/unit/files/googledrive_file_sources_conf.yml @@ -1,4 +1,8 @@ - type: googledrive id: test1 doc: Test access to a Google drive. - credentials: ${user.preferences['googledrive|credentials_json']} + token: ${user.preferences['googledrive|access_token']} + refresh_token: ${user.preferences['googledrive|refresh_token']} + token_uri: "https://www.googleapis.com/oauth2/v4/token" + client_id: ${user.preferences['googledrive|client_id']} + client_secret: ${user.preferences['googledrive|client_secret']} diff --git a/test/unit/files/test_googledrive.py b/test/unit/files/test_googledrive.py index 831b6e6407f8..4df2ce541b9e 100644 --- a/test/unit/files/test_googledrive.py +++ b/test/unit/files/test_googledrive.py @@ -12,8 +12,11 @@ FILE_SOURCES_CONF = os.path.join(SCRIPT_DIRECTORY, "googledrive_file_sources_conf.yml") skip_if_no_google_drive_access_token = pytest.mark.skipif( - not os.environ.get('GALAXY_TEST_GOOGLE_DRIVE_CREDENTIALS_JSON'), - reason="GALAXY_TEST_GOOGLE_DRIVE_CREDENTIALS_JSON not set" + not os.environ.get('GALAXY_TEST_GOOGLE_DRIVE_ACCESS_TOKEN') + and os.environ.get('GALAXY_TEST_GOOGLE_DRIVE_REFRESH_TOKEN') + and os.environ.get('GALAXY_TEST_GOOGLE_DRIVE_CLIENT_ID') + and os.environ.get('GALAXY_TEST_GOOGLE_DRIVE_CLIENT_SECRET'), + reason="GALAXY_TEST_GOOGLE_DRIVE_ACCESS_TOKEN not set" ) From b8bdb6878ed9fe41a87ced631e3067549f105a4a Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Sun, 19 Sep 2021 19:02:06 +0530 Subject: [PATCH 05/11] Added file source for google cloud storage --- lib/galaxy/dependencies/__init__.py | 3 ++ .../dependencies/conditional-requirements.txt | 1 + .../files/sources/googlecloudstorage.py | 28 ++++++++++++++++++ test/unit/files/_util.py | 2 ++ test/unit/files/gcsfs_file_sources_conf.yml | 5 ++++ test/unit/files/test_gcsfs.py | 29 +++++++++++++++++++ test/unit/files/test_googledrive.py | 2 +- 7 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 lib/galaxy/files/sources/googlecloudstorage.py create mode 100644 test/unit/files/gcsfs_file_sources_conf.yml create mode 100644 test/unit/files/test_gcsfs.py diff --git a/lib/galaxy/dependencies/__init__.py b/lib/galaxy/dependencies/__init__.py index 68af3f95f178..f569821dca97 100644 --- a/lib/galaxy/dependencies/__init__.py +++ b/lib/galaxy/dependencies/__init__.py @@ -235,6 +235,9 @@ def check_s3fs(self): def check_fs_googledrivefs(self): return 'googledrive' in self.file_sources + def check_fs_gcsfs(self): + return 'googlecloudstorage' in self.file_sources + def check_watchdog(self): install_set = {'auto', 'True', 'true', 'polling', True} return (self.config['watch_tools'] in install_set diff --git a/lib/galaxy/dependencies/conditional-requirements.txt b/lib/galaxy/dependencies/conditional-requirements.txt index 92a396ef2a09..68d8f0e8f730 100644 --- a/lib/galaxy/dependencies/conditional-requirements.txt +++ b/lib/galaxy/dependencies/conditional-requirements.txt @@ -21,6 +21,7 @@ fs-s3fs # type: s3 s3fs # type: s3fs fs.anvilfs # type: anvil fs.googledrivefs # type: googledrive +fs-gcsfs # type: googlecloudstorage # Chronos client chronos-python==1.2.1 diff --git a/lib/galaxy/files/sources/googlecloudstorage.py b/lib/galaxy/files/sources/googlecloudstorage.py new file mode 100644 index 000000000000..7eb8e034253e --- /dev/null +++ b/lib/galaxy/files/sources/googlecloudstorage.py @@ -0,0 +1,28 @@ +try: + from fs_gcsfs import GCSFS + from google.cloud.storage import Client + from google.oauth2.credentials import Credentials +except ImportError: + GCSFS = None + +from ._pyfilesystem2 import PyFilesystem2FilesSource + + +class GoogleCloudStorageFilesSource(PyFilesystem2FilesSource): + plugin_type = 'googlecloudstorage' + required_module = GCSFS + required_package = "fs-gcsfs" + + def _open_fs(self, user_context): + props = self._serialization_props(user_context) + bucket_name = props.pop('bucket_name', None) + root_path = props.pop('root_path', None) + project = props.pop('project', None) + args = {} + if props.get('token'): + args['client'] = Client(project=project, credentials=Credentials(**props)) + handle = GCSFS(bucket_name, root_path=root_path, retry=0, **args) + return handle + + +__all__ = ('GoogleCloudStorageFilesSource',) diff --git a/test/unit/files/_util.py b/test/unit/files/_util.py index 3566643d2fef..1a0547962d6f 100644 --- a/test/unit/files/_util.py +++ b/test/unit/files/_util.py @@ -59,6 +59,8 @@ def user_context_fixture(user_ftp_dir=None, role_names=None, group_names=None, i 'googledrive|refresh_token': os.environ.get('GALAXY_TEST_GOOGLE_DRIVE_REFRESH_TOKEN'), 'googledrive|client_id': os.environ.get('GALAXY_TEST_GOOGLE_DRIVE_CLIENT_ID'), 'googledrive|client_secret': os.environ.get('GALAXY_TEST_GOOGLE_DRIVE_CLIENT_SECRET'), + 'googlecloudstorage|project': os.environ.get('GALAXY_TEST_GCS_PROJECT'), + 'googlecloudstorage|bucket_name': 'genomics-public-data' }, role_names=role_names or set(), group_names=group_names or set(), diff --git a/test/unit/files/gcsfs_file_sources_conf.yml b/test/unit/files/gcsfs_file_sources_conf.yml new file mode 100644 index 000000000000..09bcb18e945b --- /dev/null +++ b/test/unit/files/gcsfs_file_sources_conf.yml @@ -0,0 +1,5 @@ +- type: googlecloudstorage + id: test1 + doc: Test access to Google Cloud Storage. + project: ${user.preferences['googlecloudstorage|project']} + bucket_name: ${user.preferences['googlecloudstorage|bucket_name']} diff --git a/test/unit/files/test_gcsfs.py b/test/unit/files/test_gcsfs.py new file mode 100644 index 000000000000..43653bb90766 --- /dev/null +++ b/test/unit/files/test_gcsfs.py @@ -0,0 +1,29 @@ +import os + +from galaxy.files import ConfiguredFileSources, ConfiguredFileSourcesConfig +from ._util import ( + assert_realizes_contains, + find, + user_context_fixture, +) +SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__)) +FILE_SOURCES_CONF = os.path.join(SCRIPT_DIRECTORY, "gcsfs_file_sources_conf.yml") + + +def test_file_source(): + user_context = user_context_fixture() + file_sources = _configured_file_sources() + file_source_pair = file_sources.get_file_source_path("gxfiles://test1") + + assert file_source_pair.path == "/" + file_source = file_source_pair.file_source + res = file_source.list("/", recursive=False, user_context=user_context) + readme_file = find(res, class_="File", name="README") + assert readme_file + + assert_realizes_contains(file_sources, "gxfiles://test1/README", "1000genomes", user_context=user_context) + + +def _configured_file_sources(conf_file=FILE_SOURCES_CONF): + file_sources_config = ConfiguredFileSourcesConfig() + return ConfiguredFileSources(file_sources_config, conf_file=conf_file) diff --git a/test/unit/files/test_googledrive.py b/test/unit/files/test_googledrive.py index 4df2ce541b9e..0e6eb1639d75 100644 --- a/test/unit/files/test_googledrive.py +++ b/test/unit/files/test_googledrive.py @@ -28,7 +28,7 @@ def test_file_source(): assert file_source_pair.path == "/" file_source = file_source_pair.file_source - res = file_source.list("/", recursive=True, user_context=user_context) + res = file_source.list("/", recursive=False, user_context=user_context) a_file = find_file_a(res) assert a_file From d1185fcd181113588e60d7e24ac5359d0c1b1c60 Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Sun, 19 Sep 2021 21:28:30 +0530 Subject: [PATCH 06/11] Added one data --- lib/galaxy/dependencies/__init__.py | 3 ++ .../dependencies/conditional-requirements.txt | 1 + lib/galaxy/files/sources/onedata.py | 20 ++++++++++ test/unit/files/_util.py | 4 +- test/unit/files/onedata_file_sources_conf.yml | 5 +++ test/unit/files/test_googledrive.py | 8 ++-- test/unit/files/test_onedata.py | 38 +++++++++++++++++++ 7 files changed, 74 insertions(+), 5 deletions(-) create mode 100644 lib/galaxy/files/sources/onedata.py create mode 100644 test/unit/files/onedata_file_sources_conf.yml create mode 100644 test/unit/files/test_onedata.py diff --git a/lib/galaxy/dependencies/__init__.py b/lib/galaxy/dependencies/__init__.py index f569821dca97..6b611393c8c9 100644 --- a/lib/galaxy/dependencies/__init__.py +++ b/lib/galaxy/dependencies/__init__.py @@ -238,6 +238,9 @@ def check_fs_googledrivefs(self): def check_fs_gcsfs(self): return 'googlecloudstorage' in self.file_sources + def check_fs_onedatafs(self): + return 'onedata' in self.file_sources + def check_watchdog(self): install_set = {'auto', 'True', 'true', 'polling', True} return (self.config['watch_tools'] in install_set diff --git a/lib/galaxy/dependencies/conditional-requirements.txt b/lib/galaxy/dependencies/conditional-requirements.txt index 68d8f0e8f730..c46b3f68a0d6 100644 --- a/lib/galaxy/dependencies/conditional-requirements.txt +++ b/lib/galaxy/dependencies/conditional-requirements.txt @@ -22,6 +22,7 @@ s3fs # type: s3fs fs.anvilfs # type: anvil fs.googledrivefs # type: googledrive fs-gcsfs # type: googlecloudstorage +fs-onedatafs # type: onedata # Chronos client chronos-python==1.2.1 diff --git a/lib/galaxy/files/sources/onedata.py b/lib/galaxy/files/sources/onedata.py new file mode 100644 index 000000000000..d9e29d33c9b3 --- /dev/null +++ b/lib/galaxy/files/sources/onedata.py @@ -0,0 +1,20 @@ +try: + from fs.onedatafs import OnedataFS +except ImportError: + OnedataFS = None + +from ._pyfilesystem2 import PyFilesystem2FilesSource + + +class OneDataFilesSource(PyFilesystem2FilesSource): + plugin_type = 'onedata' + required_module = OnedataFS + required_package = "fs-onedatafs" + + def _open_fs(self, user_context): + props = self._serialization_props(user_context) + handle = OnedataFS(**props) + return handle + + +__all__ = ('OneDataFilesSource',) diff --git a/test/unit/files/_util.py b/test/unit/files/_util.py index 1a0547962d6f..ba448db23387 100644 --- a/test/unit/files/_util.py +++ b/test/unit/files/_util.py @@ -60,7 +60,9 @@ def user_context_fixture(user_ftp_dir=None, role_names=None, group_names=None, i 'googledrive|client_id': os.environ.get('GALAXY_TEST_GOOGLE_DRIVE_CLIENT_ID'), 'googledrive|client_secret': os.environ.get('GALAXY_TEST_GOOGLE_DRIVE_CLIENT_SECRET'), 'googlecloudstorage|project': os.environ.get('GALAXY_TEST_GCS_PROJECT'), - 'googlecloudstorage|bucket_name': 'genomics-public-data' + 'googlecloudstorage|bucket_name': 'genomics-public-data', + 'onedata|provider_host': os.environ.get('GALAXY_TEST_ONEDATA_PROVIDER_HOST'), + 'onedata|access_token': os.environ.get('GALAXY_TEST_ONEDATA_ACCESS_TOKEN'), }, role_names=role_names or set(), group_names=group_names or set(), diff --git a/test/unit/files/onedata_file_sources_conf.yml b/test/unit/files/onedata_file_sources_conf.yml new file mode 100644 index 000000000000..c452acdc2994 --- /dev/null +++ b/test/unit/files/onedata_file_sources_conf.yml @@ -0,0 +1,5 @@ +- type: onedata + id: test1 + doc: Test access to a OneData host + provider_host: ${user.preferences['onedata|provider_host']} + access_token: ${user.preferences['onedata|access_token']} diff --git a/test/unit/files/test_googledrive.py b/test/unit/files/test_googledrive.py index 0e6eb1639d75..0d3118134a3d 100644 --- a/test/unit/files/test_googledrive.py +++ b/test/unit/files/test_googledrive.py @@ -13,10 +13,10 @@ skip_if_no_google_drive_access_token = pytest.mark.skipif( not os.environ.get('GALAXY_TEST_GOOGLE_DRIVE_ACCESS_TOKEN') - and os.environ.get('GALAXY_TEST_GOOGLE_DRIVE_REFRESH_TOKEN') - and os.environ.get('GALAXY_TEST_GOOGLE_DRIVE_CLIENT_ID') - and os.environ.get('GALAXY_TEST_GOOGLE_DRIVE_CLIENT_SECRET'), - reason="GALAXY_TEST_GOOGLE_DRIVE_ACCESS_TOKEN not set" + and not os.environ.get('GALAXY_TEST_GOOGLE_DRIVE_REFRESH_TOKEN') + and not os.environ.get('GALAXY_TEST_GOOGLE_DRIVE_CLIENT_ID') + and not os.environ.get('GALAXY_TEST_GOOGLE_DRIVE_CLIENT_SECRET'), + reason="GALAXY_TEST_GOOGLE_DRIVE_ACCESS_TOKEN and related vars not set" ) diff --git a/test/unit/files/test_onedata.py b/test/unit/files/test_onedata.py new file mode 100644 index 000000000000..038f82ff6667 --- /dev/null +++ b/test/unit/files/test_onedata.py @@ -0,0 +1,38 @@ +import os + +import pytest + +from galaxy.files import ConfiguredFileSources, ConfiguredFileSourcesConfig +from ._util import ( + assert_realizes_as, + find_file_a, + user_context_fixture, +) +SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__)) +FILE_SOURCES_CONF = os.path.join(SCRIPT_DIRECTORY, "onedata_file_sources_conf.yml") + +skip_if_no_onedata_access_token = pytest.mark.skipif( + not os.environ.get('GALAXY_TEST_ONEDATA_PROVIDER_HOST') + and not os.environ.get('GALAXY_TEST_ONEDATA_ACCESS_TOKEN'), + reason="GALAXY_TEST_ONEDATA_PROVIDER_HOST and GALAXY_TEST_ONEDATA_ACCESS_TOKEN not set" +) + + +@skip_if_no_onedata_access_token +def test_file_source(): + user_context = user_context_fixture() + file_sources = _configured_file_sources() + file_source_pair = file_sources.get_file_source_path("gxfiles://test1") + + assert file_source_pair.path == "/" + file_source = file_source_pair.file_source + res = file_source.list("/", recursive=False, user_context=user_context) + a_file = find_file_a(res) + assert a_file + + assert_realizes_as(file_sources, "gxfiles://test1/a", "a\n", user_context=user_context) + + +def _configured_file_sources(conf_file=FILE_SOURCES_CONF): + file_sources_config = ConfiguredFileSourcesConfig() + return ConfiguredFileSources(file_sources_config, conf_file=conf_file) From cb22970426fdf3a66239897d3c2018143780edeb Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Mon, 20 Sep 2021 08:19:19 +0530 Subject: [PATCH 07/11] Added support for illumina basespace filesource --- lib/galaxy/dependencies/__init__.py | 3 ++ .../dependencies/conditional-requirements.txt | 1 + lib/galaxy/files/sources/basespace.py | 20 +++++++++ test/unit/files/_util.py | 3 ++ .../files/basespace_file_sources_conf.yml | 7 ++++ test/unit/files/test_basespace.py | 41 +++++++++++++++++++ 6 files changed, 75 insertions(+) create mode 100644 lib/galaxy/files/sources/basespace.py create mode 100644 test/unit/files/basespace_file_sources_conf.yml create mode 100644 test/unit/files/test_basespace.py diff --git a/lib/galaxy/dependencies/__init__.py b/lib/galaxy/dependencies/__init__.py index 6b611393c8c9..c8c3f781d31b 100644 --- a/lib/galaxy/dependencies/__init__.py +++ b/lib/galaxy/dependencies/__init__.py @@ -241,6 +241,9 @@ def check_fs_gcsfs(self): def check_fs_onedatafs(self): return 'onedata' in self.file_sources + def check_fs_basespace(self): + return 'basespace' in self.file_sources + def check_watchdog(self): install_set = {'auto', 'True', 'true', 'polling', True} return (self.config['watch_tools'] in install_set diff --git a/lib/galaxy/dependencies/conditional-requirements.txt b/lib/galaxy/dependencies/conditional-requirements.txt index c46b3f68a0d6..defa88a033f0 100644 --- a/lib/galaxy/dependencies/conditional-requirements.txt +++ b/lib/galaxy/dependencies/conditional-requirements.txt @@ -23,6 +23,7 @@ fs.anvilfs # type: anvil fs.googledrivefs # type: googledrive fs-gcsfs # type: googlecloudstorage fs-onedatafs # type: onedata +fs-basespace # type: basespace # Chronos client chronos-python==1.2.1 diff --git a/lib/galaxy/files/sources/basespace.py b/lib/galaxy/files/sources/basespace.py new file mode 100644 index 000000000000..95c09b250b74 --- /dev/null +++ b/lib/galaxy/files/sources/basespace.py @@ -0,0 +1,20 @@ +try: + from fs_basespace import BASESPACEFS +except ImportError: + BASESPACEFS = None + +from ._pyfilesystem2 import PyFilesystem2FilesSource + + +class BaseSpaceFilesSource(PyFilesystem2FilesSource): + plugin_type = 'basespace' + required_module = BASESPACEFS + required_package = "fs-basespace" + + def _open_fs(self, user_context): + props = self._serialization_props(user_context) + handle = BASESPACEFS(**props) + return handle + + +__all__ = ('BaseSpaceFilesSource',) diff --git a/test/unit/files/_util.py b/test/unit/files/_util.py index ba448db23387..cb918e680d25 100644 --- a/test/unit/files/_util.py +++ b/test/unit/files/_util.py @@ -63,6 +63,9 @@ def user_context_fixture(user_ftp_dir=None, role_names=None, group_names=None, i 'googlecloudstorage|bucket_name': 'genomics-public-data', 'onedata|provider_host': os.environ.get('GALAXY_TEST_ONEDATA_PROVIDER_HOST'), 'onedata|access_token': os.environ.get('GALAXY_TEST_ONEDATA_ACCESS_TOKEN'), + 'basespace|client_id': os.environ.get('GALAXY_TEST_ONEDATA_CLIENT_ID'), + 'basespace|client_secret': os.environ.get('GALAXY_TEST_ONEDATA_CLIENT_SECRET'), + 'basespace|access_token': os.environ.get('GALAXY_TEST_ONEDATA_ACCESS_TOKEN'), }, role_names=role_names or set(), group_names=group_names or set(), diff --git a/test/unit/files/basespace_file_sources_conf.yml b/test/unit/files/basespace_file_sources_conf.yml new file mode 100644 index 000000000000..092c00c64ac0 --- /dev/null +++ b/test/unit/files/basespace_file_sources_conf.yml @@ -0,0 +1,7 @@ +- type: basespace + id: test1 + doc: Test access to Illumina BaseSpace + basespace_server: https://api.basespace.illumina.com + client_id: ${user.preferences['basespace|client_id']} + client_secret: ${user.preferences['basespace|client_secret']} + access_token: ${user.preferences['basespace|access_token']} diff --git a/test/unit/files/test_basespace.py b/test/unit/files/test_basespace.py new file mode 100644 index 000000000000..a100f8e4cb58 --- /dev/null +++ b/test/unit/files/test_basespace.py @@ -0,0 +1,41 @@ +import os + +import pytest + +from galaxy.files import ConfiguredFileSources, ConfiguredFileSourcesConfig +from ._util import ( + assert_realizes_as, + find, + user_context_fixture, +) +SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__)) +FILE_SOURCES_CONF = os.path.join(SCRIPT_DIRECTORY, "basespace_file_sources_conf.yml") + +skip_if_no_basespace_access_token = pytest.mark.skipif( + not os.environ.get('GALAXY_TEST_BASESPACE_CLIENT_ID') + and not os.environ.get('GALAXY_TEST_BASESPACE_CLIENT_SECRET') + and not os.environ.get('GALAXY_TEST_BASESPACE_ACCESS_TOKEN') + and not os.environ.get('GALAXY_TEST_BASESPACE_TEST_FILE_PATH'), + reason="GALAXY_TEST_BASESPACE_CLIENT_ID and related vars not set" +) + + +@skip_if_no_basespace_access_token +def test_file_source(): + user_context = user_context_fixture() + file_sources = _configured_file_sources() + file_source_pair = file_sources.get_file_source_path("gxfiles://test1") + + assert file_source_pair.path == "/" + file_source = file_source_pair.file_source + test_file = os.environ.get('GALAXY_TEST_BASESPACE_TEST_FILE_PATH') + res = file_source.list(os.path.dirname(test_file), recursive=False, user_context=user_context) + a_file = find(res, class_="File", name=os.path.basename(test_file)) + assert a_file + + assert_realizes_as(file_sources, a_file['uri'], "a\n", user_context=user_context) + + +def _configured_file_sources(conf_file=FILE_SOURCES_CONF): + file_sources_config = ConfiguredFileSourcesConfig() + return ConfiguredFileSources(file_sources_config, conf_file=conf_file) From 15b7b88d8b6151c2dc14d56d7ac71d4f35926e24 Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Fri, 10 Dec 2021 21:07:20 +0530 Subject: [PATCH 08/11] Streamline access tokens needed to run gcsfs and gdrive storage tests --- test/unit/files/_util.py | 10 +++++++--- test/unit/files/gcsfs_file_sources_conf.yml | 5 +++++ test/unit/files/test_basespace.py | 8 ++++---- test/unit/files/test_gcsfs.py | 11 +++++++++++ test/unit/files/test_googledrive.py | 4 +--- test/unit/files/test_onedata.py | 2 +- 6 files changed, 29 insertions(+), 11 deletions(-) diff --git a/test/unit/files/_util.py b/test/unit/files/_util.py index cb918e680d25..dfc903611f51 100644 --- a/test/unit/files/_util.py +++ b/test/unit/files/_util.py @@ -55,12 +55,16 @@ def user_context_fixture(user_ftp_dir=None, role_names=None, group_names=None, i preferences={ 'webdav|password': 'secret1234', 'dropbox|access_token': os.environ.get('GALAXY_TEST_DROPBOX_ACCESS_TOKEN'), - 'googledrive|access_token': os.environ.get('GALAXY_TEST_GOOGLE_DRIVE_ACCESS_TOKEN'), - 'googledrive|refresh_token': os.environ.get('GALAXY_TEST_GOOGLE_DRIVE_REFRESH_TOKEN'), 'googledrive|client_id': os.environ.get('GALAXY_TEST_GOOGLE_DRIVE_CLIENT_ID'), 'googledrive|client_secret': os.environ.get('GALAXY_TEST_GOOGLE_DRIVE_CLIENT_SECRET'), + 'googledrive|access_token': os.environ.get('GALAXY_TEST_GOOGLE_DRIVE_ACCESS_TOKEN'), + 'googledrive|refresh_token': os.environ.get('GALAXY_TEST_GOOGLE_DRIVE_REFRESH_TOKEN'), 'googlecloudstorage|project': os.environ.get('GALAXY_TEST_GCS_PROJECT'), - 'googlecloudstorage|bucket_name': 'genomics-public-data', + 'googlecloudstorage|bucket_name': os.environ.get('GALAXY_TEST_GCS_BUCKET'), + 'googlecloudstorage|client_id': os.environ.get('GALAXY_TEST_GCS_CLIENT_ID'), + 'googlecloudstorage|client_secret': os.environ.get('GALAXY_TEST_GCS_CLIENT_SECRET'), + 'googlecloudstorage|access_token': os.environ.get('GALAXY_TEST_GCS_ACCESS_TOKEN'), + 'googlecloudstorage|refresh_token': os.environ.get('GALAXY_TEST_GCS_REFRESH_TOKEN'), 'onedata|provider_host': os.environ.get('GALAXY_TEST_ONEDATA_PROVIDER_HOST'), 'onedata|access_token': os.environ.get('GALAXY_TEST_ONEDATA_ACCESS_TOKEN'), 'basespace|client_id': os.environ.get('GALAXY_TEST_ONEDATA_CLIENT_ID'), diff --git a/test/unit/files/gcsfs_file_sources_conf.yml b/test/unit/files/gcsfs_file_sources_conf.yml index 09bcb18e945b..5773ef0479a0 100644 --- a/test/unit/files/gcsfs_file_sources_conf.yml +++ b/test/unit/files/gcsfs_file_sources_conf.yml @@ -3,3 +3,8 @@ doc: Test access to Google Cloud Storage. project: ${user.preferences['googlecloudstorage|project']} bucket_name: ${user.preferences['googlecloudstorage|bucket_name']} + token_uri: "https://www.googleapis.com/oauth2/v4/token" + client_id: ${user.preferences['googlecloudstorage|client_id']} + client_secret: ${user.preferences['googlecloudstorage|client_secret']} + token: ${user.preferences['googlecloudstorage|access_token']} + refresh_token: ${user.preferences['googlecloudstorage|refresh_token']} diff --git a/test/unit/files/test_basespace.py b/test/unit/files/test_basespace.py index a100f8e4cb58..2a4472558d6a 100644 --- a/test/unit/files/test_basespace.py +++ b/test/unit/files/test_basespace.py @@ -13,9 +13,9 @@ skip_if_no_basespace_access_token = pytest.mark.skipif( not os.environ.get('GALAXY_TEST_BASESPACE_CLIENT_ID') - and not os.environ.get('GALAXY_TEST_BASESPACE_CLIENT_SECRET') - and not os.environ.get('GALAXY_TEST_BASESPACE_ACCESS_TOKEN') - and not os.environ.get('GALAXY_TEST_BASESPACE_TEST_FILE_PATH'), + or not os.environ.get('GALAXY_TEST_BASESPACE_CLIENT_SECRET') + or not os.environ.get('GALAXY_TEST_BASESPACE_ACCESS_TOKEN') + or not os.environ.get('GALAXY_TEST_BASESPACE_TEST_FILE_PATH'), reason="GALAXY_TEST_BASESPACE_CLIENT_ID and related vars not set" ) @@ -28,7 +28,7 @@ def test_file_source(): assert file_source_pair.path == "/" file_source = file_source_pair.file_source - test_file = os.environ.get('GALAXY_TEST_BASESPACE_TEST_FILE_PATH') + test_file = os.environ.get('GALAXY_TEST_BASESPACE_TEST_FILE_PATH', "") res = file_source.list(os.path.dirname(test_file), recursive=False, user_context=user_context) a_file = find(res, class_="File", name=os.path.basename(test_file)) assert a_file diff --git a/test/unit/files/test_gcsfs.py b/test/unit/files/test_gcsfs.py index 43653bb90766..b27282aaf155 100644 --- a/test/unit/files/test_gcsfs.py +++ b/test/unit/files/test_gcsfs.py @@ -1,5 +1,7 @@ import os +import pytest + from galaxy.files import ConfiguredFileSources, ConfiguredFileSourcesConfig from ._util import ( assert_realizes_contains, @@ -9,7 +11,16 @@ SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__)) FILE_SOURCES_CONF = os.path.join(SCRIPT_DIRECTORY, "gcsfs_file_sources_conf.yml") +skip_if_no_gcs_access_token = pytest.mark.skipif( + not os.environ.get('GALAXY_TEST_GCS_PROJECT') + or not os.environ.get('GALAXY_TEST_GCS_BUCKET') + or not os.environ.get('GALAXY_TEST_GCS_ACCESS_TOKEN') + or not os.environ.get('GALAXY_TEST_GCS_REFRESH_TOKEN'), + reason="GALAXY_TEST_GCS_ACCESS_TOKEN and related vars not set" +) + +@skip_if_no_gcs_access_token def test_file_source(): user_context = user_context_fixture() file_sources = _configured_file_sources() diff --git a/test/unit/files/test_googledrive.py b/test/unit/files/test_googledrive.py index 0d3118134a3d..8840fa63e828 100644 --- a/test/unit/files/test_googledrive.py +++ b/test/unit/files/test_googledrive.py @@ -13,9 +13,7 @@ skip_if_no_google_drive_access_token = pytest.mark.skipif( not os.environ.get('GALAXY_TEST_GOOGLE_DRIVE_ACCESS_TOKEN') - and not os.environ.get('GALAXY_TEST_GOOGLE_DRIVE_REFRESH_TOKEN') - and not os.environ.get('GALAXY_TEST_GOOGLE_DRIVE_CLIENT_ID') - and not os.environ.get('GALAXY_TEST_GOOGLE_DRIVE_CLIENT_SECRET'), + or not os.environ.get('GALAXY_TEST_GOOGLE_DRIVE_REFRESH_TOKEN'), reason="GALAXY_TEST_GOOGLE_DRIVE_ACCESS_TOKEN and related vars not set" ) diff --git a/test/unit/files/test_onedata.py b/test/unit/files/test_onedata.py index 038f82ff6667..2a7c178f7f70 100644 --- a/test/unit/files/test_onedata.py +++ b/test/unit/files/test_onedata.py @@ -13,7 +13,7 @@ skip_if_no_onedata_access_token = pytest.mark.skipif( not os.environ.get('GALAXY_TEST_ONEDATA_PROVIDER_HOST') - and not os.environ.get('GALAXY_TEST_ONEDATA_ACCESS_TOKEN'), + or not os.environ.get('GALAXY_TEST_ONEDATA_ACCESS_TOKEN'), reason="GALAXY_TEST_ONEDATA_PROVIDER_HOST and GALAXY_TEST_ONEDATA_ACCESS_TOKEN not set" ) From d1f804921c02d75b2f48f1a4cbec66294803176b Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Sun, 19 Dec 2021 23:19:06 +0530 Subject: [PATCH 09/11] Add support for anonymous gcs file source access --- lib/galaxy/files/sources/googlecloudstorage.py | 4 +++- packages/files/test-requirements.txt | 1 + test/unit/files/gcsfs_file_sources_conf.yml | 3 ++- test/unit/files/test_gcsfs.py | 11 ----------- 4 files changed, 6 insertions(+), 13 deletions(-) diff --git a/lib/galaxy/files/sources/googlecloudstorage.py b/lib/galaxy/files/sources/googlecloudstorage.py index 7eb8e034253e..e2d7ad08a87c 100644 --- a/lib/galaxy/files/sources/googlecloudstorage.py +++ b/lib/galaxy/files/sources/googlecloudstorage.py @@ -19,7 +19,9 @@ def _open_fs(self, user_context): root_path = props.pop('root_path', None) project = props.pop('project', None) args = {} - if props.get('token'): + if props.get('anonymous'): + args['client'] = Client.create_anonymous_client() + elif props.get('token'): args['client'] = Client(project=project, credentials=Credentials(**props)) handle = GCSFS(bucket_name, root_path=root_path, retry=0, **args) return handle diff --git a/packages/files/test-requirements.txt b/packages/files/test-requirements.txt index e079f8a6038d..f00855512c0b 100644 --- a/packages/files/test-requirements.txt +++ b/packages/files/test-requirements.txt @@ -1 +1,2 @@ pytest +fs-gcsfs diff --git a/test/unit/files/gcsfs_file_sources_conf.yml b/test/unit/files/gcsfs_file_sources_conf.yml index 5773ef0479a0..d823636c6505 100644 --- a/test/unit/files/gcsfs_file_sources_conf.yml +++ b/test/unit/files/gcsfs_file_sources_conf.yml @@ -2,9 +2,10 @@ id: test1 doc: Test access to Google Cloud Storage. project: ${user.preferences['googlecloudstorage|project']} - bucket_name: ${user.preferences['googlecloudstorage|bucket_name']} + bucket_name: 'genomics-public-data' token_uri: "https://www.googleapis.com/oauth2/v4/token" client_id: ${user.preferences['googlecloudstorage|client_id']} client_secret: ${user.preferences['googlecloudstorage|client_secret']} token: ${user.preferences['googlecloudstorage|access_token']} refresh_token: ${user.preferences['googlecloudstorage|refresh_token']} + anonymous: true diff --git a/test/unit/files/test_gcsfs.py b/test/unit/files/test_gcsfs.py index b27282aaf155..43653bb90766 100644 --- a/test/unit/files/test_gcsfs.py +++ b/test/unit/files/test_gcsfs.py @@ -1,7 +1,5 @@ import os -import pytest - from galaxy.files import ConfiguredFileSources, ConfiguredFileSourcesConfig from ._util import ( assert_realizes_contains, @@ -11,16 +9,7 @@ SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__)) FILE_SOURCES_CONF = os.path.join(SCRIPT_DIRECTORY, "gcsfs_file_sources_conf.yml") -skip_if_no_gcs_access_token = pytest.mark.skipif( - not os.environ.get('GALAXY_TEST_GCS_PROJECT') - or not os.environ.get('GALAXY_TEST_GCS_BUCKET') - or not os.environ.get('GALAXY_TEST_GCS_ACCESS_TOKEN') - or not os.environ.get('GALAXY_TEST_GCS_REFRESH_TOKEN'), - reason="GALAXY_TEST_GCS_ACCESS_TOKEN and related vars not set" -) - -@skip_if_no_gcs_access_token def test_file_source(): user_context = user_context_fixture() file_sources = _configured_file_sources() From 5d72ddba11c4ff4c6a740efd647bd0671d0f53e1 Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Mon, 20 Dec 2021 21:29:33 +0530 Subject: [PATCH 10/11] Make gcsfs test optional --- test/unit/files/test_gcsfs.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test/unit/files/test_gcsfs.py b/test/unit/files/test_gcsfs.py index 43653bb90766..db4a928a41c3 100644 --- a/test/unit/files/test_gcsfs.py +++ b/test/unit/files/test_gcsfs.py @@ -1,15 +1,30 @@ import os +import pytest + from galaxy.files import ConfiguredFileSources, ConfiguredFileSourcesConfig from ._util import ( assert_realizes_contains, find, user_context_fixture, ) + +try: + from fs_gcsfs import GCSFS +except ImportError: + GCSFS = None + SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__)) FILE_SOURCES_CONF = os.path.join(SCRIPT_DIRECTORY, "gcsfs_file_sources_conf.yml") +skip_if_no_gcsfs_libs = pytest.mark.skipif( + not GCSFS, + reason="Required lib to run gcs file source test: fs_gcsfs is not available" +) + + +@skip_if_no_gcsfs_libs def test_file_source(): user_context = user_context_fixture() file_sources = _configured_file_sources() From 0924628d106f13f5434ed7a689832c5baf37e201 Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Tue, 21 Dec 2021 00:48:28 +0530 Subject: [PATCH 11/11] Refactor file source tests --- test/unit/files/_util.py | 73 +++++++++++++++++++---------- test/unit/files/test_basespace.py | 9 +--- test/unit/files/test_dropbox.py | 25 ++-------- test/unit/files/test_gcsfs.py | 25 ++-------- test/unit/files/test_googledrive.py | 25 ++-------- test/unit/files/test_onedata.py | 25 ++-------- test/unit/files/test_s3.py | 27 ++--------- test/unit/files/test_webdav.py | 15 ++---- 8 files changed, 71 insertions(+), 153 deletions(-) diff --git a/test/unit/files/_util.py b/test/unit/files/_util.py index dfc903611f51..8e7e59499af1 100644 --- a/test/unit/files/_util.py +++ b/test/unit/files/_util.py @@ -4,6 +4,7 @@ from galaxy.files import ( ConfiguredFileSources, + ConfiguredFileSourcesConfig, DictFileSourcesUserContext, ) @@ -78,43 +79,43 @@ def user_context_fixture(user_ftp_dir=None, role_names=None, group_names=None, i return user_context -def assert_realizes_as(file_sources, uri, expected, user_context=None): +def realize_to_temp_file(file_sources, uri, user_context=None): file_source_path = file_sources.get_file_source_path(uri) with tempfile.NamedTemporaryFile(mode='r') as temp: file_source_path.file_source.realize_to(file_source_path.path, temp.name, user_context=user_context) with open(temp.name) as f: realized_contents = f.read() - if realized_contents != expected: - message = "Expected to realize contents at [{}] as [{}], instead found [{}]".format( - uri, - expected, - realized_contents, - ) - raise AssertionError(message) + return realized_contents + + +def assert_realizes_as(file_sources, uri, expected, user_context=None): + realized_contents = realize_to_temp_file(file_sources, uri, user_context=user_context) + if realized_contents != expected: + message = "Expected to realize contents at [{}] as [{}], instead found [{}]".format( + uri, + expected, + realized_contents, + ) + raise AssertionError(message) def assert_realizes_contains(file_sources, uri, expected, user_context=None): - file_source_path = file_sources.get_file_source_path(uri) - with tempfile.NamedTemporaryFile(mode='r') as temp: - file_source_path.file_source.realize_to(file_source_path.path, temp.name, user_context=user_context) - realized_contents = temp.read() - if expected not in realized_contents: - message = "Expected to realize contents at [{}] to contain [{}], instead found [{}]".format( - uri, - expected, - realized_contents, - ) - raise AssertionError(message) + realized_contents = realize_to_temp_file(file_sources, uri, user_context=user_context) + if expected not in realized_contents: + message = "Expected to realize contents at [{}] to contain [{}], instead found [{}]".format( + uri, + expected, + realized_contents, + ) + raise AssertionError(message) def assert_realizes_throws_exception(file_sources, uri, user_context=None) -> Exception: - file_source_path = file_sources.get_file_source_path(uri) exception = None - with tempfile.NamedTemporaryFile(mode='r') as temp: - try: - file_source_path.file_source.realize_to(file_source_path.path, temp.name, user_context=user_context) - except Exception as e: - exception = e + try: + realize_to_temp_file(file_sources, uri, user_context=user_context) + except Exception as e: + exception = e assert exception return exception @@ -125,3 +126,25 @@ def write_from(file_sources, uri, content, user_context=None): f.write(content) f.flush() file_source_path.file_source.write_from(file_source_path.path, f.name, user_context=user_context) + + +def configured_file_sources(conf_file): + file_sources_config = ConfiguredFileSourcesConfig() + return ConfiguredFileSources(file_sources_config, conf_file=conf_file) + + +def assert_simple_file_realize(conf_file, recursive=False, filename="a", contents="a\n", contains=False): + user_context = user_context_fixture() + file_sources = configured_file_sources(conf_file) + file_source_pair = file_sources.get_file_source_path("gxfiles://test1") + + assert file_source_pair.path == "/" + file_source = file_source_pair.file_source + res = file_source.list("/", recursive=recursive, user_context=user_context) + a_file = find(res, class_="File", name=filename) + assert a_file + + if contains: + assert_realizes_contains(file_sources, f"gxfiles://test1/{filename}", contents, user_context=user_context) + else: + assert_realizes_as(file_sources, f"gxfiles://test1/{filename}", contents, user_context=user_context) diff --git a/test/unit/files/test_basespace.py b/test/unit/files/test_basespace.py index 2a4472558d6a..85719d8d1f0a 100644 --- a/test/unit/files/test_basespace.py +++ b/test/unit/files/test_basespace.py @@ -2,9 +2,9 @@ import pytest -from galaxy.files import ConfiguredFileSources, ConfiguredFileSourcesConfig from ._util import ( assert_realizes_as, + configured_file_sources, find, user_context_fixture, ) @@ -23,7 +23,7 @@ @skip_if_no_basespace_access_token def test_file_source(): user_context = user_context_fixture() - file_sources = _configured_file_sources() + file_sources = configured_file_sources(FILE_SOURCES_CONF) file_source_pair = file_sources.get_file_source_path("gxfiles://test1") assert file_source_pair.path == "/" @@ -34,8 +34,3 @@ def test_file_source(): assert a_file assert_realizes_as(file_sources, a_file['uri'], "a\n", user_context=user_context) - - -def _configured_file_sources(conf_file=FILE_SOURCES_CONF): - file_sources_config = ConfiguredFileSourcesConfig() - return ConfiguredFileSources(file_sources_config, conf_file=conf_file) diff --git a/test/unit/files/test_dropbox.py b/test/unit/files/test_dropbox.py index 9d655fa8d15e..7bee738d288b 100644 --- a/test/unit/files/test_dropbox.py +++ b/test/unit/files/test_dropbox.py @@ -2,12 +2,8 @@ import pytest -from galaxy.files import ConfiguredFileSources, ConfiguredFileSourcesConfig -from ._util import ( - assert_realizes_as, - find_file_a, - user_context_fixture, -) +from ._util import assert_simple_file_realize + SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__)) FILE_SOURCES_CONF = os.path.join(SCRIPT_DIRECTORY, "dropbox_file_sources_conf.yml") @@ -19,19 +15,4 @@ @skip_if_no_dropbox_access_token def test_file_source(): - user_context = user_context_fixture() - file_sources = _configured_file_sources() - file_source_pair = file_sources.get_file_source_path("gxfiles://test1") - - assert file_source_pair.path == "/" - file_source = file_source_pair.file_source - res = file_source.list("/", recursive=True, user_context=user_context) - a_file = find_file_a(res) - assert a_file - - assert_realizes_as(file_sources, "gxfiles://test1/a", "a\n", user_context=user_context) - - -def _configured_file_sources(conf_file=FILE_SOURCES_CONF): - file_sources_config = ConfiguredFileSourcesConfig() - return ConfiguredFileSources(file_sources_config, conf_file=conf_file) + assert_simple_file_realize(FILE_SOURCES_CONF, recursive=True) diff --git a/test/unit/files/test_gcsfs.py b/test/unit/files/test_gcsfs.py index db4a928a41c3..8ad3a46cca5d 100644 --- a/test/unit/files/test_gcsfs.py +++ b/test/unit/files/test_gcsfs.py @@ -2,12 +2,7 @@ import pytest -from galaxy.files import ConfiguredFileSources, ConfiguredFileSourcesConfig -from ._util import ( - assert_realizes_contains, - find, - user_context_fixture, -) +from ._util import assert_simple_file_realize try: from fs_gcsfs import GCSFS @@ -26,19 +21,5 @@ @skip_if_no_gcsfs_libs def test_file_source(): - user_context = user_context_fixture() - file_sources = _configured_file_sources() - file_source_pair = file_sources.get_file_source_path("gxfiles://test1") - - assert file_source_pair.path == "/" - file_source = file_source_pair.file_source - res = file_source.list("/", recursive=False, user_context=user_context) - readme_file = find(res, class_="File", name="README") - assert readme_file - - assert_realizes_contains(file_sources, "gxfiles://test1/README", "1000genomes", user_context=user_context) - - -def _configured_file_sources(conf_file=FILE_SOURCES_CONF): - file_sources_config = ConfiguredFileSourcesConfig() - return ConfiguredFileSources(file_sources_config, conf_file=conf_file) + assert_simple_file_realize(FILE_SOURCES_CONF, recursive=False, filename="README", contents="1000genomes", + contains=True) diff --git a/test/unit/files/test_googledrive.py b/test/unit/files/test_googledrive.py index 8840fa63e828..8fc39c1136b2 100644 --- a/test/unit/files/test_googledrive.py +++ b/test/unit/files/test_googledrive.py @@ -2,12 +2,8 @@ import pytest -from galaxy.files import ConfiguredFileSources, ConfiguredFileSourcesConfig -from ._util import ( - assert_realizes_as, - find_file_a, - user_context_fixture, -) +from ._util import assert_simple_file_realize + SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__)) FILE_SOURCES_CONF = os.path.join(SCRIPT_DIRECTORY, "googledrive_file_sources_conf.yml") @@ -20,19 +16,4 @@ @skip_if_no_google_drive_access_token def test_file_source(): - user_context = user_context_fixture() - file_sources = _configured_file_sources() - file_source_pair = file_sources.get_file_source_path("gxfiles://test1") - - assert file_source_pair.path == "/" - file_source = file_source_pair.file_source - res = file_source.list("/", recursive=False, user_context=user_context) - a_file = find_file_a(res) - assert a_file - - assert_realizes_as(file_sources, "gxfiles://test1/a", "a\n", user_context=user_context) - - -def _configured_file_sources(conf_file=FILE_SOURCES_CONF): - file_sources_config = ConfiguredFileSourcesConfig() - return ConfiguredFileSources(file_sources_config, conf_file=conf_file) + assert_simple_file_realize(FILE_SOURCES_CONF) diff --git a/test/unit/files/test_onedata.py b/test/unit/files/test_onedata.py index 2a7c178f7f70..d78dad707c44 100644 --- a/test/unit/files/test_onedata.py +++ b/test/unit/files/test_onedata.py @@ -2,12 +2,8 @@ import pytest -from galaxy.files import ConfiguredFileSources, ConfiguredFileSourcesConfig -from ._util import ( - assert_realizes_as, - find_file_a, - user_context_fixture, -) +from ._util import assert_simple_file_realize + SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__)) FILE_SOURCES_CONF = os.path.join(SCRIPT_DIRECTORY, "onedata_file_sources_conf.yml") @@ -20,19 +16,4 @@ @skip_if_no_onedata_access_token def test_file_source(): - user_context = user_context_fixture() - file_sources = _configured_file_sources() - file_source_pair = file_sources.get_file_source_path("gxfiles://test1") - - assert file_source_pair.path == "/" - file_source = file_source_pair.file_source - res = file_source.list("/", recursive=False, user_context=user_context) - a_file = find_file_a(res) - assert a_file - - assert_realizes_as(file_sources, "gxfiles://test1/a", "a\n", user_context=user_context) - - -def _configured_file_sources(conf_file=FILE_SOURCES_CONF): - file_sources_config = ConfiguredFileSourcesConfig() - return ConfiguredFileSources(file_sources_config, conf_file=conf_file) + assert_simple_file_realize(FILE_SOURCES_CONF) diff --git a/test/unit/files/test_s3.py b/test/unit/files/test_s3.py index cbae28b7948c..f88d7905401a 100644 --- a/test/unit/files/test_s3.py +++ b/test/unit/files/test_s3.py @@ -2,12 +2,8 @@ import pytest -from galaxy.files import ConfiguredFileSources, ConfiguredFileSourcesConfig -from ._util import ( - assert_realizes_contains, - find, - user_context_fixture, -) +from ._util import assert_simple_file_realize + SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__)) FILE_SOURCES_CONF = os.path.join(SCRIPT_DIRECTORY, "s3_file_sources_conf.yml") @@ -19,20 +15,5 @@ @skip_if_not_slow def test_file_source(): - user_context = user_context_fixture() - file_sources = _configured_file_sources() - file_source_pair = file_sources.get_file_source_path("gxfiles://test1") - - assert file_source_pair.path == "/" - file_source = file_source_pair.file_source - res = file_source.list("/", recursive=False, user_context=user_context) - print(res) - dup = find(res, "File", "data_use_policies.txt") - assert dup - - assert_realizes_contains(file_sources, "gxfiles://test1/data_use_policies.txt", "DATA USE POLICIES", user_context=user_context) - - -def _configured_file_sources(conf_file=FILE_SOURCES_CONF): - file_sources_config = ConfiguredFileSourcesConfig() - return ConfiguredFileSources(file_sources_config, conf_file=conf_file) + assert_simple_file_realize(FILE_SOURCES_CONF, recursive=False, filename="data_use_policies.txt", + contents="DATA USE POLICIES", contains=True) diff --git a/test/unit/files/test_webdav.py b/test/unit/files/test_webdav.py index d91bb8cda29d..30909b99e658 100644 --- a/test/unit/files/test_webdav.py +++ b/test/unit/files/test_webdav.py @@ -4,8 +4,8 @@ import pytest -from galaxy.files import ConfiguredFileSources, ConfiguredFileSourcesConfig from ._util import ( + configured_file_sources, find, find_file_a, list_dir, @@ -27,7 +27,7 @@ @skip_if_no_webdav def test_file_source(): - file_sources = _configured_file_sources() + file_sources = configured_file_sources(FILE_SOURCES_CONF) file_source_pair = file_sources.get_file_source_path("gxfiles://test1") assert file_source_pair.path == "/" @@ -59,7 +59,7 @@ def test_file_source(): @skip_if_no_webdav def test_sniff_to_tmp(): - file_sources = _configured_file_sources() + file_sources = configured_file_sources(FILE_SOURCES_CONF) _download_and_check_file(file_sources) @@ -67,7 +67,7 @@ def test_sniff_to_tmp(): def test_serialization(): # serialize the configured file sources and rematerialize them, # ensure they still function. This is needed for uploading files. - file_sources = serialize_and_recover(_configured_file_sources()) + file_sources = serialize_and_recover(configured_file_sources(FILE_SOURCES_CONF)) res = list_root(file_sources, "gxfiles://test1", recursive=True) assert find_file_a(res) @@ -80,7 +80,7 @@ def test_serialization(): @skip_if_no_webdav def test_serialization_user(): - file_sources_o = _configured_file_sources(USER_FILE_SOURCES_CONF) + file_sources_o = configured_file_sources(USER_FILE_SOURCES_CONF) user_context = user_context_fixture() res = list_root(file_sources_o, "gxfiles://test1", recursive=True, user_context=user_context) @@ -89,8 +89,3 @@ def test_serialization_user(): file_sources = serialize_and_recover(file_sources_o, user_context=user_context) res = list_root(file_sources, "gxfiles://test1", recursive=True, user_context=None) assert find_file_a(res) - - -def _configured_file_sources(conf_file=FILE_SOURCES_CONF): - file_sources_config = ConfiguredFileSourcesConfig() - return ConfiguredFileSources(file_sources_config, conf_file=conf_file)