diff --git a/lib/galaxy/dependencies/__init__.py b/lib/galaxy/dependencies/__init__.py index 27a9ed53c8d9..c8c3f781d31b 100644 --- a/lib/galaxy/dependencies/__init__.py +++ b/lib/galaxy/dependencies/__init__.py @@ -232,6 +232,18 @@ def check_s3fs(self): # use s3fs directly (skipping pyfilesystem) for direct access to more options return 's3fs' in self.file_sources + def check_fs_googledrivefs(self): + return 'googledrive' in self.file_sources + + def check_fs_gcsfs(self): + return 'googlecloudstorage' in self.file_sources + + def check_fs_onedatafs(self): + return 'onedata' in self.file_sources + + def check_fs_basespace(self): + return 'basespace' in self.file_sources + def check_watchdog(self): install_set = {'auto', 'True', 'true', 'polling', True} return (self.config['watch_tools'] in install_set diff --git a/lib/galaxy/dependencies/conditional-requirements.txt b/lib/galaxy/dependencies/conditional-requirements.txt index 37e72d0f4686..defa88a033f0 100644 --- a/lib/galaxy/dependencies/conditional-requirements.txt +++ b/lib/galaxy/dependencies/conditional-requirements.txt @@ -20,6 +20,10 @@ fs.sshfs # type: ssh fs-s3fs # type: s3 s3fs # type: s3fs fs.anvilfs # type: anvil +fs.googledrivefs # type: googledrive +fs-gcsfs # type: googlecloudstorage +fs-onedatafs # type: onedata +fs-basespace # type: basespace # Chronos client chronos-python==1.2.1 diff --git a/lib/galaxy/files/sources/basespace.py b/lib/galaxy/files/sources/basespace.py new file mode 100644 index 000000000000..95c09b250b74 --- /dev/null +++ b/lib/galaxy/files/sources/basespace.py @@ -0,0 +1,20 @@ +try: + from fs_basespace import BASESPACEFS +except ImportError: + BASESPACEFS = None + +from ._pyfilesystem2 import PyFilesystem2FilesSource + + +class BaseSpaceFilesSource(PyFilesystem2FilesSource): + plugin_type = 'basespace' + required_module = BASESPACEFS + required_package = "fs-basespace" + + def _open_fs(self, user_context): + props = self._serialization_props(user_context) + handle = BASESPACEFS(**props) + return handle + + +__all__ = ('BaseSpaceFilesSource',) diff --git a/lib/galaxy/files/sources/googlecloudstorage.py b/lib/galaxy/files/sources/googlecloudstorage.py new file mode 100644 index 000000000000..e2d7ad08a87c --- /dev/null +++ b/lib/galaxy/files/sources/googlecloudstorage.py @@ -0,0 +1,30 @@ +try: + from fs_gcsfs import GCSFS + from google.cloud.storage import Client + from google.oauth2.credentials import Credentials +except ImportError: + GCSFS = None + +from ._pyfilesystem2 import PyFilesystem2FilesSource + + +class GoogleCloudStorageFilesSource(PyFilesystem2FilesSource): + plugin_type = 'googlecloudstorage' + required_module = GCSFS + required_package = "fs-gcsfs" + + def _open_fs(self, user_context): + props = self._serialization_props(user_context) + bucket_name = props.pop('bucket_name', None) + root_path = props.pop('root_path', None) + project = props.pop('project', None) + args = {} + if props.get('anonymous'): + args['client'] = Client.create_anonymous_client() + elif props.get('token'): + args['client'] = Client(project=project, credentials=Credentials(**props)) + handle = GCSFS(bucket_name, root_path=root_path, retry=0, **args) + return handle + + +__all__ = ('GoogleCloudStorageFilesSource',) diff --git a/lib/galaxy/files/sources/googledrive.py b/lib/galaxy/files/sources/googledrive.py new file mode 100644 index 000000000000..1c8da18849c7 --- /dev/null +++ b/lib/galaxy/files/sources/googledrive.py @@ -0,0 +1,22 @@ +try: + from fs.googledrivefs import GoogleDriveFS + from google.oauth2.credentials import Credentials +except ImportError: + GoogleDriveFS = None + +from ._pyfilesystem2 import PyFilesystem2FilesSource + + +class GoogleDriveFilesSource(PyFilesystem2FilesSource): + plugin_type = 'googledrive' + required_module = GoogleDriveFS + required_package = "fs.googledrivefs" + + def _open_fs(self, user_context): + props = self._serialization_props(user_context) + credentials = Credentials(**props) + handle = GoogleDriveFS(credentials) + return handle + + +__all__ = ('GoogleDriveFilesSource',) diff --git a/lib/galaxy/files/sources/onedata.py b/lib/galaxy/files/sources/onedata.py new file mode 100644 index 000000000000..d9e29d33c9b3 --- /dev/null +++ b/lib/galaxy/files/sources/onedata.py @@ -0,0 +1,20 @@ +try: + from fs.onedatafs import OnedataFS +except ImportError: + OnedataFS = None + +from ._pyfilesystem2 import PyFilesystem2FilesSource + + +class OneDataFilesSource(PyFilesystem2FilesSource): + plugin_type = 'onedata' + required_module = OnedataFS + required_package = "fs-onedatafs" + + def _open_fs(self, user_context): + props = self._serialization_props(user_context) + handle = OnedataFS(**props) + return handle + + +__all__ = ('OneDataFilesSource',) diff --git a/packages/files/test-requirements.txt b/packages/files/test-requirements.txt index e079f8a6038d..f00855512c0b 100644 --- a/packages/files/test-requirements.txt +++ b/packages/files/test-requirements.txt @@ -1 +1,2 @@ pytest +fs-gcsfs diff --git a/test/unit/files/_util.py b/test/unit/files/_util.py index eca9cf1fe708..8e7e59499af1 100644 --- a/test/unit/files/_util.py +++ b/test/unit/files/_util.py @@ -4,6 +4,7 @@ from galaxy.files import ( ConfiguredFileSources, + ConfiguredFileSourcesConfig, DictFileSourcesUserContext, ) @@ -55,6 +56,21 @@ def user_context_fixture(user_ftp_dir=None, role_names=None, group_names=None, i preferences={ 'webdav|password': 'secret1234', 'dropbox|access_token': os.environ.get('GALAXY_TEST_DROPBOX_ACCESS_TOKEN'), + 'googledrive|client_id': os.environ.get('GALAXY_TEST_GOOGLE_DRIVE_CLIENT_ID'), + 'googledrive|client_secret': os.environ.get('GALAXY_TEST_GOOGLE_DRIVE_CLIENT_SECRET'), + 'googledrive|access_token': os.environ.get('GALAXY_TEST_GOOGLE_DRIVE_ACCESS_TOKEN'), + 'googledrive|refresh_token': os.environ.get('GALAXY_TEST_GOOGLE_DRIVE_REFRESH_TOKEN'), + 'googlecloudstorage|project': os.environ.get('GALAXY_TEST_GCS_PROJECT'), + 'googlecloudstorage|bucket_name': os.environ.get('GALAXY_TEST_GCS_BUCKET'), + 'googlecloudstorage|client_id': os.environ.get('GALAXY_TEST_GCS_CLIENT_ID'), + 'googlecloudstorage|client_secret': os.environ.get('GALAXY_TEST_GCS_CLIENT_SECRET'), + 'googlecloudstorage|access_token': os.environ.get('GALAXY_TEST_GCS_ACCESS_TOKEN'), + 'googlecloudstorage|refresh_token': os.environ.get('GALAXY_TEST_GCS_REFRESH_TOKEN'), + 'onedata|provider_host': os.environ.get('GALAXY_TEST_ONEDATA_PROVIDER_HOST'), + 'onedata|access_token': os.environ.get('GALAXY_TEST_ONEDATA_ACCESS_TOKEN'), + 'basespace|client_id': os.environ.get('GALAXY_TEST_ONEDATA_CLIENT_ID'), + 'basespace|client_secret': os.environ.get('GALAXY_TEST_ONEDATA_CLIENT_SECRET'), + 'basespace|access_token': os.environ.get('GALAXY_TEST_ONEDATA_ACCESS_TOKEN'), }, role_names=role_names or set(), group_names=group_names or set(), @@ -63,43 +79,43 @@ def user_context_fixture(user_ftp_dir=None, role_names=None, group_names=None, i return user_context -def assert_realizes_as(file_sources, uri, expected, user_context=None): +def realize_to_temp_file(file_sources, uri, user_context=None): file_source_path = file_sources.get_file_source_path(uri) with tempfile.NamedTemporaryFile(mode='r') as temp: file_source_path.file_source.realize_to(file_source_path.path, temp.name, user_context=user_context) with open(temp.name) as f: realized_contents = f.read() - if realized_contents != expected: - message = "Expected to realize contents at [{}] as [{}], instead found [{}]".format( - uri, - expected, - realized_contents, - ) - raise AssertionError(message) + return realized_contents + + +def assert_realizes_as(file_sources, uri, expected, user_context=None): + realized_contents = realize_to_temp_file(file_sources, uri, user_context=user_context) + if realized_contents != expected: + message = "Expected to realize contents at [{}] as [{}], instead found [{}]".format( + uri, + expected, + realized_contents, + ) + raise AssertionError(message) def assert_realizes_contains(file_sources, uri, expected, user_context=None): - file_source_path = file_sources.get_file_source_path(uri) - with tempfile.NamedTemporaryFile(mode='r') as temp: - file_source_path.file_source.realize_to(file_source_path.path, temp.name, user_context=user_context) - realized_contents = temp.read() - if expected not in realized_contents: - message = "Expected to realize contents at [{}] to contain [{}], instead found [{}]".format( - uri, - expected, - realized_contents, - ) - raise AssertionError(message) + realized_contents = realize_to_temp_file(file_sources, uri, user_context=user_context) + if expected not in realized_contents: + message = "Expected to realize contents at [{}] to contain [{}], instead found [{}]".format( + uri, + expected, + realized_contents, + ) + raise AssertionError(message) def assert_realizes_throws_exception(file_sources, uri, user_context=None) -> Exception: - file_source_path = file_sources.get_file_source_path(uri) exception = None - with tempfile.NamedTemporaryFile(mode='r') as temp: - try: - file_source_path.file_source.realize_to(file_source_path.path, temp.name, user_context=user_context) - except Exception as e: - exception = e + try: + realize_to_temp_file(file_sources, uri, user_context=user_context) + except Exception as e: + exception = e assert exception return exception @@ -110,3 +126,25 @@ def write_from(file_sources, uri, content, user_context=None): f.write(content) f.flush() file_source_path.file_source.write_from(file_source_path.path, f.name, user_context=user_context) + + +def configured_file_sources(conf_file): + file_sources_config = ConfiguredFileSourcesConfig() + return ConfiguredFileSources(file_sources_config, conf_file=conf_file) + + +def assert_simple_file_realize(conf_file, recursive=False, filename="a", contents="a\n", contains=False): + user_context = user_context_fixture() + file_sources = configured_file_sources(conf_file) + file_source_pair = file_sources.get_file_source_path("gxfiles://test1") + + assert file_source_pair.path == "/" + file_source = file_source_pair.file_source + res = file_source.list("/", recursive=recursive, user_context=user_context) + a_file = find(res, class_="File", name=filename) + assert a_file + + if contains: + assert_realizes_contains(file_sources, f"gxfiles://test1/{filename}", contents, user_context=user_context) + else: + assert_realizes_as(file_sources, f"gxfiles://test1/{filename}", contents, user_context=user_context) diff --git a/test/unit/files/basespace_file_sources_conf.yml b/test/unit/files/basespace_file_sources_conf.yml new file mode 100644 index 000000000000..092c00c64ac0 --- /dev/null +++ b/test/unit/files/basespace_file_sources_conf.yml @@ -0,0 +1,7 @@ +- type: basespace + id: test1 + doc: Test access to Illumina BaseSpace + basespace_server: https://api.basespace.illumina.com + client_id: ${user.preferences['basespace|client_id']} + client_secret: ${user.preferences['basespace|client_secret']} + access_token: ${user.preferences['basespace|access_token']} diff --git a/test/unit/files/dropbox_file_sources_conf.yml b/test/unit/files/dropbox_file_sources_conf.yml index 9dac354b982d..99869accff89 100644 --- a/test/unit/files/dropbox_file_sources_conf.yml +++ b/test/unit/files/dropbox_file_sources_conf.yml @@ -1,4 +1,4 @@ - type: dropbox id: test1 - doc: Test WebDAV server. + doc: Test access to a dropbox account. accessToken: ${user.preferences['dropbox|access_token']} diff --git a/test/unit/files/gcsfs_file_sources_conf.yml b/test/unit/files/gcsfs_file_sources_conf.yml new file mode 100644 index 000000000000..d823636c6505 --- /dev/null +++ b/test/unit/files/gcsfs_file_sources_conf.yml @@ -0,0 +1,11 @@ +- type: googlecloudstorage + id: test1 + doc: Test access to Google Cloud Storage. + project: ${user.preferences['googlecloudstorage|project']} + bucket_name: 'genomics-public-data' + token_uri: "https://www.googleapis.com/oauth2/v4/token" + client_id: ${user.preferences['googlecloudstorage|client_id']} + client_secret: ${user.preferences['googlecloudstorage|client_secret']} + token: ${user.preferences['googlecloudstorage|access_token']} + refresh_token: ${user.preferences['googlecloudstorage|refresh_token']} + anonymous: true diff --git a/test/unit/files/googledrive_file_sources_conf.yml b/test/unit/files/googledrive_file_sources_conf.yml new file mode 100644 index 000000000000..809238465773 --- /dev/null +++ b/test/unit/files/googledrive_file_sources_conf.yml @@ -0,0 +1,8 @@ +- type: googledrive + id: test1 + doc: Test access to a Google drive. + token: ${user.preferences['googledrive|access_token']} + refresh_token: ${user.preferences['googledrive|refresh_token']} + token_uri: "https://www.googleapis.com/oauth2/v4/token" + client_id: ${user.preferences['googledrive|client_id']} + client_secret: ${user.preferences['googledrive|client_secret']} diff --git a/test/unit/files/onedata_file_sources_conf.yml b/test/unit/files/onedata_file_sources_conf.yml new file mode 100644 index 000000000000..c452acdc2994 --- /dev/null +++ b/test/unit/files/onedata_file_sources_conf.yml @@ -0,0 +1,5 @@ +- type: onedata + id: test1 + doc: Test access to a OneData host + provider_host: ${user.preferences['onedata|provider_host']} + access_token: ${user.preferences['onedata|access_token']} diff --git a/test/unit/files/test_basespace.py b/test/unit/files/test_basespace.py new file mode 100644 index 000000000000..85719d8d1f0a --- /dev/null +++ b/test/unit/files/test_basespace.py @@ -0,0 +1,36 @@ +import os + +import pytest + +from ._util import ( + assert_realizes_as, + configured_file_sources, + find, + user_context_fixture, +) +SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__)) +FILE_SOURCES_CONF = os.path.join(SCRIPT_DIRECTORY, "basespace_file_sources_conf.yml") + +skip_if_no_basespace_access_token = pytest.mark.skipif( + not os.environ.get('GALAXY_TEST_BASESPACE_CLIENT_ID') + or not os.environ.get('GALAXY_TEST_BASESPACE_CLIENT_SECRET') + or not os.environ.get('GALAXY_TEST_BASESPACE_ACCESS_TOKEN') + or not os.environ.get('GALAXY_TEST_BASESPACE_TEST_FILE_PATH'), + reason="GALAXY_TEST_BASESPACE_CLIENT_ID and related vars not set" +) + + +@skip_if_no_basespace_access_token +def test_file_source(): + user_context = user_context_fixture() + file_sources = configured_file_sources(FILE_SOURCES_CONF) + file_source_pair = file_sources.get_file_source_path("gxfiles://test1") + + assert file_source_pair.path == "/" + file_source = file_source_pair.file_source + test_file = os.environ.get('GALAXY_TEST_BASESPACE_TEST_FILE_PATH', "") + res = file_source.list(os.path.dirname(test_file), recursive=False, user_context=user_context) + a_file = find(res, class_="File", name=os.path.basename(test_file)) + assert a_file + + assert_realizes_as(file_sources, a_file['uri'], "a\n", user_context=user_context) diff --git a/test/unit/files/test_dropbox.py b/test/unit/files/test_dropbox.py index 9d655fa8d15e..7bee738d288b 100644 --- a/test/unit/files/test_dropbox.py +++ b/test/unit/files/test_dropbox.py @@ -2,12 +2,8 @@ import pytest -from galaxy.files import ConfiguredFileSources, ConfiguredFileSourcesConfig -from ._util import ( - assert_realizes_as, - find_file_a, - user_context_fixture, -) +from ._util import assert_simple_file_realize + SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__)) FILE_SOURCES_CONF = os.path.join(SCRIPT_DIRECTORY, "dropbox_file_sources_conf.yml") @@ -19,19 +15,4 @@ @skip_if_no_dropbox_access_token def test_file_source(): - user_context = user_context_fixture() - file_sources = _configured_file_sources() - file_source_pair = file_sources.get_file_source_path("gxfiles://test1") - - assert file_source_pair.path == "/" - file_source = file_source_pair.file_source - res = file_source.list("/", recursive=True, user_context=user_context) - a_file = find_file_a(res) - assert a_file - - assert_realizes_as(file_sources, "gxfiles://test1/a", "a\n", user_context=user_context) - - -def _configured_file_sources(conf_file=FILE_SOURCES_CONF): - file_sources_config = ConfiguredFileSourcesConfig() - return ConfiguredFileSources(file_sources_config, conf_file=conf_file) + assert_simple_file_realize(FILE_SOURCES_CONF, recursive=True) diff --git a/test/unit/files/test_gcsfs.py b/test/unit/files/test_gcsfs.py new file mode 100644 index 000000000000..8ad3a46cca5d --- /dev/null +++ b/test/unit/files/test_gcsfs.py @@ -0,0 +1,25 @@ +import os + +import pytest + +from ._util import assert_simple_file_realize + +try: + from fs_gcsfs import GCSFS +except ImportError: + GCSFS = None + +SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__)) +FILE_SOURCES_CONF = os.path.join(SCRIPT_DIRECTORY, "gcsfs_file_sources_conf.yml") + + +skip_if_no_gcsfs_libs = pytest.mark.skipif( + not GCSFS, + reason="Required lib to run gcs file source test: fs_gcsfs is not available" +) + + +@skip_if_no_gcsfs_libs +def test_file_source(): + assert_simple_file_realize(FILE_SOURCES_CONF, recursive=False, filename="README", contents="1000genomes", + contains=True) diff --git a/test/unit/files/test_googledrive.py b/test/unit/files/test_googledrive.py new file mode 100644 index 000000000000..8fc39c1136b2 --- /dev/null +++ b/test/unit/files/test_googledrive.py @@ -0,0 +1,19 @@ +import os + +import pytest + +from ._util import assert_simple_file_realize + +SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__)) +FILE_SOURCES_CONF = os.path.join(SCRIPT_DIRECTORY, "googledrive_file_sources_conf.yml") + +skip_if_no_google_drive_access_token = pytest.mark.skipif( + not os.environ.get('GALAXY_TEST_GOOGLE_DRIVE_ACCESS_TOKEN') + or not os.environ.get('GALAXY_TEST_GOOGLE_DRIVE_REFRESH_TOKEN'), + reason="GALAXY_TEST_GOOGLE_DRIVE_ACCESS_TOKEN and related vars not set" +) + + +@skip_if_no_google_drive_access_token +def test_file_source(): + assert_simple_file_realize(FILE_SOURCES_CONF) diff --git a/test/unit/files/test_onedata.py b/test/unit/files/test_onedata.py new file mode 100644 index 000000000000..d78dad707c44 --- /dev/null +++ b/test/unit/files/test_onedata.py @@ -0,0 +1,19 @@ +import os + +import pytest + +from ._util import assert_simple_file_realize + +SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__)) +FILE_SOURCES_CONF = os.path.join(SCRIPT_DIRECTORY, "onedata_file_sources_conf.yml") + +skip_if_no_onedata_access_token = pytest.mark.skipif( + not os.environ.get('GALAXY_TEST_ONEDATA_PROVIDER_HOST') + or not os.environ.get('GALAXY_TEST_ONEDATA_ACCESS_TOKEN'), + reason="GALAXY_TEST_ONEDATA_PROVIDER_HOST and GALAXY_TEST_ONEDATA_ACCESS_TOKEN not set" +) + + +@skip_if_no_onedata_access_token +def test_file_source(): + assert_simple_file_realize(FILE_SOURCES_CONF) diff --git a/test/unit/files/test_s3.py b/test/unit/files/test_s3.py index cbae28b7948c..f88d7905401a 100644 --- a/test/unit/files/test_s3.py +++ b/test/unit/files/test_s3.py @@ -2,12 +2,8 @@ import pytest -from galaxy.files import ConfiguredFileSources, ConfiguredFileSourcesConfig -from ._util import ( - assert_realizes_contains, - find, - user_context_fixture, -) +from ._util import assert_simple_file_realize + SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__)) FILE_SOURCES_CONF = os.path.join(SCRIPT_DIRECTORY, "s3_file_sources_conf.yml") @@ -19,20 +15,5 @@ @skip_if_not_slow def test_file_source(): - user_context = user_context_fixture() - file_sources = _configured_file_sources() - file_source_pair = file_sources.get_file_source_path("gxfiles://test1") - - assert file_source_pair.path == "/" - file_source = file_source_pair.file_source - res = file_source.list("/", recursive=False, user_context=user_context) - print(res) - dup = find(res, "File", "data_use_policies.txt") - assert dup - - assert_realizes_contains(file_sources, "gxfiles://test1/data_use_policies.txt", "DATA USE POLICIES", user_context=user_context) - - -def _configured_file_sources(conf_file=FILE_SOURCES_CONF): - file_sources_config = ConfiguredFileSourcesConfig() - return ConfiguredFileSources(file_sources_config, conf_file=conf_file) + assert_simple_file_realize(FILE_SOURCES_CONF, recursive=False, filename="data_use_policies.txt", + contents="DATA USE POLICIES", contains=True) diff --git a/test/unit/files/test_webdav.py b/test/unit/files/test_webdav.py index d91bb8cda29d..30909b99e658 100644 --- a/test/unit/files/test_webdav.py +++ b/test/unit/files/test_webdav.py @@ -4,8 +4,8 @@ import pytest -from galaxy.files import ConfiguredFileSources, ConfiguredFileSourcesConfig from ._util import ( + configured_file_sources, find, find_file_a, list_dir, @@ -27,7 +27,7 @@ @skip_if_no_webdav def test_file_source(): - file_sources = _configured_file_sources() + file_sources = configured_file_sources(FILE_SOURCES_CONF) file_source_pair = file_sources.get_file_source_path("gxfiles://test1") assert file_source_pair.path == "/" @@ -59,7 +59,7 @@ def test_file_source(): @skip_if_no_webdav def test_sniff_to_tmp(): - file_sources = _configured_file_sources() + file_sources = configured_file_sources(FILE_SOURCES_CONF) _download_and_check_file(file_sources) @@ -67,7 +67,7 @@ def test_sniff_to_tmp(): def test_serialization(): # serialize the configured file sources and rematerialize them, # ensure they still function. This is needed for uploading files. - file_sources = serialize_and_recover(_configured_file_sources()) + file_sources = serialize_and_recover(configured_file_sources(FILE_SOURCES_CONF)) res = list_root(file_sources, "gxfiles://test1", recursive=True) assert find_file_a(res) @@ -80,7 +80,7 @@ def test_serialization(): @skip_if_no_webdav def test_serialization_user(): - file_sources_o = _configured_file_sources(USER_FILE_SOURCES_CONF) + file_sources_o = configured_file_sources(USER_FILE_SOURCES_CONF) user_context = user_context_fixture() res = list_root(file_sources_o, "gxfiles://test1", recursive=True, user_context=user_context) @@ -89,8 +89,3 @@ def test_serialization_user(): file_sources = serialize_and_recover(file_sources_o, user_context=user_context) res = list_root(file_sources, "gxfiles://test1", recursive=True, user_context=None) assert find_file_a(res) - - -def _configured_file_sources(conf_file=FILE_SOURCES_CONF): - file_sources_config = ConfiguredFileSourcesConfig() - return ConfiguredFileSources(file_sources_config, conf_file=conf_file)