Skip to content

Commit

Permalink
Fixes and expansion of webdav unit tests.
Browse files Browse the repository at this point in the history
  • Loading branch information
jmchilton committed Oct 1, 2024
1 parent d811673 commit 5849b06
Show file tree
Hide file tree
Showing 6 changed files with 61 additions and 15 deletions.
13 changes: 9 additions & 4 deletions lib/galaxy/files/sources/_pyfilesystem2.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ class PyFilesystem2FilesSource(BaseFilesSource):
required_package: ClassVar[str]
supports_pagination = True
supports_search = True
allow_key_error_on_empty_directories = False # work around a bug in webdav

def __init__(self, **kwd: Unpack[FilesSourceProperties]):
if self.required_module is None:
Expand Down Expand Up @@ -65,10 +66,14 @@ def _list(
with self._open_fs(user_context=user_context, opts=opts) as h:
if recursive:
recursive_result: List[AnyRemoteEntry] = []
for p, dirs, files in h.walk(path, namespaces=["details"]):
to_dict = functools.partial(self._resource_info_to_dict, p)
recursive_result.extend(map(to_dict, dirs))
recursive_result.extend(map(to_dict, files))
try:
for p, dirs, files in h.walk(path, namespaces=["details"]):
to_dict = functools.partial(self._resource_info_to_dict, p)
recursive_result.extend(map(to_dict, dirs))
recursive_result.extend(map(to_dict, files))
except KeyError:
if not self.allow_key_error_on_empty_directories:
raise
return recursive_result, len(recursive_result)
else:
page = self._to_page(limit, offset)
Expand Down
1 change: 1 addition & 0 deletions lib/galaxy/files/sources/webdav.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ class WebDavFilesSource(PyFilesystem2FilesSource):
plugin_type = "webdav"
required_module = WebDAVFS
required_package = "fs.webdavfs"
allow_key_error_on_empty_directories = True

def _open_fs(self, user_context=None, opts: Optional[FilesSourceOptions] = None):
props = cast(WebDavFilesSourceProperties, self._serialization_props(user_context))
Expand Down
6 changes: 4 additions & 2 deletions test/unit/files/_util.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import os
import tempfile
from typing import Optional

from galaxy.files import (
ConfiguredFileSources,
Expand Down Expand Up @@ -156,8 +157,9 @@ def write_from(
file_source_path.file_source.write_from(file_source_path.path, f.name, user_context=user_context)


def configured_file_sources(conf_file):
file_sources_config = FileSourcePluginsConfig()
def configured_file_sources(conf_file, file_sources_config: Optional[FileSourcePluginsConfig] = None):
file_sources_config = file_sources_config or FileSourcePluginsConfig()
assert file_sources_config
if isinstance(conf_file, str):
conf = ConfiguredFileSourcesConf(conf_file=conf_file)
else:
Expand Down
43 changes: 34 additions & 9 deletions test/unit/files/test_webdav.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
# docker run -v `pwd`/test/integration/webdav/data:/media -e WEBDAV_USERNAME=alice -e WEBDAV_PASSWORD=secret1234 -p 7083:7083 jmchilton/webdavdev
# GALAXY_TEST_WEBDAV=1 pytest test/unit/files/webdav.py
# GALAXY_TEST_WEBDAV=1 pytest test/unit/files/test_webdav.py
import os

import pytest

from galaxy.files.plugins import FileSourcePluginsConfig
from ._util import (
configured_file_sources,
find,
Expand All @@ -17,6 +18,7 @@

SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__))
FILE_SOURCES_CONF = os.path.join(SCRIPT_DIRECTORY, "webdav_file_sources_conf.yml")
FILE_SOURCES_CONF_NO_USE_TEMP_FILES = os.path.join(SCRIPT_DIRECTORY, "webdav_file_sources_without_use_temp_conf.yml")
USER_FILE_SOURCES_CONF = os.path.join(SCRIPT_DIRECTORY, "webdav_user_file_sources_conf.yml")

skip_if_no_webdav = pytest.mark.skipif(not os.environ.get("GALAXY_TEST_WEBDAV"), reason="GALAXY_TEST_WEBDAV not set")
Expand Down Expand Up @@ -62,17 +64,40 @@ def test_sniff_to_tmp():

@skip_if_no_webdav
def test_serialization():
# serialize the configured file sources and rematerialize them,
# ensure they still function. This is needed for uploading files.
file_sources = serialize_and_recover(configured_file_sources(FILE_SOURCES_CONF))
configs = [FILE_SOURCES_CONF_NO_USE_TEMP_FILES, FILE_SOURCES_CONF]
for config in configs:
# serialize the configured file sources and rematerialize them,
# ensure they still function. This is needed for uploading files.
file_sources = serialize_and_recover(configured_file_sources(config))

res = list_root(file_sources, "gxfiles://test1", recursive=True)
assert find_file_a(res)
res = list_root(file_sources, "gxfiles://test1", recursive=True)
assert find_file_a(res)

res = list_root(file_sources, "gxfiles://test1", recursive=False)
assert find_file_a(res)
res = list_root(file_sources, "gxfiles://test1", recursive=False)
assert find_file_a(res)

_download_and_check_file(file_sources)
_download_and_check_file(file_sources)


@skip_if_no_webdav
def test_config_options():
file_sources = configured_file_sources(FILE_SOURCES_CONF)
fs = file_sources._file_sources[0]
user_context = user_context_fixture()
assert fs._open_fs(user_context).use_temp_files

file_sources = configured_file_sources(FILE_SOURCES_CONF_NO_USE_TEMP_FILES)
fs = file_sources._file_sources[0]
user_context = user_context_fixture()
assert not fs._open_fs(user_context).use_temp_files

disable_default_use_temp = FileSourcePluginsConfig(
webdav_use_temp_files=False,
)
file_sources = configured_file_sources(FILE_SOURCES_CONF, disable_default_use_temp)
fs = file_sources._file_sources[0]
user_context = user_context_fixture()
assert not fs._open_fs(user_context).use_temp_files


@skip_if_no_webdav
Expand Down
6 changes: 6 additions & 0 deletions test/unit/files/webdav_file_sources_conf.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
- type: webdav
id: test1
doc: Test WebDAV server.
url: http://127.0.0.1:7083
login: alice
password: secret1234
7 changes: 7 additions & 0 deletions test/unit/files/webdav_file_sources_without_use_temp_conf.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
- type: webdav
id: test1
doc: Test WebDAV server.
url: http://127.0.0.1:7083
login: alice
password: secret1234
use_temp_files: false

0 comments on commit 5849b06

Please sign in to comment.