Fixes and expansion of webdav unit tests.

lib/galaxy/files/sources/_pyfilesystem2.py

@@ -38,6 +38,7 @@ class PyFilesystem2FilesSource(BaseFilesSource):
     required_package: ClassVar[str]
     supports_pagination = True
     supports_search = True
+    allow_key_error_on_empty_directories = False  # work around a bug in webdav
 
     def __init__(self, **kwd: Unpack[FilesSourceProperties]):
         if self.required_module is None:
@@ -65,10 +66,14 @@ def _list(
             with self._open_fs(user_context=user_context, opts=opts) as h:
                 if recursive:
                     recursive_result: List[AnyRemoteEntry] = []
-                    for p, dirs, files in h.walk(path, namespaces=["details"]):
-                        to_dict = functools.partial(self._resource_info_to_dict, p)
-                        recursive_result.extend(map(to_dict, dirs))
-                        recursive_result.extend(map(to_dict, files))
+                    try:
+                        for p, dirs, files in h.walk(path, namespaces=["details"]):
+                            to_dict = functools.partial(self._resource_info_to_dict, p)
+                            recursive_result.extend(map(to_dict, dirs))
+                            recursive_result.extend(map(to_dict, files))
+                    except KeyError:
+                        if not self.allow_key_error_on_empty_directories:
+                            raise
                     return recursive_result, len(recursive_result)
                 else:
                     page = self._to_page(limit, offset)

lib/galaxy/files/sources/webdav.py

@@ -28,6 +28,7 @@ class WebDavFilesSource(PyFilesystem2FilesSource):
     plugin_type = "webdav"
     required_module = WebDAVFS
     required_package = "fs.webdavfs"
+    allow_key_error_on_empty_directories = True
 
     def _open_fs(self, user_context=None, opts: Optional[FilesSourceOptions] = None):
         props = cast(WebDavFilesSourceProperties, self._serialization_props(user_context))

test/unit/files/_util.py

@@ -2,6 +2,7 @@
 
 import os
 import tempfile
+from typing import Optional
 
 from galaxy.files import (
     ConfiguredFileSources,
@@ -156,8 +157,9 @@ def write_from(
         file_source_path.file_source.write_from(file_source_path.path, f.name, user_context=user_context)
 
 
-def configured_file_sources(conf_file):
-    file_sources_config = FileSourcePluginsConfig()
+def configured_file_sources(conf_file, file_sources_config: Optional[FileSourcePluginsConfig] = None):
+    file_sources_config = file_sources_config or FileSourcePluginsConfig()
+    assert file_sources_config
     if isinstance(conf_file, str):
         conf = ConfiguredFileSourcesConf(conf_file=conf_file)
     else:

test/unit/files/test_webdav.py

@@ -1,9 +1,10 @@
 # docker run -v `pwd`/test/integration/webdav/data:/media  -e WEBDAV_USERNAME=alice -e WEBDAV_PASSWORD=secret1234 -p 7083:7083 jmchilton/webdavdev
-# GALAXY_TEST_WEBDAV=1 pytest test/unit/files/webdav.py
+# GALAXY_TEST_WEBDAV=1 pytest test/unit/files/test_webdav.py
 import os
 
 import pytest
 
+from galaxy.files.plugins import FileSourcePluginsConfig
 from ._util import (
     configured_file_sources,
     find,
@@ -17,6 +18,7 @@
 
 SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__))
 FILE_SOURCES_CONF = os.path.join(SCRIPT_DIRECTORY, "webdav_file_sources_conf.yml")
+FILE_SOURCES_CONF_NO_USE_TEMP_FILES = os.path.join(SCRIPT_DIRECTORY, "webdav_file_sources_without_use_temp_conf.yml")
 USER_FILE_SOURCES_CONF = os.path.join(SCRIPT_DIRECTORY, "webdav_user_file_sources_conf.yml")
 
 skip_if_no_webdav = pytest.mark.skipif(not os.environ.get("GALAXY_TEST_WEBDAV"), reason="GALAXY_TEST_WEBDAV not set")
@@ -62,17 +64,40 @@ def test_sniff_to_tmp():
 
 @skip_if_no_webdav
 def test_serialization():
-    # serialize the configured file sources and rematerialize them,
-    # ensure they still function. This is needed for uploading files.
-    file_sources = serialize_and_recover(configured_file_sources(FILE_SOURCES_CONF))
+    configs = [FILE_SOURCES_CONF_NO_USE_TEMP_FILES, FILE_SOURCES_CONF]
+    for config in configs:
+        # serialize the configured file sources and rematerialize them,
+        # ensure they still function. This is needed for uploading files.
+        file_sources = serialize_and_recover(configured_file_sources(config))
 
-    res = list_root(file_sources, "gxfiles://test1", recursive=True)
-    assert find_file_a(res)
+        res = list_root(file_sources, "gxfiles://test1", recursive=True)
+        assert find_file_a(res)
 
-    res = list_root(file_sources, "gxfiles://test1", recursive=False)
-    assert find_file_a(res)
+        res = list_root(file_sources, "gxfiles://test1", recursive=False)
+        assert find_file_a(res)
 
-    _download_and_check_file(file_sources)
+        _download_and_check_file(file_sources)
+
+
+@skip_if_no_webdav
+def test_config_options():
+    file_sources = configured_file_sources(FILE_SOURCES_CONF)
+    fs = file_sources._file_sources[0]
+    user_context = user_context_fixture()
+    assert fs._open_fs(user_context).use_temp_files
+
+    file_sources = configured_file_sources(FILE_SOURCES_CONF_NO_USE_TEMP_FILES)
+    fs = file_sources._file_sources[0]
+    user_context = user_context_fixture()
+    assert not fs._open_fs(user_context).use_temp_files
+
+    disable_default_use_temp = FileSourcePluginsConfig(
+        webdav_use_temp_files=False,
+    )
+    file_sources = configured_file_sources(FILE_SOURCES_CONF, disable_default_use_temp)
+    fs = file_sources._file_sources[0]
+    user_context = user_context_fixture()
+    assert not fs._open_fs(user_context).use_temp_files
 
 
 @skip_if_no_webdav

test/unit/files/webdav_file_sources_conf.yml

@@ -0,0 +1,6 @@
+- type: webdav
+  id: test1
+  doc: Test WebDAV server.
+  url: http://127.0.0.1:7083
+  login: alice
+  password: secret1234  

test/unit/files/webdav_file_sources_without_use_temp_conf.yml

@@ -0,0 +1,7 @@
+- type: webdav
+  id: test1
+  doc: Test WebDAV server.
+  url: http://127.0.0.1:7083
+  login: alice
+  password: secret1234  
+  use_temp_files: false