Ls revamp

setup.py

@@ -35,6 +35,7 @@
     install_requires=[
         "aiohttp~=3.9.4",
         "aiosignal~=1.3.1",
+        "aiowebdav~=0.1.0rc5",
         "async-timeout~=4.0.3",
         "attrs~=23.2.0",
         "frozenlist~=1.4.1",

src/pelicanfs/core.py

@@ -24,6 +24,7 @@
 import aiohttp
 import cachetools
 import fsspec.implementations.http as fshttp
+from aiowebdav.client import Client
 from fsspec.asyn import AsyncFileSystem, sync
 from fsspec.utils import glob_translate
 
@@ -221,6 +222,9 @@ def __init__(
         self._mkdir = self.http_file_system._mkdir
         self._makedirs = self.http_file_system._makedirs
 
+        # Overwrite the httpsfs _ls_real call with ours with ours
+        self.http_file_system._ls_real = self._ls_real
+
     # Note this is a class method because it's overwriting a class method for the AbstractFileSystem
     @classmethod
     def _strip_protocol(cls, path):
@@ -384,10 +388,7 @@ async def get_origin_url(self, fileloc: str) -> str:
             raise NoAvailableSource()
         return origin
 
-    async def get_dirlist_url(self, fileloc: str) -> str:
-        """
-        Returns a dirlist host url for the given namespace locations
-        """
+    async def _set_director_url(self) -> str:
         if not self.director_url:
             metadata_json = await self._discover_federation_metadata(self.discovery_url)
             # Ensure the director url has a '/' at the end
@@ -399,6 +400,12 @@ async def get_dirlist_url(self, fileloc: str) -> str:
                 director_url = director_url + "/"
             self.director_url = director_url
 
+    async def get_dirlist_url(self, fileloc: str) -> str:
+        """
+        Returns a dirlist host url for the given namespace locations
+        """
+        await self._set_director_url()
+
         url = urllib.parse.urljoin(self.director_url, fileloc)
 
         # Timeout response in seconds - the default response is 5 minutes
@@ -470,8 +477,53 @@ async def wrapper(self, *args, **kwargs):
 
     @_dirlist_dec
     async def _ls(self, path, detail=True, **kwargs):
-        results = await self.http_file_system._ls(path, detail, **kwargs)
-        return self._remove_host_from_paths(results)
+        """
+        This _ls call will mimic the httpfs _ls call and call our version of _ls_real
+        """
+        # TODO: Add the optional listings cache for pelicanfs - removed for now in order to keep things simple for
+        # the re-implementation of ls
+        # if self.use_listings_cache and path in self.dircache:
+        #    out = self.dircache[path]
+        # else:
+        out = await self._ls_real(path, detail=detail, **kwargs)
+        # self.dircache[path] = out
+        return self._remove_host_from_paths(out)
+
+    async def _ls_real(self, url, detail=True, **kwargs):
+        """
+        This _ls_real uses a webdavclient listing rather than an https call. This lets pelicanfs identify whether an object
+        is a file or a collection. This is important for functions which are expected to recurse or walk the collection url
+        such as find/glob/walk
+        """
+        # ignoring URL-encoded arguments
+        logger.debug(url)
+        parts = urllib.parse.urlparse(url)
+        base_url = f"{parts.scheme}://{parts.netloc}"
+
+        # Create the options for the webdavclient
+
+        options = {
+            "hostname": base_url,
+        }
+
+        async with Client(options) as client:
+            remote_dir = parts.path
+            try:
+                items = await client.list(remote_dir, get_info=True)
+                if detail:
+                    return [
+                        {
+                            "name": f"{base_url}{item['path']}",  # use the base url in order for httpfs find/walk to be able to call its info
+                            "size": None,
+                            "type": "directory" if item["isdir"] else "file",
+                        }
+                        for item in items
+                    ]
+                else:
+                    return sorted([item["path"] for item in items])  # TODO: Check to see if this needs to match the name scheme
+            except Exception:
+                # TODO: Check for if the top level is a file and not a directory and handle accordingly
+                raise
 
     @_dirlist_dec
     async def _isdir(self, path):