Skip to content

Commit

Permalink
feat: unstrip protocol to make globbing work (#40)
Browse files Browse the repository at this point in the history
* save hostid to class, override parent unstrip_protocol

* test globing
  • Loading branch information
lobis authored Dec 12, 2023
1 parent 0b5a06f commit 326ed48
Show file tree
Hide file tree
Showing 2 changed files with 22 additions and 0 deletions.
4 changes: 4 additions & 0 deletions src/fsspec_xrootd/xrootd.py
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,7 @@ def __init__(
"""
super().__init__(self, asynchronous=asynchronous, loop=loop, **storage_options)
self.timeout = storage_options.get("timeout", XRootDFileSystem.default_timeout)
self.hostid = hostid
self._myclient = client.FileSystem("root://" + hostid)
if not self._myclient.url.is_valid():
raise ValueError(f"Invalid hostid: {hostid!r}")
Expand Down Expand Up @@ -207,6 +208,9 @@ def _strip_protocol(cls, path: str | list[str]) -> Any:
else:
raise ValueError("Strip protocol not given string or list")

def unstrip_protocol(self, name: str) -> str:
return f"{self.protocol}://{self.hostid}/{name}"

async def _mkdir(
self, path: str, create_parents: bool = True, **kwargs: Any
) -> None:
Expand Down
18 changes: 18 additions & 0 deletions tests/test_basicio.py
Original file line number Diff line number Diff line change
Expand Up @@ -394,3 +394,21 @@ class MockVectorReadInfo:
b"0" * 20,
b"0" * 10,
]


def test_glob_full_names(localserver, clear_server):
remoteurl, localpath = localserver
os.makedirs(localpath + "/WalkFolder")
with open(localpath + "/WalkFolder/testfile1.txt", "w") as fout:
fout.write(TESTDATA1)
with open(localpath + "/WalkFolder/testfile2.txt", "w") as fout:
fout.write(TESTDATA2)
time.sleep(sleep_time)

full_names = [
f.full_name for f in fsspec.open_files(remoteurl + "/WalkFolder/*.txt")
]

for name in full_names:
with fsspec.open(name) as f:
assert f.read() in [bytes(data, "utf-8") for data in [TESTDATA1, TESTDATA2]]

0 comments on commit 326ed48

Please sign in to comment.