From 446197137c76923a9f6f89e2c532fea0283c2420 Mon Sep 17 00:00:00 2001 From: Han Wang Date: Sat, 4 Nov 2023 22:29:25 -0700 Subject: [PATCH] Fix makedirs (#120) * Add more io utils * update * update * update * update * Add more io utils * update * update * update * update * update * update * update --- .devcontainer/devcontainer.json | 14 +++++++-- tests/utils/test_io.py | 56 +++++++++++++++++++++++++++------ triad/utils/io.py | 48 +++++++++++++++++++++++++--- 3 files changed, 100 insertions(+), 18 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 0d7f005..55677d0 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -7,6 +7,7 @@ "terminal.integrated.shell.linux": "/bin/bash", "python.pythonPath": "/usr/local/bin/python", "python.defaultInterpreterPath": "/usr/local/bin/python", + "editor.defaultFormatter": "ms-python.black-formatter", "isort.interpreter": [ "/usr/local/bin/python" ], @@ -15,6 +16,9 @@ ], "pylint.interpreter": [ "/usr/local/bin/python" + ], + "black-formatter.interpreter": [ + "/usr/local/bin/python" ] }, "extensions": [ @@ -23,6 +27,7 @@ "ms-python.flake8", "ms-python.pylint", "ms-python.mypy", + "ms-python.black-formatter", "GitHub.copilot", "njpwerner.autodocstring" ] @@ -32,7 +37,10 @@ 8888 ], "postCreateCommand": "make devenv", - "mounts": [ - "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind" - ] + "features": { + "ghcr.io/devcontainers/features/docker-in-docker:2": {}, + "ghcr.io/devcontainers/features/java:1": { + "version": "11" + } + } } diff --git a/tests/utils/test_io.py b/tests/utils/test_io.py index 8ef6d54..6ce0c15 100644 --- a/tests/utils/test_io.py +++ b/tests/utils/test_io.py @@ -1,9 +1,8 @@ import os +import sys from io import BytesIO import pytest -import sys -import os import triad.utils.io as iou @@ -38,6 +37,9 @@ def _assert(gres, expected): [os.path.join(str(tmpdir), "a", "a.txt")], ) + iou.touch("memory://gtest/m.txt", auto_mkdir=True) + assert iou.glob("memory://gtest/*.txt") == ["memory:///gtest/m.txt"] + @pytest.mark.skipif(sys.platform.startswith("win"), reason="not a test for windows") def test_join_not_win(): @@ -55,6 +57,30 @@ def test_join_is_win(): assert iou.join("c:\\a", "b", "*.parquet") == "c:\\a\\b\\*.parquet" +@pytest.mark.skipif(sys.platform.startswith("win"), reason="not a test for windows") +def test_abs_path_not_win(tmpdir): + with iou.chdir(str(tmpdir)): + assert iou.abs_path("a") == os.path.join(str(tmpdir), "a") + assert iou.abs_path("/tmp/x") == "/tmp/x" + assert iou.abs_path("file:///tmp/x") == "/tmp/x" + assert iou.abs_path("memory://tmp/x") == "memory://tmp/x" + assert iou.abs_path("memory:///tmp/x") == "memory:///tmp/x" + assert iou.abs_path("dummy:///tmp/x") == "dummy:///tmp/x" + + +@pytest.mark.skipif( + not sys.platform.startswith("win"), reason="a test only for windows" +) +def test_abs_path_is_win(tmpdir): + with iou.chdir(str(tmpdir)): + assert iou.abs_path("a") == os.path.join(str(tmpdir), "a") + assert iou.abs_path("c:\\tmp\\x") == "c:\\tmp\\x" + assert iou.abs_path("file://c:/tmp/x") == "c:\\tmp\\x" + assert iou.abs_path("memory://tmp/x") == "memory://tmp/x" + assert iou.abs_path("memory:///tmp/x") == "memory:///tmp/x" + assert iou.abs_path("dummy:///tmp/x") == "dummy:///tmp/x" + + def test_makedirs(tmpdir): path = os.path.join(str(tmpdir), "temp", "a") assert path == iou.makedirs(path, exist_ok=False) @@ -63,14 +89,12 @@ def test_makedirs(tmpdir): iou.makedirs(path, exist_ok=False) iou.makedirs(path, exist_ok=True) - op = os.getcwd() - os.chdir(str(tmpdir)) - assert os.path.join(str(tmpdir), "temp", "b") == iou.makedirs( - iou.join("temp", "b"), exist_ok=False - ) - assert iou.exists(iou.join("temp", "b")) - assert iou.exists(os.path.join(str(tmpdir), "temp", "b")) - os.chdir(op) + with iou.chdir(str(tmpdir)): + assert os.path.join(str(tmpdir), "temp", "b") == iou.makedirs( + iou.join("temp", "b"), exist_ok=False + ) + assert iou.exists(iou.join("temp", "b")) + assert iou.exists(os.path.join(str(tmpdir), "temp", "b")) path = "memory://temp/a" assert path == iou.makedirs(path, exist_ok=True) @@ -78,6 +102,12 @@ def test_makedirs(tmpdir): with pytest.raises(OSError): iou.makedirs(path, exist_ok=False) + path = os.path.join(str(tmpdir), "temp", "aa") + fs, _path = iou.url_to_fs(path) + _path = fs.unstrip_protocol(_path) # file:///... + iou.makedirs(_path, exist_ok=False) + assert iou.exists(path) + def test_exists(tmpdir): iou.write_text(os.path.join(str(tmpdir), "a.txt"), "a") @@ -88,6 +118,12 @@ def test_exists(tmpdir): assert iou.exists(os.path.join(str(tmpdir), "temp")) assert iou.exists(str(tmpdir)) + with iou.chdir(str(tmpdir)): + assert iou.exists("a.txt") + assert iou.exists(os.path.join("temp", "a.txt")) + assert iou.exists("temp") + assert iou.exists(".") + def test_is_dir_or_file(tmpdir): for base in [str(tmpdir), "memory://"]: diff --git a/triad/utils/io.py b/triad/utils/io.py index d9a3b53..6cd2c7b 100644 --- a/triad/utils/io.py +++ b/triad/utils/io.py @@ -4,7 +4,7 @@ import zipfile from contextlib import contextmanager from pathlib import Path, PurePosixPath -from typing import Any, Iterator, Tuple, List +from typing import Any, Iterator, List, Tuple import fsspec import fsspec.core as fc @@ -14,6 +14,29 @@ _SCHEME_PREFIX = re.compile(r"^[a-zA-Z0-9\-_]+:") +@contextmanager +def chdir(path: str) -> Iterator[None]: + """Change the current working directory to the given path + + :param path: the path to change to + + .. admonition:: Examples + + .. code-block:: python + + from fugue_ml.utils.io import chdir + + with chdir("/tmp"): + # do something + """ + op = os.getcwd() + os.chdir(path) + try: + yield + finally: + os.chdir(op) + + def url_to_fs(path: str, **kwargs: Any) -> Tuple[AbstractFileSystem, str]: """A wrapper of ``fsspec.core.url_to_fs`` @@ -21,6 +44,8 @@ def url_to_fs(path: str, **kwargs: Any) -> Tuple[AbstractFileSystem, str]: :param kwargs: additional arguments to ``fsspec.core.url_to_fs`` :return: the file system and the path """ + if path.startswith("file://"): + path = path[7:] return fc.url_to_fs(path, **kwargs) @@ -54,6 +79,19 @@ def isfile(path: str) -> bool: return fs.isfile(path) +def abs_path(path: str) -> str: + """Get the absolute path of a path + + :param path: the path to check + :return: the absolute path + """ + p, _path = fc.split_protocol(path) + if p is None or p == "file": # local path + # Path doesn't work with windows + return os.path.abspath(_path) + return path + + def touch(path: str, auto_mkdir: bool = False) -> None: """Create an empty file or update the timestamp of the file @@ -90,7 +128,7 @@ def makedirs(path: str, exist_ok: bool = False) -> str: fs, _path = url_to_fs(path) fs.makedirs(_path, exist_ok=exist_ok) if isinstance(fs, LocalFileSystem): - return str(Path(path).resolve()) + return str(Path(_path).resolve()) return path @@ -113,10 +151,10 @@ def glob(path: str) -> List[str]: """Glob files :param path: the path to glob - :return: the matched files + :return: the matched files (absolute paths) """ - fs, path = url_to_fs(path) - return list(fs.glob(path)) + fs, _path = url_to_fs(path) + return [fs.unstrip_protocol(x) for x in fs.glob(_path)] def write_text(path: str, contents: str) -> None: