Fix makedirs (#120)

* Add more io utils

* update

* update

* update

* update

* Add more io utils

* update

* update

* update

* update

* update

* update

* update

.devcontainer/devcontainer.json

@@ -7,6 +7,7 @@
 				"terminal.integrated.shell.linux": "/bin/bash",
 				"python.pythonPath": "/usr/local/bin/python",
 				"python.defaultInterpreterPath": "/usr/local/bin/python",
+				"editor.defaultFormatter": "ms-python.black-formatter",
 				"isort.interpreter": [
 					"/usr/local/bin/python"
 				],
@@ -15,6 +16,9 @@
 				],
 				"pylint.interpreter": [
 					"/usr/local/bin/python"
+				],
+				"black-formatter.interpreter": [
+					"/usr/local/bin/python"
 				]
 			},
 			"extensions": [
@@ -23,6 +27,7 @@
 				"ms-python.flake8",
 				"ms-python.pylint",
 				"ms-python.mypy",
+				"ms-python.black-formatter",
 				"GitHub.copilot",
 				"njpwerner.autodocstring"
 			]
@@ -32,7 +37,10 @@
 		8888
 	],
 	"postCreateCommand": "make devenv",
-	"mounts": [
-		"source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind"
-	]
+	"features": {
+		"ghcr.io/devcontainers/features/docker-in-docker:2": {},
+		"ghcr.io/devcontainers/features/java:1": {
+			"version": "11"
+		}
+	}
 }

tests/utils/test_io.py

@@ -1,9 +1,8 @@
 import os
+import sys
 from io import BytesIO
 
 import pytest
-import sys
-import os
 
 import triad.utils.io as iou
 
@@ -38,6 +37,9 @@ def _assert(gres, expected):
         [os.path.join(str(tmpdir), "a", "a.txt")],
     )
 
+    iou.touch("memory://gtest/m.txt", auto_mkdir=True)
+    assert iou.glob("memory://gtest/*.txt") == ["memory:///gtest/m.txt"]
+
 
 @pytest.mark.skipif(sys.platform.startswith("win"), reason="not a test for windows")
 def test_join_not_win():
@@ -55,6 +57,30 @@ def test_join_is_win():
     assert iou.join("c:\\a", "b", "*.parquet") == "c:\\a\\b\\*.parquet"
 
 
+@pytest.mark.skipif(sys.platform.startswith("win"), reason="not a test for windows")
+def test_abs_path_not_win(tmpdir):
+    with iou.chdir(str(tmpdir)):
+        assert iou.abs_path("a") == os.path.join(str(tmpdir), "a")
+    assert iou.abs_path("/tmp/x") == "/tmp/x"
+    assert iou.abs_path("file:///tmp/x") == "/tmp/x"
+    assert iou.abs_path("memory://tmp/x") == "memory://tmp/x"
+    assert iou.abs_path("memory:///tmp/x") == "memory:///tmp/x"
+    assert iou.abs_path("dummy:///tmp/x") == "dummy:///tmp/x"
+
+
+@pytest.mark.skipif(
+    not sys.platform.startswith("win"), reason="a test only for windows"
+)
+def test_abs_path_is_win(tmpdir):
+    with iou.chdir(str(tmpdir)):
+        assert iou.abs_path("a") == os.path.join(str(tmpdir), "a")
+    assert iou.abs_path("c:\\tmp\\x") == "c:\\tmp\\x"
+    assert iou.abs_path("file://c:/tmp/x") == "c:\\tmp\\x"
+    assert iou.abs_path("memory://tmp/x") == "memory://tmp/x"
+    assert iou.abs_path("memory:///tmp/x") == "memory:///tmp/x"
+    assert iou.abs_path("dummy:///tmp/x") == "dummy:///tmp/x"
+
+
 def test_makedirs(tmpdir):
     path = os.path.join(str(tmpdir), "temp", "a")
     assert path == iou.makedirs(path, exist_ok=False)
@@ -63,21 +89,25 @@ def test_makedirs(tmpdir):
         iou.makedirs(path, exist_ok=False)
     iou.makedirs(path, exist_ok=True)
 
-    op = os.getcwd()
-    os.chdir(str(tmpdir))
-    assert os.path.join(str(tmpdir), "temp", "b") == iou.makedirs(
-        iou.join("temp", "b"), exist_ok=False
-    )
-    assert iou.exists(iou.join("temp", "b"))
-    assert iou.exists(os.path.join(str(tmpdir), "temp", "b"))
-    os.chdir(op)
+    with iou.chdir(str(tmpdir)):
+        assert os.path.join(str(tmpdir), "temp", "b") == iou.makedirs(
+            iou.join("temp", "b"), exist_ok=False
+        )
+        assert iou.exists(iou.join("temp", "b"))
+        assert iou.exists(os.path.join(str(tmpdir), "temp", "b"))
 
     path = "memory://temp/a"
     assert path == iou.makedirs(path, exist_ok=True)
     assert iou.exists(path)
     with pytest.raises(OSError):
         iou.makedirs(path, exist_ok=False)
 
+    path = os.path.join(str(tmpdir), "temp", "aa")
+    fs, _path = iou.url_to_fs(path)
+    _path = fs.unstrip_protocol(_path)  # file:///...
+    iou.makedirs(_path, exist_ok=False)
+    assert iou.exists(path)
+
 
 def test_exists(tmpdir):
     iou.write_text(os.path.join(str(tmpdir), "a.txt"), "a")
@@ -88,6 +118,12 @@ def test_exists(tmpdir):
     assert iou.exists(os.path.join(str(tmpdir), "temp"))
     assert iou.exists(str(tmpdir))
 
+    with iou.chdir(str(tmpdir)):
+        assert iou.exists("a.txt")
+        assert iou.exists(os.path.join("temp", "a.txt"))
+        assert iou.exists("temp")
+        assert iou.exists(".")
+
 
 def test_is_dir_or_file(tmpdir):
     for base in [str(tmpdir), "memory://"]:

triad/utils/io.py

@@ -4,7 +4,7 @@
 import zipfile
 from contextlib import contextmanager
 from pathlib import Path, PurePosixPath
-from typing import Any, Iterator, Tuple, List
+from typing import Any, Iterator, List, Tuple
 
 import fsspec
 import fsspec.core as fc
@@ -14,13 +14,38 @@
 _SCHEME_PREFIX = re.compile(r"^[a-zA-Z0-9\-_]+:")
 
 
+@contextmanager
+def chdir(path: str) -> Iterator[None]:
+    """Change the current working directory to the given path
+
+    :param path: the path to change to
+
+    .. admonition:: Examples
+
+        .. code-block:: python
+
+            from fugue_ml.utils.io import chdir
+
+            with chdir("/tmp"):
+                # do something
+    """
+    op = os.getcwd()
+    os.chdir(path)
+    try:
+        yield
+    finally:
+        os.chdir(op)
+
+
 def url_to_fs(path: str, **kwargs: Any) -> Tuple[AbstractFileSystem, str]:
     """A wrapper of ``fsspec.core.url_to_fs``
 
     :param path: the path to be used
     :param kwargs: additional arguments to ``fsspec.core.url_to_fs``
     :return: the file system and the path
     """
+    if path.startswith("file://"):
+        path = path[7:]
     return fc.url_to_fs(path, **kwargs)
 
 
@@ -54,6 +79,19 @@ def isfile(path: str) -> bool:
     return fs.isfile(path)
 
 
+def abs_path(path: str) -> str:
+    """Get the absolute path of a path
+
+    :param path: the path to check
+    :return: the absolute path
+    """
+    p, _path = fc.split_protocol(path)
+    if p is None or p == "file":  # local path
+        # Path doesn't work with windows
+        return os.path.abspath(_path)
+    return path
+
+
 def touch(path: str, auto_mkdir: bool = False) -> None:
     """Create an empty file or update the timestamp of the file
 
@@ -90,7 +128,7 @@ def makedirs(path: str, exist_ok: bool = False) -> str:
     fs, _path = url_to_fs(path)
     fs.makedirs(_path, exist_ok=exist_ok)
     if isinstance(fs, LocalFileSystem):
-        return str(Path(path).resolve())
+        return str(Path(_path).resolve())
     return path
 
 
@@ -113,10 +151,10 @@ def glob(path: str) -> List[str]:
     """Glob files
 
     :param path: the path to glob
-    :return: the matched files
+    :return: the matched files (absolute paths)
     """
-    fs, path = url_to_fs(path)
-    return list(fs.glob(path))
+    fs, _path = url_to_fs(path)
+    return [fs.unstrip_protocol(x) for x in fs.glob(_path)]
 
 
 def write_text(path: str, contents: str) -> None: