diff --git a/jupyterfs/metamanager.py b/jupyterfs/metamanager.py index 97792ac..55b2dcc 100644 --- a/jupyterfs/metamanager.py +++ b/jupyterfs/metamanager.py @@ -15,7 +15,10 @@ from fs.errors import FSError from fs.opener.errors import OpenerError, ParseError from jupyter_server.base.handlers import APIHandler -from jupyter_server.services.contents.manager import AsyncContentsManager +from jupyter_server.services.contents.manager import ( + AsyncContentsManager, + ContentsManager, +) from .auth import substituteAsk, substituteEnv, substituteNone from .config import JupyterFs as JupyterFsConfig @@ -30,10 +33,10 @@ stripDrive, ) -__all__ = ["MetaManager", "MetaManagerHandler"] +__all__ = ["MetaManager", "MetaManagerHandler", "AsyncMetaManager"] -class MetaManager(AsyncContentsManager): +class MetaManagerMixin: copy_pat = re.compile(r"\-Copy\d*\.") @default("files_handler_params") @@ -153,7 +156,7 @@ def root_manager(self): def root_dir(self): return self.root_manager.root_dir - async def copy(self, from_path, to_path=None): + def copy(self, from_path, to_path=None): """Copy an existing file and return its new model. If to_path not specified, it will be the parent directory of from_path. @@ -205,32 +208,6 @@ def _getManagerForPath(self, path): return mgr, stripDrive(path) - is_hidden = path_first_arg("is_hidden", False) - dir_exists = path_first_arg("dir_exists", False) - file_exists = path_kwarg("file_exists", "", False) - exists = path_first_arg("exists", False) - - save = path_second_arg("save", "model", True) - rename = path_old_new("rename", False) - - get = path_first_arg("get", True) - delete = path_first_arg("delete", False) - - get_kernel_path = path_first_arg("get_kernel_path", False, sync=True) - - create_checkpoint = path_first_arg("create_checkpoint", False) - list_checkpoints = path_first_arg("list_checkpoints", False) - restore_checkpoint = path_second_arg( - "restore_checkpoint", - "checkpoint_id", - False, - ) - delete_checkpoint = path_second_arg( - "delete_checkpoint", - "checkpoint_id", - False, - ) - class MetaManagerHandler(APIHandler): _jupyterfsConfig = None @@ -261,7 +238,7 @@ def _validate_resource(self, resource): return True @web.authenticated - async def get(self): + def get(self): """Returns all the available contents manager prefixes e.g. if the contents manager configuration is something like: @@ -280,7 +257,7 @@ async def get(self): self.finish(json.dumps(self.contents_manager.resources)) @web.authenticated - async def post(self): + def post(self): # will be a list of resource dicts body = self.get_json_body() options = body["options"] @@ -300,3 +277,57 @@ async def post(self): self.finish( json.dumps(self.contents_manager.initResource(*resources, options=options)) ) + + +class AsyncMetaManager(MetaManagerMixin, AsyncContentsManager): + async def copy(self, from_path, to_path=None): + return super().copy(from_path=from_path, to_path=to_path) + + is_hidden = path_first_arg("is_hidden", False, sync=False) + dir_exists = path_first_arg("dir_exists", False, sync=False) + file_exists = path_kwarg("file_exists", "", False, sync=False) + exists = path_first_arg("exists", False, sync=False) + + save = path_second_arg("save", "model", True, sync=False) + rename = path_old_new("rename", False, sync=False) + + get = path_first_arg("get", True, sync=False) + delete = path_first_arg("delete", False, sync=False) + + get_kernel_path = path_first_arg("get_kernel_path", False, sync=False) + + create_checkpoint = path_first_arg("create_checkpoint", False, sync=False) + list_checkpoints = path_first_arg("list_checkpoints", False, sync=False) + restore_checkpoint = path_second_arg( + "restore_checkpoint", "checkpoint_id", False, sync=False + ) + delete_checkpoint = path_second_arg( + "delete_checkpoint", "checkpoint_id", False, sync=False + ) + + +class MetaManager(MetaManagerMixin, ContentsManager): + def copy(self, from_path, to_path=None): + return super().copy(from_path=from_path, to_path=to_path) + + is_hidden = path_first_arg("is_hidden", False, sync=True) + dir_exists = path_first_arg("dir_exists", False, sync=True) + file_exists = path_kwarg("file_exists", "", False, sync=True) + exists = path_first_arg("exists", False, sync=True) + + save = path_second_arg("save", "model", True, sync=True) + rename = path_old_new("rename", False, sync=True) + + get = path_first_arg("get", True, sync=True) + delete = path_first_arg("delete", False, sync=True) + + get_kernel_path = path_first_arg("get_kernel_path", False, sync=True) + + create_checkpoint = path_first_arg("create_checkpoint", False, sync=True) + list_checkpoints = path_first_arg("list_checkpoints", False, sync=True) + restore_checkpoint = path_second_arg( + "restore_checkpoint", "checkpoint_id", False, sync=True + ) + delete_checkpoint = path_second_arg( + "delete_checkpoint", "checkpoint_id", False, sync=True + ) diff --git a/jupyterfs/pathutils.py b/jupyterfs/pathutils.py index 65b8be7..0e88d6e 100644 --- a/jupyterfs/pathutils.py +++ b/jupyterfs/pathutils.py @@ -112,58 +112,98 @@ async def _wrapper(self, *args, **kwargs): return _wrapper -def path_second_arg(method_name, first_argname, returns_model): +def path_second_arg(method_name, first_argname, returns_model, sync=False): """Decorator for methods that accept path as a second argument. e.g. manager.save(model, path, ...)""" - async def _wrapper(self, *args, **kwargs): - other, args = _get_arg(first_argname, args, kwargs) - path, args = _get_arg("path", args, kwargs) - _, mgr, mgr_path = _resolve_path(path, self._managers) - result = getattr(mgr, method_name)(other, mgr_path, *args, **kwargs) - return result + if sync: + + def _wrapper(self, *args, **kwargs): + other, args = _get_arg(first_argname, args, kwargs) + path, args = _get_arg("path", args, kwargs) + _, mgr, mgr_path = _resolve_path(path, self._managers) + result = getattr(mgr, method_name)(other, mgr_path, *args, **kwargs) + return result + + else: + + async def _wrapper(self, *args, **kwargs): + other, args = _get_arg(first_argname, args, kwargs) + path, args = _get_arg("path", args, kwargs) + _, mgr, mgr_path = _resolve_path(path, self._managers) + result = getattr(mgr, method_name)(other, mgr_path, *args, **kwargs) + return result return _wrapper -def path_kwarg(method_name, path_default, returns_model): +def path_kwarg(method_name, path_default, returns_model, sync=False): """Parameterized decorator for methods that accept path as a second argument. e.g. manager.file_exists(path='') """ + if sync: - async def _wrapper(self, path=path_default, **kwargs): - _, mgr, mgr_path = _resolve_path(path, self._managers) - result = getattr(mgr, method_name)(path=mgr_path, **kwargs) - return result + def _wrapper(self, path=path_default, **kwargs): + _, mgr, mgr_path = _resolve_path(path, self._managers) + result = getattr(mgr, method_name)(path=mgr_path, **kwargs) + return result + + else: + + async def _wrapper(self, path=path_default, **kwargs): + _, mgr, mgr_path = _resolve_path(path, self._managers) + result = getattr(mgr, method_name)(path=mgr_path, **kwargs) + return result return _wrapper -def path_old_new(method_name, returns_model): +def path_old_new(method_name, returns_model, sync=False): """Decorator for methods accepting old_path and new_path. e.g. manager.rename(old_path, new_path) """ + if sync: - async def _wrapper(self, old_path, new_path, *args, **kwargs): - old_prefix, old_mgr, old_mgr_path = _resolve_path(old_path, self._managers) - new_prefix, new_mgr, new_mgr_path = _resolve_path(new_path, self._managers) - if old_mgr is not new_mgr: - # TODO: Consider supporting this via get+save+delete. - raise HTTPError( - 400, - "Can't move files between backends yet ({old} -> {new})".format( - old=old_path, - new=new_path, - ), + def _wrapper(self, old_path, new_path, *args, **kwargs): + old_prefix, old_mgr, old_mgr_path = _resolve_path(old_path, self._managers) + new_prefix, new_mgr, new_mgr_path = _resolve_path(new_path, self._managers) + if old_mgr is not new_mgr: + # TODO: Consider supporting this via get+save+delete. + raise HTTPError( + 400, + "Can't move files between backends yet ({old} -> {new})".format( + old=old_path, + new=new_path, + ), + ) + assert new_prefix == old_prefix + result = getattr(new_mgr, method_name)( + old_mgr_path, new_mgr_path, *args, **kwargs ) - assert new_prefix == old_prefix - result = getattr(new_mgr, method_name)( - old_mgr_path, new_mgr_path, *args, **kwargs - ) - return result + return result + + else: + + async def _wrapper(self, old_path, new_path, *args, **kwargs): + old_prefix, old_mgr, old_mgr_path = _resolve_path(old_path, self._managers) + new_prefix, new_mgr, new_mgr_path = _resolve_path(new_path, self._managers) + if old_mgr is not new_mgr: + # TODO: Consider supporting this via get+save+delete. + raise HTTPError( + 400, + "Can't move files between backends yet ({old} -> {new})".format( + old=old_path, + new=new_path, + ), + ) + assert new_prefix == old_prefix + result = getattr(new_mgr, method_name)( + old_mgr_path, new_mgr_path, *args, **kwargs + ) + return result return _wrapper diff --git a/jupyterfs/tests/test_asyncmetamanager.py b/jupyterfs/tests/test_asyncmetamanager.py new file mode 100644 index 0000000..71e3f32 --- /dev/null +++ b/jupyterfs/tests/test_asyncmetamanager.py @@ -0,0 +1,151 @@ +# ***************************************************************************** +# +# Copyright (c) 2019, the jupyter-fs authors. +# +# This file is part of the jupyter-fs library, distributed under the terms of +# the Apache License 2.0. The full license can be found in the LICENSE file. + +import pytest +from traitlets.config import Config + +from .utils.client import ContentsClient + + +# base config +base_config = { + "ServerApp": { + "jpserver_extensions": {"jupyterfs.extension": True}, + "contents_manager_class": "jupyterfs.metamanager.AsyncMetaManager", + }, + "JupyterFs": {}, +} + +deny_client_config = { + "JupyterFs": { + "allow_user_resources": False, + } +} + + +@pytest.fixture +def tmp_osfs_resource(): + """parametrize if we want tmp resource""" + return False + + +@pytest.fixture +def our_config(): + """parametrize if we want custom config""" + return {} + + +@pytest.fixture +def jp_server_config(tmp_path, tmp_osfs_resource, our_config): + c = Config(base_config) + c.JupyterFs.setdefault("resources", []) + if tmp_osfs_resource: + c.JupyterFs.resources.append( + { + "name": "test-server-config", + "url": f"osfs://{tmp_path.as_posix()}", + } + ) + c.merge(Config(our_config)) + return c + + +@pytest.mark.parametrize("our_config", [deny_client_config]) +async def test_client_creation_disallowed(tmp_path, jp_fetch, jp_server_config): + cc = ContentsClient(jp_fetch) + resources = await cc.set_resources( + [{"name": "test-2", "url": f"osfs://{tmp_path.as_posix()}"}] + ) + assert resources == [] + + +@pytest.mark.parametrize("our_config", [deny_client_config]) +@pytest.mark.parametrize("tmp_osfs_resource", [True]) +async def test_client_creation_disallowed_retains_server_config( + tmp_path, jp_fetch, jp_server_config +): + cc = ContentsClient(jp_fetch) + resources = await cc.set_resources( + [{"name": "test-2", "url": f"osfs://{tmp_path.as_posix()}"}] + ) + names = set(map(lambda r: r["name"], resources)) + assert names == {"test-server-config"} + + +@pytest.mark.parametrize( + "our_config", + [ + { + "JupyterFs": { + "resource_validators": [ + r"osfs://.*/test-valid-A.*", + r".*://.*/test-valid-B", + ] + } + } + ], +) +async def test_resource_validators(tmp_path, jp_fetch, jp_server_config): + cc = ContentsClient(jp_fetch) + (tmp_path / "test-valid-A").mkdir() + (tmp_path / "test-valid-B").mkdir() + (tmp_path / "test-invalid-A").mkdir() + (tmp_path / "test-invalid-B").mkdir() + (tmp_path / "invalid-C").mkdir() + resources = await cc.set_resources( + [ + {"name": "valid-1", "url": f"osfs://{tmp_path.as_posix()}/test-valid-A"}, + {"name": "valid-2", "url": f"osfs://{tmp_path.as_posix()}/test-valid-B"}, + { + "name": "invalid-1", + "url": f"osfs://{tmp_path.as_posix()}/test-invalid-A", + }, + { + "name": "invalid-2", + "url": f"osfs://{tmp_path.as_posix()}/test-invalid-B", + }, + {"name": "invalid-3", "url": f"osfs://{tmp_path.as_posix()}/invalid-C"}, + {"name": "invalid-4", "url": f"osfs://{tmp_path.as_posix()}/foo"}, + { + "name": "invalid-5", + "url": f"osfs://{tmp_path.as_posix()}/test-valid-A/non-existant", + }, + { + "name": "invalid-6", + "url": f"non-existant://{tmp_path.as_posix()}/test-valid-B", + }, + ] + ) + names = {r["name"] for r in resources if r["init"]} + assert names == {"valid-1", "valid-2"} + + +@pytest.mark.parametrize( + "our_config", + [ + { + "JupyterFs": { + "resource_validators": [ + r"osfs://([^@]*|[^:]*[:][@].*)", # no auth, or at least no password + r"osfs://", # sanity check that this doesn't change the result + ] + } + } + ], +) +async def test_resource_validators_no_auth(tmp_path, jp_fetch, jp_server_config): + cc = ContentsClient(jp_fetch) + resources = await cc.set_resources( + [ + {"name": "valid-1", "url": f"osfs://{tmp_path.as_posix()}"}, + {"name": "valid-2", "url": f"osfs://username:@{tmp_path.as_posix()}"}, + {"name": "invalid-1", "url": f"osfs://username:pwd@{tmp_path.as_posix()}"}, + {"name": "invalid-2", "url": f"osfs://:pwd@{tmp_path.as_posix()}"}, + ] + ) + names = set(map(lambda r: r["name"], resources)) + assert names == {"valid-1", "valid-2"}