Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[SVCS-132] Factor out move/copy code #212

Open
wants to merge 10 commits into
base: develop
Choose a base branch
from
156 changes: 71 additions & 85 deletions waterbutler/core/provider.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,49 @@ def build_url(base, *segments, **query):
return url.url


def move_or_copy_validation(func):
async def wrapper(self,
dest_provider,
src_path,
dest_path,
rename=None,
conflict='replace',
handle_naming=True):

self.provider_metrics.add(func.__name__, {
'got_handle_naming': handle_naming,
'conflict': conflict,
'got_rename': rename is not None,
})

if handle_naming:
dest_path = await dest_provider.handle_naming(
src_path,
dest_path,
rename=rename,
conflict=conflict,
)

# files and folders shouldn't overwrite themselves
if (
self.shares_storage_root(dest_provider) and
src_path.materialized_path == dest_path.materialized_path
):
raise exceptions.OverwriteSelfError(src_path)

intra_move_or_copy = getattr(self, 'intra_{}'.format(func.__name__))
can_intra_move_or_copy = getattr(self, 'can_intra_{}'.format(func.__name__))

if can_intra_move_or_copy(dest_provider, src_path):
self.provider_metrics.add('{0}.can_intra_{0}'.format(func.__name__), True)
return await intra_move_or_copy(dest_provider, src_path, dest_path)
else:
self.provider_metrics.add('{0}.can_intra_{0}'.format(func.__name__), False)

return await func(self, dest_provider, src_path, dest_path)
return wrapper


class BaseProvider(metaclass=abc.ABCMeta):
"""The base class for all providers. Every provider must, at the least, implement all abstract
methods in this class.
Expand Down Expand Up @@ -197,103 +240,46 @@ async def make_request(self, method: str, url: str, *args, **kwargs) -> aiohttp.
def request(self, *args, **kwargs):
return RequestHandlerContext(self.make_request(*args, **kwargs))

async def move(self,
dest_provider: 'BaseProvider',
src_path: wb_path.WaterButlerPath,
dest_path: wb_path.WaterButlerPath,
rename: str=None,
conflict: str='replace',
handle_naming: bool=True) -> typing.Tuple[wb_metadata.BaseMetadata, bool]:
@move_or_copy_validation # Takes kwargs rename, conflict and handle_naming
async def move(self, dest_provider, src_path, dest_path):
"""Moves a file or folder from the current provider to the specified one
Performs a copy and then a delete.
Calls :func:`BaseProvider.intra_move` if possible.

:param dest_provider: ( :class:`.BaseProvider` ) The provider to move to
:param src_path: ( :class:`.WaterButlerPath` ) Path to where the resource can be found
:param dest_path: ( :class:`.WaterButlerPath` ) Path to where the resource will be moved
:param rename: ( :class:`str` ) The desired name of the resulting path, may be incremented
:param conflict: ( :class:`str` ) What to do in the event of a name conflict, ``replace`` or ``keep``
:param handle_naming: ( :class:`bool` ) If a naming conflict is detected, should it be automatically handled?
:param BaseProvider dest_provider: The provider to move to
:param wb_path.WaterButlerPath dest_path: A path object to be sent to either
:func:`BaseProvider.intra_copy`
:param wb_path.WaterButlerPath src_path: A path object to be sent to either
:func:`BaseProvider.intra_move`
or :func:`BaseProvider.copy` and :func:`BaseProvider.delete`
:param dict dest_options: A dict to be sent to either :func:`BaseProvider.intra_move`
or :func:`BaseProvider.copy`
"""
args = (dest_provider, src_path, dest_path)
kwargs = {'rename': rename, 'conflict': conflict}

self.provider_metrics.add('move', {
'got_handle_naming': handle_naming,
'conflict': conflict,
'got_rename': rename is not None,
})

if handle_naming:
dest_path = await dest_provider.handle_naming(
src_path,
dest_path,
rename=rename,
conflict=conflict,
)
args = (dest_provider, src_path, dest_path)
kwargs = {}

# files and folders shouldn't overwrite themselves
if (
self.shares_storage_root(dest_provider) and
src_path.materialized_path == dest_path.materialized_path
):
raise exceptions.OverwriteSelfError(src_path)

self.provider_metrics.add('move.can_intra_move', False)
if self.can_intra_move(dest_provider, src_path):
self.provider_metrics.add('move.can_intra_move', True)
return await self.intra_move(*args)

if src_path.is_dir:
meta_data, created = await self._folder_file_op(self.move, *args, **kwargs) # type: ignore
metadata, created = await self._folder_file_op(self.move, dest_provider, src_path,
dest_path)
else:
meta_data, created = await self.copy(*args, handle_naming=False, **kwargs) # type: ignore
metadata, created = await self.copy(dest_provider, src_path, dest_path,
handle_naming=False)

await self.delete(src_path)

return meta_data, created

async def copy(self,
dest_provider: 'BaseProvider',
src_path: wb_path.WaterButlerPath,
dest_path: wb_path.WaterButlerPath,
rename: str=None, conflict: str='replace',
handle_naming: bool=True) \
-> typing.Tuple[wb_metadata.BaseMetadata, bool]:
args = (dest_provider, src_path, dest_path)
kwargs = {'rename': rename, 'conflict': conflict, 'handle_naming': handle_naming}

self.provider_metrics.add('copy', {
'got_handle_naming': handle_naming,
'conflict': conflict,
'got_rename': rename is not None,
})
if handle_naming:
dest_path = await dest_provider.handle_naming(
src_path,
dest_path,
rename=rename,
conflict=conflict,
)
args = (dest_provider, src_path, dest_path)
kwargs = {}

# files and folders shouldn't overwrite themselves
if (
self.shares_storage_root(dest_provider) and
src_path.materialized_path == dest_path.materialized_path
):
raise exceptions.OverwriteSelfError(src_path)

self.provider_metrics.add('copy.can_intra_copy', False)
if self.can_intra_copy(dest_provider, src_path):
self.provider_metrics.add('copy.can_intra_copy', True)
return await self.intra_copy(*args)
return metadata, created

@move_or_copy_validation # Takes kwargs rename, conflict and handle_naming
async def copy(self, dest_provider, src_path, dest_path):
"""Copies a file or folder from the current provider to the specified one by simply
downloading the file and uploading it to the destination directory, naming conflicts
intra-provider moves and metrics are dealt with in the move_or_copy_validation decorator.
:param BaseProvider dest_provider: The provider to move to
:param wb_path.WaterButlerPath src_path: A path object to be sent to either
:func:`BaseProvider.intra_copy`
or :func:`BaseProvider.copy`
:param dict dest_path: A path object to be sent to :func:`BaseProvider.intra_copy`
"""

if src_path.is_dir:
return await self._folder_file_op(self.copy, *args, **kwargs) # type: ignore
return await self._folder_file_op(self.copy, dest_provider, src_path, dest_path)

download_stream = await self.download(src_path)

Expand Down