Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add WalkerBase #513

Draft
wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

Add WalkerBase #513

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions fs/compress.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
from .path import relpath
from .time import datetime_to_epoch
from .errors import NoSysPath, MissingInfoNamespace
from .walk import Walker
from .walk import Walker, WalkerBase

if typing.TYPE_CHECKING:
from typing import BinaryIO, Optional, Text, Tuple, Union
Expand All @@ -34,7 +34,7 @@ def write_zip(
file, # type: Union[Text, BinaryIO]
compression=zipfile.ZIP_DEFLATED, # type: int
encoding="utf-8", # type: Text
walker=None, # type: Optional[Walker]
walker=None, # type: Optional[WalkerBase]
):
# type: (...) -> None
"""Write the contents of a filesystem to a zip file.
Expand Down Expand Up @@ -110,7 +110,7 @@ def write_tar(
file, # type: Union[Text, BinaryIO]
compression=None, # type: Optional[Text]
encoding="utf-8", # type: Text
walker=None, # type: Optional[Walker]
walker=None, # type: Optional[WalkerBase]
):
# type: (...) -> None
"""Write the contents of a filesystem to a tar file.
Expand Down
18 changes: 9 additions & 9 deletions fs/copy.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
from .opener import manage_fs
from .path import abspath, combine, frombase, normpath
from .tools import is_thread_safe
from .walk import Walker
from .walk import Walker, WalkerBase

if typing.TYPE_CHECKING:
from typing import Callable, Optional, Text, Union
Expand All @@ -22,7 +22,7 @@
def copy_fs(
src_fs, # type: Union[FS, Text]
dst_fs, # type: Union[FS, Text]
walker=None, # type: Optional[Walker]
walker=None, # type: Optional[WalkerBase]
on_copy=None, # type: Optional[_OnCopy]
workers=0, # type: int
preserve_time=False, # type: bool
Expand Down Expand Up @@ -53,7 +53,7 @@ def copy_fs(
def copy_fs_if_newer(
src_fs, # type: Union[FS, Text]
dst_fs, # type: Union[FS, Text]
walker=None, # type: Optional[Walker]
walker=None, # type: Optional[WalkerBase]
on_copy=None, # type: Optional[_OnCopy]
workers=0, # type: int
preserve_time=False, # type: bool
Expand All @@ -77,7 +77,7 @@ def copy_fs_if(
src_fs, # type: Union[FS, Text]
dst_fs, # type: Union[FS, Text]
condition="always", # type: Text
walker=None, # type: Optional[Walker]
walker=None, # type: Optional[WalkerBase]
on_copy=None, # type: Optional[_OnCopy]
workers=0, # type: int
preserve_time=False, # type: bool
Expand Down Expand Up @@ -282,7 +282,7 @@ def _copy_locked():
def copy_structure(
src_fs, # type: Union[FS, Text]
dst_fs, # type: Union[FS, Text]
walker=None, # type: Optional[Walker]
walker=None, # type: Optional[WalkerBase]
src_root="/", # type: Text
dst_root="/", # type: Text
):
Expand Down Expand Up @@ -316,7 +316,7 @@ def copy_dir(
src_path, # type: Text
dst_fs, # type: Union[FS, Text]
dst_path, # type: Text
walker=None, # type: Optional[Walker]
walker=None, # type: Optional[WalkerBase]
on_copy=None, # type: Optional[_OnCopy]
workers=0, # type: int
preserve_time=False, # type: bool
Expand Down Expand Up @@ -359,7 +359,7 @@ def copy_dir_if_newer(
src_path, # type: Text
dst_fs, # type: Union[FS, Text]
dst_path, # type: Text
walker=None, # type: Optional[Walker]
walker=None, # type: Optional[WalkerBase]
on_copy=None, # type: Optional[_OnCopy]
workers=0, # type: int
preserve_time=False, # type: bool
Expand Down Expand Up @@ -393,7 +393,7 @@ def copy_dir_if(
dst_fs, # type: Union[FS, Text]
dst_path, # type: Text
condition, # type: Text
walker=None, # type: Optional[Walker]
walker=None, # type: Optional[WalkerBase]
on_copy=None, # type: Optional[_OnCopy]
workers=0, # type: int
preserve_time=False, # type: bool
Expand All @@ -407,7 +407,7 @@ def copy_dir_if(
dst_fs (FS or str): Destination filesystem (instance or URL).
dst_path (str): Path to a directory on the destination filesystem.
condition (str): Name of the condition to check for each file.
walker (~fs.walk.Walker, optional): A walker object that will be
walker (~fs.walk.WalkerBase, optional): A walker object that will be
used to scan for files in ``src_fs``. Set this if you only want
to consider a sub-set of the resources in ``src_fs``.
on_copy (callable):A function callback called after a single file copy
Expand Down
6 changes: 3 additions & 3 deletions fs/mirror.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
from .errors import ResourceNotFound
from .opener import manage_fs
from .tools import is_thread_safe
from .walk import Walker
from .walk import Walker, WalkerBase

if typing.TYPE_CHECKING:
from typing import Callable, Optional, Text, Union
Expand Down Expand Up @@ -54,7 +54,7 @@ def _compare(info1, info2):
def mirror(
src_fs, # type: Union[FS, Text]
dst_fs, # type: Union[FS, Text]
walker=None, # type: Optional[Walker]
walker=None, # type: Optional[WalkerBase]
copy_if_newer=True, # type: bool
workers=0, # type: int
preserve_time=False, # type: bool
Expand Down Expand Up @@ -104,7 +104,7 @@ def dst():
def _mirror(
src_fs, # type: FS
dst_fs, # type: FS
walker=None, # type: Optional[Walker]
walker=None, # type: Optional[WalkerBase]
copy_if_newer=True, # type: bool
copy_file=copy_file_internal, # type: Callable[[FS, str, FS, str, bool], None]
preserve_time=False, # type: bool
Expand Down
87 changes: 84 additions & 3 deletions fs/walk.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,14 @@

from __future__ import unicode_literals

import abc
import typing
from collections import defaultdict
from collections import deque
from collections import namedtuple

import six

from ._repr import make_repr
from .errors import FSError
from .path import abspath
Expand Down Expand Up @@ -45,11 +48,89 @@
"""


# TODO(@althonos): It could be a good idea to create an Abstract Base Class
# BaseWalker (with methods walk, files, dirs and info) ?
@six.add_metaclass(abc.ABCMeta)
class WalkerBase(object):
"""A walker object recursively lists directories in a filesystem."""

@abc.abstractmethod
def walk(
self,
fs, # type: FS
path="/", # type: Text
namespaces=None, # type: Optional[Collection[Text]]
):
# type: (...) -> Iterator[Step]
"""Walk the directory structure of a filesystem.

Arguments:
fs (FS): A filesystem instance.
path (str): A path to a directory on the filesystem.
namespaces (list, optional): A list of additional namespaces
to add to the `Info` objects.

Returns:
collections.Iterator: an iterator of `~fs.walk.Step` instances.

The return value is an iterator of ``(<path>, <dirs>, <files>)``
named tuples, where ``<path>`` is an absolute path to a
directory, and ``<dirs>`` and ``<files>`` are a list of
`~fs.info.Info` objects for directories and files in ``<path>``.

"""

@abc.abstractmethod
def files(self, fs, path="/"):
# type: (FS, Text) -> Iterator[Text]
"""Walk a filesystem, yielding absolute paths to files.

Arguments:
fs (FS): A filesystem instance.
path (str): A path to a directory on the filesystem.

Yields:
str: absolute path to files on the filesystem found
recursively within the given directory.

"""

@abc.abstractmethod
def dirs(self, fs, path="/"):
# type: (FS, Text) -> Iterator[Text]
"""Walk a filesystem, yielding absolute paths to directories.

Arguments:
fs (FS): A filesystem instance.
path (str): A path to a directory on the filesystem.

Yields:
str: absolute path to directories on the filesystem found
recursively within the given directory.

"""

@abc.abstractmethod
def info(
self,
fs, # type: FS
path="/", # type: Text
namespaces=None, # type: Optional[Collection[Text]]
):
# type: (...) -> Iterator[Tuple[Text, Info]]
"""Walk a filesystem, yielding tuples of ``(<path>, <info>)``.

Arguments:
fs (FS): A filesystem instance.
path (str): A path to a directory on the filesystem.
namespaces (list, optional): A list of additional namespaces
to add to the `Info` objects.

Yields:
(str, Info): a tuple of ``(<absolute path>, <resource info>)``.

"""


class Walker(object):
class Walker(WalkerBase):
"""A walker object recursively lists directories in a filesystem."""

def __init__(
Expand Down