Skip to content

Commit

Permalink
[pre-commit.ci] auto fixes from pre-commit.com hooks
Browse files Browse the repository at this point in the history
for more information, see https://pre-commit.ci
  • Loading branch information
pre-commit-ci[bot] authored and GeigerJ2 committed Dec 11, 2024
1 parent 0e8e1cb commit 8bcf0f8
Show file tree
Hide file tree
Showing 3 changed files with 47 additions and 54 deletions.
20 changes: 10 additions & 10 deletions src/aiida/cmdline/commands/cmd_data/cmd_remote.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,21 +90,21 @@ def remote_show(datum):
echo.echo(f'- Remote folder full path: {datum.get_remote_path()}')


@remote.command("size")
@remote.command('size')
@arguments.NODE()
@click.option(
"-m",
"--method",
'-m',
'--method',
type=click.STRING,
default="du",
help="The method that should be used to evaluate the size (either ``du`` or ``lstat``.)",
default='du',
help='The method that should be used to evaluate the size (either ``du`` or ``lstat``.)',
)
@click.option(
"-p",
"--path",
'-p',
'--path',
type=click.Path(),
default=None,
help="Relative path of the object of the ``RemoteData`` node for which the size should be evaluated.",
help='Relative path of the object of the ``RemoteData`` node for which the size should be evaluated.',
)
def remote_size(node, method, path):
"""Print the total size of a RemoteData object."""
Expand All @@ -113,8 +113,8 @@ def remote_size(node, method, path):
remote_path = Path(node.get_remote_path())
full_path = remote_path / path if path is not None else remote_path
echo.echo(

Check warning on line 115 in src/aiida/cmdline/commands/cmd_data/cmd_remote.py

View check run for this annotation

Codecov / codecov/patch

src/aiida/cmdline/commands/cmd_data/cmd_remote.py#L111-L115

Added lines #L111 - L115 were not covered by tests
f"Estimated total size of directory `{full_path}` on the Computer "
f"`{node.computer.label}` obtained via `{method}`: {total_size}"
f'Estimated total size of directory `{full_path}` on the Computer '
f'`{node.computer.label}` obtained via `{method}`: {total_size}'
)
except FileNotFoundError as exc:
echo.echo_critical(str(exc))

Check warning on line 120 in src/aiida/cmdline/commands/cmd_data/cmd_remote.py

View check run for this annotation

Codecov / codecov/patch

src/aiida/cmdline/commands/cmd_data/cmd_remote.py#L119-L120

Added lines #L119 - L120 were not covered by tests
62 changes: 28 additions & 34 deletions src/aiida/orm/nodes/data/remote/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,7 @@ def _validate(self):
def get_authinfo(self):
return AuthInfo.get_collection(self.backend).get(dbcomputer=self.computer, aiidauser=self.user)

def get_size_on_disk(self, relpath: Path | None = None, method: str = "du") -> str:
def get_size_on_disk(self, relpath: Path | None = None, method: str = 'du') -> str:
"""
Connects to the remote folder and returns the total size of all files in the directory recursively in a
human-readable format.
Expand All @@ -211,55 +211,53 @@ def get_size_on_disk(self, relpath: Path | None = None, method: str = "du") -> s
from aiida.common.utils import format_directory_size

if relpath is None:
relpath = Path(".")
relpath = Path('.')

authinfo = self.get_authinfo()
full_path = Path(self.get_remote_path()) / relpath
computer_label = self.computer.label if self.computer is not None else ""
computer_label = self.computer.label if self.computer is not None else ''

with authinfo.get_transport() as transport:
if not transport.path_exists(str(full_path)):
exc_message = (
f"The required remote folder {full_path} on Computer <{computer_label}> "
"does not exist, is not a directory or has been deleted."
f'The required remote folder {full_path} on Computer <{computer_label}> '
'does not exist, is not a directory or has been deleted.'
)
raise FileNotFoundError(exc_message)

if method not in ("du", "lstat"):
if method not in ('du', 'lstat'):
raise NotImplementedError(

Check warning on line 229 in src/aiida/orm/nodes/data/remote/base.py

View check run for this annotation

Codecov / codecov/patch

src/aiida/orm/nodes/data/remote/base.py#L229

Added line #L229 was not covered by tests
f"Specified method `{method}` for evaluating the size on disk not implemented."
f'Specified method `{method}` for evaluating the size on disk not implemented.'
)

if method == "du":
if method == 'du':
try:
total_size: int = self._get_size_on_disk_du(full_path, transport)

except (RuntimeError, NotImplementedError):
lstat_warn = (

Check warning on line 238 in src/aiida/orm/nodes/data/remote/base.py

View check run for this annotation

Codecov / codecov/patch

src/aiida/orm/nodes/data/remote/base.py#L237-L238

Added lines #L237 - L238 were not covered by tests
"Problem executing `du` command. Will return total file size based on `lstat`. "
"Take the result with a grain of salt, as `lstat` does not consider the file system block size,"
" but instead returns the true size of the files in bytes, which differs from the actual space"
"requirements on disk."
'Problem executing `du` command. Will return total file size based on `lstat`. '
'Take the result with a grain of salt, as `lstat` does not consider the file system block size,'
' but instead returns the true size of the files in bytes, which differs from the actual space'
'requirements on disk.'
)

_logger.warning(lstat_warn)
method = "lstat"
method = 'lstat'

Check warning on line 246 in src/aiida/orm/nodes/data/remote/base.py

View check run for this annotation

Codecov / codecov/patch

src/aiida/orm/nodes/data/remote/base.py#L245-L246

Added lines #L245 - L246 were not covered by tests

else:
_logger.report("Obtained size on the remote using `du`.")
_logger.report('Obtained size on the remote using `du`.')

# No elif here, but another if, to allow that the method is internally changed to `lstat`, if `du` fails
if method == "lstat":
if method == 'lstat':
try:
total_size: int = self._get_size_on_disk_lstat(full_path, transport)
print(f"TOTAL_SIZE: {total_size}")
print(f'TOTAL_SIZE: {total_size}')

except OSError:
_logger.critical(
"Could not evaluate directory size using either `du` or `lstat`."
)
_logger.critical('Could not evaluate directory size using either `du` or `lstat`.')

Check warning on line 258 in src/aiida/orm/nodes/data/remote/base.py

View check run for this annotation

Codecov / codecov/patch

src/aiida/orm/nodes/data/remote/base.py#L257-L258

Added lines #L257 - L258 were not covered by tests
else:
_logger.report("Obtained size on the remote using `lstat`.")
_logger.report('Obtained size on the remote using `lstat`.')

return format_directory_size(size_in_bytes=total_size), method

Expand All @@ -275,19 +273,15 @@ def _get_size_on_disk_du(self, full_path: Path, transport: Transport) -> int:
"""

try:
retval, stdout, stderr = transport.exec_command_wait(
f"du -s --bytes {full_path}"
)
retval, stdout, stderr = transport.exec_command_wait(f'du -s --bytes {full_path}')
if not stderr and retval == 0:
total_size: int = int(stdout.split("\t")[0])
total_size: int = int(stdout.split('\t')[0])
return total_size
else:
raise RuntimeError(f"Error executing `du` command: {stderr}")
raise RuntimeError(f'Error executing `du` command: {stderr}')

except NotImplementedError as exc:
raise NotImplementedError(
"`exec_command_wait` not implemented for the current transport plugin."
) from exc
raise NotImplementedError('`exec_command_wait` not implemented for the current transport plugin.') from exc

def _get_size_on_disk_lstat(self, full_path: Path, transport: Transport) -> int:
"""
Expand All @@ -302,21 +296,21 @@ def _get_size_on_disk_lstat(self, full_path: Path, transport: Transport) -> int:
:raises OSError: When directory given by ``full_path`` not existing or not a directory.
:return: Total size of directory in bytes (including all its contents).
"""
def _get_size_on_disk_lstat_recursive(full_path, transport):

def _get_size_on_disk_lstat_recursive(full_path, transport):
current_size = 0

contents = self.listdir_withattributes(full_path)

for item in contents:
item_path = full_path / item["name"]
item_path = full_path / item['name']
# Add size of current item (file or directory metadata)
# breakpoint()
# print(f'ITEM: {item["name"]}, {item["attributes"]["st_size"]}({item_path})')
current_size += item["attributes"]["st_size"]
current_size += item['attributes']['st_size']

# If it's a directory, recursively get size of contents
if item["isdir"]:
if item['isdir']:
# print(item["name"])
current_size += _get_size_on_disk_lstat_recursive(item_path, transport)

Expand All @@ -333,8 +327,8 @@ def _get_size_on_disk_lstat_recursive(full_path, transport):
if exception.errno in (2, 20):
# directory not existing or not a directory
exc = OSError(

Check warning on line 329 in src/aiida/orm/nodes/data/remote/base.py

View check run for this annotation

Codecov / codecov/patch

src/aiida/orm/nodes/data/remote/base.py#L329

Added line #L329 was not covered by tests
f"The required remote folder {full_path} on {self.computer.label} does not exist, is not a "
"directory or has been deleted."
f'The required remote folder {full_path} on {self.computer.label} does not exist, is not a '
'directory or has been deleted.'
)
exc.errno = exception.errno
raise exc from exception

Check warning on line 334 in src/aiida/orm/nodes/data/remote/base.py

View check run for this annotation

Codecov / codecov/patch

src/aiida/orm/nodes/data/remote/base.py#L333-L334

Added lines #L333 - L334 were not covered by tests
Expand Down
19 changes: 9 additions & 10 deletions tests/orm/nodes/data/test_remote.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ def test_get_size_on_disk(request, fixture):
with pytest.raises(FileNotFoundError, match='.*does not exist, is not a directory.*'):
remote_data.get_size_on_disk(relpath=Path('non-existent'))


@pytest.mark.parametrize(
'num_char, relpath, sizes',
(
Expand All @@ -94,30 +95,28 @@ def test_get_size_on_disk(request, fixture):
),
)
def test_get_size_on_disk_nested(aiida_localhost, tmp_path, num_char, relpath, sizes):

sub_dir1 = tmp_path / "subdir1"
sub_dir1 = tmp_path / 'subdir1'
sub_dir1.mkdir()

sub_dir2 = tmp_path / "subdir1" / "subdir2"
sub_dir2 = tmp_path / 'subdir1' / 'subdir2'
sub_dir2.mkdir()

# Create some files with known sizes
file1 = sub_dir1 / "file1.txt"
file1.write_text("a"*num_char)
file1 = sub_dir1 / 'file1.txt'
file1.write_text('a' * num_char)

file2 = sub_dir2 / "file2.bin"
file2.write_bytes(b"a" * num_char)
file2 = sub_dir2 / 'file2.bin'
file2.write_bytes(b'a' * num_char)

file3 = tmp_path / "file3.txt"
file3.write_text("a" * num_char)
file3 = tmp_path / 'file3.txt'
file3.write_text('a' * num_char)

remote_data = RemoteData(computer=aiida_localhost, remote_path=tmp_path)

authinfo = remote_data.get_authinfo()
full_path = Path(remote_data.get_remote_path()) / relpath

with authinfo.get_transport() as transport:

size_on_disk_du = remote_data._get_size_on_disk_du(transport=transport, full_path=full_path)
size_on_disk_lstat = remote_data._get_size_on_disk_lstat(transport=transport, full_path=full_path)

Expand Down

0 comments on commit 8bcf0f8

Please sign in to comment.