diff --git a/src/aiida/cmdline/commands/cmd_data/cmd_remote.py b/src/aiida/cmdline/commands/cmd_data/cmd_remote.py index 653b0f09a..d6e138d74 100644 --- a/src/aiida/cmdline/commands/cmd_data/cmd_remote.py +++ b/src/aiida/cmdline/commands/cmd_data/cmd_remote.py @@ -90,21 +90,21 @@ def remote_show(datum): echo.echo(f'- Remote folder full path: {datum.get_remote_path()}') -@remote.command("size") +@remote.command('size') @arguments.NODE() @click.option( - "-m", - "--method", + '-m', + '--method', type=click.STRING, - default="du", - help="The method that should be used to evaluate the size (either ``du`` or ``lstat``.)", + default='du', + help='The method that should be used to evaluate the size (either ``du`` or ``lstat``.)', ) @click.option( - "-p", - "--path", + '-p', + '--path', type=click.Path(), default=None, - help="Relative path of the object of the ``RemoteData`` node for which the size should be evaluated.", + help='Relative path of the object of the ``RemoteData`` node for which the size should be evaluated.', ) def remote_size(node, method, path): """Print the total size of a RemoteData object.""" @@ -113,8 +113,8 @@ def remote_size(node, method, path): remote_path = Path(node.get_remote_path()) full_path = remote_path / path if path is not None else remote_path echo.echo( - f"Estimated total size of directory `{full_path}` on the Computer " - f"`{node.computer.label}` obtained via `{method}`: {total_size}" + f'Estimated total size of directory `{full_path}` on the Computer ' + f'`{node.computer.label}` obtained via `{method}`: {total_size}' ) except FileNotFoundError as exc: echo.echo_critical(str(exc)) diff --git a/src/aiida/orm/nodes/data/remote/base.py b/src/aiida/orm/nodes/data/remote/base.py index c7bc7e450..57d868854 100644 --- a/src/aiida/orm/nodes/data/remote/base.py +++ b/src/aiida/orm/nodes/data/remote/base.py @@ -195,7 +195,7 @@ def _validate(self): def get_authinfo(self): return AuthInfo.get_collection(self.backend).get(dbcomputer=self.computer, aiidauser=self.user) - def get_size_on_disk(self, relpath: Path | None = None, method: str = "du") -> str: + def get_size_on_disk(self, relpath: Path | None = None, method: str = 'du') -> str: """ Connects to the remote folder and returns the total size of all files in the directory recursively in a human-readable format. @@ -211,55 +211,53 @@ def get_size_on_disk(self, relpath: Path | None = None, method: str = "du") -> s from aiida.common.utils import format_directory_size if relpath is None: - relpath = Path(".") + relpath = Path('.') authinfo = self.get_authinfo() full_path = Path(self.get_remote_path()) / relpath - computer_label = self.computer.label if self.computer is not None else "" + computer_label = self.computer.label if self.computer is not None else '' with authinfo.get_transport() as transport: if not transport.path_exists(str(full_path)): exc_message = ( - f"The required remote folder {full_path} on Computer <{computer_label}> " - "does not exist, is not a directory or has been deleted." + f'The required remote folder {full_path} on Computer <{computer_label}> ' + 'does not exist, is not a directory or has been deleted.' ) raise FileNotFoundError(exc_message) - if method not in ("du", "lstat"): + if method not in ('du', 'lstat'): raise NotImplementedError( - f"Specified method `{method}` for evaluating the size on disk not implemented." + f'Specified method `{method}` for evaluating the size on disk not implemented.' ) - if method == "du": + if method == 'du': try: total_size: int = self._get_size_on_disk_du(full_path, transport) except (RuntimeError, NotImplementedError): lstat_warn = ( - "Problem executing `du` command. Will return total file size based on `lstat`. " - "Take the result with a grain of salt, as `lstat` does not consider the file system block size," - " but instead returns the true size of the files in bytes, which differs from the actual space" - "requirements on disk." + 'Problem executing `du` command. Will return total file size based on `lstat`. ' + 'Take the result with a grain of salt, as `lstat` does not consider the file system block size,' + ' but instead returns the true size of the files in bytes, which differs from the actual space' + 'requirements on disk.' ) _logger.warning(lstat_warn) - method = "lstat" + method = 'lstat' else: - _logger.report("Obtained size on the remote using `du`.") + _logger.report('Obtained size on the remote using `du`.') # No elif here, but another if, to allow that the method is internally changed to `lstat`, if `du` fails - if method == "lstat": + if method == 'lstat': try: total_size: int = self._get_size_on_disk_lstat(full_path, transport) - print(f"TOTAL_SIZE: {total_size}") + print(f'TOTAL_SIZE: {total_size}') except OSError: - _logger.critical( - "Could not evaluate directory size using either `du` or `lstat`." - ) + _logger.critical('Could not evaluate directory size using either `du` or `lstat`.') else: - _logger.report("Obtained size on the remote using `lstat`.") + _logger.report('Obtained size on the remote using `lstat`.') return format_directory_size(size_in_bytes=total_size), method @@ -275,19 +273,15 @@ def _get_size_on_disk_du(self, full_path: Path, transport: Transport) -> int: """ try: - retval, stdout, stderr = transport.exec_command_wait( - f"du -s --bytes {full_path}" - ) + retval, stdout, stderr = transport.exec_command_wait(f'du -s --bytes {full_path}') if not stderr and retval == 0: - total_size: int = int(stdout.split("\t")[0]) + total_size: int = int(stdout.split('\t')[0]) return total_size else: - raise RuntimeError(f"Error executing `du` command: {stderr}") + raise RuntimeError(f'Error executing `du` command: {stderr}') except NotImplementedError as exc: - raise NotImplementedError( - "`exec_command_wait` not implemented for the current transport plugin." - ) from exc + raise NotImplementedError('`exec_command_wait` not implemented for the current transport plugin.') from exc def _get_size_on_disk_lstat(self, full_path: Path, transport: Transport) -> int: """ @@ -302,21 +296,21 @@ def _get_size_on_disk_lstat(self, full_path: Path, transport: Transport) -> int: :raises OSError: When directory given by ``full_path`` not existing or not a directory. :return: Total size of directory in bytes (including all its contents). """ - def _get_size_on_disk_lstat_recursive(full_path, transport): + def _get_size_on_disk_lstat_recursive(full_path, transport): current_size = 0 contents = self.listdir_withattributes(full_path) for item in contents: - item_path = full_path / item["name"] + item_path = full_path / item['name'] # Add size of current item (file or directory metadata) # breakpoint() # print(f'ITEM: {item["name"]}, {item["attributes"]["st_size"]}({item_path})') - current_size += item["attributes"]["st_size"] + current_size += item['attributes']['st_size'] # If it's a directory, recursively get size of contents - if item["isdir"]: + if item['isdir']: # print(item["name"]) current_size += _get_size_on_disk_lstat_recursive(item_path, transport) @@ -333,8 +327,8 @@ def _get_size_on_disk_lstat_recursive(full_path, transport): if exception.errno in (2, 20): # directory not existing or not a directory exc = OSError( - f"The required remote folder {full_path} on {self.computer.label} does not exist, is not a " - "directory or has been deleted." + f'The required remote folder {full_path} on {self.computer.label} does not exist, is not a ' + 'directory or has been deleted.' ) exc.errno = exception.errno raise exc from exception diff --git a/tests/orm/nodes/data/test_remote.py b/tests/orm/nodes/data/test_remote.py index c853c974a..848165f5a 100644 --- a/tests/orm/nodes/data/test_remote.py +++ b/tests/orm/nodes/data/test_remote.py @@ -82,6 +82,7 @@ def test_get_size_on_disk(request, fixture): with pytest.raises(FileNotFoundError, match='.*does not exist, is not a directory.*'): remote_data.get_size_on_disk(relpath=Path('non-existent')) + @pytest.mark.parametrize( 'num_char, relpath, sizes', ( @@ -94,22 +95,21 @@ def test_get_size_on_disk(request, fixture): ), ) def test_get_size_on_disk_nested(aiida_localhost, tmp_path, num_char, relpath, sizes): - - sub_dir1 = tmp_path / "subdir1" + sub_dir1 = tmp_path / 'subdir1' sub_dir1.mkdir() - sub_dir2 = tmp_path / "subdir1" / "subdir2" + sub_dir2 = tmp_path / 'subdir1' / 'subdir2' sub_dir2.mkdir() # Create some files with known sizes - file1 = sub_dir1 / "file1.txt" - file1.write_text("a"*num_char) + file1 = sub_dir1 / 'file1.txt' + file1.write_text('a' * num_char) - file2 = sub_dir2 / "file2.bin" - file2.write_bytes(b"a" * num_char) + file2 = sub_dir2 / 'file2.bin' + file2.write_bytes(b'a' * num_char) - file3 = tmp_path / "file3.txt" - file3.write_text("a" * num_char) + file3 = tmp_path / 'file3.txt' + file3.write_text('a' * num_char) remote_data = RemoteData(computer=aiida_localhost, remote_path=tmp_path) @@ -117,7 +117,6 @@ def test_get_size_on_disk_nested(aiida_localhost, tmp_path, num_char, relpath, s full_path = Path(remote_data.get_remote_path()) / relpath with authinfo.get_transport() as transport: - size_on_disk_du = remote_data._get_size_on_disk_du(transport=transport, full_path=full_path) size_on_disk_lstat = remote_data._get_size_on_disk_lstat(transport=transport, full_path=full_path)